apify-test-tools 0.9.1-beta.6 → 0.9.1-beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/actionlint.yaml +12 -0
- package/.github/actions/checkout-restore-dependencies/action.yaml +120 -0
- package/.github/review-prompt.md +154 -0
- package/.github/scripts/check-major-tag-refs.mjs +72 -0
- package/.github/scripts/check-package-version-bump.mjs +74 -0
- package/.github/scripts/run-with-apify-tokens.mjs +104 -0
- package/.github/workflows/_check_code.yaml +16 -0
- package/.github/workflows/_move_major_tag.yaml +100 -0
- package/.github/workflows/_update_release_metadata.yaml +12 -0
- package/.github/workflows/manual_move_major_tag.yaml +26 -0
- package/.github/workflows/manual_release_stable.yaml +12 -0
- package/.github/workflows/on_master.yaml +11 -0
- package/.github/workflows/on_pull_request.yaml +48 -0
- package/.github/workflows/public_claude.yaml +101 -0
- package/.github/workflows/public_platform-tests-claude-investigate-and-fix.yaml +208 -0
- package/.github/workflows/public_platform-tests.yaml +75 -0
- package/.github/workflows/public_pr-build-test.yaml +197 -0
- package/.github/workflows/public_push-build-latest.yaml +57 -0
- package/.github/workflows/public_review.yaml +114 -0
- package/.github/workflows-package-version +1 -0
- package/.prettierignore +6 -0
- package/CHANGELOG.md +1 -0
- package/CONTRIBUTING.md +47 -2
- package/README.md +147 -5
- package/package.json +1 -1
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
name: Move major version tag
|
|
2
|
+
|
|
3
|
+
# Consumer repos track a floating major tag (`...@v0`), not `master`, so merging to master does not
|
|
4
|
+
# ship anything by itself. This workflow is what ships: it moves that tag forward, but only once the
|
|
5
|
+
# package version the workflows pin in .github/workflows-package-version is actually on npm.
|
|
6
|
+
#
|
|
7
|
+
# That gate is the whole point. The workflows and the npm package release on their own schedules, so
|
|
8
|
+
# the only combination that can break consumers is a workflow that calls a CLI feature which has not
|
|
9
|
+
# been published yet. In that case the tag simply stays where it is and production keeps running the
|
|
10
|
+
# previous workflows until someone cuts a release.
|
|
11
|
+
|
|
12
|
+
on:
|
|
13
|
+
workflow_call:
|
|
14
|
+
inputs:
|
|
15
|
+
ref:
|
|
16
|
+
description: Commit to move the tag to. Defaults to the commit that triggered the caller.
|
|
17
|
+
type: string
|
|
18
|
+
default: ''
|
|
19
|
+
|
|
20
|
+
permissions:
|
|
21
|
+
contents: read
|
|
22
|
+
|
|
23
|
+
concurrency:
|
|
24
|
+
group: move-major-tag
|
|
25
|
+
cancel-in-progress: false
|
|
26
|
+
|
|
27
|
+
jobs:
|
|
28
|
+
move_major_tag:
|
|
29
|
+
name: Move major version tag
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
permissions:
|
|
32
|
+
contents: write
|
|
33
|
+
env:
|
|
34
|
+
# The major version consumers pin, e.g.
|
|
35
|
+
# `uses: apify/apify-test-tools/.github/workflows/public_pr-build-test.yaml@v0`.
|
|
36
|
+
# Bump this only for a breaking change to a workflow's inputs, secrets or behaviour.
|
|
37
|
+
# The previous tag then stops moving and keeps working, so repos migrate when they get
|
|
38
|
+
# to it instead of all at once.
|
|
39
|
+
#
|
|
40
|
+
# This tracks the workflows' own contract, not the npm package version — the two move
|
|
41
|
+
# independently by design, so `v0` here is expected to outlive the package reaching 1.0.
|
|
42
|
+
# `uses:` cannot take an expression, so every ref into this repo hardcodes this same tag;
|
|
43
|
+
# check-major-tag-refs.mjs fails the build if one of them drifts.
|
|
44
|
+
MAJOR_TAG: v0
|
|
45
|
+
steps:
|
|
46
|
+
- name: Checkout repository
|
|
47
|
+
uses: actions/checkout@v6
|
|
48
|
+
with:
|
|
49
|
+
ref: ${{ inputs.ref || github.sha }}
|
|
50
|
+
|
|
51
|
+
- name: Check the required package version is published
|
|
52
|
+
id: check
|
|
53
|
+
run: |
|
|
54
|
+
pinned_version=$(cat .github/workflows-package-version)
|
|
55
|
+
echo "pinned_version=$pinned_version" >> "$GITHUB_OUTPUT"
|
|
56
|
+
|
|
57
|
+
# The publish that satisfies a pin bump can still be in flight when this runs, so
|
|
58
|
+
# give it a short grace period before deciding the tag has to stay put.
|
|
59
|
+
for attempt in 1 2 3 4 5 6; do
|
|
60
|
+
if npm view "apify-test-tools@$pinned_version" version >/dev/null 2>&1; then
|
|
61
|
+
echo "apify-test-tools@$pinned_version is on npm"
|
|
62
|
+
echo "published=true" >> "$GITHUB_OUTPUT"
|
|
63
|
+
exit 0
|
|
64
|
+
fi
|
|
65
|
+
echo "apify-test-tools@$pinned_version is not on npm yet (attempt $attempt/6)"
|
|
66
|
+
if [ "$attempt" -lt 6 ]; then sleep 20; fi
|
|
67
|
+
done
|
|
68
|
+
echo "published=false" >> "$GITHUB_OUTPUT"
|
|
69
|
+
|
|
70
|
+
- name: Move tag
|
|
71
|
+
if: steps.check.outputs.published == 'true'
|
|
72
|
+
env:
|
|
73
|
+
TARGET_SHA: ${{ inputs.ref || github.sha }}
|
|
74
|
+
run: |
|
|
75
|
+
git tag -f "$MAJOR_TAG" "$TARGET_SHA"
|
|
76
|
+
git push -f origin "refs/tags/$MAJOR_TAG"
|
|
77
|
+
{
|
|
78
|
+
echo "### \`$MAJOR_TAG\` moved"
|
|
79
|
+
echo
|
|
80
|
+
echo "Now points at \`$TARGET_SHA\`."
|
|
81
|
+
echo "Pins \`apify-test-tools@${{ steps.check.outputs.pinned_version }}\`, which is published."
|
|
82
|
+
} >> "$GITHUB_STEP_SUMMARY"
|
|
83
|
+
|
|
84
|
+
# Not a failure. Holding the tag is the designed outcome of merging a workflow change
|
|
85
|
+
# that needs an unreleased package feature, and consumers are unaffected while it waits.
|
|
86
|
+
- name: Report held tag
|
|
87
|
+
if: steps.check.outputs.published != 'true'
|
|
88
|
+
env:
|
|
89
|
+
TARGET_SHA: ${{ inputs.ref || github.sha }}
|
|
90
|
+
run: |
|
|
91
|
+
{
|
|
92
|
+
echo "### \`$MAJOR_TAG\` held"
|
|
93
|
+
echo
|
|
94
|
+
echo "\`$TARGET_SHA\` pins \`apify-test-tools@${{ steps.check.outputs.pinned_version }}\`, which is not on npm."
|
|
95
|
+
echo "Consumers keep running the workflows \`$MAJOR_TAG\` already points at."
|
|
96
|
+
echo
|
|
97
|
+
echo "Publish that version with the **Stable release** workflow. The tag moves on the"
|
|
98
|
+
echo "next master push, or immediately via the **Move major version tag** workflow."
|
|
99
|
+
} >> "$GITHUB_STEP_SUMMARY"
|
|
100
|
+
echo "::warning::$MAJOR_TAG not moved: apify-test-tools@${{ steps.check.outputs.pinned_version }} is not published yet."
|
|
@@ -77,6 +77,18 @@ jobs:
|
|
|
77
77
|
write-mode: overwrite
|
|
78
78
|
contents: ${{ needs.release_metadata.outputs.changelog }}
|
|
79
79
|
|
|
80
|
+
# The reusable workflows install this exact version, so the commit that releases a
|
|
81
|
+
# version is the commit that pins it. That is what lets a frozen major tag keep
|
|
82
|
+
# installing the library it was tested against instead of drifting to whatever is
|
|
83
|
+
# newest.
|
|
84
|
+
#
|
|
85
|
+
# Skipped for prereleases: master pushes release a `-beta` that consumer repos must
|
|
86
|
+
# never install. Their pin stays on the last stable, and the beta stays reachable
|
|
87
|
+
# through the lockfile path in the setup action, which is how branch testing works.
|
|
88
|
+
- name: Update the version the workflows pin
|
|
89
|
+
if: inputs.release_type != 'prerelease'
|
|
90
|
+
run: echo "${{ needs.release_metadata.outputs.version_number }}" > .github/workflows-package-version
|
|
91
|
+
|
|
80
92
|
- name: Commit changes
|
|
81
93
|
id: commit
|
|
82
94
|
uses: apify/actions/signed-commit@v1.0.0
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: Move major version tag
|
|
2
|
+
|
|
3
|
+
# Escape hatch for the two cases the automatic move does not cover: shipping a workflow change right
|
|
4
|
+
# after the release that unblocked it, without waiting for the next master push, and rolling the tag
|
|
5
|
+
# back to an earlier commit when a shipped workflow turns out to be broken.
|
|
6
|
+
|
|
7
|
+
on:
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
inputs:
|
|
10
|
+
ref:
|
|
11
|
+
description: Commit, branch or tag to move the major tag to (defaults to master)
|
|
12
|
+
required: false
|
|
13
|
+
type: string
|
|
14
|
+
default: ''
|
|
15
|
+
|
|
16
|
+
permissions:
|
|
17
|
+
contents: read
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
move_major_tag:
|
|
21
|
+
name: Move major version tag
|
|
22
|
+
permissions:
|
|
23
|
+
contents: write
|
|
24
|
+
uses: ./.github/workflows/_move_major_tag.yaml
|
|
25
|
+
with:
|
|
26
|
+
ref: ${{ inputs.ref }}
|
|
@@ -79,3 +79,15 @@ jobs:
|
|
|
79
79
|
"ref": "${{ needs.release_metadata.outputs.changelog_commitish }}",
|
|
80
80
|
"tag": "latest"
|
|
81
81
|
}
|
|
82
|
+
|
|
83
|
+
# This release is what unblocks a major tag being held for an unpublished package version, so
|
|
84
|
+
# try the move again now rather than making someone wait for the next master push. A no-op when
|
|
85
|
+
# the tag was not being held.
|
|
86
|
+
move_major_tag:
|
|
87
|
+
name: Move major version tag
|
|
88
|
+
needs: [release_metadata, npm_publish]
|
|
89
|
+
permissions:
|
|
90
|
+
contents: write
|
|
91
|
+
uses: ./.github/workflows/_move_major_tag.yaml
|
|
92
|
+
with:
|
|
93
|
+
ref: ${{ needs.release_metadata.outputs.changelog_commitish }}
|
|
@@ -25,6 +25,17 @@ jobs:
|
|
|
25
25
|
name: Tests
|
|
26
26
|
uses: ./.github/workflows/_tests.yaml
|
|
27
27
|
|
|
28
|
+
# Ships any workflow change in this push to consumer repos. Deliberately independent of the npm
|
|
29
|
+
# release below: a workflow-only change needs no release, and a package-only release leaves the
|
|
30
|
+
# workflows byte-identical. `code_checks` runs actionlint, which is what validates the workflows;
|
|
31
|
+
# `tests` covers the npm package and is skipped for docs commits, so it is not a dependency here.
|
|
32
|
+
move_major_tag:
|
|
33
|
+
name: Move major version tag
|
|
34
|
+
needs: [code_checks]
|
|
35
|
+
permissions:
|
|
36
|
+
contents: write
|
|
37
|
+
uses: ./.github/workflows/_move_major_tag.yaml
|
|
38
|
+
|
|
28
39
|
release_metadata:
|
|
29
40
|
if: >-
|
|
30
41
|
startsWith(github.event.head_commit.message, 'feat') ||
|
|
@@ -21,3 +21,51 @@ jobs:
|
|
|
21
21
|
tests:
|
|
22
22
|
name: Tests
|
|
23
23
|
uses: ./.github/workflows/_tests.yaml
|
|
24
|
+
|
|
25
|
+
# Catches the one combination that breaks consumers: a workflow starting to use a CLI feature
|
|
26
|
+
# from the same PR. Merging ships the workflow immediately, so the pinned version has to move in
|
|
27
|
+
# the same change or the workflow goes live calling something that is not published.
|
|
28
|
+
package_version_bump_needed:
|
|
29
|
+
name: Package version bump needed
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
steps:
|
|
32
|
+
- name: Checkout repository
|
|
33
|
+
uses: actions/checkout@v6
|
|
34
|
+
with:
|
|
35
|
+
fetch-depth: 0
|
|
36
|
+
ref: ${{ github.event.pull_request.head.sha }}
|
|
37
|
+
|
|
38
|
+
- name: Check whether this PR needs a version bump
|
|
39
|
+
env:
|
|
40
|
+
BASE_REF: ${{ github.base_ref }}
|
|
41
|
+
PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }}
|
|
42
|
+
run: |
|
|
43
|
+
git fetch --no-tags origin "$BASE_REF"
|
|
44
|
+
git diff --name-only FETCH_HEAD...HEAD | node .github/scripts/check-package-version-bump.mjs
|
|
45
|
+
|
|
46
|
+
# Reports whether merging this PR will ship the workflows or hold the major tag until a release.
|
|
47
|
+
# Never fails: pinning a version ahead of the release that publishes it is a supported way to
|
|
48
|
+
# land the package change and the workflow change that needs it in one PR.
|
|
49
|
+
workflows_package_version:
|
|
50
|
+
name: Workflows package version
|
|
51
|
+
runs-on: ubuntu-latest
|
|
52
|
+
steps:
|
|
53
|
+
- name: Checkout repository
|
|
54
|
+
uses: actions/checkout@v6
|
|
55
|
+
|
|
56
|
+
- name: Check the pinned package version is published
|
|
57
|
+
run: |
|
|
58
|
+
pinned_version=$(cat .github/workflows-package-version)
|
|
59
|
+
if npm view "apify-test-tools@$pinned_version" version >/dev/null 2>&1; then
|
|
60
|
+
echo "Workflows pin apify-test-tools@$pinned_version, which is published."
|
|
61
|
+
echo "The major tag moves on merge and these workflows go live." >> "$GITHUB_STEP_SUMMARY"
|
|
62
|
+
else
|
|
63
|
+
{
|
|
64
|
+
echo "### Major tag will be held on merge"
|
|
65
|
+
echo
|
|
66
|
+
echo "These workflows pin \`apify-test-tools@$pinned_version\`, which is not on npm yet."
|
|
67
|
+
echo "Merging is fine: consumers keep running the current workflows until you cut a"
|
|
68
|
+
echo "stable release, and the tag moves once that version is published."
|
|
69
|
+
} >> "$GITHUB_STEP_SUMMARY"
|
|
70
|
+
echo "::notice::Merging will not ship these workflows until apify-test-tools $pinned_version is released."
|
|
71
|
+
fi
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
name: Claude Code
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_call:
|
|
5
|
+
inputs:
|
|
6
|
+
model:
|
|
7
|
+
default: sonnet
|
|
8
|
+
type: string
|
|
9
|
+
description: Model to use for Claude Code (best, sonnet, opus, haiku, sonnet[1m], opus[1m], opusplan)
|
|
10
|
+
required: false
|
|
11
|
+
extra_allowed_tools:
|
|
12
|
+
default: ''
|
|
13
|
+
type: string
|
|
14
|
+
description: Comma-separated list of additional tool patterns to append to --allowedTools (e.g. "Bash(npm run e2e),Bash(npm run typecheck)")
|
|
15
|
+
required: false
|
|
16
|
+
secrets:
|
|
17
|
+
ANTHROPIC_API_KEY:
|
|
18
|
+
required: true
|
|
19
|
+
description: Claude API key
|
|
20
|
+
|
|
21
|
+
jobs:
|
|
22
|
+
claude:
|
|
23
|
+
runs-on: ubuntu-latest
|
|
24
|
+
permissions:
|
|
25
|
+
contents: write
|
|
26
|
+
pull-requests: write
|
|
27
|
+
issues: write
|
|
28
|
+
id-token: write
|
|
29
|
+
actions: read
|
|
30
|
+
if: |
|
|
31
|
+
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
|
32
|
+
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
|
33
|
+
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
|
34
|
+
(github.event_name == 'pull_request_review' && github.event.review.state == 'changes_requested' && github.event.pull_request.user.login == 'claude[bot]') ||
|
|
35
|
+
(github.event_name == 'issues' && github.event.action == 'labeled' && github.event.label.name == 'claude' && !contains(github.event.issue.labels.*.name, 'claude:done')) ||
|
|
36
|
+
(github.event_name == 'issues' && github.event.action != 'labeled' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
|
37
|
+
steps:
|
|
38
|
+
- name: Checkout repository
|
|
39
|
+
uses: actions/checkout@v6
|
|
40
|
+
with:
|
|
41
|
+
fetch-depth: 1
|
|
42
|
+
|
|
43
|
+
- name: Compute allowed tools
|
|
44
|
+
id: tools
|
|
45
|
+
env:
|
|
46
|
+
EXTRA_ALLOWED_TOOLS: ${{ inputs.extra_allowed_tools }}
|
|
47
|
+
run: |
|
|
48
|
+
BASE_TOOLS="Bash(gh pr*),Bash(gh api*),Bash(gh issue*),Bash(npm ci)"
|
|
49
|
+
if [ -n "$EXTRA_ALLOWED_TOOLS" ]; then
|
|
50
|
+
ALLOWED_TOOLS="${BASE_TOOLS},${EXTRA_ALLOWED_TOOLS}"
|
|
51
|
+
else
|
|
52
|
+
ALLOWED_TOOLS="${BASE_TOOLS}"
|
|
53
|
+
fi
|
|
54
|
+
echo "allowed_tools=${ALLOWED_TOOLS}" >> "$GITHUB_OUTPUT"
|
|
55
|
+
|
|
56
|
+
- name: Compute PR instruction
|
|
57
|
+
id: pr_instruction
|
|
58
|
+
env:
|
|
59
|
+
EVENT_NAME: ${{ github.event_name }}
|
|
60
|
+
IS_PR_COMMENT: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request != null }}
|
|
61
|
+
run: |
|
|
62
|
+
if [ "$EVENT_NAME" = "pull_request_review_comment" ] || [ "$EVENT_NAME" = "pull_request_review" ] || [ "$IS_PR_COMMENT" = "true" ]; then
|
|
63
|
+
INSTRUCTION="After completing the requested changes, push them to the existing pull request's branch. Do not open a new pull request."
|
|
64
|
+
else
|
|
65
|
+
INSTRUCTION="After completing the requested changes, always open a pull request with your work. Do not just push commits or leave a comment without opening a PR."
|
|
66
|
+
fi
|
|
67
|
+
echo "instruction=${INSTRUCTION}" >> "$GITHUB_OUTPUT"
|
|
68
|
+
|
|
69
|
+
- name: Run Claude Code
|
|
70
|
+
id: claude
|
|
71
|
+
uses: anthropics/claude-code-action@v1
|
|
72
|
+
with:
|
|
73
|
+
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
74
|
+
track_progress: true
|
|
75
|
+
use_commit_signing: true
|
|
76
|
+
claude_args: |
|
|
77
|
+
--model "${{ inputs.model }}"
|
|
78
|
+
--allowedTools "${{ steps.tools.outputs.allowed_tools }}"
|
|
79
|
+
--append-system-prompt "${{ steps.pr_instruction.outputs.instruction }}"
|
|
80
|
+
|
|
81
|
+
- name: Mark issue as processed by Claude
|
|
82
|
+
if: |
|
|
83
|
+
github.event_name == 'issues' &&
|
|
84
|
+
github.event.action == 'labeled' &&
|
|
85
|
+
github.event.label.name == 'claude' &&
|
|
86
|
+
steps.claude.conclusion == 'success'
|
|
87
|
+
uses: actions/github-script@v7
|
|
88
|
+
with:
|
|
89
|
+
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
90
|
+
script: |
|
|
91
|
+
try {
|
|
92
|
+
await github.rest.issues.addLabels({
|
|
93
|
+
owner: context.repo.owner,
|
|
94
|
+
repo: context.repo.repo,
|
|
95
|
+
issue_number: context.issue.number,
|
|
96
|
+
labels: ['claude:done']
|
|
97
|
+
});
|
|
98
|
+
console.log('Label claude:done added to issue.');
|
|
99
|
+
} catch (e) {
|
|
100
|
+
console.log('Label claude:done does not exist. Skipping.');
|
|
101
|
+
}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
name: 'Platform tests: Claude failure investigation and fix'
|
|
2
|
+
|
|
3
|
+
# Notion page docs: https://app.notion.com/p/apify/374f39950a2280d99c89f27fc9fb49c5
|
|
4
|
+
|
|
5
|
+
# Reusable workflow that uses Claude Code to investigate a failing E2E platform test and
|
|
6
|
+
# attempt an automated fix.
|
|
7
|
+
#
|
|
8
|
+
# Two-phase execution:
|
|
9
|
+
# 1. investigate: Claude reviews test logs, recent workflow history, and existing issues,
|
|
10
|
+
# then opens a GitHub issue with its findings. Skips silently if a duplicate exists.
|
|
11
|
+
# On success, adds a "claude" label to the issue for visibility.
|
|
12
|
+
# 2. fix: Runs only when investigate created a new issue. Claude reads the issue,
|
|
13
|
+
# implements a fix, and opens a PR. On success, adds a "claude:done" label to the issue.
|
|
14
|
+
#
|
|
15
|
+
# Note: this workflow is intended to be called from a scheduled workflow. Scheduled runs use
|
|
16
|
+
# GITHUB_TOKEN with reduced permissions that cannot trigger other workflow runs. For this reason
|
|
17
|
+
# the PR phase is chained as a direct job dependency rather than relying on a label-based trigger.
|
|
18
|
+
#
|
|
19
|
+
# Example usage:
|
|
20
|
+
#
|
|
21
|
+
# jobs:
|
|
22
|
+
# platformTestsCore:
|
|
23
|
+
# uses: apify/apify-test-tools/.github/workflows/public_platform-tests.yaml@v0
|
|
24
|
+
# with:
|
|
25
|
+
# subtest: core
|
|
26
|
+
# secrets: inherit
|
|
27
|
+
#
|
|
28
|
+
# handle-failure:
|
|
29
|
+
# needs: platformTestsCore
|
|
30
|
+
# if: ${{ failure() }}
|
|
31
|
+
# uses: apify/apify-test-tools/.github/workflows/public_platform-tests-claude-investigate-and-fix.yaml@v0
|
|
32
|
+
# secrets: inherit
|
|
33
|
+
|
|
34
|
+
on:
|
|
35
|
+
workflow_call:
|
|
36
|
+
inputs:
|
|
37
|
+
model:
|
|
38
|
+
required: false
|
|
39
|
+
type: string
|
|
40
|
+
default: sonnet
|
|
41
|
+
description: Claude model to use for investigation and fix
|
|
42
|
+
investigate_prompt:
|
|
43
|
+
required: false
|
|
44
|
+
type: string
|
|
45
|
+
default: |
|
|
46
|
+
Investigate the failing E2E platform test and open a GitHub issue documenting your findings. Start by reviewing the test logs, error messages, and recent workflow run history to determine the root cause.
|
|
47
|
+
|
|
48
|
+
The failure will typically fall into one of these categories:
|
|
49
|
+
- Flaky test: The test is non-deterministic or depends on timing/external state and fails intermittently. Check past workflow runs to see if this test has failed before.
|
|
50
|
+
- Real bug: Genuine broken functionality — either a pre-existing issue or an environment/dependency change outside the codebase (e.g. a website changed its structure, an external API changed behavior).
|
|
51
|
+
|
|
52
|
+
Use available tools to check previous workflow runs, git history, and existing GitHub issues to gather context and avoid creating duplicate issues. For deeper context,
|
|
53
|
+
use tools to investigate the run on Apify platform, review its logs, datasets, key-value stores.
|
|
54
|
+
|
|
55
|
+
Look specifically for debug traces, not just the top-level error:
|
|
56
|
+
- Debug snapshots: often stored as a field on dataset items (e.g. `_debugSnapshots`) or as records in a key-value store, keyed so they can be matched to the log line for the failing request.
|
|
57
|
+
- Any other debug artifacts the Actor produced (extra logs, error/state fields on output items) that narrow down the failing request.
|
|
58
|
+
|
|
59
|
+
When you create the issue, write it like an incident postmortem: a short summary and impact up top for anyone skimming, full technical detail below for whoever fixes it. Don't water down the engineering detail to make the top readable.
|
|
60
|
+
- Title: short and clean, describing the actual problem in plain terms — no dates, run numbers, or other noise. Prefix it with a conventional-commit-style type, e.g. "fix: blocking issue on detail endpoint".
|
|
61
|
+
- Body, section 1 ("## Summary"): one line stating what's broken plus its impact, in plain language a PM can understand at a glance — no jargon, internal field names, or stack traces (e.g. "Scraping product detail pages fails when a listing has no reviews, affecting N% of runs").
|
|
62
|
+
- A `---` divider, then section 2 ("## Details"): the engineering bug report — root cause, relevant log lines, links to the failing run, and any other internals an engineer needs. If a single item/request failed among otherwise-good ones, show or link that specific one (e.g. the dataset item index/id, or its matching debug snapshot) rather than just describing the run as failed.
|
|
63
|
+
|
|
64
|
+
If you create a new issue, you MUST write its issue number (just the number, nothing else) to $GITHUB_WORKSPACE/issue-number.txt as the very last thing you do. If you skip creating one because a duplicate already exists, do not create the file.
|
|
65
|
+
description: Prompt given to Claude for the investigation phase. Override to customize how issues are investigated, titled, and described.
|
|
66
|
+
secrets:
|
|
67
|
+
ANTHROPIC_API_KEY:
|
|
68
|
+
required: true
|
|
69
|
+
description: Claude API key
|
|
70
|
+
TESTER_APIFY_TOKEN_READ_ONLY:
|
|
71
|
+
required: true
|
|
72
|
+
description: Apify token used only for read-only investigation — reading the failing run, its log and its storages.
|
|
73
|
+
|
|
74
|
+
jobs:
|
|
75
|
+
investigate:
|
|
76
|
+
runs-on: ubuntu-latest
|
|
77
|
+
permissions:
|
|
78
|
+
contents: write
|
|
79
|
+
pull-requests: read
|
|
80
|
+
issues: write
|
|
81
|
+
actions: read
|
|
82
|
+
outputs:
|
|
83
|
+
issue_number: ${{ steps.issue.outputs.number }}
|
|
84
|
+
steps:
|
|
85
|
+
- name: Checkout repository
|
|
86
|
+
uses: actions/checkout@v5
|
|
87
|
+
|
|
88
|
+
- name: Run Claude Code
|
|
89
|
+
id: claude
|
|
90
|
+
uses: anthropics/claude-code-action@8251c103ac8c1d761882c86aba1412c7f583c844 # v1.0.213
|
|
91
|
+
env:
|
|
92
|
+
TESTER_APIFY_TOKEN_READ_ONLY: ${{ secrets.TESTER_APIFY_TOKEN_READ_ONLY }}
|
|
93
|
+
with:
|
|
94
|
+
prompt: |
|
|
95
|
+
The failing E2E platform test ran in workflow run ${{ github.run_id }} of this repo.
|
|
96
|
+
|
|
97
|
+
${{ inputs.investigate_prompt }}
|
|
98
|
+
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
99
|
+
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
100
|
+
# The `tools` query parameter below selects which tools the Apify MCP server
|
|
101
|
+
# exposes: reading an Actor run — its status, exit code, storage ids and log — and
|
|
102
|
+
# reading that run's dataset items and its key-value store keys and records, e.g.
|
|
103
|
+
# INPUT, OUTPUT and error snapshots. WARN: the server adds `abort-actor-run` on top
|
|
104
|
+
# even when not listed here, see https://github.com/apify/apify-mcp-server
|
|
105
|
+
claude_args: |
|
|
106
|
+
--mcp-config '{"mcpServers":{"apify":{"type":"http","url":"https://mcp.apify.com/?tools=get-actor-run,get-actor-log,get-dataset-list,get-dataset,get-dataset-items,get-key-value-store-list,get-key-value-store,get-key-value-store-keys,get-key-value-store-record","headers":{"Authorization":"Bearer ${TESTER_APIFY_TOKEN_READ_ONLY}"}}}}'
|
|
107
|
+
--model ${{ inputs.model }}
|
|
108
|
+
--dangerously-skip-permissions
|
|
109
|
+
|
|
110
|
+
- name: Upload Claude output (for debugging)
|
|
111
|
+
if: always()
|
|
112
|
+
uses: actions/upload-artifact@v6
|
|
113
|
+
with:
|
|
114
|
+
name: claude-output-investigate
|
|
115
|
+
path: /home/runner/work/_temp/claude-execution-output.json
|
|
116
|
+
|
|
117
|
+
- name: Read created issue
|
|
118
|
+
id: issue
|
|
119
|
+
# Claude writes the issue number to a file because there is no direct way to pass
|
|
120
|
+
# structured output from a Claude Code action step to subsequent steps.
|
|
121
|
+
run: |
|
|
122
|
+
if [ -f "$GITHUB_WORKSPACE/issue-number.txt" ]; then
|
|
123
|
+
ISSUE_NUMBER=$(cat "$GITHUB_WORKSPACE/issue-number.txt")
|
|
124
|
+
echo "number=$ISSUE_NUMBER" >> "$GITHUB_OUTPUT"
|
|
125
|
+
echo "Issue created: #$ISSUE_NUMBER"
|
|
126
|
+
else
|
|
127
|
+
echo "number=" >> "$GITHUB_OUTPUT"
|
|
128
|
+
echo "No issue created (duplicate skipped)"
|
|
129
|
+
fi
|
|
130
|
+
|
|
131
|
+
# Adds the "claude" label for visibility. Normally this would trigger the claude.yaml
|
|
132
|
+
# workflow, but scheduled runs lack the permissions to trigger other workflows, so the
|
|
133
|
+
# PR phase is driven by the fix job dependency instead.
|
|
134
|
+
- name: Add Claude label to issue
|
|
135
|
+
if: steps.issue.outputs.number != ''
|
|
136
|
+
uses: actions/github-script@v7
|
|
137
|
+
with:
|
|
138
|
+
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
139
|
+
script: |
|
|
140
|
+
try {
|
|
141
|
+
await github.rest.issues.createLabel({
|
|
142
|
+
owner: context.repo.owner,
|
|
143
|
+
repo: context.repo.repo,
|
|
144
|
+
name: "claude",
|
|
145
|
+
color: "d93f0b",
|
|
146
|
+
});
|
|
147
|
+
} catch (e) {
|
|
148
|
+
// Label already exists
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
await github.rest.issues.addLabels({
|
|
152
|
+
owner: context.repo.owner,
|
|
153
|
+
repo: context.repo.repo,
|
|
154
|
+
issue_number: parseInt('${{ steps.issue.outputs.number }}'),
|
|
155
|
+
labels: ["claude"],
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
fix:
|
|
159
|
+
needs: investigate
|
|
160
|
+
if: needs.investigate.outputs.issue_number != ''
|
|
161
|
+
runs-on: ubuntu-latest
|
|
162
|
+
permissions:
|
|
163
|
+
contents: write
|
|
164
|
+
pull-requests: write
|
|
165
|
+
issues: write
|
|
166
|
+
id-token: write
|
|
167
|
+
actions: read
|
|
168
|
+
steps:
|
|
169
|
+
- name: Checkout repository
|
|
170
|
+
uses: actions/checkout@v5
|
|
171
|
+
|
|
172
|
+
- name: Run Claude Code
|
|
173
|
+
id: claude
|
|
174
|
+
uses: anthropics/claude-code-action@8251c103ac8c1d761882c86aba1412c7f583c844 # v1.0.213
|
|
175
|
+
with:
|
|
176
|
+
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
177
|
+
use_commit_signing: true
|
|
178
|
+
prompt: 'Open a pull request to address GitHub issue #${{ needs.investigate.outputs.issue_number }}. Use `gh issue view ${{ needs.investigate.outputs.issue_number }}` to read the issue details first, then implement the fix and create a PR.'
|
|
179
|
+
claude_args: |
|
|
180
|
+
--model ${{ inputs.model }}
|
|
181
|
+
--dangerously-skip-permissions
|
|
182
|
+
|
|
183
|
+
- name: Upload Claude output (for debugging)
|
|
184
|
+
if: always()
|
|
185
|
+
uses: actions/upload-artifact@v6
|
|
186
|
+
with:
|
|
187
|
+
name: claude-output-fix
|
|
188
|
+
path: /home/runner/work/_temp/claude-execution-output.json
|
|
189
|
+
|
|
190
|
+
# Adds "claude:done" to the issue to signal that a fix PR was opened. Pure status
|
|
191
|
+
# tracking — does not trigger any further automation.
|
|
192
|
+
- name: Mark issue as processed by Claude
|
|
193
|
+
if: steps.claude.conclusion == 'success'
|
|
194
|
+
uses: actions/github-script@v7
|
|
195
|
+
with:
|
|
196
|
+
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
197
|
+
script: |
|
|
198
|
+
try {
|
|
199
|
+
await github.rest.issues.addLabels({
|
|
200
|
+
owner: context.repo.owner,
|
|
201
|
+
repo: context.repo.repo,
|
|
202
|
+
issue_number: parseInt('${{ needs.investigate.outputs.issue_number }}'),
|
|
203
|
+
labels: ['claude:done']
|
|
204
|
+
});
|
|
205
|
+
console.log('Label claude:done added to issue.');
|
|
206
|
+
} catch (e) {
|
|
207
|
+
console.log('Label claude:done does not exist. Skipping.');
|
|
208
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
name: Platform tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_call:
|
|
5
|
+
inputs:
|
|
6
|
+
# Deprecated, use `test-files-glob`
|
|
7
|
+
subtest:
|
|
8
|
+
type: string
|
|
9
|
+
default: ''
|
|
10
|
+
required: false
|
|
11
|
+
# Specify what tests you want to run, relative to `test/platform`
|
|
12
|
+
test-files-glob:
|
|
13
|
+
type: string
|
|
14
|
+
default: ''
|
|
15
|
+
required: false
|
|
16
|
+
working-directory:
|
|
17
|
+
type: string
|
|
18
|
+
default: .
|
|
19
|
+
required: false
|
|
20
|
+
additional-working-directory:
|
|
21
|
+
type: string
|
|
22
|
+
required: false
|
|
23
|
+
# Defaults to #notif-<repo-name> when empty, matching the previous behavior.
|
|
24
|
+
report-slack-channel:
|
|
25
|
+
type: string
|
|
26
|
+
default: ''
|
|
27
|
+
required: false
|
|
28
|
+
|
|
29
|
+
jobs:
|
|
30
|
+
scheduledPlatformTests:
|
|
31
|
+
name: Scheduled Platform tests
|
|
32
|
+
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main'
|
|
33
|
+
runs-on: ubuntu-latest
|
|
34
|
+
defaults:
|
|
35
|
+
run:
|
|
36
|
+
working-directory: ${{ inputs.working-directory }}
|
|
37
|
+
steps:
|
|
38
|
+
- name: Setup repository and dependencies
|
|
39
|
+
# For testing, you have to temporarily change the branch
|
|
40
|
+
uses: apify/apify-test-tools/.github/actions/checkout-restore-dependencies@v0
|
|
41
|
+
with:
|
|
42
|
+
working-directory: ${{ inputs.working-directory }}
|
|
43
|
+
additional-working-directory: ${{ inputs.additional-working-directory }}
|
|
44
|
+
npm-token: ${{ secrets.NPM_TOKEN }}
|
|
45
|
+
|
|
46
|
+
- name: Test
|
|
47
|
+
env:
|
|
48
|
+
TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
|
|
49
|
+
SLACK_TOKEN_TESTS_BOT: ${{ secrets.SLACK_TOKEN_TESTS_BOT }}
|
|
50
|
+
run: |
|
|
51
|
+
set +e
|
|
52
|
+
export RUN_ALL_PLATFORM_TESTS=1
|
|
53
|
+
npx vitest ./test/platform/${{ inputs.test-files-glob || inputs.subtest }} \
|
|
54
|
+
--run \
|
|
55
|
+
--reporter=default \
|
|
56
|
+
--reporter=json \
|
|
57
|
+
--outputFile=./test-output.json \
|
|
58
|
+
--maxConcurrency 20 \
|
|
59
|
+
--fileParallelism=true \
|
|
60
|
+
--maxWorkers 100
|
|
61
|
+
return_code=$?
|
|
62
|
+
npx apify-test-tools report-tests \
|
|
63
|
+
--report-file ./test-output.json \
|
|
64
|
+
--workspace . \
|
|
65
|
+
--workflow-name "${{ github.workflow }}" \
|
|
66
|
+
--job-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
|
|
67
|
+
--report-slack-channel "${{ inputs.report-slack-channel || format('#notif-{0}', github.event.repository.name) }}"
|
|
68
|
+
exit $return_code
|
|
69
|
+
|
|
70
|
+
- name: Upload Vitest JSON artifact
|
|
71
|
+
uses: actions/upload-artifact@v6
|
|
72
|
+
if: always()
|
|
73
|
+
with:
|
|
74
|
+
name: vitest-results
|
|
75
|
+
path: ./test-output.json
|