apify-test-tools 0.10.2-beta.2 → 0.10.2-beta.20

package/.github/workflows/_check_code.yaml

@@ -26,22 +26,6 @@ jobs:
     #         - name: Check spelling with typos
     #           uses: crate-ci/typos@v1
 
-    major_tag_refs:
-        name: Major tag refs
-        runs-on: ubuntu-latest
-        steps:
-            - name: Checkout repository
-              uses: actions/checkout@v6
-            - name: Use Node.js
-              uses: actions/setup-node@v6
-              with:
-                  node-version: 24
-            # `uses:` takes no expressions, so the major tag is repeated in every ref into this repo.
-            # Missing one during a bump is silent while both tags exist: the new workflows quietly
-            # run the old composite action.
-            - name: Check refs match MAJOR_TAG
-              run: node .github/scripts/check-major-tag-refs.mjs
-
     lint_check:
         name: Lint check
         runs-on: ubuntu-latest
@@ -93,10 +77,10 @@ jobs:
             - name: Build
               run: npm run build
 
-    # The composite action runs the CLI from .github/cli/apify-test-tools.mjs, which the pre-commit
-    # hook rebuilds. This catches commits that skipped the hook, so a stale bundle never ships.
+    # The composite action runs the CLI and workflow helper bundles, which the pre-commit hook rebuilds.
+    # This catches commits that skipped the hook, so stale bundles never ship.
     cli_bundle:
-        name: CLI bundle
+        name: CLI and workflow bundles
         runs-on: ubuntu-latest
         steps:
             - name: Checkout repository
@@ -112,8 +96,8 @@ jobs:
             - name: Check the bundle is up to date
               run: |
                   npm run bundle
-                  if ! git diff --quiet -- .github/cli/; then
-                      echo "::error::.github/cli/apify-test-tools.mjs is out of date. Run \`npm run bundle\` and commit the result."
+                  if ! git diff --quiet -- .github/cli/ .github/scripts/run-with-apify-tokens.mjs; then
+                      echo "::error::A CLI or workflow script bundle is out of date. Run \`npm run bundle\` and commit the result."
                       exit 1
                   fi
                   node .github/cli/apify-test-tools.mjs --help > /dev/null

package/.github/workflows/_move_major_tag.yaml

@@ -42,8 +42,8 @@ jobs:
             # it. Depending on a regex in another repo to keep releases working is not a dependency
             # worth having, and `v1` beside a package `v1.0.0` would be worse still.
             #
-            # `uses:` cannot take an expression, so every ref into this repo hardcodes this same tag;
-            # check-major-tag-refs.mjs fails the build if one of them drifts.
+            # The workflows load this repo with `$/`, so they never name the tag. Only the usage
+            # examples in the workflows, README and docs do: update them by hand when bumping it.
             MAJOR_TAG: workflows-v0
         steps:
             - name: Checkout repository

package/.github/workflows/public_platform-tests-claude-investigate-and-fix.yaml

@@ -22,14 +22,19 @@ name: 'Platform tests: Claude failure investigation and fix'
 #     platformTestsCore:
 #         uses: apify/apify-test-tools/.github/workflows/public_platform-tests.yaml@workflows-v0
 #         with:
-#             subtest: core
-#         secrets: inherit
+#             test-files-glob: 'test/platform/hourly/**'
+#             report-slack-channel: '#notif-my-repo'
+#         secrets:
+#             TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
+#             SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 #
 #     handle-failure:
 #         needs: platformTestsCore
 #         if: ${{ failure() }}
 #         uses: apify/apify-test-tools/.github/workflows/public_platform-tests-claude-investigate-and-fix.yaml@workflows-v0
-#         secrets: inherit
+#         secrets:
+#             ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+#             TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
 
 on:
     workflow_call:
@@ -39,6 +44,11 @@ on:
                 type: string
                 default: sonnet
                 description: Claude model to use for investigation and fix
+            effort:
+                required: false
+                type: string
+                default: ''
+                description: Claude effort level (low, medium, high, xhigh, max) to use for investigation and fix. Uses the Claude Code default when empty.
             max_flakiness_runs:
                 required: false
                 type: number
@@ -59,14 +69,34 @@ on:
 
                     Before you create a new issue, check whether the `platform-tests-investigate-and-fix:measure-flakiness` skill applies, and follow it if it does.
 
+                    At the end of your investigation, before you create a new issue, follow the `platform-tests-investigate-and-fix:assess-impact` skill.
+
                     When you create the issue, write it like an incident postmortem: a short summary and impact up top for anyone skimming, full technical detail below for whoever fixes it. Don't water down the engineering detail to make the top readable.
                     - Title: short and clean, describing the actual problem in plain terms — no dates, run numbers, or other noise. Prefix it with a conventional-commit-style type, e.g. "fix: blocking issue on detail endpoint".
-                    - Body, section 1 ("## Summary"): one line stating what's broken plus its impact, in plain language a PM can understand at a glance — no jargon, internal field names, or stack traces (e.g. "Scraping product detail pages fails when a listing has no reviews, affecting N% of runs").
+                    - Body, section 1 ("## Summary"): one line stating what's broken plus its impact, in plain language a PM can understand at a glance — no jargon, internal field names, or stack traces (e.g. "Scraping product detail pages fails when a listing has no reviews").
                     - If you measured flakiness, a "## Flakiness" section right after the Summary, as the `measure-flakiness` skill describes.
+                    - An "## Impact" section under the Flakiness section (or under the Summary if there's none), as the `assess-impact` skill describes.
                     - A `---` divider, then section 2 ("## Details"): the engineering bug report — root cause, relevant log lines, links to the failing run, and any other internals an engineer needs. If a single item/request failed among otherwise-good ones, show or link that specific one (e.g. the dataset item index/id, or its matching debug snapshot) rather than just describing the run as failed.
 
                     If you create a new issue, you MUST write its issue number (just the number, nothing else) to $GITHUB_WORKSPACE/issue-number.txt as the very last thing you do. If you skip creating one because a duplicate already exists, do not create the file.
                 description: Prompt given to Claude for the investigation phase. Override to customize how issues are investigated, titled, and described.
+            priority_prompt:
+                required: false
+                type: string
+                # The prio prompt here is purposefully vague; you can adjust it to
+                # use custom thresholds based on measured impact metrics.
+                default: |
+                    After you create a new issue and before you write its issue number to $GITHUB_WORKSPACE/issue-number.txt, set its Priority field from the failure's impact and flakiness:
+
+                    To do so, exercise your best judgement. Issues marked with higher priority will be reacted to first.
+                    Here are guidelines:
+                    - Determine priority based on combined severity and revenue actually at risk: a less severe bug can outrank a more severe one if much more revenue is actually at risk, and vice versa.
+                    - If the Impact section considers the bug from several points of view, each in its own sub-section, base the priority on the most impactful one.
+                    - If it helps you, you can base your assessment off priorities of already existing bugs in the repo.
+                    - Invisible to the user or Test severity always receives lowest priority.
+
+                    Priority is an organization-level issue field. If the revenue actually at risk can't be determined, e.g. the Impact section says "Cannot be determined", leave Priority unset, unless the relative severity is Invisible to the user or Test.
+                description: Prompt given to Claude for setting a new issue's Priority field from its impact and flakiness. Override to customize how priority is judged, e.g. to set revenue thresholds.
         secrets:
             ANTHROPIC_API_KEY:
                 required: true
@@ -77,6 +107,12 @@ on:
             NPM_TOKEN:
                 required: false
                 description: Read-only token for installing private npm packages before Claude runs.
+            REDASH_API_KEY:
+                required: false
+                description: Redash API key for measuring the failure's impact — affected runs, high-touch customers and revenue — from Snowflake.
+            AGENT_SKILLS_INTERNAL_GITHUB_TOKEN:
+                required: false
+                description: GitHub token with read access to apify/agent-skills-internal, for installing its redash-analyst plugin.
 
 jobs:
     investigate:
@@ -92,29 +128,71 @@ jobs:
             - name: Checkout repository
               uses: actions/checkout@v5
 
+            - name: Check which optional secrets are set
+              id: optional-secrets
+              run: |
+                  echo "agent-skills-token=${{ secrets.AGENT_SKILLS_INTERNAL_GITHUB_TOKEN != '' }}" >> "$GITHUB_OUTPUT"
+                  echo "redash-api-key=${{ secrets.REDASH_API_KEY != '' }}" >> "$GITHUB_OUTPUT"
+
+            - name: Checkout internal agent skills
+              id: agent-skills-internal
+              if: steps.optional-secrets.outputs.agent-skills-token == 'true'
+              continue-on-error: true
+              uses: actions/checkout@v5
+              with:
+                  repository: apify/agent-skills-internal
+                  token: ${{ secrets.AGENT_SKILLS_INTERNAL_GITHUB_TOKEN }}
+                  path: .agent-skills-internal
+                  sparse-checkout: |
+                      .claude-plugin
+                      skills/redash-analyst
+                  persist-credentials: false
+
+            - uses: actions/setup-node@v6
+              if: steps.optional-secrets.outputs.redash-api-key == 'true'
+              with:
+                  node-version: 24
+
+            - name: Install Redash MCP server
+              if: steps.optional-secrets.outputs.redash-api-key == 'true'
+              continue-on-error: true
+              working-directory: ${{ runner.temp }}
+              run: npm install --prefix redash-mcp --ignore-scripts --no-audit --no-fund @suthio/redash-mcp@0.1.0
+
             - name: Run Claude Code
               id: claude
               uses: anthropics/claude-code-action@8251c103ac8c1d761882c86aba1412c7f583c844 # v1.0.213
               env:
                   TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
+                  REDASH_API_KEY: ${{ secrets.REDASH_API_KEY }}
                   MAX_FLAKINESS_RUNS: ${{ inputs.max_flakiness_runs }}
               with:
                   prompt: |
                       The failing E2E platform test ran in workflow run ${{ github.run_id }} of this repo.
 
                       ${{ inputs.investigate_prompt }}
+
+                      ${{ inputs.priority_prompt }}
                   github_token: ${{ secrets.GITHUB_TOKEN }}
                   anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
-                  plugin_marketplaces: https://github.com/apify/apify-test-tools.git
-                  plugins: platform-tests-investigate-and-fix@apify-test-tools
+                  plugin_marketplaces: |
+                      https://github.com/apify/apify-test-tools.git
+                      ${{ steps.agent-skills-internal.outcome == 'success' && format('{0}/.agent-skills-internal', github.workspace) || '' }}
+                  plugins: |
+                      platform-tests-investigate-and-fix@apify-test-tools
+                      ${{ steps.agent-skills-internal.outcome == 'success' && 'redash-analyst@apify-agent-skills-internal' || '' }}
                   # The `tools` query parameter below selects which tools the Apify MCP server
                   # exposes: reading an Actor run — its status, exit code, storage ids and log — and
                   # reading that run's dataset items and its key-value store keys and records, e.g.
                   # INPUT, OUTPUT and error snapshots. WARN: the server adds `abort-actor-run` on top
                   # even when not listed here, see https://github.com/apify/apify-mcp-server
+                  # `--disallowedTools` removes the Redash MCP server's tools that change Redash: creating,
+                  # editing, sharing or deleting queries, visualizations, dashboards, widgets, alerts and snippets.
                   claude_args: |
-                      --mcp-config '{"mcpServers":{"apify":{"type":"http","url":"https://mcp.apify.com/?tools=get-actor-run,get-actor-run-log,get-dataset-list,get-dataset,get-dataset-items,get-key-value-store-list,get-key-value-store,get-key-value-store-keys,get-key-value-store-record","headers":{"Authorization":"Bearer ${TESTER_APIFY_TOKEN}"}}}}'
+                      --mcp-config '{"mcpServers":{"apify":{"type":"http","url":"https://mcp.apify.com/?tools=get-actor-run,get-actor-run-log,get-dataset-list,get-dataset,get-dataset-items,get-key-value-store-list,get-key-value-store,get-key-value-store-keys,get-key-value-store-record","headers":{"Authorization":"Bearer ${TESTER_APIFY_TOKEN}"}},"redash":{"type":"stdio","command":"${{ runner.temp }}/redash-mcp/node_modules/.bin/redash-mcp","env":{"REDASH_URL":"https://charts.apify.com/","REDASH_API_KEY":"${REDASH_API_KEY}"}}}}'
+                      --disallowedTools mcp__redash__create_query,mcp__redash__update_query,mcp__redash__update_query_parameters,mcp__redash__archive_query,mcp__redash__fork_query,mcp__redash__add_query_favorite,mcp__redash__remove_query_favorite,mcp__redash__create_visualization,mcp__redash__update_visualization,mcp__redash__update_chart_visualization,mcp__redash__delete_visualization,mcp__redash__create_dashboard,mcp__redash__update_dashboard,mcp__redash__update_dashboard_parameters,mcp__redash__archive_dashboard,mcp__redash__fork_dashboard,mcp__redash__share_dashboard,mcp__redash__unshare_dashboard,mcp__redash__add_dashboard_favorite,mcp__redash__remove_dashboard_favorite,mcp__redash__create_alert,mcp__redash__update_alert,mcp__redash__delete_alert,mcp__redash__mute_alert,mcp__redash__add_alert_subscription,mcp__redash__remove_alert_subscription,mcp__redash__create_widget,mcp__redash__update_widget,mcp__redash__update_widget_layout,mcp__redash__update_dashboard_layout,mcp__redash__update_widget_parameter_mappings,mcp__redash__delete_widget,mcp__redash__create_query_snippet,mcp__redash__update_query_snippet,mcp__redash__delete_query_snippet
                       --model ${{ inputs.model }}
+                      ${{ inputs.effort && format('--effort {0}', inputs.effort) || '' }}
                       --dangerously-skip-permissions
 
             - name: Upload Claude output (for debugging)
@@ -177,7 +255,9 @@ jobs:
             actions: read
         steps:
             - name: Setup repository and dependencies
-              uses: apify/apify-test-tools/.github/actions/checkout-restore-dependencies@workflows-v0
+              # `$/` loads the action from this repo at the commit this workflow was called at, so a caller
+              # pinned to a SHA or a branch gets that commit's action and CLI, not whatever the major tag is on.
+              uses: $/.github/actions/checkout-restore-dependencies
               with:
                   npm-token: ${{ secrets.NPM_TOKEN }}
 
@@ -190,6 +270,7 @@ jobs:
                   prompt: 'Open a pull request to address GitHub issue #${{ needs.investigate.outputs.issue_number }}. Use `gh issue view ${{ needs.investigate.outputs.issue_number }}` to read the issue details first, then implement the fix and create a PR.'
                   claude_args: |
                       --model ${{ inputs.model }}
+                      ${{ inputs.effort && format('--effort {0}', inputs.effort) || '' }}
                       --dangerously-skip-permissions
 
             - name: Upload Claude output (for debugging)

package/.github/workflows/public_platform-tests.yaml

@@ -3,21 +3,27 @@ name: Platform tests
 on:
     workflow_call:
         inputs:
-            # Deprecated, use `test-files-glob`
-            subtest:
-                type: string
-                default: ''
-                required: false
-            # Specify what tests you want to run, relative to `test/platform`
+            # Glob of the test files to run, from the repo root, e.g. `test/platform/{hourly,hourly-without-pr}/**`
             test-files-glob:
                 type: string
-                default: ''
+                default: 'test/platform/**'
                 required: false
-            # Defaults to #notif-<repo-name> when empty, matching the previous behavior.
+            # Slack channel for failed tests. Empty skips Slack.
             report-slack-channel:
                 type: string
                 default: ''
                 required: false
+        # Only TESTER_APIFY_TOKEN is required, every test needs it. See the Secrets section of docs/workflows.md.
+        secrets:
+            NPM_TOKEN:
+                required: false
+                description: Read-only token for installing private npm packages.
+            TESTER_APIFY_TOKEN:
+                required: true
+                description: Apify token the platform tests run Actors with.
+            SLACK_BOT_TOKEN:
+                required: false
+                description: Slack token for reporting test results.
 
 jobs:
     scheduledPlatformTests:
@@ -26,37 +32,69 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - name: Setup repository and dependencies
-              # For testing, you have to temporarily change the branch
-              uses: apify/apify-test-tools/.github/actions/checkout-restore-dependencies@workflows-v0
+              # `$/` loads the action from this repo at the commit this workflow was called at, so a caller
+              # pinned to a SHA or a branch gets that commit's action and CLI, not whatever the major tag is on.
+              uses: $/.github/actions/checkout-restore-dependencies
               with:
                   npm-token: ${{ secrets.NPM_TOKEN }}
 
+            # Fails here with a clear message rather than deep in vitest. `required: true` alone doesn't
+            # catch it: a caller can list the secret while the repo doesn't have it, and PRs from forks
+            # never get secrets.
+            - name: Check TESTER_APIFY_TOKEN
+              env:
+                  TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
+              run: |
+                  if [ -z "$TESTER_APIFY_TOKEN" ]; then
+                      echo "::error::TESTER_APIFY_TOKEN is empty. Pass it in the secrets: block of the job that calls this workflow."
+                      exit 1
+                  fi
+
+            # Runs the repo's test code and its whole import graph, so it gets the tester token only.
+            # Fails on failing tests, but writes the report anyway for the notification below.
+            # Every Actor in the config, on its default build: the one users run
             - name: Test
+              id: test
+              # bash rather than the default shell, for pipefail
+              shell: bash
               env:
                   TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
-                  SLACK_TOKEN_TESTS_BOT: ${{ secrets.SLACK_TOKEN_TESTS_BOT }}
+                  # Passed through the environment and quoted, so the shell doesn't expand the glob itself
+                  # An empty input, e.g. from an expression that picks the glob, runs everything, like the default
+                  TEST_FILES_GLOB: ${{ inputs.test-files-glob || 'test/platform/**' }}
+              run: |
+                  apify-test-tools get-actor-configs \
+                    | apify-test-tools test --test-files-glob "$TEST_FILES_GLOB" \
+                        --reporter github-actions --vitest-output-file ./test-output.json > test-report.json
+
+            # Posts only when the tests failed, but checks the Slack token on every run that has a channel. Runs
+            # whenever the tests did, also after a failure, but not when the token check stopped the job before them.
+            - name: Notify
+              if: ${{ !cancelled() && steps.test.outcome != 'skipped' }}
+              env:
+                  SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+                  # Passed through the environment, so a quote in the workflow name can't break the command
+                  WORKFLOW_NAME: ${{ github.workflow }}
+                  JOB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+                  # Empty makes notify skip Slack
+                  SLACK_CHANNEL: ${{ inputs.report-slack-channel }}
               run: |
-                  set +e
-                  export RUN_ALL_PLATFORM_TESTS=1
-                  npx vitest ./test/platform/${{ inputs.test-files-glob || inputs.subtest }} \
-                    --run \
-                    --reporter=default \
-                    --reporter=json \
-                    --outputFile=./test-output.json \
-                    --maxConcurrency 20 \
-                    --fileParallelism=true \
-                    --maxWorkers 100
-                  return_code=$?
-                  apify-test-tools report-tests \
-                    --report-file ./test-output.json \
-                    --workflow-name "${{ github.workflow }}" \
-                    --job-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
-                    --report-slack-channel "${{ inputs.report-slack-channel || format('#notif-{0}', github.event.repository.name) }}"
-                  exit $return_code
+                  # Empty when `test` itself failed before running vitest, its error is in the Test step
+                  if [ ! -s test-report.json ]; then
+                      echo "No test report, so there is nothing to notify about"
+                      exit 0
+                  fi
+                  apify-test-tools notify \
+                    --slack-channel "$SLACK_CHANNEL" \
+                    --title "$WORKFLOW_NAME" \
+                    --job-url "$JOB_URL" \
+                    < test-report.json
 
-            - name: Upload Vitest JSON artifact
+            - name: Upload test reports
               uses: actions/upload-artifact@v6
               if: always()
               with:
                   name: vitest-results
-                  path: ./test-output.json
+                  path: |
+                      ./test-output.json
+                      ./test-report.json

package/.github/workflows/public_pr-build-test.yaml

@@ -7,11 +7,33 @@ on:
                 type: boolean
                 default: false
                 required: false
-            # Specify what tests you want to run, relative to `test/platform`
+            # Glob of the test files to run, from the repo root, e.g. `test/platform/{daily,hourly}/**`. Which changed
+            # files count as test changes is `platformTestsGlob` in apify-test-tools.config.json, not this.
             test-files-glob:
                 type: string
-                default: ''
+                default: 'test/platform/**'
                 required: false
+            # Fail the build unless the env var names of version 0.99 match the default version's
+            require-env-vars-in-sync:
+                type: boolean
+                default: true
+                required: false
+        # Every secret is optional. The ones this workflow reads are declared by name. The secrets that
+        # apify-test-tools.config.json names (e.g. each Actor's `tokenEnvVar`) differ per repo, so they
+        # come packed in CONFIG_SECRETS. See the Secrets section of docs/workflows.md.
+        secrets:
+            NPM_TOKEN:
+                required: false
+                description: Read-only token for installing private npm packages.
+            TESTER_APIFY_TOKEN:
+                required: false
+                description: Apify token the platform tests run Actors with.
+            CONFIG_SECRETS:
+                required: false
+                description: >-
+                    JSON object of the secrets apify-test-tools.config.json names, secret name -> value.
+                    Today that is each Actor's `tokenEnvVar`. Passing `toJSON(secrets)` works too, only
+                    the names the config declares are used.
 
 concurrency:
     group: ${{ github.workflow }}-${{ github.ref }}
@@ -24,11 +46,25 @@ jobs:
         steps:
             - name: Setup repository and dependencies
               id: setup
-              # For testing, you have to temporarily change the branch
-              uses: apify/apify-test-tools/.github/actions/checkout-restore-dependencies@workflows-v0
+              # `$/` loads the action from this repo at the commit this workflow was called at, so a caller
+              # pinned to a SHA or a branch gets that commit's action and CLI, not whatever the major tag is on.
+              uses: $/.github/actions/checkout-restore-dependencies
               with:
                   # Some repos use private npm packages, they need to pass NPM_TOKEN secret to this workflow
                   npm-token: ${{ secrets.NPM_TOKEN }}
+                  config-secrets: ${{ secrets.CONFIG_SECRETS }}
+
+            # Fails here with a clear message rather than deep in vitest. `required: true` alone doesn't
+            # catch it: a caller can list the secret while the repo doesn't have it, and PRs from forks
+            # never get secrets.
+            - name: Check TESTER_APIFY_TOKEN
+              env:
+                  TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
+              run: |
+                  if [ -z "$TESTER_APIFY_TOKEN" ]; then
+                      echo "::error::TESTER_APIFY_TOKEN is empty. Pass it in the secrets: block of the job that calls this workflow."
+                      exit 1
+                  fi
 
             - name: Pick last validated commit from cache
               id: base_commit
@@ -41,54 +77,56 @@ jobs:
                   # This is needed to force overriding the cache with new entry at the end
                   restore-keys: base_commit-
 
-            # run-with-apify-tokens.mjs reads the token names from in apify-test-tools.config.json
-            # and hands the build those secrets and nothing else, so the tokens stay in this one command instead of the whole job.
-            # The library never sees secrets it doesn't need
-            - name: Build
-              id: build
+            # Needs no tokens, it only reads git and apify-test-tools.config.json. The setup action checks out
+            # the PR branch itself (not GitHub's merge commit), so the range ends at the branch's HEAD.
+            - name: Find affected Actors
               env:
-                  ALL_SECRETS: ${{ toJSON(secrets) }}
                   # Branch names are attacker-controlled on fork PRs, so they reach the script as
                   # environment variables rather than being interpolated into it.
-                  HEAD_REF: ${{ github.head_ref }}
                   BASE_REF: ${{ github.base_ref }}
-                  SCRIPTS_PATH: ${{ steps.setup.outputs.scripts-path }}
               run: |
                   base_commit=$(cat base_commit.txt 2>/dev/null || echo "")
                   echo "Last validated (base) commit to be used for build & test: $base_commit"
-                  args=(
-                    --source-branch "origin/$HEAD_REF"
-                    --target-branch "origin/$BASE_REF"
-                    # TODO: Temporary hotfix until build glob matching aligns with Vitest's test selection glob.
-                    --test-files-glob 'test/platform/**'
-                    --use-docker-cache
-                  )
-                  [ -n "$base_commit" ] && args+=(--base-commit "$base_commit")
-                  actor_builds=$(node "$SCRIPTS_PATH/run-with-apify-tokens.mjs" \
-                    apify-test-tools build "${args[@]}")
-                  echo "actor_builds=$actor_builds" | tee -a "$GITHUB_OUTPUT"
+                  args=(--base "origin/$BASE_REF")
+                  [ -n "$base_commit" ] && args+=(--since "$base_commit")
+                  apify-test-tools get-affected-actors "${args[@]}" > affected.json
+
+            # Cosmetic-only changes need no tests, so only the Actors with functional or platform test changes are
+            # built, into the test version 0.99 that users never run.
+            # run-with-apify-tokens.mjs reads the token names from in apify-test-tools.config.json
+            # and hands the build those secrets and nothing else, so the tokens stay in this one command instead of the whole job.
+            # The library never sees secrets it doesn't need
+            - name: Build
+              # bash rather than the default shell, for pipefail
+              shell: bash
+              env:
+                  WORKFLOW_SECRETS: ${{ toJSON(secrets) }}
+                  SCRIPTS_PATH: ${{ steps.setup.outputs.scripts-path }}
+                  REQUIRE_ENV_VARS_IN_SYNC: ${{ inputs.require-env-vars-in-sync }}
+              run: |
+                  jq '{ actors: (.functionallyChangedActors + .testChangedActors) }' affected.json \
+                    | node "$SCRIPTS_PATH/run-with-apify-tokens.mjs" apify-test-tools build --version-number 0.99 \
+                        --require-env-vars-in-sync="$REQUIRE_ENV_VARS_IN_SYNC" \
+                    | tee builds.json
 
             # Runs the repo's test code and its whole import graph, so it gets the tester token only.
             - name: Test
               env:
-                  ACTOR_BUILDS: ${{ steps.build.outputs.actor_builds }}
                   TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
-              run: npx vitest ./test/platform/${{ inputs.test-files-glob }} --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100
+                  # Passed through the environment and quoted, so the shell doesn't expand the glob itself
+                  # An empty input, e.g. from an expression that picks the glob, runs everything, like the default
+                  TEST_FILES_GLOB: ${{ inputs.test-files-glob || 'test/platform/**' }}
+              # The github-actions reporter annotates the PR with the failing tests
+              run: |
+                  apify-test-tools test --test-files-glob "$TEST_FILES_GLOB" --reporter github-actions \
+                    < builds.json > test-report.json
 
             # NOTE: This is an optimization that if we did functional changes and later only cosmetic changes (e.g. dev readme), we will compare changes files only for the last commit. This must run after tests because we want to cache the latest commit only if the tests are successful
             - name: Store last validated commit to cache
-              env:
-                  HEAD_REF: ${{ github.head_ref }}
-                  BASE_REF: ${{ github.base_ref }}
               run: |
-                  base_commit=$(cat base_commit.txt 2>/dev/null || echo "")
-                  echo "Old last validated (base) commit: $base_commit"
-                  args=(
-                    --source-branch "origin/$HEAD_REF"
-                    --target-branch "origin/$BASE_REF"
-                  )
-                  [ -n "$base_commit" ] && args+=(--base-commit "$base_commit")
-                  apify-test-tools get-latest-commit "${args[@]}" > ./base_commit.txt
+                  echo "Old last validated (base) commit: $(cat base_commit.txt 2>/dev/null || echo "")"
+                  # The HEAD that was just built and tested
+                  jq -r .head affected.json > base_commit.txt
                   echo "New last validated (base) commit: $(cat base_commit.txt)"
 
     # Pure static checks against the repo. Needs no Apify or Slack credentials, only the npm
@@ -98,8 +136,9 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - name: Setup repository and dependencies
-              # For testing, you have to temporarily change the branch
-              uses: apify/apify-test-tools/.github/actions/checkout-restore-dependencies@workflows-v0
+              # `$/` loads the action from this repo at the commit this workflow was called at, so a caller
+              # pinned to a SHA or a branch gets that commit's action and CLI, not whatever the major tag is on.
+              uses: $/.github/actions/checkout-restore-dependencies
               with:
                   npm-token: ${{ secrets.NPM_TOKEN }}
 

package/.github/workflows/public_push-build-latest.yaml

@@ -3,11 +3,27 @@ name: Build latest and report to slack
 on:
     workflow_call:
         inputs:
-            # Defaults to #notif-<repo-name> when empty, matching the previous behavior.
+            # Slack channel for release announcements. Empty skips Slack.
             report-slack-channel:
                 type: string
                 default: ''
                 required: false
+        # Every secret is optional. The ones this workflow reads are declared by name. The secrets that
+        # apify-test-tools.config.json names (e.g. each Actor's `tokenEnvVar`) differ per repo, so they
+        # come packed in CONFIG_SECRETS. See the Secrets section of docs/workflows.md.
+        secrets:
+            NPM_TOKEN:
+                required: false
+                description: Read-only token for installing private npm packages.
+            SLACK_BOT_TOKEN:
+                required: false
+                description: Slack token for announcing releases.
+            CONFIG_SECRETS:
+                required: false
+                description: >-
+                    JSON object of the secrets apify-test-tools.config.json names, secret name -> value.
+                    Today that is each Actor's `tokenEnvVar`. Passing `toJSON(secrets)` works too, only
+                    the names the config declares are used.
 
 jobs:
     pushBuildLatest:
@@ -20,27 +36,49 @@ jobs:
         steps:
             - name: Setup repository and dependencies
               id: setup
-              # For testing, you have to temporarily change the branch
-              uses: apify/apify-test-tools/.github/actions/checkout-restore-dependencies@workflows-v0
+              # `$/` loads the action from this repo at the commit this workflow was called at, so a caller
+              # pinned to a SHA or a branch gets that commit's action and CLI, not whatever the major tag is on.
+              uses: $/.github/actions/checkout-restore-dependencies
               with:
                   npm-token: ${{ secrets.NPM_TOKEN }}
+                  config-secrets: ${{ secrets.CONFIG_SECRETS }}
+
+            # --since is the last commit that is already released: the push's "before" commit.
+            # Everything after it up to HEAD gets released. See the "Releasing Actors" section of docs/cli.md.
+            # Needs no tokens, it only reads git and apify-test-tools.config.json.
+            - name: Find changes since the last release
+              run: |
+                  apify-test-tools get-affected-actors --since ${{ github.event.before }} > affected.json
 
-            # --base-commit is the last commit that is already released: the push's "before" commit.
-            # Everything after it up to HEAD gets released. See the "Releasing Actors" section of the README.
+            # A release ships cosmetic changes too (README, input schema descriptions), so both lists are built,
+            # into each Actor's default version: the one its default build tag points to, which users run.
+            # testChangedActors is left out: a change to platform tests alone needs no release.
             # run-with-apify-tokens.mjs reads the token names from in apify-test-tools.config.json
             # and hands the build those secrets and nothing else, so the tokens stay in this one command instead of the whole job.
             # The library never sees secrets it doesn't need
-            - name: Release Actors and notify
+            - name: Build the default versions
+              # bash rather than the default shell, for pipefail
+              shell: bash
               env:
-                  ALL_SECRETS: ${{ toJSON(secrets) }}
-                  SLACK_TOKEN_RELEASES_BOT: ${{ secrets.SLACK_TOKEN_RELEASES_BOT }}
+                  WORKFLOW_SECRETS: ${{ toJSON(secrets) }}
               run: |
-                  node "${{ steps.setup.outputs.scripts-path }}/run-with-apify-tokens.mjs" \
-                    apify-test-tools release --base-commit ${{ github.event.before }} --release-slack-channel "#delivery-public-actors" --report-slack-channel "${{ inputs.report-slack-channel || format('#notif-{0}', github.event.repository.name) }}" --use-docker-cache
+                  jq '{ actors: (.functionallyChangedActors + .cosmeticallyChangedActors) }' affected.json \
+                    | node "${{ steps.setup.outputs.scripts-path }}/run-with-apify-tokens.mjs" \
+                        apify-test-tools build --version-number default
+
+            # Skips the announcement when the push brought no new commits, or when no channel is set
+            - name: Announce the release on Slack
+              env:
+                  SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+                  SLACK_CHANNEL: ${{ inputs.report-slack-channel }}
+              run: |
+                  apify-test-tools notify \
+                    --slack-channel "$SLACK_CHANNEL" \
+                    < affected.json
 
             - name: Delete old builds
               env:
-                  ALL_SECRETS: ${{ toJSON(secrets) }}
+                  WORKFLOW_SECRETS: ${{ toJSON(secrets) }}
               run: |
                   node "${{ steps.setup.outputs.scripts-path }}/run-with-apify-tokens.mjs" \
                     apify-test-tools delete-old-builds