apify-test-tools 0.10.2-beta.2 → 0.10.2-beta.20

package/.github/workflows/public_review.yaml

@@ -14,12 +14,12 @@ on:
                 description: Label added after a successful review and never removed. Set to an empty string to disable.
                 required: false
             prompt-ref:
-                default: workflows-v0
+                default: ''
                 type: string
                 description: >-
                     Ref of this repository to fetch the review instructions from. Defaults to the
-                    major tag, matching the ref consumers call this workflow at. Override it only to
-                    test a prompt change from a branch.
+                    commit this workflow was called at, so the instructions always match it. Override
+                    it only to test a prompt change from a branch.
                 required: false
             model:
                 default: sonnet
@@ -55,10 +55,11 @@ jobs:
                   # head_ref must be used for pull_request but for push and schedule events we have to use ref to get the branch name
                   ref: ${{ github.head_ref || github.ref }}
 
+            # job.workflow_* is this reusable workflow's own repo and commit; the github context is the caller's.
             - name: 'Fetch review instructions'
               run: |
                   curl -sfL \
-                    "https://raw.githubusercontent.com/apify/apify-test-tools/${{ inputs.prompt-ref }}/.github/review-prompt.md" \
+                    "https://raw.githubusercontent.com/${{ job.workflow_repository }}/${{ inputs.prompt-ref || job.workflow_sha }}/.github/review-prompt.md" \
                     -o "${{ github.workspace }}/.claude-review-prompt.md"
 
             - name: 'Review PR against guidelines'

package/.github/workflows/public_stale-issues.yaml

@@ -0,0 +1,51 @@
+name: Close stale issues
+
+on:
+    workflow_call:
+        inputs:
+            stale-months:
+                description: Months without activity before warning issue owners (one month = 30 days). Issues close 30 days after the warning.
+                type: number
+                default: 6
+
+jobs:
+    staleIssues:
+        runs-on: ubuntu-latest
+        timeout-minutes: 30
+        permissions:
+            issues: write
+        concurrency:
+            group: stale-issues-${{ github.repository }}
+            cancel-in-progress: false
+        steps:
+            - name: Convert months to days
+              id: threshold
+              uses: actions/github-script@v9
+              env:
+                  STALE_MONTHS: ${{ inputs.stale-months }}
+              with:
+                  script: |
+                      const months = Number(process.env.STALE_MONTHS);
+                      if (!Number.isSafeInteger(months) || months < 1 || !Number.isSafeInteger(months * 30)) {
+                          throw new Error('stale-months must be a positive whole number whose day count is a safe integer.');
+                      }
+                      if (months > 12) {
+                          throw new Error('stale-months must be less than or equal to 12.');
+                      }
+                      return months * 30;
+
+            - name: Warn and close stale issues
+              uses: actions/stale@v11
+              with:
+                  days-before-issue-stale: ${{ steps.threshold.outputs.result }}
+                  days-before-issue-close: 30
+                  days-before-pr-stale: ${{ steps.threshold.outputs.result }}
+                  days-before-pr-close: 30
+                  stale-issue-label: stale
+                  stale-issue-message: >-
+                      This issue has had no activity for ${{ inputs.stale-months }} month(s).
+                      It will be closed as not planned in 30 days unless it is updated or commented on.
+                  close-issue-message: Closed as not planned after 30 days without activity following the stale warning.
+                  close-issue-reason: not_planned
+                  remove-stale-when-updated: true
+                  operations-per-run: 1000

package/.husky/pre-commit

@@ -1,10 +1,10 @@
 echo "pre-commit: typechecking, linting, testing, checking code formatting, checking for unused exports & validating schemas"
 
 
-# The workflows run the CLI from this bundle, so it ships with every commit that changes bin/ or its
-# dependencies. Rebuilding takes well under a second.
+# The workflows run the CLI and token wrapper from these bundles, so they ship with every commit
+# that changes their sources or dependencies.
 npm run -s bundle
-git add .github/cli/apify-test-tools.mjs
+git add .github/cli/apify-test-tools.mjs .github/scripts/run-with-apify-tokens.mjs
 
 npm test
 npm run format:check
@@ -15,4 +15,4 @@ npx lint-staged
 # Activate once we resolv unused delete old builds
 # npm run check-unused
 
-echo "pre-commit: passed"
+echo "pre-commit: passed"

package/.prettierignore

@@ -1,5 +1,5 @@
 CHANGELOG.md
-
+.cli-test-*
 # Operating instructions read by Claude, not prose for humans. Prettier collapses the nested bullet
 # list under "do not visit, fetch, infer, or evaluate the following external links" into one run-on
 # line, which changes what the model is told. Keeping it byte-identical also makes re-syncing it
@@ -11,3 +11,4 @@ test/fixtures
 
 # Generated by `npm run bundle`
 .github/cli/
+.github/scripts/run-with-apify-tokens.mjs

package/CHANGELOG.md

@@ -8,11 +8,33 @@ All notable changes to this project will be documented in this file.
 ### 🚀 Features
 
 - **ci:** Ship the workflows' CLI as a bundle in the action instead of installing it ([#200](https://github.com/apify/apify-test-tools/pull/200)) ([a0ef975](https://github.com/apify/apify-test-tools/commit/a0ef9756123a30a8210590b080a2a289c6a9c17c)) by [@metalwarrior665](https://github.com/metalwarrior665)
+- [**breaking**] Revamp get-affected-actors to be a core logic command ([#190](https://github.com/apify/apify-test-tools/pull/190)) ([ea5f244](https://github.com/apify/apify-test-tools/commit/ea5f244cfd42bf3fa9ffd38cd2ad499dde2e83f8)) by [@metalwarrior665](https://github.com/metalwarrior665)
+- [**breaking**] Add logic-less build command and use command chaining in workflows ([#193](https://github.com/apify/apify-test-tools/pull/193)) ([e5b0b86](https://github.com/apify/apify-test-tools/commit/e5b0b86ee06fca51d4be0ddc81b0a4a469d31603)) by [@metalwarrior665](https://github.com/metalwarrior665)
+- **get-affected-actors:** Three-dot diff, narrowed per Actor with --since ([#204](https://github.com/apify/apify-test-tools/pull/204)) ([fa9a0e7](https://github.com/apify/apify-test-tools/commit/fa9a0e7518764dd4722ec9ecbf67b2a26a5a062e)) by [@metalwarrior665](https://github.com/metalwarrior665)
+- Add effort input to Claude investigate-and-fix workflow ([#221](https://github.com/apify/apify-test-tools/pull/221)) ([c2ac078](https://github.com/apify/apify-test-tools/commit/c2ac078aec9ef02337af0e5c06db1b947b6c969b)) by [@foxt451](https://github.com/foxt451)
+- **ci:** Support pnpm ([#214](https://github.com/apify/apify-test-tools/pull/214)) ([ee7d734](https://github.com/apify/apify-test-tools/commit/ee7d7340564c5b8657a8a1f0cf920d67b40ae013)) by [@ruocco-l](https://github.com/ruocco-l)
+- Support flexible secret passing methods in workflows ([#225](https://github.com/apify/apify-test-tools/pull/225)) ([22f7e80](https://github.com/apify/apify-test-tools/commit/22f7e807ca3949656c15b79f848a192d71b15946)) by [@metalwarrior665](https://github.com/metalwarrior665), closes [#205](https://github.com/apify/apify-test-tools/issues/205)
+- [**breaking**] Require env vars of test versions in sync with the default version ([#217](https://github.com/apify/apify-test-tools/pull/217)) ([a2481dd](https://github.com/apify/apify-test-tools/commit/a2481ddf09004308cb54e49b783c8b21a25dcbe4)) by [@metalwarrior665](https://github.com/metalwarrior665)
+- Add impact & priority assessment to claude investigator ([#216](https://github.com/apify/apify-test-tools/pull/216)) ([fd16622](https://github.com/apify/apify-test-tools/commit/fd16622b910ec6e9b5515ea0bd26b0a41c22bd23)) by [@foxt451](https://github.com/foxt451), closes [#208](https://github.com/apify/apify-test-tools/issues/208)
+- Stale issue/pr workflow ([#236](https://github.com/apify/apify-test-tools/pull/236)) ([aff1fa7](https://github.com/apify/apify-test-tools/commit/aff1fa778adb846a60fdacdd26aa13a71729613d)) by [@JuanGalilea](https://github.com/JuanGalilea), closes [#35](https://github.com/apify/apify-test-tools/issues/35)
+- **workflows:** [**breaking**] Skip Slack when report-slack-channel is empty ([#238](https://github.com/apify/apify-test-tools/pull/238)) ([9a15b56](https://github.com/apify/apify-test-tools/commit/9a15b56de9ca42fb5ea753dc478116f9cd9c2c94)) by [@metalwarrior665](https://github.com/metalwarrior665)
+- [**breaking**] Repo-root test-files-glob, platformTestsGlob in the config, and drop subtest ([#239](https://github.com/apify/apify-test-tools/pull/239)) ([640bc97](https://github.com/apify/apify-test-tools/commit/640bc97d9f16cfa07501bc4eae400cd2b1bd0c9c)) by [@metalwarrior665](https://github.com/metalwarrior665), closes [#192](https://github.com/apify/apify-test-tools/issues/192)
 
 ### 🐛 Bug Fixes
 
 - **toBeObject:** Reject null ([#191](https://github.com/apify/apify-test-tools/pull/191)) ([6bdae49](https://github.com/apify/apify-test-tools/commit/6bdae4905756e0ae8dd6146942e6fc04dbead48e)) by [@JuanGalilea](https://github.com/JuanGalilea), closes [#88](https://github.com/apify/apify-test-tools/issues/88)
 - Build and test on test-only changes even when .dockerignore lists the tests ([#177](https://github.com/apify/apify-test-tools/pull/177)) ([5cb1b5a](https://github.com/apify/apify-test-tools/commit/5cb1b5ab5f6f43ec279f9624b402e73d55a5974a)) by [@artogahr](https://github.com/artogahr), closes [#145](https://github.com/apify/apify-test-tools/issues/145)
+- **git:** List both paths of moved files when detecting changes ([#215](https://github.com/apify/apify-test-tools/pull/215)) ([44ba565](https://github.com/apify/apify-test-tools/commit/44ba565242c1ac3daea281a648acb28712bb4363)) by [@metalwarrior665](https://github.com/metalwarrior665)
+- **workflows:** Load setup action and review prompt from the called commit ([#226](https://github.com/apify/apify-test-tools/pull/226)) ([7524711](https://github.com/apify/apify-test-tools/commit/752471135d4d147f6f6492a26c08af388d7ccf72)) by [@metalwarrior665](https://github.com/metalwarrior665)
+- **test:** Clean up compiled CLI dir when build fails ([#224](https://github.com/apify/apify-test-tools/pull/224)) ([f9f2f5c](https://github.com/apify/apify-test-tools/commit/f9f2f5c64cfe936ee0b72a0fbe7408282c68fbac)) by [@JuanGalilea](https://github.com/JuanGalilea)
+- **cli-test:** Look up actor raw IDs for tests that reference them ([#234](https://github.com/apify/apify-test-tools/pull/234)) ([315baf4](https://github.com/apify/apify-test-tools/commit/315baf495dae984cbcd7056a101935536d7b0011)) by [@metalwarrior665](https://github.com/metalwarrior665)
+- **cli-test:** Warn instead of failing on Actors the tester token can't see ([#237](https://github.com/apify/apify-test-tools/pull/237)) ([1d339ab](https://github.com/apify/apify-test-tools/commit/1d339ab04d3190191cffa3766e86a7c74c309985)) by [@metalwarrior665](https://github.com/metalwarrior665)
+- **run-with-apify-token:** Support all config modes ([#219](https://github.com/apify/apify-test-tools/pull/219)) ([4b9e95b](https://github.com/apify/apify-test-tools/commit/4b9e95b8328144eb77805ddd7ae1176ee14b84dc)) by [@JuanGalilea](https://github.com/JuanGalilea)
+
+### 🚜 Refactor
+
+- [**breaking**] Remove build-for-test and release composite commands ([#203](https://github.com/apify/apify-test-tools/pull/203)) ([85efc54](https://github.com/apify/apify-test-tools/commit/85efc54697e98208c827c4f450906e00eebcd9ca)) by [@metalwarrior665](https://github.com/metalwarrior665)
+- [**breaking**] Remove legacy get-commits, get-latest-commit and get-changed-files ([#228](https://github.com/apify/apify-test-tools/pull/228)) ([d254a6d](https://github.com/apify/apify-test-tools/commit/d254a6dd14ac9cb027a26ec73ca20a6ee10460ec)) by [@metalwarrior665](https://github.com/metalwarrior665)
 
 
 <!-- git-cliff-unreleased-end -->

package/CONTRIBUTING.md

@@ -11,8 +11,10 @@ The package consists of three parts:
 - `bin/build.ts` actor building
 - `bin/git.ts` git wrappers for getting relevant git commits and changed files
 - `bin/main.ts` entrypoing
-- `bin/slack.ts` sending notifications to slack
-- `bin/test-report.ts` processing vitest's test reports
+- `bin/run-tests.ts` running the repository's vitest for the `test` command
+- `bin/test-report.ts` turning vitest's JSON report into the report `test` prints
+- `bin/notify.ts` the `notify` command: what it says about test runs and releases, and its targets (Slack for now)
+- `bin/slack.ts` the Slack client
 
 CLI command results use `writeJson` from `bin/output.ts` to write newline-terminated JSON to stdout.
 Use the CLI `logger` from `bin/logger.ts` for diagnostics; every level writes to stderr.
@@ -53,9 +55,11 @@ To work on the library, run the cli from the root of your testing repo:
 ```sh
 cd /path/to/testing-repo-for-github-actions
 npx tsx /path/to/apify-test-tools/bin/main.ts --help
-npx tsx /path/to/apify-test-tools/bin/main.ts get-commits --target-branch master --source-branch feat/testing-feature-branch
+npx tsx /path/to/apify-test-tools/bin/main.ts get-affected-actors --base master
 ```
 
+Or clone a test repo into `local-clone/` inside this repo and run `npm run start:dev -- <command>`.
+
 #### Working on the library
 
 You need to istall the local version of `apify-test-tools` in your cloned `testing-repo-for-github-actions`:
@@ -69,7 +73,7 @@ You need to run `npm run build` inside `apify-test-tools` repo everytime you wan
 ## Reusable workflows
 
 The `public_`-prefixed workflows are the ones consumer repos call: `public_pr-build-test`,
-`public_platform-tests`, `public_push-build-latest`, `public_claude`, `public_review` and
+`public_platform-tests`, `public_push-build-latest`, `public_claude`, `public_review`, `public_stale-issues` and
 `public_platform-tests-claude-investigate-and-fix`. They live here because they call this package's
 CLI, so a change to both is one PR. GitHub only reads workflow files at the top level of
 `.github/workflows`, so they sit next to this repo's own CI, and the prefix is what separates the
@@ -82,26 +86,122 @@ renaming one is a breaking change that needs a major tag bump, not a tidy-up.
 The tag is not semver, so `git-cliff` would abort on it. It is skipped only because
 `apify/actions`' cliff.toml sets `tag_pattern = "v[0-9]+\."` — a release failing with
 `Semver error: unexpected character` means that pattern changed, not that anything here is wrong.
-The action's "Nothing to release" guard resolves to this tag and no longer fires, so an empty stable
+The action's "Nothing to release" guard resolves to this tag and never fires, so an empty stable
 release fails at `npm publish` instead of erroring cleanly.
 
-Consumers pin `@workflows-v0`, not `@master`. See
-[Versioning and releases](./README.md#versioning-and-releases) in the README for how the tag and the
-npm release relate — the short version:
+Consumers pin `@workflows-v0`, not `@master`. See [Versioning and releases](#versioning-and-releases) below.
 
-- the workflows and the CLI ship together on every master push, through the tag; no npm release needed
-- the lib ships through a stable npm release
-- the CLI runs from `.github/cli/apify-test-tools.mjs`, a bundle the pre-commit hook rebuilds with
-  `npm run bundle`. If you commit with `--no-verify`, run it yourself: the `CLI bundle` check fails
-  on a stale bundle, and a stale bundle would hold the tag.
-- `npm test` covers the bundle without network (`test/bundle/`). `npm run test:e2e` runs it against two
+- `npm test` covers the bundle without network (`test/bundle/`). `npm run test:e2e` runs it against
   dummy Actors in an isolated sandbox account (`test/e2e/`) and skips itself unless
   `E2E_TEST_APIFY_TOKEN_DBYPODOIHUZABNQTR` is set. CI passes that secret only to the `E2E tests` job.
+  The git `build` and release pipeline tests clone
+  [testing-repo-for-github-actions](https://github.com/apify-store/testing-repo-for-github-actions) and
+  use its `e2e/base` and `e2e/feature` branches as fixtures, so never rebase or delete those.
 
 `npm run lint` and `actionlint` (via the `Code checks` workflow) both gate master, so run them before
 pushing workflow changes.
 
+The reusable workflows load the composite action with `uses: $/.github/actions/checkout-restore-dependencies`, never
+`apify/apify-test-tools/...@workflows-v0`. `$/` is the
+[self-repository syntax](https://github.blog/changelog/2026-07-30-reference-same-repository-actions-with-self-repository-syntax/):
+it resolves to this repo at the commit the workflow was called at. A hardcoded tag would hand a caller pinned to an
+older SHA, or to a branch, the action and CLI from the tag instead of from the commit they asked for.
+
 `public_review` is the odd one out: it fetches `.github/review-prompt.md` over HTTP at run time, because a
-reusable workflow runs with the caller's repo checked out and never gets its own. Its `prompt-ref`
-input defaults to `workflows-v0` so the instructions come from the same release as the workflow — leaving it
-at `master` would run released workflows against unreleased instructions.
+reusable workflow runs with the caller's repo checked out and never gets its own. With `prompt-ref` left empty it
+fetches from `job.workflow_repository` at `job.workflow_sha`, the commit the workflow itself came from, so the
+instructions always match the workflow.
+
+## Versioning and releases
+
+The workflows and the npm package live in one repo but ship on their own schedules:
+
+| What                                         | Consumers get it through                   | Ships when              |
+| -------------------------------------------- | ------------------------------------------ | ----------------------- |
+| the workflows and the CLI they call          | the `@workflows-v0` tag in `uses:`         | every master push       |
+| the lib (`testActor`, `describe`, matchers…) | `apify-test-tools` in their `package.json` | a stable release on npm |
+
+The CLI is bundled with its dependencies into `.github/cli/apify-test-tools.mjs` (`npm run bundle`, run by the
+pre-commit hook and checked in CI), and the setup action runs it from there. Nothing is installed for it, so a tag is
+a complete statement: these workflows and this CLI, with the exact dependency versions from `package-lock.json` at
+that commit. The consumer's own `node_modules` is left alone, so their tests keep the lib version they locked.
+
+- **Workflow or CLI change** — merge it. `workflows-v0` moves once checks and tests pass, and it goes live. No
+  release needed, even when a workflow starts using a new CLI feature: both are in the same commit.
+- **Lib change** — merge it, then cut a release when you want it out. Consumers pick it up when they update
+  `apify-test-tools`.
+
+The CLI and the lib in one job can be different versions now, so the formats between them are a contract: the
+`ACTOR_BUILDS` JSON that `build` writes and the lib reads, and the vitest JSON report (with the run each test started on its `meta`) that `test` reads. Add
+fields freely; renaming or removing one needs a new major tag.
+
+`workflows-v0` moving on every master push means `@workflows-v0` is as live as `@master` was — there's no staging
+step. What the tag buys you is a `workflows-v1` for breaking workflow changes, so repos migrate one at a time instead
+of all at once, and a way to roll back by pointing the tag at an earlier commit (**Move major version tag**
+workflow). Bump `MAJOR_TAG` in `.github/workflows/_move_major_tag.yaml` to cut the next major; the old tag then
+freezes where it is and keeps working, CLI included, since the bundle is part of the frozen commit.
+
+The tag tracks the **workflows'** contract, not the npm package version. They move independently on purpose, so
+`@workflows-v0` is expected to stay `@workflows-v0` after the package reaches 1.0 — bump it when a workflow breaks
+its callers, not when the package does. The usage examples in the workflows, README and docs repeat the tag
+literally, so update them by hand when you bump it.
+
+## Testing workflow changes
+
+- Point [testing-repo-for-github-actions](https://github.com/apify-store/testing-repo-for-github-actions) at your
+  branch (`uses: ...@your-branch`). It has real attached Actors and tests. The workflows load the composite action
+  with `$/` and the branch carries its own CLI bundle, so one branch tests the workflows, the action and the CLI
+  together, with nothing to repoint.
+- Make sure the shell code actually works on your laptop first.
+- After merging, watch the workflow on a real project before moving on.
+
+## How the workflows scope secrets
+
+The public side of this is the [Secrets](./docs/workflows.md#secrets) table.
+
+The Actor tokens are the one set that cannot be listed in the workflow, because each Actor names its own token via
+`tokenEnvVar` in `apify-test-tools.config.json`, and a reusable workflow only accepts secrets it declares by name.
+Callers pack them into the `CONFIG_SECRETS` secret. Those steps pass `${{ toJSON(secrets) }}` as `WORKFLOW_SECRETS`
+and run the command through `.github/scripts/run-with-apify-tokens.mjs`, which reads that same config file to decide
+which secrets to pass on. It looks each one up in `CONFIG_SECRETS` first, then among the secrets `secrets: inherit`
+brought:
+
+```yaml
+- name: Build
+  shell: bash
+  env:
+      WORKFLOW_SECRETS: ${{ toJSON(secrets) }}
+  run: |
+      jq '{ actors: (.functionallyChangedActors + .testChangedActors) }' affected.json \
+        | node "${{ steps.setup.outputs.scripts-path }}/run-with-apify-tokens.mjs" \
+          apify-test-tools build --version-number 0.99
+```
+
+The wrapper passes only the tokens the config declares and drops `WORKFLOW_SECRETS`, so neither the CLI nor anything
+under `node_modules` sees the blob. Nothing is written to `$GITHUB_ENV`, so the tokens stay inside that one command
+rather than leaking into later steps. Reading the same file `apify-test-tools` reads means the two can't drift, and a
+secret that merely looks like an Actor token is not passed just because of its name.
+
+A token the config declares but the repo hasn't set is a warning, not a failure: a repo can carry an Actor whose
+token isn't configured and still build fine as long as that Actor never changes, and `apify-test-tools` raises a
+precise error naming the Actor at the point it actually needs the token.
+
+The runner masks a secret by its exact value, so it only knows `CONFIG_SECRETS` as a whole blob. The setup action
+runs `.github/scripts/mask-config-secrets.sh` first, which registers each value inside it with `::add-mask::`, so a
+token taken out of the blob is masked too.
+
+`scripts-path` comes from the setup action (give the step `id: setup`) and points at this repo's `.github/scripts/`
+directory inside the runner's action checkout, so workflows can run these helpers without checking this repo out
+again. The caller's workspace holds the caller's repo, not this one.
+
+Two tidier-looking alternatives don't work, so don't reach for them:
+
+- **Exporting to `$GITHUB_ENV`** would let the steps call the CLI directly with no wrapper, but `$GITHUB_ENV`
+  applies to every later step in the job. In `pr-build-test` the test step runs after the build, so it would
+  inherit Actor tokens it has no use for.
+- **Returning the tokens as a step output** would be scoped correctly, but the runner refuses to set an output whose
+  value contains a registered secret. It logs `Skip output <name> since it may contain secret` and leaves the output
+  empty, so anything reading it downstream gets nothing.
+
+The `unitTest` job runs static checks and needs `NPM_TOKEN` only. No job runs `npm ci` or `pnpm install` with Apify
+or Slack credentials in scope, so a postinstall script in the dependency tree cannot read them.