apify-test-tools 0.10.2-beta.13 → 0.10.2-beta.14

package/docs/cli.md

@@ -10,15 +10,15 @@ npx apify-test-tools <command> [flags]
 Run every command from the root of the Actor repository; it reads `apify-test-tools.config.json` from there.
 Results are written to stdout as JSON, diagnostics to stderr, so commands can be piped into each other and into `jq`.
 
-| Command                                                               | Does                                                        | Needs                      |
-| --------------------------------------------------------------------- | ----------------------------------------------------------- | -------------------------- |
-| [`get-affected-actors`](#finding-affected-actors-get-affected-actors) | Lists the Actors changed in a commit range                  | git only                   |
-| [`build`](#building-actors-build)                                     | Builds the Actors given on stdin from the pushed branch     | Actor tokens               |
-| [`build-from-local`](#build-from-local-source-build-from-local)       | Builds Actors from local files                              | Actor tokens               |
-| [`notify-release`](#announcing-a-release-notify-release)              | Announces a release on Slack                                | `SLACK_TOKEN_RELEASES_BOT` |
-| [`report-tests`](#reporting-test-results-report-tests)                | Summarizes a vitest JSON report and posts failures to Slack | `SLACK_TOKEN_TESTS_BOT`    |
-| [`delete-old-builds`](#deleting-old-builds-delete-old-builds)         | Deletes old builds of the Actors                            | Actor tokens               |
-| [`get-actor-configs`](#listing-actors-get-actor-configs)              | Prints the resolved config                                  | nothing                    |
+| Command                                                               | Does                                                    | Needs                |
+| --------------------------------------------------------------------- | ------------------------------------------------------- | -------------------- |
+| [`get-affected-actors`](#finding-affected-actors-get-affected-actors) | Lists the Actors changed in a commit range              | git only             |
+| [`build`](#building-actors-build)                                     | Builds the Actors given on stdin from the pushed branch | Actor tokens         |
+| [`build-from-local`](#build-from-local-source-build-from-local)       | Builds Actors from local files                          | Actor tokens         |
+| [`test`](#running-tests-test)                                         | Runs the platform tests on the builds given on stdin    | `TESTER_APIFY_TOKEN` |
+| [`notify`](#notifying-notify)                                         | Posts a test report or a release to Slack               | a Slack token        |
+| [`delete-old-builds`](#deleting-old-builds-delete-old-builds)         | Deletes old builds of the Actors                        | Actor tokens         |
+| [`get-actor-configs`](#listing-actors-get-actor-configs)              | Prints the resolved config                              | nothing              |
 
 "Actor tokens" are the environment variables named by each Actor's `tokenEnvVar` in the config. A command only needs
 the tokens of the Actors it touches.
@@ -37,8 +37,8 @@ name is an error rather than silently selecting nothing.
 
 ## Dry runs (`--dry-run`)
 
-Every command that changes something (`build`, `build-from-local`, `notify-release`, `delete-old-builds`,
-`report-tests`) takes `--dry-run`. A dry run:
+Every command that changes something (`build`, `build-from-local`, `notify`, `delete-old-builds`) takes
+`--dry-run`. A dry run:
 
 - **Changes nothing.** No versions or builds are created, nothing is uploaded or deleted, and no Slack message is
   posted.
@@ -67,9 +67,8 @@ changed files. For each changed file, for each Actor:
 4. **Hardcoded ignore list, context-aware** — the file path is first "hoisted" relative to the context path it
    matched (e.g. a standalone Actor's own `.eslintrc` is checked as just `.eslintrc`, not the full
    repo-root-relative path), then checked against repo-level dev file patterns (`.vscode/`, `.gitignore`, `.husky/`,
-   `.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`). There's no hardcoded special-casing for legacy
-   `code/`/`shared/` layouts — repos that need those directories treated as top-level must list them explicitly in
-   `overrideActorContext`.
+   `.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`). Directories like `code/` or `shared/`
+   get no special treatment: to treat them as top-level, list them in `overrideActorContext`.
 5. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the Actor's `dockerContextDir`,
    matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior.
    Test files are the exception: `get-affected-actors --test-files-glob <glob>` (relative to the repo root) skips
@@ -238,9 +237,11 @@ npx apify-test-tools get-affected-actors --since <sha> > affected.json
 jq '{ actors: (.functionallyChangedActors + .cosmeticallyChangedActors) }' affected.json \
   | npx apify-test-tools build --version-number default --dry-run
 
-npx apify-test-tools notify-release --release-slack-channel "#releases" --dry-run < affected.json
+npx apify-test-tools notify --slack-channel "#notif-my-repo" --dry-run < affected.json
 ```
 
+[`notify`](#notifying-notify) says what it posts for a release.
+
 ### The last released commit (`--since`)
 
 The last commit that is **already released**. Everything after it, up to `HEAD`, is released. Git alone can't tell
@@ -273,38 +274,123 @@ Actors are reported; it never forces a build, so on its own it doesn't get past
 A failed release isn't retried by the next push: that push's base commit is the failed one's head, so its changes
 are skipped. Re-run the failed workflow run, or release the Actors manually.
 
-## Announcing a release (`notify-release`)
+## Running tests (`test`)
+
+`test` runs the repository's own vitest (the version in its `node_modules`, so the tests keep the lib version
+the repository locked) on the Actors it is given on stdin, and prints a report of the run for
+[`notify`](#notifying-notify). Only the tests of those Actors run.
+
+```bash
+# A PR: test what was just built
+... | npx apify-test-tools build --version-number 0.99 | npx apify-test-tools test > test-report.json
+
+# Scheduled tests: every Actor in the config, on its default build
+npx apify-test-tools get-actor-configs | npx apify-test-tools test test/platform/core > test-report.json
+
+# Only some of them
+npx apify-test-tools get-actor-configs --actors john.doe/my-actor | npx apify-test-tools test > test-report.json
+```
+
+The input is a JSON array of Actors, each given by its full name, its raw ID or both:
+`[{ "actorFullName": "<owner/name>" }]` or `[{ "actorRawId": "<id>" }]`. An Actor's tests run against the build given
+with it (`buildId` and `buildNumber`, as `build` and `build-from-local` print them), or against its default build
+without one. Other fields are ignored, so the output of [`get-actor-configs`](#listing-actors-get-actor-configs) works
+as it is. `test` looks up the ID an Actor is listed without with `TESTER_APIFY_TOKEN`, so its tests run whichever ID
+they name it by. It fails if the token can't see the Actor. A dry-run plan, empty or invalid input is an
+error, as with `build`. So is a missing `TESTER_APIFY_TOKEN`: without it the lib skips every platform test, and the
+run would pass having tested nothing. `test` passes the Actors to vitest as `ACTOR_BUILDS`, in place of any
+`ACTOR_BUILDS` in its own environment.
+
+It runs vitest with `--maxConcurrency=20 --fileParallelism --maxWorkers=100`, since platform tests mostly wait on
+Actor runs rather than the CPU, and with the default reporter plus the JSON one the report is read from. These
+replace any `reporters` in the repository's vitest config; add more with `--reporter`, e.g. `github-actions` to
+annotate the failing tests in a GitHub Actions run. It takes no other vitest options; for those, e.g. `-t` to run one
+test by name, run vitest directly with the builds in `ACTOR_BUILDS` (see
+[Running tests locally](lib.md#running-tests-locally)).
+
+| Argument / flag        | Default         | Meaning                                                                                                                         |
+| ---------------------- | --------------- | ------------------------------------------------------------------------------------------------------------------------------- |
+| `[filters..]`          | `test/platform` | Test files to run, as vitest filters                                                                                            |
+| `--vitest-output-file` |                 | Also keep vitest's own JSON report there, e.g. to upload it                                                                     |
+| `--reporter`           |                 | [Vitest reporter](https://vitest.dev/guide/reporters) to add to `default` and `json`, e.g. `github-actions`. Repeat it for more |
+
+Vitest's own output goes to stderr, followed by a summary that lists every test with the link to its run. Stdout
+gets only the report:
 
-`notify-release` reads the output of `get-affected-actors` on stdin. It posts the changelog additions to
-`--release-slack-channel` and the commits and changed files to `--report-slack-channel`, and posts nothing when there
-are no new commits. It names the release after the `origin` remote (or `--repo-url`) and the author of the newest
-commit, and needs `SLACK_TOKEN_RELEASES_BOT`.
+```json
+{
+    "kind": "test-report",
+    "version": 1,
+    "success": false,
+    "summary": { "total": 3, "passed": 1, "failed": 1, "skipped": 1 },
+    "tests": [
+        {
+            "name": "john.doe/my-actor: finishes",
+            "file": "test/platform/my-actor.test.ts",
+            "status": "failed",
+            "durationMs": 61234,
+            "failureMessages": ["AssertionError: expected 'FAILED' to be 'SUCCEEDED'"],
+            "run": { "actorId": "john.doe/my-actor", "runId": "…", "runLink": "https://console.apify.com/view/runs/…" }
+        }
+    ],
+    "errors": ["test/platform/other.test.ts: Cannot find package 'foo'"]
+}
+```
 
-| Flag                      | Meaning                                                |
-| ------------------------- | ------------------------------------------------------ |
-| `--release-slack-channel` | Channel for the changelog additions only               |
-| `--report-slack-channel`  | Channel for the full report: commits and changed files |
-| `--repo-url`              | Repository to name the release after                   |
-| `--dry-run`               | See [Dry runs](#dry-runs---dry-run)                    |
+`failureMessages` keeps the first line of each failure; diffs and stack traces are in the vitest output. `run` is
+`null` for a test that started no run. `errors` lists what failed outside of any test: a test file that failed to
+load, or vitest exiting without a report. `test` exits with 1 whenever `success` is false, and prints the report
+anyway, so `notify` still hears about the failure.
 
-## Reporting test results (`report-tests`)
+## Notifying (`notify`)
 
-Reads a vitest JSON report (`--reporter=json --outputFile=<file>`), logs the passed and failed tests with their run
-links, and posts the failures to Slack. Without `--report-slack-channel` it only logs; with it,
-`SLACK_TOKEN_TESTS_BOT` must be set, even when no test failed.
+`notify` tells one target about what it reads on stdin: a test run, as [`test`](#running-tests-test) reports it,
+or a release, as [`get-affected-actors`](#finding-affected-actors-get-affected-actors) reports it. It tells the two
+apart by the report's `kind` and the release's `commits`, so either can be piped in as it is. Slack is the only target for now. To post to more
+places, call it once for each.
 
 ```bash
-npx vitest --run --reporter=default --reporter=json --outputFile=./test-output.json test/platform
-npx apify-test-tools report-tests --report-file ./test-output.json --report-slack-channel "#my-channel"
+# A test run: posts only when the tests failed, the failure counts and the first failure, linked to its run,
+# with the other failures in the message's thread
+npx apify-test-tools notify --slack-channel '#notif-my-repo' < test-report.json
+npx apify-test-tools notify --slack-channel '#my-team' < test-report.json
+
+# Or straight from test. With `set -o pipefail` (`shell: bash` in GitHub Actions), failed tests still fail the pipe
+npx apify-test-tools get-actor-configs | npx apify-test-tools test | npx apify-test-tools notify --slack-channel '#notif-my-repo'
+
+# A release, see Releasing Actors
+npx apify-test-tools get-affected-actors --since <sha> | npx apify-test-tools notify --slack-channel '#notif-my-repo'
 ```
 
-| Flag                     | Default  | Meaning                                       |
-| ------------------------ | -------- | --------------------------------------------- |
-| `--report-file`          | required | Path to the vitest JSON report                |
-| `--report-slack-channel` |          | Channel to post the failures to               |
-| `--job-url`              |          | Link to the CI job, included in the message   |
-| `--workflow-name`        |          | Name of the workflow, included in the message |
-| `--dry-run`              | `false`  | See [Dry runs](#dry-runs---dry-run)           |
+| Flag              | Input   | Default             | Meaning                                                           |
+| ----------------- | ------- | ------------------- | ----------------------------------------------------------------- |
+| `--slack-channel` | both    |                     | Slack channel to post to                                          |
+| `--on`            | tests   | `failure`           | `failure` posts only when the tests failed, `always` on every run |
+| `--title`         | tests   | `-`                 | What the message calls the run, e.g. the CI workflow name         |
+| `--job-url`       | tests   |                     | Link to the CI job, included in the message                       |
+| `--repo-url`      | release | the `origin` remote | The repository the release is named after and its commits link to |
+
+A flag the input has no use for, e.g. `--repo-url` with a test report, is an error rather than ignored. Both kinds
+of notification post with `SLACK_BOT_TOKEN`.
+
+Without a target, `notify` posts nothing and needs no token. With one, it checks the token on every run, also when
+there is nothing to post, so a broken token shows up on a green run rather than on the day tests fail. A release
+with no new commits is the exception: there is nothing to announce, so it needs no token either.
+
+For a release, `notify` posts the changelog additions, then the commits, oldest first. Each commit is its title (cut
+at 200 characters) linked to its page on GitHub, and its author. A merge commit or a squash merge is one commit; a
+rebase merge lists every commit it brought. A repository hosted elsewhere gets the titles without links. The release
+is named after the `origin` remote (or `--repo-url`) and the author of the newest commit. Nothing is posted when there
+are no new commits:
+
+```text
+*my-repo* – New release (by Jane Doe):
+
+- Fixed pagination on the search page
+
+*Commits*:
+1. <https://github.com/my-org/my-repo/commit/1a2b3c…|fix: pagination on the search page> – Jane Doe
+```
 
 ## Deleting old builds (`delete-old-builds`)
 
@@ -332,8 +418,8 @@ To test changed Actor source, you need builds of it. The quickest way is
 [`build-from-local`](#build-from-local-source-build-from-local), which needs no push:
 
 ```bash
-BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
-    npx apify-test-tools build-from-local --actors john.doe/my-actor)
+APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
+    npx apify-test-tools build-from-local --actors john.doe/my-actor > builds.json
 ```
 
 To build from git, like CI does:
@@ -345,9 +431,9 @@ To build from git, like CI does:
 5. Run tests against those builds. You can change tests and run on the same builds.
 
 ```bash
-BUILDS=$(npx apify-test-tools get-affected-actors --base origin/master --test-files-glob 'test/platform/**' \
+npx apify-test-tools get-affected-actors --base origin/master --test-files-glob 'test/platform/**' \
     | jq '{ actors: .functionallyChangedActors }' \
-    | APIFY_TOKEN_JOHN_DOE=apify_api_xxx npx apify-test-tools build --version-number 0.99)
+    | APIFY_TOKEN_JOHN_DOE=apify_api_xxx npx apify-test-tools build --version-number 0.99 > builds.json
 ```
 
 Run it with your dummy branch checked out: it builds the Actors that branch changed functionally, from the branch as
@@ -358,16 +444,8 @@ Actors in `ACTOR_BUILDS`, so pass `--test-files-glob` to count test changes even
 If you want to test against existing builds instead, skip the build and write the build array by hand, only for the
 Actors you need to test.
 
-Then run the tests against the builds:
+Then run the tests against the builds with [`test`](#running-tests-test), as often as you like:
 
 ```bash
-ACTOR_BUILDS="$BUILDS" \
-TESTER_APIFY_TOKEN=apify_api_yyy \
-  npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform
+TESTER_APIFY_TOKEN=apify_api_yyy npx apify-test-tools test < builds.json
 ```
-
-## Legacy git commands
-
-`get-commits`, `get-latest-commit` and `get-changed-files` print the commits, the newest commit, or the changed files
-on `--source-branch` since it diverged from `--target-branch` (optionally only after `--base-commit`). They predate
-`get-affected-actors`, which reports the same data, and are kept for older workflows.

package/docs/lib.md

@@ -10,11 +10,10 @@ Requires `vitest` `3.2.4` or later as a peer dependency.
 
 The library reads these when the test file is loaded:
 
-| Variable                 | Description                                                                                                                                                                                                                                   |
-| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `TESTER_APIFY_TOKEN`     | Token the tests start runs and read their data with. Without it (and without `RUN_ALL_PLATFORM_TESTS`), every `describe` block is skipped, so a plain `npm test` stays unaffected.                                                            |
-| `ACTOR_BUILDS`           | JSON array of builds, as printed by the CLI's [`build`](cli.md#building-actors-build) and [`build-from-local`](cli.md#build-from-local-source-build-from-local). Only tests of Actors listed here run, and they run against the listed build. |
-| `RUN_ALL_PLATFORM_TESTS` | Run every test, not only those of Actors in `ACTOR_BUILDS`. Actors missing from `ACTOR_BUILDS` run their default build. The scheduled platform tests set it.                                                                                  |
+| Variable             | Description                                                                                                                                                                                                                                                                                            |
+| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `TESTER_APIFY_TOKEN` | Token the tests start runs and read their data with. Without it, every `describe` block is skipped, so a plain `npm test` stays unaffected.                                                                                                                                                            |
+| `ACTOR_BUILDS`       | JSON array of Actors to test, e.g. the builds the CLI's [`build`](cli.md#building-actors-build) and [`build-from-local`](cli.md#build-from-local-source-build-from-local) print. Only tests of Actors listed here run, against the listed build, or the default build for an Actor listed without one. |
 
 `ACTOR_BUILDS` entries look like this, and are matched to tests by `actorFullName` or `actorRawId`:
 
@@ -24,13 +23,13 @@ The library reads these when the test file is loaded:
 
 ## `describe(name, fn, options?)`
 
-Same as vitest's `describe`, but skipped unless `TESTER_APIFY_TOKEN` or `RUN_ALL_PLATFORM_TESTS` is set. Its tests
+Same as vitest's `describe`, but skipped unless `TESTER_APIFY_TOKEN` is set. Its tests
 run concurrently by default. Use it instead of vitest's `describe` for platform tests.
 
 ## `testActor(actorId, testName, fn, options?)`
 
 Registers a test for an Actor. `actorId` is the Actor's full name (`owner/name`) or its raw platform ID. The test
-only runs when the Actor is in `ACTOR_BUILDS` or `RUN_ALL_PLATFORM_TESTS` is set. Its name is
+only runs when the Actor is in `ACTOR_BUILDS`. Its name is
 `<actorId>: <testName>`.
 
 `fn` gets the vitest test context, with `expect` extended by the [custom matchers](#custom-matchers), and a `run`
@@ -247,25 +246,31 @@ export const validateItem = (expect: ExpectStatic, item: any) => {
 
 Run every command from the root of the Actor repository.
 
-**Against the current default builds.** Enough when you only change the tests:
+**Against the current default builds.** Enough when you only change the tests. List the Actors with
+[`get-actor-configs`](cli.md#listing-actors-get-actor-configs), all of them or a few with `--actors`, and pass them to
+[`test`](cli.md#running-tests-test), which tests each on its default build:
 
 ```bash
-RUN_ALL_PLATFORM_TESTS=1 TESTER_APIFY_TOKEN=<token> npx vitest --run test/platform
+npx apify-test-tools get-actor-configs --actors john.doe/my-actor | TESTER_APIFY_TOKEN=<token> npx apify-test-tools test
 ```
 
+This also runs a test that reuses an existing run with [`runId`](#testing-an-existing-run), as long as its Actor is
+listed.
+
 **Against builds of your changes.** Build the Actors with the CLI, either from local files with
 [`build-from-local`](cli.md#build-from-local-source-build-from-local) or from a pushed branch with
-[`build`](cli.md#building-actors-build), and pass the output as `ACTOR_BUILDS`:
+[`build`](cli.md#building-actors-build), and pass the output to [`test`](cli.md#running-tests-test):
 
 ```bash
-BUILDS=$(APIFY_TOKEN_JOHN_DOE=<token> npx apify-test-tools build-from-local --actors john.doe/my-actor)
+APIFY_TOKEN_JOHN_DOE=<token> npx apify-test-tools build-from-local --actors john.doe/my-actor > builds.json
 
-ACTOR_BUILDS="$BUILDS" \
-TESTER_APIFY_TOKEN=<token> \
-  npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform
+TESTER_APIFY_TOKEN=<token> npx apify-test-tools test < builds.json
 ```
 
+`test` passes the builds to vitest as `ACTOR_BUILDS`, so plain vitest works too:
+`ACTOR_BUILDS="$(cat builds.json)" TESTER_APIFY_TOKEN=<token> npx vitest run test/platform`.
+
 You can change the tests and re-run them against the same builds.
 
-`TESTER_APIFY_TOKEN` can be the token of any account with enough memory for the runs. To run only some tests, point vitest at a specific file instead of `test/platform`, or filter
-by name with `-t`. See [Running locally](cli.md#running-locally) for the full flow.
+`TESTER_APIFY_TOKEN` can be the token of any account with enough memory for the runs. To run only some tests, pass a specific file instead of `test/platform`, or filter by
+name with vitest's `-t <name>` when running vitest directly. See [Running locally](cli.md#running-locally) for the full flow.

package/docs/workflows.md

@@ -42,8 +42,8 @@ Validates a pull request. It runs two jobs in parallel.
    `.dockerignore` lists them.
 2. Builds the functionally changed Actors into the test version `0.99` with [`build`](cli.md#building-actors-build).
    Cosmetic-only changes (README, CHANGELOG, schema formatting) need no tests, so they're not built.
-3. Runs `npx vitest ./test/platform/<test-files-glob>` with those builds in `ACTOR_BUILDS`, so only the tests of
-   the built Actors run.
+3. Runs [`test`](cli.md#running-tests-test) on `./test/platform/<test-files-glob>` with those builds, so only the
+   tests of the built Actors run. Failing tests are annotated on the PR.
 4. On success, caches the tested commit as the "last validated commit". The next push to the PR then only builds
    and tests the Actors that changed since it (see `--since` in
    [`get-affected-actors`](cli.md#finding-affected-actors-get-affected-actors)).
@@ -101,8 +101,8 @@ Releases the Actors a push changed. Skipped when the head commit message contain
    [Releasing Actors](cli.md#releasing-actors) for how the released range is determined.
 2. `build --version-number default` builds both the functionally and the cosmetically changed Actors into their
    default version, the one users run.
-3. `notify-release` posts the changelog additions to `#delivery-public-actors`, and the commits and changed files to
-   the report channel. It posts nothing when the push brought no new commits.
+3. [`notify`](cli.md#notifying-notify) posts the release (its changelog additions and its commits, linked to
+   GitHub) to the report channel. It posts nothing when the push brought no new commits.
 4. `delete-old-builds` deletes old builds of every Actor in the config (see
    [`delete-old-builds`](cli.md#deleting-old-builds-delete-old-builds)).
 
@@ -110,7 +110,7 @@ Releases the Actors a push changed. Skipped when the head commit message contain
 | ---------------------- | ------ | ------- | ------------------------------------------------------------- |
 | `report-slack-channel` | string | `''`    | Slack channel for the report. Empty uses `#notif-<repo-name>` |
 
-Secrets: `SLACK_TOKEN_RELEASES_BOT`, the Actor tokens in `CONFIG_SECRETS`, and optionally `NPM_TOKEN`.
+Secrets: `SLACK_BOT_TOKEN`, the Actor tokens in `CONFIG_SECRETS`, and optionally `NPM_TOKEN`.
 
 A failed release isn't retried by the next push: that push's "before" commit is the failed one's head, so its changes
 are skipped. Re-run the failed workflow run, or release the Actors manually.
@@ -126,7 +126,7 @@ jobs:
     buildLatest:
         uses: apify/apify-test-tools/.github/workflows/public_push-build-latest.yaml@workflows-v0
         secrets:
-            SLACK_TOKEN_RELEASES_BOT: ${{ secrets.SLACK_TOKEN_RELEASES_BOT }}
+            SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
             # One entry per secret apify-test-tools.config.json names, today each `tokenEnvVar`
             CONFIG_SECRETS: |
                 {
@@ -137,10 +137,11 @@ jobs:
 
 ## `public_platform-tests.yaml`
 
-Runs the platform tests against the Actors' current default builds, with `RUN_ALL_PLATFORM_TESTS=1` so every test
-runs, not only those of built Actors. It then reports the results to Slack with
-[`report-tests`](cli.md#reporting-test-results-report-tests) and uploads the vitest JSON report as the
-`vitest-results` artifact. Runs only on `master` or `main`.
+Runs the platform tests of every Actor in the config against its current default build, by piping
+[`get-actor-configs`](cli.md#listing-actors-get-actor-configs) into [`test`](cli.md#running-tests-test). A separate step then posts failures to Slack with
+[`notify`](cli.md#notifying-notify), named after the calling workflow and linking to the run, so the Slack token never
+reaches the tests. Both the report of `test` and
+vitest's own JSON report are uploaded as the `vitest-results` artifact. Runs only on `master` or `main`.
 
 | Input                  | Type   | Default | Description                                                                  |
 | ---------------------- | ------ | ------- | ---------------------------------------------------------------------------- |
@@ -148,7 +149,7 @@ runs, not only those of built Actors. It then reports the results to Slack with
 | `subtest`              | string | `''`    | Deprecated, use `test-files-glob`                                            |
 | `report-slack-channel` | string | `''`    | Slack channel for the results. Empty uses `#notif-<repo-name>`               |
 
-Secrets: `TESTER_APIFY_TOKEN` (required), `SLACK_TOKEN_TESTS_BOT`, and optionally `NPM_TOKEN`.
+Secrets: `TESTER_APIFY_TOKEN` (required), `SLACK_BOT_TOKEN`, and optionally `NPM_TOKEN`.
 
 How often to run which tests is up to you. For example, every test once a day:
 
@@ -166,7 +167,7 @@ jobs:
         uses: apify/apify-test-tools/.github/workflows/public_platform-tests.yaml@workflows-v0
         secrets:
             TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
-            SLACK_TOKEN_TESTS_BOT: ${{ secrets.SLACK_TOKEN_TESTS_BOT }}
+            SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 ```
 
 and a few critical tests, kept in e.g. `test/platform/hourly`, every hour:
@@ -187,7 +188,7 @@ jobs:
             test-files-glob: hourly
         secrets:
             TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
-            SLACK_TOKEN_TESTS_BOT: ${{ secrets.SLACK_TOKEN_TESTS_BOT }}
+            SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 ```
 
 ## `public_review.yaml`
@@ -286,7 +287,7 @@ jobs:
             test-files-glob: hourly
         secrets:
             TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
-            SLACK_TOKEN_TESTS_BOT: ${{ secrets.SLACK_TOKEN_TESTS_BOT }}
+            SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
     handle-failure:
         needs: platformTestsHourly
@@ -335,15 +336,15 @@ out of it.
 The workflows do not turn the secrets they receive into job-wide environment variables, so a secret is only visible
 to the step that needs it:
 
-| Secret                                                                    | Reaches                                                                                                                                                                                                                      |
-| ------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `NPM_TOKEN`                                                               | dependency install steps only, as both `NPM_TOKEN` and `NODE_AUTH_TOKEN`. Use a read-only token: npm granular tokens can be read-only, classic automation tokens can publish.                                                |
-| the Actor tokens named by `tokenEnvVar` in `apify-test-tools.config.json` | the build and `delete-old-builds` steps only                                                                                                                                                                                 |
-| `TESTER_APIFY_TOKEN`                                                      | the vitest step, and the Claude investigation step: the Apify MCP server's bearer token for reading the failing run, its log and its storages, and Claude's token for re-running it to measure how often the failure happens |
-| `SLACK_TOKEN_TESTS_BOT` / `SLACK_TOKEN_RELEASES_BOT`                      | the reporting and announcement steps only                                                                                                                                                                                    |
-| `ANTHROPIC_API_KEY`                                                       | the Claude steps of the Claude workflows only                                                                                                                                                                                |
-| `REDASH_API_KEY`                                                          | the Claude investigation step: the Redash MCP server's key for measuring the failure's impact from Snowflake. Optional                                                                                                       |
-| `AGENT_SKILLS_INTERNAL_GITHUB_TOKEN`                                      | the investigation job's checkout of `apify/agent-skills-internal` only, for the `redash-analyst` plugin. Optional                                                                                                            |
+| Secret                                                                    | Reaches                                                                                                                                                                                                                    |
+| ------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `NPM_TOKEN`                                                               | dependency install steps only, as both `NPM_TOKEN` and `NODE_AUTH_TOKEN`. Use a read-only token: npm granular tokens can be read-only, classic automation tokens can publish.                                              |
+| the Actor tokens named by `tokenEnvVar` in `apify-test-tools.config.json` | the build and `delete-old-builds` steps only                                                                                                                                                                               |
+| `TESTER_APIFY_TOKEN`                                                      | the test step, and the Claude investigation step: the Apify MCP server's bearer token for reading the failing run, its log and its storages, and Claude's token for re-running it to measure how often the failure happens |
+| `SLACK_BOT_TOKEN`                                                         | the `notify` steps only                                                                                                                                                                                                    |
+| `ANTHROPIC_API_KEY`                                                       | the Claude steps of the Claude workflows only                                                                                                                                                                              |
+| `REDASH_API_KEY`                                                          | the Claude investigation step: the Redash MCP server's key for measuring the failure's impact from Snowflake. Optional                                                                                                     |
+| `AGENT_SKILLS_INTERNAL_GITHUB_TOKEN`                                      | the investigation job's checkout of `apify/agent-skills-internal` only, for the `redash-analyst` plugin. Optional                                                                                                          |
 
 The Actor tokens reach the build only for the Actors your config declares: a secret that merely looks like an Actor
 token is not passed on. A token the config declares but the repo hasn't set is a warning, not a failure: a repo can

package/lib/lib.ts

@@ -31,7 +31,7 @@ const config = actorBuilds.reduce<Map<string, ActorBuild>>((map, cfg) => {
 
 export { ExpectStatic };
 
-const { TESTER_APIFY_TOKEN, RUN_ALL_PLATFORM_TESTS } = process.env;
+const { TESTER_APIFY_TOKEN } = process.env;
 const apifyClient = new ApifyClient({ token: TESTER_APIFY_TOKEN });
 
 const DEFAULT_TEST_OPTIONS: ActorTestOptions = {
@@ -44,16 +44,14 @@ const DEFAULT_TEST_OPTIONS: ActorTestOptions = {
 
 /**
  * Platform tests need `TESTER_APIFY_TOKEN` to talk to the platform, so without it we skip them altogether.
- *
- * `RUN_ALL_PLATFORM_TESTS` enables them too because locally we can test against a hardcoded `runId`,
- * which doesn't need the tester token.
  */
 export const describe = (name: string, fn?: SuiteFactory<object>, options: ActorTestOptions = DEFAULT_TEST_OPTIONS) => {
-    vitestDescribe.runIf(!!TESTER_APIFY_TOKEN || !!RUN_ALL_PLATFORM_TESTS)(name, options, fn);
+    vitestDescribe.runIf(!!TESTER_APIFY_TOKEN)(name, options, fn);
 };
 
 /**
- * Registers a vitest test for an Actor, which only runs when that Actor was built (or for the scheduled runs).
+ * Registers a vitest test for an Actor, which only runs when the Actor is listed in `ACTOR_BUILDS`: against the build
+ * listed with it, or its default build without one.
  * `body` gets the test context with the soft `expect` and the full test name.
  */
 const defineActorTest = (
@@ -64,10 +62,7 @@ const defineActorTest = (
 ) => {
     const options = { ...DEFAULT_TEST_OPTIONS, ...testOptions };
     const name = `${actorId}: ${testName}`;
-    // `RUN_ALL_PLATFORM_TESTS` is needed for the scheduled tests, which have no `ACTOR_BUILDS` to match the
-    // tests against - without it, every test would be filtered out as an actor we didn't build.
-    const shouldRun = !!RUN_ALL_PLATFORM_TESTS || config.has(actorId);
-    vitestTest.runIf(shouldRun)(name, options, async (context) => {
+    vitestTest.runIf(config.has(actorId))(name, options, async (context) => {
         await body({ ...context, expect: extendExpect(context.expect) }, name);
     });
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "apify-test-tools",
-  "version": "0.10.2-beta.13",
+  "version": "0.10.2-beta.14",
   "type": "module",
   "description": "TBD",
   "repository": {