apify-test-tools 0.10.2-beta.13 → 0.10.2-beta.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/cli/apify-test-tools.mjs +123 -128
- package/.github/workflows/public_platform-tests-claude-investigate-and-fix.yaml +1 -1
- package/.github/workflows/public_platform-tests.yaml +36 -22
- package/.github/workflows/public_pr-build-test.yaml +7 -6
- package/.github/workflows/public_push-build-latest.yaml +6 -4
- package/.github/workflows/public_stale-issues.yaml +51 -0
- package/CHANGELOG.md +3 -0
- package/CONTRIBUTING.md +9 -7
- package/README.md +11 -12
- package/bin/actor-ids.ts +38 -0
- package/bin/build.ts +1 -1
- package/bin/git.ts +20 -92
- package/bin/main.ts +87 -73
- package/bin/notify.ts +198 -0
- package/bin/run-tests.ts +141 -0
- package/bin/slack.ts +16 -87
- package/bin/stdin-input.ts +98 -7
- package/bin/test-report.ts +141 -140
- package/bin/types.ts +0 -9
- package/dist/bin/actor-ids.d.ts +16 -0
- package/dist/bin/actor-ids.d.ts.map +1 -0
- package/dist/bin/actor-ids.js +24 -0
- package/dist/bin/actor-ids.js.map +1 -0
- package/dist/bin/build.js +1 -1
- package/dist/bin/build.js.map +1 -1
- package/dist/bin/git.d.ts +6 -16
- package/dist/bin/git.d.ts.map +1 -1
- package/dist/bin/git.js +15 -75
- package/dist/bin/git.js.map +1 -1
- package/dist/bin/main.js +80 -51
- package/dist/bin/main.js.map +1 -1
- package/dist/bin/notify.d.ts +35 -0
- package/dist/bin/notify.d.ts.map +1 -0
- package/dist/bin/notify.js +131 -0
- package/dist/bin/notify.js.map +1 -0
- package/dist/bin/run-tests.d.ts +21 -0
- package/dist/bin/run-tests.d.ts.map +1 -0
- package/dist/bin/run-tests.js +116 -0
- package/dist/bin/run-tests.js.map +1 -0
- package/dist/bin/slack.d.ts +2 -17
- package/dist/bin/slack.d.ts.map +1 -1
- package/dist/bin/slack.js +17 -58
- package/dist/bin/slack.js.map +1 -1
- package/dist/bin/stdin-input.d.ts +71 -9
- package/dist/bin/stdin-input.d.ts.map +1 -1
- package/dist/bin/stdin-input.js +76 -6
- package/dist/bin/stdin-input.js.map +1 -1
- package/dist/bin/test-report.d.ts +69 -8
- package/dist/bin/test-report.d.ts.map +1 -1
- package/dist/bin/test-report.js +76 -67
- package/dist/bin/test-report.js.map +1 -1
- package/dist/bin/types.d.ts +0 -8
- package/dist/bin/types.d.ts.map +1 -1
- package/dist/lib/lib.d.ts +0 -3
- package/dist/lib/lib.d.ts.map +1 -1
- package/dist/lib/lib.js +5 -10
- package/dist/lib/lib.js.map +1 -1
- package/dist/test/bundle/cli-bundle.test.js +40 -48
- package/dist/test/bundle/cli-bundle.test.js.map +1 -1
- package/dist/test/e2e/cli-bundle.e2e.test.js +15 -16
- package/dist/test/e2e/cli-bundle.e2e.test.js.map +1 -1
- package/dist/test/e2e/git-build-release.e2e.test.js +2 -2
- package/dist/test/e2e/git-build-release.e2e.test.js.map +1 -1
- package/dist/test/unit/bin/actor-ids.test.d.ts +2 -0
- package/dist/test/unit/bin/actor-ids.test.d.ts.map +1 -0
- package/dist/test/unit/bin/actor-ids.test.js +32 -0
- package/dist/test/unit/bin/actor-ids.test.js.map +1 -0
- package/dist/test/unit/bin/build-cli.test.js +8 -3
- package/dist/test/unit/bin/build-cli.test.js.map +1 -1
- package/dist/test/unit/bin/diff-changes.test.js +1 -1
- package/dist/test/unit/bin/diff-changes.test.js.map +1 -1
- package/dist/test/unit/bin/git.test.js +4 -142
- package/dist/test/unit/bin/git.test.js.map +1 -1
- package/dist/test/unit/bin/notify.test.d.ts +2 -0
- package/dist/test/unit/bin/notify.test.d.ts.map +1 -0
- package/dist/test/unit/bin/notify.test.js +222 -0
- package/dist/test/unit/bin/notify.test.js.map +1 -0
- package/dist/test/unit/bin/stdin-input.test.js +51 -9
- package/dist/test/unit/bin/stdin-input.test.js.map +1 -1
- package/dist/test/unit/bin/test-cli.test.d.ts +2 -0
- package/dist/test/unit/bin/test-cli.test.d.ts.map +1 -0
- package/dist/test/unit/bin/test-cli.test.js +115 -0
- package/dist/test/unit/bin/test-cli.test.js.map +1 -0
- package/dist/test/unit/bin/test-report.test.d.ts +2 -0
- package/dist/test/unit/bin/test-report.test.d.ts.map +1 -0
- package/dist/test/unit/bin/test-report.test.js +100 -0
- package/dist/test/unit/bin/test-report.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/docs/cli.md +131 -53
- package/docs/lib.md +21 -16
- package/docs/workflows.md +63 -23
- package/lib/lib.ts +5 -10
- package/package.json +1 -1
- package/test/bundle/cli-bundle.test.ts +47 -60
- package/test/e2e/cli-bundle.e2e.test.ts +18 -25
- package/test/e2e/git-build-release.e2e.test.ts +2 -2
- package/test/unit/bin/actor-ids.test.ts +43 -0
- package/test/unit/bin/build-cli.test.ts +9 -3
- package/test/unit/bin/diff-changes.test.ts +1 -1
- package/test/unit/bin/git.test.ts +3 -197
- package/test/unit/bin/notify.test.ts +299 -0
- package/test/unit/bin/stdin-input.test.ts +68 -9
- package/test/unit/bin/test-cli.test.ts +137 -0
- package/test/unit/bin/test-report.test.ts +118 -0
- package/dist/test/unit/bin/logging.test.d.ts +0 -2
- package/dist/test/unit/bin/logging.test.d.ts.map +0 -1
- package/dist/test/unit/bin/logging.test.js +0 -37
- package/dist/test/unit/bin/logging.test.js.map +0 -1
- package/dist/test/unit/bin/slack.test.d.ts +0 -2
- package/dist/test/unit/bin/slack.test.d.ts.map +0 -1
- package/dist/test/unit/bin/slack.test.js +0 -71
- package/dist/test/unit/bin/slack.test.js.map +0 -1
- package/test/unit/bin/logging.test.ts +0 -48
- package/test/unit/bin/slack.test.ts +0 -84
package/docs/cli.md
CHANGED
|
@@ -10,15 +10,15 @@ npx apify-test-tools <command> [flags]
|
|
|
10
10
|
Run every command from the root of the Actor repository; it reads `apify-test-tools.config.json` from there.
|
|
11
11
|
Results are written to stdout as JSON, diagnostics to stderr, so commands can be piped into each other and into `jq`.
|
|
12
12
|
|
|
13
|
-
| Command | Does
|
|
14
|
-
| --------------------------------------------------------------------- |
|
|
15
|
-
| [`get-affected-actors`](#finding-affected-actors-get-affected-actors) | Lists the Actors changed in a commit range
|
|
16
|
-
| [`build`](#building-actors-build) | Builds the Actors given on stdin from the pushed branch
|
|
17
|
-
| [`build-from-local`](#build-from-local-source-build-from-local) | Builds Actors from local files
|
|
18
|
-
| [`
|
|
19
|
-
| [`
|
|
20
|
-
| [`delete-old-builds`](#deleting-old-builds-delete-old-builds) | Deletes old builds of the Actors
|
|
21
|
-
| [`get-actor-configs`](#listing-actors-get-actor-configs) | Prints the resolved config
|
|
13
|
+
| Command | Does | Needs |
|
|
14
|
+
| --------------------------------------------------------------------- | ------------------------------------------------------- | -------------------- |
|
|
15
|
+
| [`get-affected-actors`](#finding-affected-actors-get-affected-actors) | Lists the Actors changed in a commit range | git only |
|
|
16
|
+
| [`build`](#building-actors-build) | Builds the Actors given on stdin from the pushed branch | Actor tokens |
|
|
17
|
+
| [`build-from-local`](#build-from-local-source-build-from-local) | Builds Actors from local files | Actor tokens |
|
|
18
|
+
| [`test`](#running-tests-test) | Runs the platform tests on the builds given on stdin | `TESTER_APIFY_TOKEN` |
|
|
19
|
+
| [`notify`](#notifying-notify) | Posts a test report or a release to Slack | a Slack token |
|
|
20
|
+
| [`delete-old-builds`](#deleting-old-builds-delete-old-builds) | Deletes old builds of the Actors | Actor tokens |
|
|
21
|
+
| [`get-actor-configs`](#listing-actors-get-actor-configs) | Prints the resolved config | nothing |
|
|
22
22
|
|
|
23
23
|
"Actor tokens" are the environment variables named by each Actor's `tokenEnvVar` in the config. A command only needs
|
|
24
24
|
the tokens of the Actors it touches.
|
|
@@ -37,8 +37,8 @@ name is an error rather than silently selecting nothing.
|
|
|
37
37
|
|
|
38
38
|
## Dry runs (`--dry-run`)
|
|
39
39
|
|
|
40
|
-
Every command that changes something (`build`, `build-from-local`, `notify
|
|
41
|
-
|
|
40
|
+
Every command that changes something (`build`, `build-from-local`, `notify`, `delete-old-builds`) takes
|
|
41
|
+
`--dry-run`. A dry run:
|
|
42
42
|
|
|
43
43
|
- **Changes nothing.** No versions or builds are created, nothing is uploaded or deleted, and no Slack message is
|
|
44
44
|
posted.
|
|
@@ -67,9 +67,8 @@ changed files. For each changed file, for each Actor:
|
|
|
67
67
|
4. **Hardcoded ignore list, context-aware** — the file path is first "hoisted" relative to the context path it
|
|
68
68
|
matched (e.g. a standalone Actor's own `.eslintrc` is checked as just `.eslintrc`, not the full
|
|
69
69
|
repo-root-relative path), then checked against repo-level dev file patterns (`.vscode/`, `.gitignore`, `.husky/`,
|
|
70
|
-
`.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`).
|
|
71
|
-
|
|
72
|
-
`overrideActorContext`.
|
|
70
|
+
`.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`). Directories like `code/` or `shared/`
|
|
71
|
+
get no special treatment: to treat them as top-level, list them in `overrideActorContext`.
|
|
73
72
|
5. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the Actor's `dockerContextDir`,
|
|
74
73
|
matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior.
|
|
75
74
|
Test files are the exception: `get-affected-actors --test-files-glob <glob>` (relative to the repo root) skips
|
|
@@ -238,9 +237,11 @@ npx apify-test-tools get-affected-actors --since <sha> > affected.json
|
|
|
238
237
|
jq '{ actors: (.functionallyChangedActors + .cosmeticallyChangedActors) }' affected.json \
|
|
239
238
|
| npx apify-test-tools build --version-number default --dry-run
|
|
240
239
|
|
|
241
|
-
npx apify-test-tools notify
|
|
240
|
+
npx apify-test-tools notify --slack-channel "#notif-my-repo" --dry-run < affected.json
|
|
242
241
|
```
|
|
243
242
|
|
|
243
|
+
[`notify`](#notifying-notify) says what it posts for a release.
|
|
244
|
+
|
|
244
245
|
### The last released commit (`--since`)
|
|
245
246
|
|
|
246
247
|
The last commit that is **already released**. Everything after it, up to `HEAD`, is released. Git alone can't tell
|
|
@@ -273,38 +274,123 @@ Actors are reported; it never forces a build, so on its own it doesn't get past
|
|
|
273
274
|
A failed release isn't retried by the next push: that push's base commit is the failed one's head, so its changes
|
|
274
275
|
are skipped. Re-run the failed workflow run, or release the Actors manually.
|
|
275
276
|
|
|
276
|
-
##
|
|
277
|
+
## Running tests (`test`)
|
|
278
|
+
|
|
279
|
+
`test` runs the repository's own vitest (the version in its `node_modules`, so the tests keep the lib version
|
|
280
|
+
the repository locked) on the Actors it is given on stdin, and prints a report of the run for
|
|
281
|
+
[`notify`](#notifying-notify). Only the tests of those Actors run.
|
|
282
|
+
|
|
283
|
+
```bash
|
|
284
|
+
# A PR: test what was just built
|
|
285
|
+
... | npx apify-test-tools build --version-number 0.99 | npx apify-test-tools test > test-report.json
|
|
286
|
+
|
|
287
|
+
# Scheduled tests: every Actor in the config, on its default build
|
|
288
|
+
npx apify-test-tools get-actor-configs | npx apify-test-tools test test/platform/core > test-report.json
|
|
289
|
+
|
|
290
|
+
# Only some of them
|
|
291
|
+
npx apify-test-tools get-actor-configs --actors john.doe/my-actor | npx apify-test-tools test > test-report.json
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
The input is a JSON array of Actors, each given by its full name, its raw ID or both:
|
|
295
|
+
`[{ "actorFullName": "<owner/name>" }]` or `[{ "actorRawId": "<id>" }]`. An Actor's tests run against the build given
|
|
296
|
+
with it (`buildId` and `buildNumber`, as `build` and `build-from-local` print them), or against its default build
|
|
297
|
+
without one. Other fields are ignored, so the output of [`get-actor-configs`](#listing-actors-get-actor-configs) works
|
|
298
|
+
as it is. `test` looks up the ID an Actor is listed without with `TESTER_APIFY_TOKEN`, so its tests run whichever ID
|
|
299
|
+
they name it by. It fails if the token can't see the Actor. A dry-run plan, empty or invalid input is an
|
|
300
|
+
error, as with `build`. So is a missing `TESTER_APIFY_TOKEN`: without it the lib skips every platform test, and the
|
|
301
|
+
run would pass having tested nothing. `test` passes the Actors to vitest as `ACTOR_BUILDS`, in place of any
|
|
302
|
+
`ACTOR_BUILDS` in its own environment.
|
|
303
|
+
|
|
304
|
+
It runs vitest with `--maxConcurrency=20 --fileParallelism --maxWorkers=100`, since platform tests mostly wait on
|
|
305
|
+
Actor runs rather than the CPU, and with the default reporter plus the JSON one the report is read from. These
|
|
306
|
+
replace any `reporters` in the repository's vitest config; add more with `--reporter`, e.g. `github-actions` to
|
|
307
|
+
annotate the failing tests in a GitHub Actions run. It takes no other vitest options; for those, e.g. `-t` to run one
|
|
308
|
+
test by name, run vitest directly with the builds in `ACTOR_BUILDS` (see
|
|
309
|
+
[Running tests locally](lib.md#running-tests-locally)).
|
|
310
|
+
|
|
311
|
+
| Argument / flag | Default | Meaning |
|
|
312
|
+
| ---------------------- | --------------- | ------------------------------------------------------------------------------------------------------------------------------- |
|
|
313
|
+
| `[filters..]` | `test/platform` | Test files to run, as vitest filters |
|
|
314
|
+
| `--vitest-output-file` | | Also keep vitest's own JSON report there, e.g. to upload it |
|
|
315
|
+
| `--reporter` | | [Vitest reporter](https://vitest.dev/guide/reporters) to add to `default` and `json`, e.g. `github-actions`. Repeat it for more |
|
|
316
|
+
|
|
317
|
+
Vitest's own output goes to stderr, followed by a summary that lists every test with the link to its run. Stdout
|
|
318
|
+
gets only the report:
|
|
277
319
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
320
|
+
```json
|
|
321
|
+
{
|
|
322
|
+
"kind": "test-report",
|
|
323
|
+
"version": 1,
|
|
324
|
+
"success": false,
|
|
325
|
+
"summary": { "total": 3, "passed": 1, "failed": 1, "skipped": 1 },
|
|
326
|
+
"tests": [
|
|
327
|
+
{
|
|
328
|
+
"name": "john.doe/my-actor: finishes",
|
|
329
|
+
"file": "test/platform/my-actor.test.ts",
|
|
330
|
+
"status": "failed",
|
|
331
|
+
"durationMs": 61234,
|
|
332
|
+
"failureMessages": ["AssertionError: expected 'FAILED' to be 'SUCCEEDED'"],
|
|
333
|
+
"run": { "actorId": "john.doe/my-actor", "runId": "…", "runLink": "https://console.apify.com/view/runs/…" }
|
|
334
|
+
}
|
|
335
|
+
],
|
|
336
|
+
"errors": ["test/platform/other.test.ts: Cannot find package 'foo'"]
|
|
337
|
+
}
|
|
338
|
+
```
|
|
282
339
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
| `--repo-url` | Repository to name the release after |
|
|
288
|
-
| `--dry-run` | See [Dry runs](#dry-runs---dry-run) |
|
|
340
|
+
`failureMessages` keeps the first line of each failure; diffs and stack traces are in the vitest output. `run` is
|
|
341
|
+
`null` for a test that started no run. `errors` lists what failed outside of any test: a test file that failed to
|
|
342
|
+
load, or vitest exiting without a report. `test` exits with 1 whenever `success` is false, and prints the report
|
|
343
|
+
anyway, so `notify` still hears about the failure.
|
|
289
344
|
|
|
290
|
-
##
|
|
345
|
+
## Notifying (`notify`)
|
|
291
346
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
`
|
|
347
|
+
`notify` tells one target about what it reads on stdin: a test run, as [`test`](#running-tests-test) reports it,
|
|
348
|
+
or a release, as [`get-affected-actors`](#finding-affected-actors-get-affected-actors) reports it. It tells the two
|
|
349
|
+
apart by the report's `kind` and the release's `commits`, so either can be piped in as it is. Slack is the only target for now. To post to more
|
|
350
|
+
places, call it once for each.
|
|
295
351
|
|
|
296
352
|
```bash
|
|
297
|
-
|
|
298
|
-
|
|
353
|
+
# A test run: posts only when the tests failed, the failure counts and the first failure, linked to its run,
|
|
354
|
+
# with the other failures in the message's thread
|
|
355
|
+
npx apify-test-tools notify --slack-channel '#notif-my-repo' < test-report.json
|
|
356
|
+
npx apify-test-tools notify --slack-channel '#my-team' < test-report.json
|
|
357
|
+
|
|
358
|
+
# Or straight from test. With `set -o pipefail` (`shell: bash` in GitHub Actions), failed tests still fail the pipe
|
|
359
|
+
npx apify-test-tools get-actor-configs | npx apify-test-tools test | npx apify-test-tools notify --slack-channel '#notif-my-repo'
|
|
360
|
+
|
|
361
|
+
# A release, see Releasing Actors
|
|
362
|
+
npx apify-test-tools get-affected-actors --since <sha> | npx apify-test-tools notify --slack-channel '#notif-my-repo'
|
|
299
363
|
```
|
|
300
364
|
|
|
301
|
-
| Flag
|
|
302
|
-
|
|
|
303
|
-
| `--
|
|
304
|
-
| `--
|
|
305
|
-
| `--
|
|
306
|
-
| `--
|
|
307
|
-
| `--
|
|
365
|
+
| Flag | Input | Default | Meaning |
|
|
366
|
+
| ----------------- | ------- | ------------------- | ----------------------------------------------------------------- |
|
|
367
|
+
| `--slack-channel` | both | | Slack channel to post to |
|
|
368
|
+
| `--on` | tests | `failure` | `failure` posts only when the tests failed, `always` on every run |
|
|
369
|
+
| `--title` | tests | `-` | What the message calls the run, e.g. the CI workflow name |
|
|
370
|
+
| `--job-url` | tests | | Link to the CI job, included in the message |
|
|
371
|
+
| `--repo-url` | release | the `origin` remote | The repository the release is named after and its commits link to |
|
|
372
|
+
|
|
373
|
+
A flag the input has no use for, e.g. `--repo-url` with a test report, is an error rather than ignored. Both kinds
|
|
374
|
+
of notification post with `SLACK_BOT_TOKEN`.
|
|
375
|
+
|
|
376
|
+
Without a target, `notify` posts nothing and needs no token. With one, it checks the token on every run, also when
|
|
377
|
+
there is nothing to post, so a broken token shows up on a green run rather than on the day tests fail. A release
|
|
378
|
+
with no new commits is the exception: there is nothing to announce, so it needs no token either.
|
|
379
|
+
|
|
380
|
+
For a release, `notify` posts the changelog additions, then the commits, oldest first. Each commit is its title (cut
|
|
381
|
+
at 200 characters) linked to its page on GitHub, and its author. A merge commit or a squash merge is one commit; a
|
|
382
|
+
rebase merge lists every commit it brought. A repository hosted elsewhere gets the titles without links. The release
|
|
383
|
+
is named after the `origin` remote (or `--repo-url`) and the author of the newest commit. Nothing is posted when there
|
|
384
|
+
are no new commits:
|
|
385
|
+
|
|
386
|
+
```text
|
|
387
|
+
*my-repo* – New release (by Jane Doe):
|
|
388
|
+
|
|
389
|
+
- Fixed pagination on the search page
|
|
390
|
+
|
|
391
|
+
*Commits*:
|
|
392
|
+
1. <https://github.com/my-org/my-repo/commit/1a2b3c…|fix: pagination on the search page> – Jane Doe
|
|
393
|
+
```
|
|
308
394
|
|
|
309
395
|
## Deleting old builds (`delete-old-builds`)
|
|
310
396
|
|
|
@@ -332,8 +418,8 @@ To test changed Actor source, you need builds of it. The quickest way is
|
|
|
332
418
|
[`build-from-local`](#build-from-local-source-build-from-local), which needs no push:
|
|
333
419
|
|
|
334
420
|
```bash
|
|
335
|
-
|
|
336
|
-
npx apify-test-tools build-from-local --actors john.doe/my-actor
|
|
421
|
+
APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
|
|
422
|
+
npx apify-test-tools build-from-local --actors john.doe/my-actor > builds.json
|
|
337
423
|
```
|
|
338
424
|
|
|
339
425
|
To build from git, like CI does:
|
|
@@ -345,9 +431,9 @@ To build from git, like CI does:
|
|
|
345
431
|
5. Run tests against those builds. You can change tests and run on the same builds.
|
|
346
432
|
|
|
347
433
|
```bash
|
|
348
|
-
|
|
434
|
+
npx apify-test-tools get-affected-actors --base origin/master --test-files-glob 'test/platform/**' \
|
|
349
435
|
| jq '{ actors: .functionallyChangedActors }' \
|
|
350
|
-
| APIFY_TOKEN_JOHN_DOE=apify_api_xxx npx apify-test-tools build --version-number 0.99
|
|
436
|
+
| APIFY_TOKEN_JOHN_DOE=apify_api_xxx npx apify-test-tools build --version-number 0.99 > builds.json
|
|
351
437
|
```
|
|
352
438
|
|
|
353
439
|
Run it with your dummy branch checked out: it builds the Actors that branch changed functionally, from the branch as
|
|
@@ -358,16 +444,8 @@ Actors in `ACTOR_BUILDS`, so pass `--test-files-glob` to count test changes even
|
|
|
358
444
|
If you want to test against existing builds instead, skip the build and write the build array by hand, only for the
|
|
359
445
|
Actors you need to test.
|
|
360
446
|
|
|
361
|
-
Then run the tests against the builds:
|
|
447
|
+
Then run the tests against the builds with [`test`](#running-tests-test), as often as you like:
|
|
362
448
|
|
|
363
449
|
```bash
|
|
364
|
-
|
|
365
|
-
TESTER_APIFY_TOKEN=apify_api_yyy \
|
|
366
|
-
npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform
|
|
450
|
+
TESTER_APIFY_TOKEN=apify_api_yyy npx apify-test-tools test < builds.json
|
|
367
451
|
```
|
|
368
|
-
|
|
369
|
-
## Legacy git commands
|
|
370
|
-
|
|
371
|
-
`get-commits`, `get-latest-commit` and `get-changed-files` print the commits, the newest commit, or the changed files
|
|
372
|
-
on `--source-branch` since it diverged from `--target-branch` (optionally only after `--base-commit`). They predate
|
|
373
|
-
`get-affected-actors`, which reports the same data, and are kept for older workflows.
|
package/docs/lib.md
CHANGED
|
@@ -10,11 +10,10 @@ Requires `vitest` `3.2.4` or later as a peer dependency.
|
|
|
10
10
|
|
|
11
11
|
The library reads these when the test file is loaded:
|
|
12
12
|
|
|
13
|
-
| Variable
|
|
14
|
-
|
|
|
15
|
-
| `TESTER_APIFY_TOKEN`
|
|
16
|
-
| `ACTOR_BUILDS`
|
|
17
|
-
| `RUN_ALL_PLATFORM_TESTS` | Run every test, not only those of Actors in `ACTOR_BUILDS`. Actors missing from `ACTOR_BUILDS` run their default build. The scheduled platform tests set it. |
|
|
13
|
+
| Variable | Description |
|
|
14
|
+
| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
15
|
+
| `TESTER_APIFY_TOKEN` | Token the tests start runs and read their data with. Without it, every `describe` block is skipped, so a plain `npm test` stays unaffected. |
|
|
16
|
+
| `ACTOR_BUILDS` | JSON array of Actors to test, e.g. the builds the CLI's [`build`](cli.md#building-actors-build) and [`build-from-local`](cli.md#build-from-local-source-build-from-local) print. Only tests of Actors listed here run, against the listed build, or the default build for an Actor listed without one. |
|
|
18
17
|
|
|
19
18
|
`ACTOR_BUILDS` entries look like this, and are matched to tests by `actorFullName` or `actorRawId`:
|
|
20
19
|
|
|
@@ -24,13 +23,13 @@ The library reads these when the test file is loaded:
|
|
|
24
23
|
|
|
25
24
|
## `describe(name, fn, options?)`
|
|
26
25
|
|
|
27
|
-
Same as vitest's `describe`, but skipped unless `TESTER_APIFY_TOKEN`
|
|
26
|
+
Same as vitest's `describe`, but skipped unless `TESTER_APIFY_TOKEN` is set. Its tests
|
|
28
27
|
run concurrently by default. Use it instead of vitest's `describe` for platform tests.
|
|
29
28
|
|
|
30
29
|
## `testActor(actorId, testName, fn, options?)`
|
|
31
30
|
|
|
32
31
|
Registers a test for an Actor. `actorId` is the Actor's full name (`owner/name`) or its raw platform ID. The test
|
|
33
|
-
only runs when the Actor is in `ACTOR_BUILDS
|
|
32
|
+
only runs when the Actor is in `ACTOR_BUILDS`. Its name is
|
|
34
33
|
`<actorId>: <testName>`.
|
|
35
34
|
|
|
36
35
|
`fn` gets the vitest test context, with `expect` extended by the [custom matchers](#custom-matchers), and a `run`
|
|
@@ -247,25 +246,31 @@ export const validateItem = (expect: ExpectStatic, item: any) => {
|
|
|
247
246
|
|
|
248
247
|
Run every command from the root of the Actor repository.
|
|
249
248
|
|
|
250
|
-
**Against the current default builds.** Enough when you only change the tests
|
|
249
|
+
**Against the current default builds.** Enough when you only change the tests. List the Actors with
|
|
250
|
+
[`get-actor-configs`](cli.md#listing-actors-get-actor-configs), all of them or a few with `--actors`, and pass them to
|
|
251
|
+
[`test`](cli.md#running-tests-test), which tests each on its default build:
|
|
251
252
|
|
|
252
253
|
```bash
|
|
253
|
-
|
|
254
|
+
npx apify-test-tools get-actor-configs --actors john.doe/my-actor | TESTER_APIFY_TOKEN=<token> npx apify-test-tools test
|
|
254
255
|
```
|
|
255
256
|
|
|
257
|
+
This also runs a test that reuses an existing run with [`runId`](#testing-an-existing-run), as long as its Actor is
|
|
258
|
+
listed.
|
|
259
|
+
|
|
256
260
|
**Against builds of your changes.** Build the Actors with the CLI, either from local files with
|
|
257
261
|
[`build-from-local`](cli.md#build-from-local-source-build-from-local) or from a pushed branch with
|
|
258
|
-
[`build`](cli.md#building-actors-build), and pass the output
|
|
262
|
+
[`build`](cli.md#building-actors-build), and pass the output to [`test`](cli.md#running-tests-test):
|
|
259
263
|
|
|
260
264
|
```bash
|
|
261
|
-
|
|
265
|
+
APIFY_TOKEN_JOHN_DOE=<token> npx apify-test-tools build-from-local --actors john.doe/my-actor > builds.json
|
|
262
266
|
|
|
263
|
-
|
|
264
|
-
TESTER_APIFY_TOKEN=<token> \
|
|
265
|
-
npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform
|
|
267
|
+
TESTER_APIFY_TOKEN=<token> npx apify-test-tools test < builds.json
|
|
266
268
|
```
|
|
267
269
|
|
|
270
|
+
`test` passes the builds to vitest as `ACTOR_BUILDS`, so plain vitest works too:
|
|
271
|
+
`ACTOR_BUILDS="$(cat builds.json)" TESTER_APIFY_TOKEN=<token> npx vitest run test/platform`.
|
|
272
|
+
|
|
268
273
|
You can change the tests and re-run them against the same builds.
|
|
269
274
|
|
|
270
|
-
`TESTER_APIFY_TOKEN` can be the token of any account with enough memory for the runs. To run only some tests,
|
|
271
|
-
|
|
275
|
+
`TESTER_APIFY_TOKEN` can be the token of any account with enough memory for the runs. To run only some tests, pass a specific file instead of `test/platform`, or filter by
|
|
276
|
+
name with vitest's `-t <name>` when running vitest directly. See [Running locally](cli.md#running-locally) for the full flow.
|
package/docs/workflows.md
CHANGED
|
@@ -26,6 +26,7 @@ The build, release and test workflows below list the CLI commands they run.
|
|
|
26
26
|
| [`public_platform-tests.yaml`](#public_platform-testsyaml) | `schedule` | Runs platform tests against the released builds and reports results to Slack |
|
|
27
27
|
| [`public_review.yaml`](#public_reviewyaml) | `pull_request` (labeled) | Claude reviews the PR against the shared guidelines |
|
|
28
28
|
| [`public_claude.yaml`](#public_claudeyaml) | issue and PR comments, issue labels | Claude acts on `@claude` mentions |
|
|
29
|
+
| [`public_stale-issues.yaml`](#public_stale-issuesyaml) | `schedule` | Warns owners of inactive issues, then closes them a month later |
|
|
29
30
|
| [`public_platform-tests-claude-investigate-and-fix.yaml`](#public_platform-tests-claude-investigate-and-fixyaml) | after a failed platform test job | Claude investigates a failing platform test, opens an issue, and tries to fix it |
|
|
30
31
|
|
|
31
32
|
All of them run on `ubuntu-latest` with Node.js 24. The ones that need the repo's dependencies install them with
|
|
@@ -42,8 +43,8 @@ Validates a pull request. It runs two jobs in parallel.
|
|
|
42
43
|
`.dockerignore` lists them.
|
|
43
44
|
2. Builds the functionally changed Actors into the test version `0.99` with [`build`](cli.md#building-actors-build).
|
|
44
45
|
Cosmetic-only changes (README, CHANGELOG, schema formatting) need no tests, so they're not built.
|
|
45
|
-
3. Runs `
|
|
46
|
-
the built Actors run.
|
|
46
|
+
3. Runs [`test`](cli.md#running-tests-test) on `./test/platform/<test-files-glob>` with those builds, so only the
|
|
47
|
+
tests of the built Actors run. Failing tests are annotated on the PR.
|
|
47
48
|
4. On success, caches the tested commit as the "last validated commit". The next push to the PR then only builds
|
|
48
49
|
and tests the Actors that changed since it (see `--since` in
|
|
49
50
|
[`get-affected-actors`](cli.md#finding-affected-actors-get-affected-actors)).
|
|
@@ -101,8 +102,8 @@ Releases the Actors a push changed. Skipped when the head commit message contain
|
|
|
101
102
|
[Releasing Actors](cli.md#releasing-actors) for how the released range is determined.
|
|
102
103
|
2. `build --version-number default` builds both the functionally and the cosmetically changed Actors into their
|
|
103
104
|
default version, the one users run.
|
|
104
|
-
3. `notify-
|
|
105
|
-
the report channel. It posts nothing when the push brought no new commits.
|
|
105
|
+
3. [`notify`](cli.md#notifying-notify) posts the release (its changelog additions and its commits, linked to
|
|
106
|
+
GitHub) to the report channel. It posts nothing when the push brought no new commits.
|
|
106
107
|
4. `delete-old-builds` deletes old builds of every Actor in the config (see
|
|
107
108
|
[`delete-old-builds`](cli.md#deleting-old-builds-delete-old-builds)).
|
|
108
109
|
|
|
@@ -110,7 +111,7 @@ Releases the Actors a push changed. Skipped when the head commit message contain
|
|
|
110
111
|
| ---------------------- | ------ | ------- | ------------------------------------------------------------- |
|
|
111
112
|
| `report-slack-channel` | string | `''` | Slack channel for the report. Empty uses `#notif-<repo-name>` |
|
|
112
113
|
|
|
113
|
-
Secrets: `
|
|
114
|
+
Secrets: `SLACK_BOT_TOKEN`, the Actor tokens in `CONFIG_SECRETS`, and optionally `NPM_TOKEN`.
|
|
114
115
|
|
|
115
116
|
A failed release isn't retried by the next push: that push's "before" commit is the failed one's head, so its changes
|
|
116
117
|
are skipped. Re-run the failed workflow run, or release the Actors manually.
|
|
@@ -126,7 +127,7 @@ jobs:
|
|
|
126
127
|
buildLatest:
|
|
127
128
|
uses: apify/apify-test-tools/.github/workflows/public_push-build-latest.yaml@workflows-v0
|
|
128
129
|
secrets:
|
|
129
|
-
|
|
130
|
+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
130
131
|
# One entry per secret apify-test-tools.config.json names, today each `tokenEnvVar`
|
|
131
132
|
CONFIG_SECRETS: |
|
|
132
133
|
{
|
|
@@ -137,10 +138,11 @@ jobs:
|
|
|
137
138
|
|
|
138
139
|
## `public_platform-tests.yaml`
|
|
139
140
|
|
|
140
|
-
Runs the platform tests
|
|
141
|
-
|
|
142
|
-
[`
|
|
143
|
-
|
|
141
|
+
Runs the platform tests of every Actor in the config against its current default build, by piping
|
|
142
|
+
[`get-actor-configs`](cli.md#listing-actors-get-actor-configs) into [`test`](cli.md#running-tests-test). A separate step then posts failures to Slack with
|
|
143
|
+
[`notify`](cli.md#notifying-notify), named after the calling workflow and linking to the run, so the Slack token never
|
|
144
|
+
reaches the tests. Both the report of `test` and
|
|
145
|
+
vitest's own JSON report are uploaded as the `vitest-results` artifact. Runs only on `master` or `main`.
|
|
144
146
|
|
|
145
147
|
| Input | Type | Default | Description |
|
|
146
148
|
| ---------------------- | ------ | ------- | ---------------------------------------------------------------------------- |
|
|
@@ -148,7 +150,7 @@ runs, not only those of built Actors. It then reports the results to Slack with
|
|
|
148
150
|
| `subtest` | string | `''` | Deprecated, use `test-files-glob` |
|
|
149
151
|
| `report-slack-channel` | string | `''` | Slack channel for the results. Empty uses `#notif-<repo-name>` |
|
|
150
152
|
|
|
151
|
-
Secrets: `TESTER_APIFY_TOKEN` (required), `
|
|
153
|
+
Secrets: `TESTER_APIFY_TOKEN` (required), `SLACK_BOT_TOKEN`, and optionally `NPM_TOKEN`.
|
|
152
154
|
|
|
153
155
|
How often to run which tests is up to you. For example, every test once a day:
|
|
154
156
|
|
|
@@ -166,7 +168,7 @@ jobs:
|
|
|
166
168
|
uses: apify/apify-test-tools/.github/workflows/public_platform-tests.yaml@workflows-v0
|
|
167
169
|
secrets:
|
|
168
170
|
TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
|
|
169
|
-
|
|
171
|
+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
170
172
|
```
|
|
171
173
|
|
|
172
174
|
and a few critical tests, kept in e.g. `test/platform/hourly`, every hour:
|
|
@@ -187,7 +189,7 @@ jobs:
|
|
|
187
189
|
test-files-glob: hourly
|
|
188
190
|
secrets:
|
|
189
191
|
TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
|
|
190
|
-
|
|
192
|
+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
191
193
|
```
|
|
192
194
|
|
|
193
195
|
## `public_review.yaml`
|
|
@@ -286,7 +288,7 @@ jobs:
|
|
|
286
288
|
test-files-glob: hourly
|
|
287
289
|
secrets:
|
|
288
290
|
TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
|
|
289
|
-
|
|
291
|
+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
290
292
|
|
|
291
293
|
handle-failure:
|
|
292
294
|
needs: platformTestsHourly
|
|
@@ -297,6 +299,44 @@ jobs:
|
|
|
297
299
|
TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
|
|
298
300
|
```
|
|
299
301
|
|
|
302
|
+
## `public_stale-issues.yaml`
|
|
303
|
+
|
|
304
|
+
Optional. Wraps [`actions/stale`](https://github.com/actions/stale) to warn on inactive GitHub issues, then close them
|
|
305
|
+
as **not planned** 30 days later if still inactive. Locked issues are skipped. Pull requests use the same stale and
|
|
306
|
+
close thresholds, with the `Stale` label and no warning or closing comments.
|
|
307
|
+
|
|
308
|
+
| Input | Type | Default | Description |
|
|
309
|
+
| -------------- | ------ | ------- | -------------------------------------------------------------------- |
|
|
310
|
+
| `stale-months` | number | `6` | Optional whole number of inactive months (1–12); one month = 30 days |
|
|
311
|
+
|
|
312
|
+
```yaml
|
|
313
|
+
name: Close stale issues
|
|
314
|
+
|
|
315
|
+
on:
|
|
316
|
+
schedule:
|
|
317
|
+
- cron: '0 3 * * *'
|
|
318
|
+
workflow_dispatch:
|
|
319
|
+
|
|
320
|
+
jobs:
|
|
321
|
+
staleIssues:
|
|
322
|
+
permissions:
|
|
323
|
+
issues: write
|
|
324
|
+
uses: apify/apify-test-tools/.github/workflows/public_stale-issues.yaml@workflows-v0
|
|
325
|
+
with:
|
|
326
|
+
stale-months: 6
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
With `stale-months: 6`, an issue is labeled `stale` and warned after 180 days without activity, then closed after
|
|
330
|
+
another 30 inactive days. Existing older issues also get the full 30-day warning when adopting the workflow.
|
|
331
|
+
Actions happen on a scheduled run after the threshold; large backlogs are processed over successive runs.
|
|
332
|
+
|
|
333
|
+
The warning comment uses [GitHub's normal issue notifications](https://docs.github.com/en/subscriptions-and-notifications/reference/inbox-filters),
|
|
334
|
+
including notifications for assigned issues, subject to each user's notification settings. It does not add explicit
|
|
335
|
+
@mentions. Subsequent activity causes `actions/stale` to remove the stale label and restart the inactivity period.
|
|
336
|
+
|
|
337
|
+
The caller must grant `issues: write` as shown above. The workflow uses the automatic `GITHUB_TOKEN`; no additional
|
|
338
|
+
secrets, checkout, project dependencies, or Apify configuration are needed.
|
|
339
|
+
|
|
300
340
|
## Secrets
|
|
301
341
|
|
|
302
342
|
The workflows take two kinds of secrets:
|
|
@@ -335,15 +375,15 @@ out of it.
|
|
|
335
375
|
The workflows do not turn the secrets they receive into job-wide environment variables, so a secret is only visible
|
|
336
376
|
to the step that needs it:
|
|
337
377
|
|
|
338
|
-
| Secret | Reaches
|
|
339
|
-
| ------------------------------------------------------------------------- |
|
|
340
|
-
| `NPM_TOKEN` | dependency install steps only, as both `NPM_TOKEN` and `NODE_AUTH_TOKEN`. Use a read-only token: npm granular tokens can be read-only, classic automation tokens can publish.
|
|
341
|
-
| the Actor tokens named by `tokenEnvVar` in `apify-test-tools.config.json` | the build and `delete-old-builds` steps only
|
|
342
|
-
| `TESTER_APIFY_TOKEN` | the
|
|
343
|
-
| `
|
|
344
|
-
| `ANTHROPIC_API_KEY` | the Claude steps of the Claude workflows only
|
|
345
|
-
| `REDASH_API_KEY` | the Claude investigation step: the Redash MCP server's key for measuring the failure's impact from Snowflake. Optional
|
|
346
|
-
| `AGENT_SKILLS_INTERNAL_GITHUB_TOKEN` | the investigation job's checkout of `apify/agent-skills-internal` only, for the `redash-analyst` plugin. Optional
|
|
378
|
+
| Secret | Reaches |
|
|
379
|
+
| ------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
380
|
+
| `NPM_TOKEN` | dependency install steps only, as both `NPM_TOKEN` and `NODE_AUTH_TOKEN`. Use a read-only token: npm granular tokens can be read-only, classic automation tokens can publish. |
|
|
381
|
+
| the Actor tokens named by `tokenEnvVar` in `apify-test-tools.config.json` | the build and `delete-old-builds` steps only |
|
|
382
|
+
| `TESTER_APIFY_TOKEN` | the test step, and the Claude investigation step: the Apify MCP server's bearer token for reading the failing run, its log and its storages, and Claude's token for re-running it to measure how often the failure happens |
|
|
383
|
+
| `SLACK_BOT_TOKEN` | the `notify` steps only |
|
|
384
|
+
| `ANTHROPIC_API_KEY` | the Claude steps of the Claude workflows only |
|
|
385
|
+
| `REDASH_API_KEY` | the Claude investigation step: the Redash MCP server's key for measuring the failure's impact from Snowflake. Optional |
|
|
386
|
+
| `AGENT_SKILLS_INTERNAL_GITHUB_TOKEN` | the investigation job's checkout of `apify/agent-skills-internal` only, for the `redash-analyst` plugin. Optional |
|
|
347
387
|
|
|
348
388
|
The Actor tokens reach the build only for the Actors your config declares: a secret that merely looks like an Actor
|
|
349
389
|
token is not passed on. A token the config declares but the repo hasn't set is a warning, not a failure: a repo can
|
package/lib/lib.ts
CHANGED
|
@@ -31,7 +31,7 @@ const config = actorBuilds.reduce<Map<string, ActorBuild>>((map, cfg) => {
|
|
|
31
31
|
|
|
32
32
|
export { ExpectStatic };
|
|
33
33
|
|
|
34
|
-
const { TESTER_APIFY_TOKEN
|
|
34
|
+
const { TESTER_APIFY_TOKEN } = process.env;
|
|
35
35
|
const apifyClient = new ApifyClient({ token: TESTER_APIFY_TOKEN });
|
|
36
36
|
|
|
37
37
|
const DEFAULT_TEST_OPTIONS: ActorTestOptions = {
|
|
@@ -44,16 +44,14 @@ const DEFAULT_TEST_OPTIONS: ActorTestOptions = {
|
|
|
44
44
|
|
|
45
45
|
/**
|
|
46
46
|
* Platform tests need `TESTER_APIFY_TOKEN` to talk to the platform, so without it we skip them altogether.
|
|
47
|
-
*
|
|
48
|
-
* `RUN_ALL_PLATFORM_TESTS` enables them too because locally we can test against a hardcoded `runId`,
|
|
49
|
-
* which doesn't need the tester token.
|
|
50
47
|
*/
|
|
51
48
|
export const describe = (name: string, fn?: SuiteFactory<object>, options: ActorTestOptions = DEFAULT_TEST_OPTIONS) => {
|
|
52
|
-
vitestDescribe.runIf(!!TESTER_APIFY_TOKEN
|
|
49
|
+
vitestDescribe.runIf(!!TESTER_APIFY_TOKEN)(name, options, fn);
|
|
53
50
|
};
|
|
54
51
|
|
|
55
52
|
/**
|
|
56
|
-
* Registers a vitest test for an Actor, which only runs when
|
|
53
|
+
* Registers a vitest test for an Actor, which only runs when the Actor is listed in `ACTOR_BUILDS`: against the build
|
|
54
|
+
* listed with it, or its default build without one.
|
|
57
55
|
* `body` gets the test context with the soft `expect` and the full test name.
|
|
58
56
|
*/
|
|
59
57
|
const defineActorTest = (
|
|
@@ -64,10 +62,7 @@ const defineActorTest = (
|
|
|
64
62
|
) => {
|
|
65
63
|
const options = { ...DEFAULT_TEST_OPTIONS, ...testOptions };
|
|
66
64
|
const name = `${actorId}: ${testName}`;
|
|
67
|
-
|
|
68
|
-
// tests against - without it, every test would be filtered out as an actor we didn't build.
|
|
69
|
-
const shouldRun = !!RUN_ALL_PLATFORM_TESTS || config.has(actorId);
|
|
70
|
-
vitestTest.runIf(shouldRun)(name, options, async (context) => {
|
|
65
|
+
vitestTest.runIf(config.has(actorId))(name, options, async (context) => {
|
|
71
66
|
await body({ ...context, expect: extendExpect(context.expect) }, name);
|
|
72
67
|
});
|
|
73
68
|
};
|