apify-test-tools 0.10.2-beta.2 → 0.10.2-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/cli/apify-test-tools.mjs +140 -140
- package/.github/scripts/run-with-apify-tokens.mjs +1 -1
- package/.github/workflows/public_pr-build-test.yaml +25 -25
- package/.github/workflows/public_push-build-latest.yaml +20 -4
- package/CHANGELOG.md +2 -0
- package/README.md +119 -14
- package/bin/affected-actors.ts +157 -0
- package/bin/build.ts +55 -38
- package/bin/diff-changes.ts +25 -17
- package/bin/git.ts +44 -44
- package/bin/main.ts +132 -106
- package/bin/slack.ts +24 -0
- package/bin/stdin-input.ts +65 -0
- package/dist/bin/affected-actors.d.ts +45 -0
- package/dist/bin/affected-actors.d.ts.map +1 -0
- package/dist/bin/affected-actors.js +87 -0
- package/dist/bin/affected-actors.js.map +1 -0
- package/dist/bin/build.d.ts +12 -5
- package/dist/bin/build.d.ts.map +1 -1
- package/dist/bin/build.js +45 -23
- package/dist/bin/build.js.map +1 -1
- package/dist/bin/diff-changes.d.ts +8 -2
- package/dist/bin/diff-changes.d.ts.map +1 -1
- package/dist/bin/diff-changes.js +16 -16
- package/dist/bin/diff-changes.js.map +1 -1
- package/dist/bin/git.d.ts +19 -14
- package/dist/bin/git.d.ts.map +1 -1
- package/dist/bin/git.js +37 -35
- package/dist/bin/git.js.map +1 -1
- package/dist/bin/main.js +110 -86
- package/dist/bin/main.js.map +1 -1
- package/dist/bin/slack.d.ts +8 -0
- package/dist/bin/slack.d.ts.map +1 -1
- package/dist/bin/slack.js +18 -0
- package/dist/bin/slack.js.map +1 -1
- package/dist/bin/stdin-input.d.ts +17 -0
- package/dist/bin/stdin-input.d.ts.map +1 -0
- package/dist/bin/stdin-input.js +54 -0
- package/dist/bin/stdin-input.js.map +1 -0
- package/dist/test/bundle/cli-bundle.test.js +36 -6
- package/dist/test/bundle/cli-bundle.test.js.map +1 -1
- package/dist/test/e2e/cli-bundle.e2e.test.js +16 -9
- package/dist/test/e2e/cli-bundle.e2e.test.js.map +1 -1
- package/dist/test/unit/bin/affected-actors.test.d.ts +2 -0
- package/dist/test/unit/bin/affected-actors.test.d.ts.map +1 -0
- package/dist/test/unit/bin/affected-actors.test.js +154 -0
- package/dist/test/unit/bin/affected-actors.test.js.map +1 -0
- package/dist/test/unit/bin/build-cli.test.d.ts +2 -0
- package/dist/test/unit/bin/build-cli.test.d.ts.map +1 -0
- package/dist/test/unit/bin/build-cli.test.js +52 -0
- package/dist/test/unit/bin/build-cli.test.js.map +1 -0
- package/dist/test/unit/bin/build.test.js +61 -4
- package/dist/test/unit/bin/build.test.js.map +1 -1
- package/dist/test/unit/bin/diff-changes.test.js +78 -49
- package/dist/test/unit/bin/diff-changes.test.js.map +1 -1
- package/dist/test/unit/bin/git.test.js +39 -54
- package/dist/test/unit/bin/git.test.js.map +1 -1
- package/dist/test/unit/bin/slack.test.js +39 -1
- package/dist/test/unit/bin/slack.test.js.map +1 -1
- package/dist/test/unit/bin/stdin-input.test.d.ts +2 -0
- package/dist/test/unit/bin/stdin-input.test.d.ts.map +1 -0
- package/dist/test/unit/bin/stdin-input.test.js +50 -0
- package/dist/test/unit/bin/stdin-input.test.js.map +1 -0
- package/dist/test/unit/should-built-and-test.test.js +23 -17
- package/dist/test/unit/should-built-and-test.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/test/bundle/cli-bundle.test.ts +59 -6
- package/test/e2e/cli-bundle.e2e.test.ts +26 -11
- package/test/unit/bin/affected-actors.test.ts +184 -0
- package/test/unit/bin/build-cli.test.ts +61 -0
- package/test/unit/bin/build.test.ts +71 -3
- package/test/unit/bin/diff-changes.test.ts +87 -49
- package/test/unit/bin/git.test.ts +44 -55
- package/test/unit/bin/slack.test.ts +44 -1
- package/test/unit/bin/stdin-input.test.ts +61 -0
- package/test/unit/should-built-and-test.test.ts +27 -17
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// Runs apify-test-tools and passes it only secrets needed for the Actors, no other secrets are leaked.
|
|
3
3
|
//
|
|
4
|
-
// node .github/scripts/run-with-apify-tokens.mjs apify-test-tools build --target-branch ...
|
|
4
|
+
// node .github/scripts/run-with-apify-tokens.mjs apify-test-tools build-for-test --target-branch ...
|
|
5
5
|
//
|
|
6
6
|
// The step passes the whole secrets map as ALL_SECRETS. The token names come from `tokenEnvVar` in
|
|
7
7
|
// the repo's apify-test-tools.config.json, which is the same file apify-test-tools itself reads, so
|
|
@@ -41,31 +41,39 @@ jobs:
|
|
|
41
41
|
# This is needed to force overriding the cache with new entry at the end
|
|
42
42
|
restore-keys: base_commit-
|
|
43
43
|
|
|
44
|
-
#
|
|
45
|
-
#
|
|
46
|
-
|
|
47
|
-
- name: Build
|
|
48
|
-
id: build
|
|
44
|
+
# Needs no tokens, it only reads git and apify-test-tools.config.json. The setup action checks out
|
|
45
|
+
# the PR branch itself (not GitHub's merge commit), so the range ends at the branch's HEAD.
|
|
46
|
+
- name: Find affected Actors
|
|
49
47
|
env:
|
|
50
|
-
ALL_SECRETS: ${{ toJSON(secrets) }}
|
|
51
48
|
# Branch names are attacker-controlled on fork PRs, so they reach the script as
|
|
52
49
|
# environment variables rather than being interpolated into it.
|
|
53
|
-
HEAD_REF: ${{ github.head_ref }}
|
|
54
50
|
BASE_REF: ${{ github.base_ref }}
|
|
55
|
-
SCRIPTS_PATH: ${{ steps.setup.outputs.scripts-path }}
|
|
56
51
|
run: |
|
|
57
52
|
base_commit=$(cat base_commit.txt 2>/dev/null || echo "")
|
|
58
53
|
echo "Last validated (base) commit to be used for build & test: $base_commit"
|
|
59
54
|
args=(
|
|
60
|
-
--
|
|
61
|
-
--target-branch "origin/$BASE_REF"
|
|
55
|
+
--base "origin/$BASE_REF"
|
|
62
56
|
# TODO: Temporary hotfix until build glob matching aligns with Vitest's test selection glob.
|
|
63
57
|
--test-files-glob 'test/platform/**'
|
|
64
|
-
--use-docker-cache
|
|
65
58
|
)
|
|
66
|
-
[ -n "$base_commit" ] && args+=(--
|
|
67
|
-
|
|
68
|
-
|
|
59
|
+
[ -n "$base_commit" ] && args+=(--since "$base_commit")
|
|
60
|
+
apify-test-tools get-affected-actors "${args[@]}" > affected.json
|
|
61
|
+
|
|
62
|
+
# Cosmetic-only changes need no tests, so only the functionally changed Actors are built, into the
|
|
63
|
+
# test version 0.99 that users never run.
|
|
64
|
+
# run-with-apify-tokens.mjs reads the token names from in apify-test-tools.config.json
|
|
65
|
+
# and hands the build those secrets and nothing else, so the tokens stay in this one command instead of the whole job.
|
|
66
|
+
# The library never sees secrets it doesn't need
|
|
67
|
+
- name: Build
|
|
68
|
+
id: build
|
|
69
|
+
# bash rather than the default shell, for pipefail
|
|
70
|
+
shell: bash
|
|
71
|
+
env:
|
|
72
|
+
ALL_SECRETS: ${{ toJSON(secrets) }}
|
|
73
|
+
SCRIPTS_PATH: ${{ steps.setup.outputs.scripts-path }}
|
|
74
|
+
run: |
|
|
75
|
+
actor_builds=$(jq '{ actors: .functionallyChangedActors }' affected.json \
|
|
76
|
+
| node "$SCRIPTS_PATH/run-with-apify-tokens.mjs" apify-test-tools build --version-number 0.99)
|
|
69
77
|
echo "actor_builds=$actor_builds" | tee -a "$GITHUB_OUTPUT"
|
|
70
78
|
|
|
71
79
|
# Runs the repo's test code and its whole import graph, so it gets the tester token only.
|
|
@@ -77,18 +85,10 @@ jobs:
|
|
|
77
85
|
|
|
78
86
|
# NOTE: This is an optimization that if we did functional changes and later only cosmetic changes (e.g. dev readme), we will compare changes files only for the last commit. This must run after tests because we want to cache the latest commit only if the tests are successful
|
|
79
87
|
- name: Store last validated commit to cache
|
|
80
|
-
env:
|
|
81
|
-
HEAD_REF: ${{ github.head_ref }}
|
|
82
|
-
BASE_REF: ${{ github.base_ref }}
|
|
83
88
|
run: |
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
--source-branch "origin/$HEAD_REF"
|
|
88
|
-
--target-branch "origin/$BASE_REF"
|
|
89
|
-
)
|
|
90
|
-
[ -n "$base_commit" ] && args+=(--base-commit "$base_commit")
|
|
91
|
-
apify-test-tools get-latest-commit "${args[@]}" > ./base_commit.txt
|
|
89
|
+
echo "Old last validated (base) commit: $(cat base_commit.txt 2>/dev/null || echo "")"
|
|
90
|
+
# The HEAD that was just built and tested
|
|
91
|
+
jq -r .head affected.json > base_commit.txt
|
|
92
92
|
echo "New last validated (base) commit: $(cat base_commit.txt)"
|
|
93
93
|
|
|
94
94
|
# Pure static checks against the repo. Needs no Apify or Slack credentials, only the npm
|
|
@@ -25,18 +25,34 @@ jobs:
|
|
|
25
25
|
with:
|
|
26
26
|
npm-token: ${{ secrets.NPM_TOKEN }}
|
|
27
27
|
|
|
28
|
-
# --
|
|
28
|
+
# --since is the last commit that is already released: the push's "before" commit.
|
|
29
29
|
# Everything after it up to HEAD gets released. See the "Releasing Actors" section of the README.
|
|
30
|
+
# Needs no tokens, it only reads git and apify-test-tools.config.json.
|
|
31
|
+
- name: Find changes since the last release
|
|
32
|
+
run: |
|
|
33
|
+
apify-test-tools get-affected-actors --since ${{ github.event.before }} > affected.json
|
|
34
|
+
|
|
35
|
+
# A release ships cosmetic changes too (README, input schema descriptions), so both lists are built,
|
|
36
|
+
# into each Actor's default version: the one its default build tag points to, which users run.
|
|
30
37
|
# run-with-apify-tokens.mjs reads the token names from in apify-test-tools.config.json
|
|
31
38
|
# and hands the build those secrets and nothing else, so the tokens stay in this one command instead of the whole job.
|
|
32
39
|
# The library never sees secrets it doesn't need
|
|
33
|
-
- name:
|
|
40
|
+
- name: Build the default versions
|
|
41
|
+
# bash rather than the default shell, for pipefail
|
|
42
|
+
shell: bash
|
|
34
43
|
env:
|
|
35
44
|
ALL_SECRETS: ${{ toJSON(secrets) }}
|
|
45
|
+
run: |
|
|
46
|
+
jq '{ actors: (.functionallyChangedActors + .cosmeticallyChangedActors) }' affected.json \
|
|
47
|
+
| node "${{ steps.setup.outputs.scripts-path }}/run-with-apify-tokens.mjs" \
|
|
48
|
+
apify-test-tools build --version-number default
|
|
49
|
+
|
|
50
|
+
# Skips the announcement when the push brought no new commits
|
|
51
|
+
- name: Announce the release on Slack
|
|
52
|
+
env:
|
|
36
53
|
SLACK_TOKEN_RELEASES_BOT: ${{ secrets.SLACK_TOKEN_RELEASES_BOT }}
|
|
37
54
|
run: |
|
|
38
|
-
|
|
39
|
-
apify-test-tools release --base-commit ${{ github.event.before }} --release-slack-channel "#delivery-public-actors" --report-slack-channel "${{ inputs.report-slack-channel || format('#notif-{0}', github.event.repository.name) }}" --use-docker-cache
|
|
55
|
+
apify-test-tools notify-release --release-slack-channel "#delivery-public-actors" --report-slack-channel "${{ inputs.report-slack-channel || format('#notif-{0}', github.event.repository.name) }}" < affected.json
|
|
40
56
|
|
|
41
57
|
- name: Delete old builds
|
|
42
58
|
env:
|
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,8 @@ All notable changes to this project will be documented in this file.
|
|
|
8
8
|
### 🚀 Features
|
|
9
9
|
|
|
10
10
|
- **ci:** Ship the workflows' CLI as a bundle in the action instead of installing it ([#200](https://github.com/apify/apify-test-tools/pull/200)) ([a0ef975](https://github.com/apify/apify-test-tools/commit/a0ef9756123a30a8210590b080a2a289c6a9c17c)) by [@metalwarrior665](https://github.com/metalwarrior665)
|
|
11
|
+
- [**breaking**] Revamp get-affected-actors to be a core logic command ([#190](https://github.com/apify/apify-test-tools/pull/190)) ([ea5f244](https://github.com/apify/apify-test-tools/commit/ea5f244cfd42bf3fa9ffd38cd2ad499dde2e83f8)) by [@metalwarrior665](https://github.com/metalwarrior665)
|
|
12
|
+
- [**breaking**] Add logic-less build command and use command chaining in workflows ([#193](https://github.com/apify/apify-test-tools/pull/193)) ([e5b0b86](https://github.com/apify/apify-test-tools/commit/e5b0b86ee06fca51d4be0ddc81b0a4a469d31603)) by [@metalwarrior665](https://github.com/metalwarrior665)
|
|
11
13
|
|
|
12
14
|
### 🐛 Bug Fixes
|
|
13
15
|
|
package/README.md
CHANGED
|
@@ -57,7 +57,7 @@ When a PR is opened or code is pushed, the tool determines which actors need to
|
|
|
57
57
|
2. **CHANGELOG classification** — a `CHANGELOG.md` file is always `cosmetic` (only triggers a release build, not tests), for every actor, regardless of context or folder. (See [issue #106](https://github.com/apify/apify-test-tools/issues/106).)
|
|
58
58
|
3. **Context matching** — the file must fall within one of the actor's context paths (`dockerContextDir` from `actor.json` by default, or `overrideActorContext` from config if set). Files outside every context path are skipped.
|
|
59
59
|
4. **Hardcoded ignore list, context-aware** — the file path is first "hoisted" relative to the context path it matched (e.g. a standalone actor's own `.eslintrc` is checked as just `.eslintrc`, not the full repo-root-relative path), then checked against repo-level dev file patterns (`.vscode/`, `.gitignore`, `.husky/`, `.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`). There's no hardcoded special-casing for legacy `code/`/`shared/` layouts — repos that need those directories treated as top-level must list them explicitly in `overrideActorContext`.
|
|
60
|
-
5. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the actor's `dockerContextDir`, matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior. Test files are the exception: `build --test-files-glob <glob>` (relative to the repo root) skips this check for matching files, so a changed test triggers a build, and with it its tests, even when `.dockerignore` lists it. The PR workflow passes `test/platform/**`. Context matching still applies, so tests outside an actor's `dockerContextDir` (e.g. a root `test/` with standalone actors) need to be listed in its `overrideActorContext`; `build` logs a warning for changed test files outside every actor's context.
|
|
60
|
+
5. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the actor's `dockerContextDir`, matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior. Test files are the exception: `build-for-test --test-files-glob <glob>` (relative to the repo root, also taken by `get-affected-actors`) skips this check for matching files, so a changed test triggers a build, and with it its tests, even when `.dockerignore` lists it. The PR workflow passes `test/platform/**`. Context matching still applies, so tests outside an actor's `dockerContextDir` (e.g. a root `test/` with standalone actors) need to be listed in its `overrideActorContext`; `build-for-test` logs a warning for changed test files outside every actor's context.
|
|
61
61
|
6. **README classification** — a `README.md` file is `cosmetic` (only triggers a release build, not tests) if it lives inside the actor's own `folder`; otherwise it's ignored entirely, since it isn't documentation for this actor.
|
|
62
62
|
7. **Cosmetic JSON classification** — `.json` files inside the actor's own `.actor/` directory with only cosmetic schema changes (whitespace, key ordering) only trigger a release build.
|
|
63
63
|
8. **Functional** — everything else triggers both build and tests.
|
|
@@ -226,9 +226,10 @@ jobs:
|
|
|
226
226
|
secrets: inherit
|
|
227
227
|
```
|
|
228
228
|
|
|
229
|
-
This workflow
|
|
230
|
-
|
|
231
|
-
|
|
229
|
+
This workflow does what `apify-test-tools release --base-commit ${{ github.event.before }}` does, one
|
|
230
|
+
step per command: `get-affected-actors --since ${{ github.event.before }}`, then `build --version-number default`
|
|
231
|
+
on every changed Actor, then `notify-release`. See [Releasing Actors](#releasing-actors-release) for how the
|
|
232
|
+
released range is determined and what to pass if you call the CLI from somewhere else.
|
|
232
233
|
|
|
233
234
|
### `claude-review.yaml`
|
|
234
235
|
|
|
@@ -261,9 +262,9 @@ environment variables, so a secret is only visible to the step that needs it:
|
|
|
261
262
|
| Secret | Reaches |
|
|
262
263
|
| ------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
263
264
|
| `NPM_TOKEN` | dependency install steps only, as both `NPM_TOKEN` and `NODE_AUTH_TOKEN`. Use a read-only token: npm granular tokens can be read-only, classic automation tokens can publish. |
|
|
264
|
-
| the Actor tokens named by `tokenEnvVar` in `apify-test-tools.config.json` | the
|
|
265
|
+
| the Actor tokens named by `tokenEnvVar` in `apify-test-tools.config.json` | the build and `delete-old-builds` steps only |
|
|
265
266
|
| `TESTER_APIFY_TOKEN` | the vitest step, and the Claude investigation step: the Apify MCP server's bearer token for reading the failing run, its log and its storages, and Claude's token for re-running it to measure how often the failure happens |
|
|
266
|
-
| `SLACK_TOKEN_TESTS_BOT` / `SLACK_TOKEN_RELEASES_BOT` | the reporting and
|
|
267
|
+
| `SLACK_TOKEN_TESTS_BOT` / `SLACK_TOKEN_RELEASES_BOT` | the reporting and announcement steps only |
|
|
267
268
|
|
|
268
269
|
The Actor tokens are the one set that cannot be listed in the workflow, because each Actor names its
|
|
269
270
|
own token via `tokenEnvVar` in `apify-test-tools.config.json`. Those steps pass
|
|
@@ -277,7 +278,7 @@ secrets to pass on:
|
|
|
277
278
|
ALL_SECRETS: ${{ toJSON(secrets) }}
|
|
278
279
|
run: |
|
|
279
280
|
node "${{ steps.setup.outputs.scripts-path }}/run-with-apify-tokens.mjs" \
|
|
280
|
-
npx apify-test-tools build --target-branch ...
|
|
281
|
+
npx apify-test-tools build-for-test --target-branch ...
|
|
281
282
|
```
|
|
282
283
|
|
|
283
284
|
The wrapper passes only the tokens the config declares and drops `ALL_SECRETS`, so neither `npx` nor
|
|
@@ -568,12 +569,12 @@ The package includes a CLI binary used by CI workflows to build Actors, detect c
|
|
|
568
569
|
|
|
569
570
|
### Dry runs (`--dry-run`)
|
|
570
571
|
|
|
571
|
-
Every command that changes something (`build`, `build-from-local`, `release`, `delete-old-builds`, `report-tests`) takes `--dry-run`. A dry run:
|
|
572
|
+
Every command that changes something (`build`, `build-for-test`, `build-from-local`, `release`, `notify-release`, `delete-old-builds`, `report-tests`) takes `--dry-run`. A dry run:
|
|
572
573
|
|
|
573
574
|
- **Changes nothing.** No versions or builds are created, nothing is uploaded or deleted, and no Slack message is posted.
|
|
574
575
|
- **Does every read and check the real run does**, so it fails wherever the real run would: change detection, the Apify token and Actor lookup, the [repository check](#repository-check), collecting and zipping local sources, and the Slack token (verified with Slack's read-only `auth.test`).
|
|
575
576
|
- **Needs the same tokens as the real run.** Read-only tokens are enough, since nothing is written. A dry run can't tell whether a token is also allowed to write, because the Apify API has no dry-run mode of its own.
|
|
576
|
-
- **Prints a plan, not results.** Logs start with `[DRY RUN] Would …`. `build` and `build-from-local` write `{ "dryRun": true, "builds": [...] }` to stdout instead of the build array, so the plan can't be passed to tests as `ACTOR_BUILDS` by mistake.
|
|
577
|
+
- **Prints a plan, not results.** Logs start with `[DRY RUN] Would …`. `build`, `build-for-test` and `build-from-local` write `{ "dryRun": true, "builds": [...] }` to stdout instead of the build array, so the plan can't be passed to tests as `ACTOR_BUILDS` by mistake.
|
|
577
578
|
|
|
578
579
|
### Running locally
|
|
579
580
|
|
|
@@ -599,13 +600,15 @@ Requires `APIFY_TOKEN_<USERNAME>` for all Apify users that own your Actors (e.g.
|
|
|
599
600
|
|
|
600
601
|
```bash
|
|
601
602
|
APIFY_TOKEN_JOHN_DOE=<token> \
|
|
602
|
-
npx apify-test-tools build \
|
|
603
|
+
npx apify-test-tools build-for-test \
|
|
603
604
|
--target-branch origin/master \
|
|
604
|
-
--source-branch origin/my-dummy-branch \
|
|
605
605
|
--dry-run
|
|
606
606
|
```
|
|
607
607
|
|
|
608
|
-
|
|
608
|
+
Run it with your dummy branch checked out: it builds the Actors that branch changed, from the branch as it is on
|
|
609
|
+
the remote. With `--dry-run`, it prints the plan (the version and Git URL each Actor would be built from). Remove
|
|
610
|
+
`--dry-run` to actually trigger builds and update the target branch name. The command outputs a JSON array of build
|
|
611
|
+
objects to stdout:
|
|
609
612
|
|
|
610
613
|
```json
|
|
611
614
|
[{ "buildId": "...", "actorRawId": "...", "buildNumber": "...", "actorFullName": "john.doe/my-actor" }]
|
|
@@ -649,9 +652,8 @@ Tests only run for Actors in `ACTOR_BUILDS`, so a custom workflow has to build (
|
|
|
649
652
|
```bash
|
|
650
653
|
# Build and capture output
|
|
651
654
|
BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
|
|
652
|
-
npx apify-test-tools build \
|
|
655
|
+
npx apify-test-tools build-for-test \
|
|
653
656
|
--target-branch origin/master \
|
|
654
|
-
--source-branch origin/my-dummy-branch \
|
|
655
657
|
--test-files-glob 'test/platform/**')
|
|
656
658
|
|
|
657
659
|
# Run tests with the builds
|
|
@@ -660,6 +662,100 @@ TESTER_APIFY_TOKEN=apify_api_yyy \
|
|
|
660
662
|
npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform
|
|
661
663
|
```
|
|
662
664
|
|
|
665
|
+
### Finding affected Actors (`get-affected-actors`)
|
|
666
|
+
|
|
667
|
+
`get-affected-actors` lists the Actors whose context changed in a commit range, using the rules in
|
|
668
|
+
[Change detection](#change-detection). It only reads git and the config file, so it needs no tokens.
|
|
669
|
+
The range always ends at the checked-out `HEAD`, so check out the branch you want to inspect first.
|
|
670
|
+
|
|
671
|
+
```bash
|
|
672
|
+
# A PR: the branch's own commits, after its last validated commit
|
|
673
|
+
npx apify-test-tools get-affected-actors --base origin/master --since <sha>
|
|
674
|
+
|
|
675
|
+
# A release: everything after the last released commit
|
|
676
|
+
npx apify-test-tools get-affected-actors --since <sha>
|
|
677
|
+
```
|
|
678
|
+
|
|
679
|
+
| Flag | Default | Meaning |
|
|
680
|
+
| ------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
681
|
+
| `--base` | | Only commits on `HEAD` that are not on this branch count |
|
|
682
|
+
| `--since` | | Only commits after this one count (see below) |
|
|
683
|
+
| `--test-files-glob` | | Test files (glob from the repo root) that count as changes even when `.dockerignore` lists them, see [Change detection](#change-detection) |
|
|
684
|
+
| `--actors` | all | Only report these Actors |
|
|
685
|
+
| `--ignore` | none | Never report these Actors |
|
|
686
|
+
|
|
687
|
+
Pass `--base`, `--since`, or both:
|
|
688
|
+
|
|
689
|
+
- **`--base` only** inspects the whole branch. If the branch merged `--base` in and has no functional changes
|
|
690
|
+
of its own, no Actor is reported as functional, since `--base` is already validated (and none as cosmetic
|
|
691
|
+
either, if the branch has no changes of its own at all).
|
|
692
|
+
- **`--base` and `--since`** inspects the branch after its last validated commit. If `--since` is not on the
|
|
693
|
+
branch (force-push), or it is the head (a rerun), the whole branch is inspected instead.
|
|
694
|
+
- **`--since` only** inspects everything after it up to `HEAD`. It must be an ancestor of `HEAD` and fails
|
|
695
|
+
otherwise, exactly like [`--base-commit` of `release`](#--base-commit). If `HEAD` is `--since`, nothing is
|
|
696
|
+
reported.
|
|
697
|
+
|
|
698
|
+
The output is a JSON document:
|
|
699
|
+
|
|
700
|
+
```json
|
|
701
|
+
{
|
|
702
|
+
"version": 1,
|
|
703
|
+
"head": "<sha of HEAD>",
|
|
704
|
+
"commits": [{ "sha": "…", "author": "…", "date": "…", "message": "…" }],
|
|
705
|
+
"changedFiles": ["src/main.ts", "CHANGELOG.md"],
|
|
706
|
+
"changelog": "Lines added to CHANGELOG.md, or null",
|
|
707
|
+
"functionallyChangedActors": [{ "actorFullName": "john.doe/my-actor" }],
|
|
708
|
+
"cosmeticallyChangedActors": [{ "actorFullName": "john.doe/other-actor" }]
|
|
709
|
+
}
|
|
710
|
+
```
|
|
711
|
+
|
|
712
|
+
Each changed Actor is in exactly one of the lists. `functionallyChangedActors` have at least one functional change, so
|
|
713
|
+
they need a new build and tests. `cosmeticallyChangedActors` only have cosmetic changes (see
|
|
714
|
+
[Change detection](#change-detection)), so they need a new build only when released: `build-for-test` builds
|
|
715
|
+
`functionallyChangedActors`, `release` builds both.
|
|
716
|
+
|
|
717
|
+
`build-for-test` and `release` use the same detection internally.
|
|
718
|
+
|
|
719
|
+
### Building Actors (`build`)
|
|
720
|
+
|
|
721
|
+
`build` builds exactly the Actors it is given on stdin, from the checked-out branch as it is on the remote, and
|
|
722
|
+
prints the builds as the same JSON array `build-for-test` and `build-from-local` print. A detached HEAD is an
|
|
723
|
+
error, since there is no branch to build from. It doesn't look at git history: which Actors to build is
|
|
724
|
+
up to you, usually picked from the output of [`get-affected-actors`](#finding-affected-actors-get-affected-actors).
|
|
725
|
+
|
|
726
|
+
```bash
|
|
727
|
+
# What build-for-test does: the Actors a branch changed functionally, built into the test version
|
|
728
|
+
npx apify-test-tools get-affected-actors --base origin/master \
|
|
729
|
+
| jq '{ actors: .functionallyChangedActors }' \
|
|
730
|
+
| npx apify-test-tools build --version-number 0.99
|
|
731
|
+
|
|
732
|
+
# What release builds: every Actor changed since the last release, into the version users run
|
|
733
|
+
npx apify-test-tools get-affected-actors --since <sha> \
|
|
734
|
+
| jq '{ actors: (.functionallyChangedActors + .cosmeticallyChangedActors) }' \
|
|
735
|
+
| npx apify-test-tools build --version-number default
|
|
736
|
+
|
|
737
|
+
# One Actor, whatever changed
|
|
738
|
+
echo '{ "actors": [{ "actorFullName": "john.doe/my-actor" }] }' | npx apify-test-tools build --version-number 0.99
|
|
739
|
+
```
|
|
740
|
+
|
|
741
|
+
The input is `{ "actors": [{ "actorFullName": "<owner/name>" }] }`; other fields are ignored. An empty `actors` list builds nothing and prints `[]`. Empty or invalid input is an error,
|
|
742
|
+
so a failed command earlier in the pipe can't turn into a successful run that built nothing. In GitHub
|
|
743
|
+
Actions, give such a step `shell: bash` so the pipe also fails on its own (the default shell has no `pipefail`).
|
|
744
|
+
|
|
745
|
+
| Flag | Default | Meaning |
|
|
746
|
+
| ------------------ | ------------------- | ---------------------------------------------------------------------------------------------------------------------- |
|
|
747
|
+
| `--version-number` | required | Version to build, like `0.99`, or `default` for each Actor's default version (the one its default build tag points to) |
|
|
748
|
+
| `--repo-url` | the `origin` remote | Repository to build from, see [Repository check](#repository-check) |
|
|
749
|
+
|
|
750
|
+
`build` never sets or changes build tags. A build gets the tag its version is configured with on the platform,
|
|
751
|
+
so building the default version moves its tag (usually `latest`) to the new build, and building a version
|
|
752
|
+
without a tag leaves every tag where it is. Builds always use the Docker cache.
|
|
753
|
+
|
|
754
|
+
`build-for-test` and `release` are shortcuts for the two pipelines above, with the flags they always had.
|
|
755
|
+
`build-for-test` takes `--target-branch`, `--base-commit` and `--test-files-glob`, inspects and builds the
|
|
756
|
+
checked-out branch, and builds into version `0.99`. `release` takes `--base-commit`, and announces the release on Slack after building, the same
|
|
757
|
+
way `notify-release` does.
|
|
758
|
+
|
|
663
759
|
### Releasing Actors (`release`)
|
|
664
760
|
|
|
665
761
|
`release` builds the `latest` (default) version of every Actor changed since the last release and
|
|
@@ -671,6 +767,15 @@ the `origin` remote.
|
|
|
671
767
|
npx apify-test-tools release --base-commit <sha> --release-slack-channel "#releases" --dry-run
|
|
672
768
|
```
|
|
673
769
|
|
|
770
|
+
The announcement is also its own command, `notify-release`, which reads the output of `get-affected-actors` on
|
|
771
|
+
stdin. It posts the changelog additions to `--release-slack-channel` and the commits and changed files to
|
|
772
|
+
`--report-slack-channel`, and posts nothing when there are no new commits. It needs `SLACK_TOKEN_RELEASES_BOT`.
|
|
773
|
+
|
|
774
|
+
```bash
|
|
775
|
+
npx apify-test-tools get-affected-actors --since <sha> \
|
|
776
|
+
| npx apify-test-tools notify-release --release-slack-channel "#releases" --dry-run
|
|
777
|
+
```
|
|
778
|
+
|
|
674
779
|
#### `--base-commit`
|
|
675
780
|
|
|
676
781
|
The last commit that is **already released**. Everything after it, up to `HEAD`, is released. It is
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { type ChangedActors, getChangedActors } from './diff-changes.js';
|
|
2
|
+
import {
|
|
3
|
+
getBranchOnlyChangedFiles,
|
|
4
|
+
getChangedFilesBetween,
|
|
5
|
+
getChangelogAdditions,
|
|
6
|
+
getCommits,
|
|
7
|
+
getCommitsBetween,
|
|
8
|
+
hasMergeFromTarget,
|
|
9
|
+
resolveCommitSha,
|
|
10
|
+
resolveReleaseBaseCommit,
|
|
11
|
+
} from './git.js';
|
|
12
|
+
import { logger } from './logger.js';
|
|
13
|
+
import type { ActorConfig, Commit } from './types.js';
|
|
14
|
+
import { readConfigFile } from './utils/config/load-config.js';
|
|
15
|
+
|
|
16
|
+
// The range always ends at the checked-out commit: check out what you want to inspect
|
|
17
|
+
const HEAD = 'HEAD';
|
|
18
|
+
|
|
19
|
+
interface AffectedActorsOptions {
|
|
20
|
+
/** Only the commits on HEAD that are not on base count, i.e. the branch's own changes. */
|
|
21
|
+
base?: string;
|
|
22
|
+
/**
|
|
23
|
+
* Only changes after this commit count. With `base` it's the last validated commit of the branch and is
|
|
24
|
+
* ignored when it isn't on the branch. Without `base` it must be an ancestor of HEAD (e.g. the last release).
|
|
25
|
+
*/
|
|
26
|
+
since?: string;
|
|
27
|
+
/** Glob relative to the repo root. Matching files count as changes even when .dockerignore lists them. */
|
|
28
|
+
testFilesGlob?: string;
|
|
29
|
+
actors: string[];
|
|
30
|
+
ignore: string[];
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
interface AffectedActors {
|
|
34
|
+
headSha: string;
|
|
35
|
+
commits: Commit[];
|
|
36
|
+
changedFiles: string[];
|
|
37
|
+
changelog: string | null;
|
|
38
|
+
/** Actors with at least one functional change. They need a new build and tests. */
|
|
39
|
+
functionallyChangedActors: ActorConfig[];
|
|
40
|
+
/** Actors whose changes are all cosmetic (README, CHANGELOG, cosmetic .actor JSON). They only need a release build. */
|
|
41
|
+
cosmeticallyChangedActors: ActorConfig[];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
interface CommitRange {
|
|
45
|
+
commits: Commit[];
|
|
46
|
+
fromRef: string;
|
|
47
|
+
toRef: string;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** The branch's own commits, after `since` when it is on the branch (see getCommits). */
|
|
51
|
+
const getBranchRange = async (base: string, since: string | undefined): Promise<CommitRange> => {
|
|
52
|
+
const commits = await getCommits({ sourceBranch: HEAD, targetBranch: base, baseCommit: since });
|
|
53
|
+
return { commits, fromRef: `${commits[0].sha}~`, toRef: commits[commits.length - 1].sha };
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
/** Everything after `since` up to HEAD. Unlike on a branch, `since` must be an ancestor of HEAD. */
|
|
57
|
+
const getRangeSince = async (headSha: string, since: string): Promise<CommitRange> => {
|
|
58
|
+
const sinceSha = await resolveReleaseBaseCommit(since);
|
|
59
|
+
const commits = sinceSha === headSha ? [] : await getCommitsBetween(sinceSha, headSha);
|
|
60
|
+
return { commits, fromRef: sinceSha, toRef: headSha };
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* This is an optimization for the common case where a branch only has cosmetic changes but had to merge in
|
|
65
|
+
* functional changes from base (being up-to-date is a CI requirement). Base is already validated, and
|
|
66
|
+
* since the branch has no functional changes of its own, there is nothing new to validate.
|
|
67
|
+
* Exception: if the branch has any functional changes alongside the merge, we must re-test — even
|
|
68
|
+
* individually validated changes can have novel interactions when combined.
|
|
69
|
+
* Returns the Actors the branch's own commits changed, or undefined when it didn't merge base in.
|
|
70
|
+
*
|
|
71
|
+
* TODO: Apply this per Actor instead of to the whole branch. Changes merged in from base only interact with the
|
|
72
|
+
* branch's own changes in Actors whose context both touch, so an Actor that only base changed is already validated
|
|
73
|
+
* and can be skipped even when the branch changes other Actors functionally. Today any functional change of the
|
|
74
|
+
* branch's own keeps every Actor, and the log messages below describe the whole branch.
|
|
75
|
+
*/
|
|
76
|
+
const getOwnChangesIfMergedFromBase = async (
|
|
77
|
+
base: string,
|
|
78
|
+
actorConfigs: ActorConfig[],
|
|
79
|
+
testFilesGlob: string | undefined,
|
|
80
|
+
): Promise<ChangedActors | undefined> => {
|
|
81
|
+
if (!(await hasMergeFromTarget(HEAD, base))) return undefined;
|
|
82
|
+
|
|
83
|
+
logger.info(
|
|
84
|
+
'[MERGE-FROM-TARGET-OPTIMIZATION]: There is merge from target branch, checking if there are no functional changes in our own branch. If so, we can skip tests',
|
|
85
|
+
);
|
|
86
|
+
const own = await getChangedActors({
|
|
87
|
+
filepathsChanged: await getBranchOnlyChangedFiles(HEAD, base),
|
|
88
|
+
actorConfigs,
|
|
89
|
+
// Omit since to get full branch history. Validated functional commits can still interact with merged ones
|
|
90
|
+
commits: await getCommits({ sourceBranch: HEAD, targetBranch: base }),
|
|
91
|
+
testFilesGlob,
|
|
92
|
+
});
|
|
93
|
+
logger.info(
|
|
94
|
+
own.functional.length > 0
|
|
95
|
+
? `[MERGE-FROM-TARGET-OPTIMIZATION]: Branch has ${own.functional.length} functional changes, cannot optimize, we continue with full check`
|
|
96
|
+
: '[MERGE-FROM-TARGET-OPTIMIZATION]: Branch itself has no functional changes, skipping tests',
|
|
97
|
+
);
|
|
98
|
+
return own;
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Finds the Actors whose context changed in the given range. `base` selects the branch's own commits
|
|
103
|
+
* (a PR), `since` the commits after a known one (a release, or the last validated commit of a PR).
|
|
104
|
+
*/
|
|
105
|
+
export const getAffectedActors = async ({
|
|
106
|
+
base,
|
|
107
|
+
since: sinceOption,
|
|
108
|
+
testFilesGlob,
|
|
109
|
+
actors,
|
|
110
|
+
ignore,
|
|
111
|
+
}: AffectedActorsOptions): Promise<AffectedActors> => {
|
|
112
|
+
// An empty --since (e.g. an empty cache file) means the same as not passing it
|
|
113
|
+
const since = sinceOption || undefined;
|
|
114
|
+
if (!base && !since) {
|
|
115
|
+
throw new Error('Pass --base, --since, or both to select the commits to inspect.');
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const headSha = await resolveCommitSha(HEAD);
|
|
119
|
+
const { commits, fromRef, toRef } = base ? await getBranchRange(base, since) : await getRangeSince(headSha, since!);
|
|
120
|
+
const actorConfigs = await readConfigFile({ actors, ignore });
|
|
121
|
+
|
|
122
|
+
// What the branch only merged in from base is already validated. Without functional changes of its own
|
|
123
|
+
// the branch has nothing to test, and without any changes of its own nothing to release either.
|
|
124
|
+
const own = base ? await getOwnChangesIfMergedFromBase(base, actorConfigs, testFilesGlob) : undefined;
|
|
125
|
+
const changedFiles = await getChangedFilesBetween(fromRef, toRef);
|
|
126
|
+
const changed = await getChangedActors({ filepathsChanged: changedFiles, actorConfigs, commits, testFilesGlob });
|
|
127
|
+
|
|
128
|
+
return {
|
|
129
|
+
headSha,
|
|
130
|
+
commits,
|
|
131
|
+
changedFiles,
|
|
132
|
+
changelog: await getChangelogAdditions(fromRef, toRef, changedFiles),
|
|
133
|
+
functionallyChangedActors: own && own.functional.length === 0 ? [] : changed.functional,
|
|
134
|
+
cosmeticallyChangedActors: own && own.functional.length + own.cosmetic.length === 0 ? [] : changed.cosmetic,
|
|
135
|
+
};
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
/** The JSON document `get-affected-actors` prints. Versioned, since other commands and workflows consume it. */
|
|
139
|
+
export const toAffectedActorsDocument = ({
|
|
140
|
+
headSha,
|
|
141
|
+
commits,
|
|
142
|
+
changedFiles,
|
|
143
|
+
changelog,
|
|
144
|
+
functionallyChangedActors,
|
|
145
|
+
cosmeticallyChangedActors,
|
|
146
|
+
}: AffectedActors) => {
|
|
147
|
+
const names = (configs: ActorConfig[]) => configs.map(({ actorFullName }) => ({ actorFullName }));
|
|
148
|
+
return {
|
|
149
|
+
version: 1,
|
|
150
|
+
head: headSha,
|
|
151
|
+
commits,
|
|
152
|
+
changedFiles,
|
|
153
|
+
changelog,
|
|
154
|
+
functionallyChangedActors: names(functionallyChangedActors),
|
|
155
|
+
cosmeticallyChangedActors: names(cosmeticallyChangedActors),
|
|
156
|
+
};
|
|
157
|
+
};
|