apify-test-tools 0.9.1-beta.0 → 0.9.1-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/CODEOWNERS +1 -2
- package/.github/actionlint.yaml +12 -0
- package/.github/actions/checkout-restore-dependencies/action.yaml +94 -0
- package/.github/review-prompt.md +154 -0
- package/.github/scripts/check-major-tag-refs.mjs +72 -0
- package/.github/scripts/check-package-version-bump.mjs +74 -0
- package/.github/scripts/run-with-apify-tokens.mjs +104 -0
- package/.github/workflows/_check_code.yaml +16 -0
- package/.github/workflows/_move_major_tag.yaml +108 -0
- package/.github/workflows/_update_release_metadata.yaml +12 -0
- package/.github/workflows/manual_move_major_tag.yaml +26 -0
- package/.github/workflows/manual_release_stable.yaml +12 -0
- package/.github/workflows/on_master.yaml +11 -0
- package/.github/workflows/on_pull_request.yaml +49 -1
- package/.github/workflows/public_claude.yaml +101 -0
- package/.github/workflows/public_platform-tests-claude-investigate-and-fix.yaml +208 -0
- package/.github/workflows/public_platform-tests.yaml +63 -0
- package/.github/workflows/public_pr-build-test.yaml +175 -0
- package/.github/workflows/public_push-build-latest.yaml +44 -0
- package/.github/workflows/public_review.yaml +107 -0
- package/.github/workflows-package-version +1 -0
- package/.prettierignore +9 -0
- package/.vscode/settings.json +2 -1
- package/CHANGELOG.md +14 -0
- package/CONTRIBUTING.md +53 -2
- package/README.md +151 -9
- package/bin/build-from-local.ts +23 -28
- package/bin/build.ts +82 -70
- package/bin/main.ts +2 -1
- package/bin/types.ts +0 -11
- package/bin/utils/config/load-config.ts +133 -0
- package/bin/utils/config/parser.ts +88 -0
- package/bin/utils/config/structures/base.ts +61 -0
- package/bin/utils/config/structures/legacy.ts +20 -0
- package/bin/utils/json-file.ts +55 -0
- package/bin/utils.ts +5 -150
- package/dist/bin/build-from-local.d.ts +2 -2
- package/dist/bin/build-from-local.d.ts.map +1 -1
- package/dist/bin/build-from-local.js +20 -19
- package/dist/bin/build-from-local.js.map +1 -1
- package/dist/bin/build.d.ts +7 -11
- package/dist/bin/build.d.ts.map +1 -1
- package/dist/bin/build.js +51 -50
- package/dist/bin/build.js.map +1 -1
- package/dist/bin/main.d.ts.map +1 -1
- package/dist/bin/main.js +2 -1
- package/dist/bin/main.js.map +1 -1
- package/dist/bin/types.d.ts +0 -9
- package/dist/bin/types.d.ts.map +1 -1
- package/dist/bin/utils/config/load-config.d.ts +13 -0
- package/dist/bin/utils/config/load-config.d.ts.map +1 -0
- package/dist/bin/utils/config/load-config.js +99 -0
- package/dist/bin/utils/config/load-config.js.map +1 -0
- package/dist/bin/utils/config/parser.d.ts +19 -0
- package/dist/bin/utils/config/parser.d.ts.map +1 -0
- package/dist/bin/utils/config/parser.js +72 -0
- package/dist/bin/utils/config/parser.js.map +1 -0
- package/dist/bin/utils/config/structures/base.d.ts +35 -0
- package/dist/bin/utils/config/structures/base.d.ts.map +1 -0
- package/dist/bin/utils/config/structures/base.js +40 -0
- package/dist/bin/utils/config/structures/base.js.map +1 -0
- package/dist/bin/utils/config/structures/legacy.d.ts +6 -0
- package/dist/bin/utils/config/structures/legacy.d.ts.map +1 -0
- package/dist/bin/utils/config/structures/legacy.js +14 -0
- package/dist/bin/utils/config/structures/legacy.js.map +1 -0
- package/dist/bin/utils/json-file.d.ts +20 -0
- package/dist/bin/utils/json-file.d.ts.map +1 -0
- package/dist/bin/utils/json-file.js +42 -0
- package/dist/bin/utils/json-file.js.map +1 -0
- package/dist/bin/utils.d.ts +5 -8
- package/dist/bin/utils.d.ts.map +1 -1
- package/dist/bin/utils.js +4 -107
- package/dist/bin/utils.js.map +1 -1
- package/dist/lib/consts.d.ts +1 -0
- package/dist/lib/consts.d.ts.map +1 -1
- package/dist/lib/consts.js +3 -0
- package/dist/lib/consts.js.map +1 -1
- package/dist/lib/lib.d.ts +10 -0
- package/dist/lib/lib.d.ts.map +1 -1
- package/dist/lib/lib.js +24 -18
- package/dist/lib/lib.js.map +1 -1
- package/dist/lib/types.d.ts +11 -3
- package/dist/lib/types.d.ts.map +1 -1
- package/dist/test/unit/bin/utils/config/load-config.test.d.ts +2 -0
- package/dist/test/unit/bin/utils/config/load-config.test.d.ts.map +1 -0
- package/dist/test/unit/bin/utils/config/load-config.test.js +138 -0
- package/dist/test/unit/bin/utils/config/load-config.test.js.map +1 -0
- package/dist/test/unit/bin/utils/config/parser.test.d.ts +2 -0
- package/dist/test/unit/bin/utils/config/parser.test.d.ts.map +1 -0
- package/dist/test/unit/bin/utils/config/parser.test.js +111 -0
- package/dist/test/unit/bin/utils/config/parser.test.js.map +1 -0
- package/dist/test/unit/bin/utils/config/structures/base.test.d.ts +2 -0
- package/dist/test/unit/bin/utils/config/structures/base.test.d.ts.map +1 -0
- package/dist/test/unit/bin/utils/config/structures/base.test.js +74 -0
- package/dist/test/unit/bin/utils/config/structures/base.test.js.map +1 -0
- package/dist/test/unit/bin/utils/config/structures/legacy.test.d.ts +2 -0
- package/dist/test/unit/bin/utils/config/structures/legacy.test.d.ts.map +1 -0
- package/dist/test/unit/bin/utils/config/structures/legacy.test.js +63 -0
- package/dist/test/unit/bin/utils/config/structures/legacy.test.js.map +1 -0
- package/dist/test/unit/bin/utils/files.test.d.ts +2 -0
- package/dist/test/unit/bin/utils/files.test.d.ts.map +1 -0
- package/dist/test/unit/bin/utils/files.test.js +117 -0
- package/dist/test/unit/bin/utils/files.test.js.map +1 -0
- package/dist/test/unit/lib.test.d.ts +2 -0
- package/dist/test/unit/lib.test.d.ts.map +1 -0
- package/dist/test/unit/lib.test.js +37 -0
- package/dist/test/unit/lib.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/lib/consts.ts +4 -0
- package/lib/lib.ts +32 -20
- package/lib/types.ts +11 -3
- package/package.json +5 -2
- package/test/fixtures/bin/utils/files/array.json +1 -0
- package/test/fixtures/bin/utils/files/comments.jsonc +4 -0
- package/test/fixtures/bin/utils/files/directory.json/.gitkeep +0 -0
- package/test/fixtures/bin/utils/files/empty-object.json +1 -0
- package/test/fixtures/bin/utils/files/empty.json +0 -0
- package/test/fixtures/bin/utils/files/malformed.json +1 -0
- package/test/fixtures/bin/utils/files/nested.json +6 -0
- package/test/fixtures/bin/utils/files/no-json-extension.actorrc +1 -0
- package/test/fixtures/bin/utils/files/null.json +1 -0
- package/test/fixtures/bin/utils/files/number.json +1 -0
- package/test/fixtures/bin/utils/files/unreadable.json +1 -0
- package/test/tsconfig.json +4 -2
- package/test/unit/bin/utils/config/load-config.test.ts +179 -0
- package/test/unit/bin/utils/config/parser.test.ts +137 -0
- package/test/unit/bin/utils/config/structures/base.test.ts +83 -0
- package/test/unit/bin/utils/config/structures/legacy.test.ts +78 -0
- package/test/unit/bin/utils/files.test.ts +158 -0
- package/test/unit/lib.test.ts +53 -0
- package/dist/test/unit/bin/utils.test.d.ts +0 -2
- package/dist/test/unit/bin/utils.test.d.ts.map +0 -1
- package/dist/test/unit/bin/utils.test.js +0 -321
- package/dist/test/unit/bin/utils.test.js.map +0 -1
- package/test/unit/bin/utils.test.ts +0 -381
package/README.md
CHANGED
|
@@ -99,7 +99,12 @@ See the [GitHub workflows](#github-worklows) section below.
|
|
|
99
99
|
|
|
100
100
|
## Github worklows
|
|
101
101
|
|
|
102
|
-
|
|
102
|
+
The reusable workflows live in this repo, alongside the package they call. They are the
|
|
103
|
+
`public_`-prefixed files in `.github/workflows`; everything else there is this repo's own CI.
|
|
104
|
+
Reference them at the `@workflows-v0` major tag, never at `@master` — see
|
|
105
|
+
[Versioning and releases](#versioning-and-releases).
|
|
106
|
+
|
|
107
|
+
There should be 4 GH workflow files in `.github/workflows`, plus an optional fifth for Claude reviews.
|
|
103
108
|
|
|
104
109
|
### `platform-tests-core.yaml`
|
|
105
110
|
|
|
@@ -114,7 +119,7 @@ on:
|
|
|
114
119
|
|
|
115
120
|
jobs:
|
|
116
121
|
platformTestsCore:
|
|
117
|
-
uses: apify
|
|
122
|
+
uses: apify/apify-test-tools/.github/workflows/public_platform-tests.yaml@workflows-v0
|
|
118
123
|
with:
|
|
119
124
|
subtest: core
|
|
120
125
|
secrets: inherit
|
|
@@ -133,7 +138,7 @@ on:
|
|
|
133
138
|
|
|
134
139
|
jobs:
|
|
135
140
|
platformTestsDaily:
|
|
136
|
-
uses: apify
|
|
141
|
+
uses: apify/apify-test-tools/.github/workflows/public_platform-tests.yaml@workflows-v0
|
|
137
142
|
secrets: inherit
|
|
138
143
|
```
|
|
139
144
|
|
|
@@ -148,7 +153,7 @@ on:
|
|
|
148
153
|
|
|
149
154
|
jobs:
|
|
150
155
|
buildDevelAndTest:
|
|
151
|
-
uses: apify
|
|
156
|
+
uses: apify/apify-test-tools/.github/workflows/public_pr-build-test.yaml@workflows-v0
|
|
152
157
|
secrets: inherit
|
|
153
158
|
```
|
|
154
159
|
|
|
@@ -163,10 +168,149 @@ on:
|
|
|
163
168
|
|
|
164
169
|
jobs:
|
|
165
170
|
buildLatest:
|
|
166
|
-
uses: apify
|
|
171
|
+
uses: apify/apify-test-tools/.github/workflows/public_push-build-latest.yaml@workflows-v0
|
|
167
172
|
secrets: inherit
|
|
168
173
|
```
|
|
169
174
|
|
|
175
|
+
### `claude-review.yaml`
|
|
176
|
+
|
|
177
|
+
Optional. Reviews a PR against the shared guidelines when you add the trigger label, and again on
|
|
178
|
+
every push while that label is on.
|
|
179
|
+
|
|
180
|
+
```yaml
|
|
181
|
+
name: Claude review
|
|
182
|
+
|
|
183
|
+
on:
|
|
184
|
+
pull_request:
|
|
185
|
+
types: [labeled, synchronize]
|
|
186
|
+
|
|
187
|
+
jobs:
|
|
188
|
+
review:
|
|
189
|
+
uses: apify/apify-test-tools/.github/workflows/public_review.yaml@workflows-v0
|
|
190
|
+
secrets: inherit
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
The review instructions live in `.github/review-prompt.md` in this repo and are fetched at run time,
|
|
194
|
+
because a reusable workflow doesn't get its own repo checked out. `prompt-ref` selects which ref to
|
|
195
|
+
fetch them from and defaults to `workflows-v0`, so the instructions match the workflow you're calling — point
|
|
196
|
+
it at a branch only to test a prompt change.
|
|
197
|
+
|
|
198
|
+
### Secrets
|
|
199
|
+
|
|
200
|
+
Callers pass `secrets: inherit`. The workflows do not turn every inherited secret into job-wide
|
|
201
|
+
environment variables, so a secret is only visible to the step that needs it:
|
|
202
|
+
|
|
203
|
+
| Secret | Reaches |
|
|
204
|
+
| ------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
205
|
+
| `NPM_TOKEN` | dependency install steps only, as both `NPM_TOKEN` and `NODE_AUTH_TOKEN`. Use a read-only token: npm granular tokens can be read-only, classic automation tokens can publish. |
|
|
206
|
+
| the Actor tokens named by `tokenEnvVar` in `apify-test-tools.config.json` | the `build`, `release`, and `delete-old-builds` steps only |
|
|
207
|
+
| `TESTER_APIFY_TOKEN` | the vitest step only |
|
|
208
|
+
| `SLACK_TOKEN_TESTS_BOT` / `SLACK_TOKEN_RELEASES_BOT` | the reporting and release steps only |
|
|
209
|
+
| `TESTER_APIFY_TOKEN_READ_ONLY` | the Claude investigation step only, as the Apify MCP server's bearer token. Required by `public_platform-tests-claude-investigate-and-fix`. Use a read-only token: it reads the failing run, its log and its storages. |
|
|
210
|
+
|
|
211
|
+
The Actor tokens are the one set that cannot be listed in the workflow, because each Actor names its
|
|
212
|
+
own token via `tokenEnvVar` in `apify-test-tools.config.json`. Those steps pass
|
|
213
|
+
`${{ toJSON(secrets) }}` as `ALL_SECRETS` and run the command through
|
|
214
|
+
`.github/scripts/run-with-apify-tokens.mjs`, which reads that same config file to decide which
|
|
215
|
+
secrets to pass on:
|
|
216
|
+
|
|
217
|
+
```yaml
|
|
218
|
+
- name: Build
|
|
219
|
+
env:
|
|
220
|
+
ALL_SECRETS: ${{ toJSON(secrets) }}
|
|
221
|
+
run: |
|
|
222
|
+
node "${{ steps.setup.outputs.scripts-path }}/run-with-apify-tokens.mjs" \
|
|
223
|
+
npx apify-test-tools build --target-branch ...
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
The wrapper passes only the tokens the config declares and drops `ALL_SECRETS`, so neither `npx` nor
|
|
227
|
+
anything under `node_modules` sees the blob. Nothing is written to `$GITHUB_ENV`, so the tokens stay
|
|
228
|
+
inside that one command rather than leaking into later steps. Reading the same file
|
|
229
|
+
`apify-test-tools` reads means the two can't drift, and a secret that merely looks like an Actor
|
|
230
|
+
token is not passed just because of its name.
|
|
231
|
+
|
|
232
|
+
A token the config declares but the repo hasn't set is a warning, not a failure: a repo can carry an
|
|
233
|
+
Actor whose token isn't configured and still build fine as long as that Actor never changes, and
|
|
234
|
+
`apify-test-tools` raises a precise error naming the Actor at the point it actually needs the token.
|
|
235
|
+
|
|
236
|
+
`scripts-path` comes from the setup action (give the step `id: setup`) and points at this repo's
|
|
237
|
+
`.github/scripts/` directory inside the runner's action checkout, so workflows can run these helpers
|
|
238
|
+
without checking this repo out again. The caller's workspace holds the caller's repo, not this one.
|
|
239
|
+
|
|
240
|
+
Two tidier-looking alternatives don't work, so don't reach for them:
|
|
241
|
+
|
|
242
|
+
- **Exporting to `$GITHUB_ENV`** would let the steps call `npx` directly with no wrapper, but
|
|
243
|
+
`$GITHUB_ENV` applies to every later step in the job. In `pr-build-test` the vitest step runs after
|
|
244
|
+
the build, so it would inherit Actor tokens it has no use for.
|
|
245
|
+
- **Returning the tokens as a step output** would be scoped correctly, but the runner refuses to set
|
|
246
|
+
an output whose value contains a registered secret. It logs `Skip output <name> since it may
|
|
247
|
+
contain secret` and leaves the output empty, so anything reading it downstream gets nothing.
|
|
248
|
+
|
|
249
|
+
The `unitTest` job runs static checks and needs `NPM_TOKEN` only. No job runs `npm ci` with Apify or
|
|
250
|
+
Slack credentials in scope, so a postinstall script in the dependency tree cannot read them.
|
|
251
|
+
|
|
252
|
+
## Versioning and releases
|
|
253
|
+
|
|
254
|
+
The workflows and the npm package live in one repo but ship on their own schedules. Two pointers
|
|
255
|
+
decide what a consumer repo actually runs:
|
|
256
|
+
|
|
257
|
+
| Pointer | What it selects | Moves when |
|
|
258
|
+
| ----------------------------------- | ---------------------------------------------- | -------------------------------------------------------- |
|
|
259
|
+
| the `@workflows-v0` tag in `uses:` | which workflows run | a master push, once the version below is published |
|
|
260
|
+
| `.github/workflows-package-version` | which `apify-test-tools` the workflows install | a stable release writes it; you may set it ahead of time |
|
|
261
|
+
|
|
262
|
+
The setup action installs that **exact** version — not a range. A tag is therefore a complete
|
|
263
|
+
statement: these workflows _and_ this library. The stable release writes the file into the same
|
|
264
|
+
commit that bumps `package.json` and `CHANGELOG.md`, so a released commit always names the version
|
|
265
|
+
it published, and rollout latency is unchanged: the release publishes and moves the tag in one run.
|
|
266
|
+
|
|
267
|
+
Prereleases are excluded. Master pushes publish a `-beta` that consumer repos must never install, so
|
|
268
|
+
the pin only moves on a stable release; betas stay reachable through the lockfile path in the setup
|
|
269
|
+
action, which is how branch testing works.
|
|
270
|
+
|
|
271
|
+
Nothing is coupled that doesn't need to be:
|
|
272
|
+
|
|
273
|
+
- **Workflow-only change** — merge it. `workflows-v0` moves, it goes live, no release needed.
|
|
274
|
+
- **Package-only change** — merge it, then cut a release when you want it out. The workflows are
|
|
275
|
+
unchanged, so consumers see nothing until the release lands.
|
|
276
|
+
- **A workflow that calls a new CLI feature** — the one case that can break consumers, and the only
|
|
277
|
+
one with any ceremony. Put the package change, the workflow change, and the new pin in one PR. On
|
|
278
|
+
merge the tag is **held**: the CI job reports that the pinned version isn't on npm and leaves `workflows-v0`
|
|
279
|
+
where it is, so consumers keep running the previous workflows. Cut a stable release, and the tag
|
|
280
|
+
moves on its own. Run **Move major version tag** if you don't want to wait for the next master
|
|
281
|
+
push.
|
|
282
|
+
|
|
283
|
+
`workflows-v0` moving on every master push means `@workflows-v0` is as live as `@master` was —
|
|
284
|
+
there's no staging step, just a gate on the package version. What the tag buys you is a
|
|
285
|
+
`workflows-v1` for breaking workflow changes, so repos migrate one at a time instead of all at once,
|
|
286
|
+
and a way to roll back by pointing the tag at an earlier commit. Bump `MAJOR_TAG` in
|
|
287
|
+
`.github/workflows/_move_major_tag.yaml` to cut the next major; the old tag then freezes where it is
|
|
288
|
+
and keeps working.
|
|
289
|
+
|
|
290
|
+
Freezing is real, which is the whole reason the version is pinned rather than a floor. A frozen `workflows-v0`
|
|
291
|
+
points at a commit whose pinned version never changes, so it keeps installing the library it was
|
|
292
|
+
tested against no matter how far the package moves on. Had the workflows installed `>=<floor>`, a
|
|
293
|
+
frozen `workflows-v0` would still resolve to whatever is newest — and since the release that enables
|
|
294
|
+
`workflows-v1` is usually the same release that breaks `workflows-v0`, the migration window would
|
|
295
|
+
have been zero.
|
|
296
|
+
|
|
297
|
+
The tag tracks the **workflows'** contract, not the npm package version. They move independently on
|
|
298
|
+
purpose, so `@workflows-v0` is expected to stay `@workflows-v0` after the package reaches 1.0 —
|
|
299
|
+
bump it when a workflow breaks its callers, not when the package does. Because `uses:` cannot take an expression, every ref
|
|
300
|
+
into this repo repeats the tag literally; `check-major-tag-refs.mjs` fails the build if `MAJOR_TAG`
|
|
301
|
+
and those refs disagree, which is the mistake that would otherwise ship silently during a bump.
|
|
302
|
+
|
|
303
|
+
### Testing workflow changes
|
|
304
|
+
|
|
305
|
+
- Point [testing-repo-for-github-actions](https://github.com/apify-store/testing-repo-for-github-actions)
|
|
306
|
+
at your branch (`uses: ...@your-branch`). It has real attached Actors and tests. Because the
|
|
307
|
+
package lives here too, a master push publishes a `beta`, and the lockfile-beta path in the setup
|
|
308
|
+
action installs that exact version — so one branch tests both halves of a change together.
|
|
309
|
+
- To change the composite action itself, repoint the `uses:` refs inside the reusable workflows at
|
|
310
|
+
your branch as well, and change them back before merging.
|
|
311
|
+
- Make sure the shell code actually works on your laptop first.
|
|
312
|
+
- After merging, watch the workflow on a real project before moving on.
|
|
313
|
+
|
|
170
314
|
## Writing tests
|
|
171
315
|
|
|
172
316
|
### Test structure
|
|
@@ -418,7 +562,7 @@ Remove `--dry-run` to actually trigger builds and update the branch names/ The c
|
|
|
418
562
|
|
|
419
563
|
#### Build from local source (no push needed)
|
|
420
564
|
|
|
421
|
-
If you don't want to push a dummy branch just to test a change and wait for all the tests to finish, `build-from-local` builds Actors directly from your local files (zipped and
|
|
565
|
+
If you don't want to push a dummy branch just to test a change and wait for all the tests to finish, `build-from-local` builds Actors directly from your local files (zipped into the Actor's `actor-<id>-source` key-value store and built as `TARBALL`, like `apify push`), skipping steps 1-4 above.
|
|
422
566
|
|
|
423
567
|
```bash
|
|
424
568
|
APIFY_TOKEN_JOHN_DOE=<token> \
|
|
@@ -439,14 +583,13 @@ Since you already scoped the build to just the Actor(s) you care about, point vi
|
|
|
439
583
|
|
|
440
584
|
#### 5. Run tests against the builds
|
|
441
585
|
|
|
442
|
-
Pass the build output as `ACTOR_BUILDS` and provide `TESTER_APIFY_TOKEN`. The token can point to your own account (if you have enough memory) or you can use the testing account (xRGg9iAfJSymqartk).
|
|
586
|
+
Pass the build output as `ACTOR_BUILDS` and provide `TESTER_APIFY_TOKEN`. The token can point to your own account (if you have enough memory) or you can use the testing account (xRGg9iAfJSymqartk). Platform test suites are skipped unless `TESTER_APIFY_TOKEN` is set, so regular unit test runs stay unaffected.
|
|
443
587
|
|
|
444
588
|
If you want to run only certain tests, change the `test/platform` to be more specific.
|
|
445
589
|
|
|
446
590
|
```bash
|
|
447
591
|
ACTOR_BUILDS='<JSON output from build command>' \
|
|
448
592
|
TESTER_APIFY_TOKEN=<token> \
|
|
449
|
-
RUN_PLATFORM_TESTS=1 \
|
|
450
593
|
npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform
|
|
451
594
|
```
|
|
452
595
|
|
|
@@ -463,7 +606,6 @@ BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
|
|
|
463
606
|
# Run tests with the builds
|
|
464
607
|
ACTOR_BUILDS="$BUILDS" \
|
|
465
608
|
TESTER_APIFY_TOKEN=apify_api_yyy \
|
|
466
|
-
RUN_PLATFORM_TESTS=1 \
|
|
467
609
|
npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform
|
|
468
610
|
```
|
|
469
611
|
|
package/bin/build-from-local.ts
CHANGED
|
@@ -2,13 +2,16 @@ import fs from 'node:fs/promises';
|
|
|
2
2
|
import os from 'node:os';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import AdmZip from 'adm-zip';
|
|
6
|
+
import type { ActorVersion } from 'apify-client';
|
|
7
|
+
import { ActorSourceType } from 'apify-client';
|
|
6
8
|
|
|
7
|
-
import {
|
|
9
|
+
import { dryRunBuildData, LOCAL_SOURCE_VERSION_NUMBER, runAndSummarizeBuilds } from './build.js';
|
|
8
10
|
import { buildDockerIgnoreMatcher } from './dockerignore.js';
|
|
9
11
|
import { isPathWithinScope } from './path-utils.js';
|
|
10
12
|
import type { ActorConfig, BuildData } from './types.js';
|
|
11
|
-
import {
|
|
13
|
+
import type { SourceFile } from './utils.js';
|
|
14
|
+
import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, readSourceFile } from './utils.js';
|
|
12
15
|
|
|
13
16
|
// JUST IN CASE. File patterns that commonly hold credentials — never ship these into a build, regardless
|
|
14
17
|
// of sourceType or of whether the repo's .gitignore happens to list them. Everything else that should be
|
|
@@ -17,7 +20,7 @@ import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, toActorVersionSour
|
|
|
17
20
|
const SKIP_FILE_PATTERNS = [/^\.env(\..+)?$/, /\.pem$/, /\.key$/, /\.pfx$/, /\.p12$/];
|
|
18
21
|
const isSecretFile = (fileName: string): boolean => SKIP_FILE_PATTERNS.some((pattern) => pattern.test(fileName));
|
|
19
22
|
|
|
20
|
-
export const collectSourceFiles = async (actorName: string, actorDir: string): Promise<
|
|
23
|
+
export const collectSourceFiles = async (actorName: string, actorDir: string): Promise<SourceFile[]> => {
|
|
21
24
|
const repoRoot = process.cwd();
|
|
22
25
|
const absActorDir = path.resolve(actorDir);
|
|
23
26
|
|
|
@@ -34,9 +37,7 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P
|
|
|
34
37
|
const keptFilePaths = collectNonIgnoredFiles(dockerContextDirAbs, repoRoot);
|
|
35
38
|
|
|
36
39
|
if (!isMonorepoActor) {
|
|
37
|
-
return Promise.all(
|
|
38
|
-
keptFilePaths.map(async (filePath) => toActorVersionSourceFile(filePath, dockerContextDirAbs)),
|
|
39
|
-
);
|
|
40
|
+
return Promise.all(keptFilePaths.map(async (filePath) => readSourceFile(filePath, dockerContextDirAbs)));
|
|
40
41
|
}
|
|
41
42
|
|
|
42
43
|
const { tempDir, filePaths } = await flattenMonorepoContext(
|
|
@@ -47,7 +48,7 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P
|
|
|
47
48
|
keptFilePaths,
|
|
48
49
|
);
|
|
49
50
|
try {
|
|
50
|
-
return await Promise.all(filePaths.map(async (filePath) =>
|
|
51
|
+
return await Promise.all(filePaths.map(async (filePath) => readSourceFile(filePath, tempDir)));
|
|
51
52
|
} finally {
|
|
52
53
|
// Only the flattened copy is temporary — never delete the actor's own directory.
|
|
53
54
|
await fs.rm(tempDir, { recursive: true, force: true });
|
|
@@ -197,26 +198,20 @@ export const runBuildsFromLocal = async ({
|
|
|
197
198
|
for (const { actorFullName, folder } of actorConfigs) {
|
|
198
199
|
console.error(` ${actorFullName} (${folder})`);
|
|
199
200
|
}
|
|
200
|
-
return actorConfigs.map(({ actorFullName }) => (
|
|
201
|
-
buildId: 'dry-run',
|
|
202
|
-
actorRawId: 'dry-run',
|
|
203
|
-
buildNumber: '0.98.0',
|
|
204
|
-
actorFullName,
|
|
205
|
-
}));
|
|
201
|
+
return actorConfigs.map(({ actorFullName }) => dryRunBuildData(actorFullName, LOCAL_SOURCE_VERSION_NUMBER));
|
|
206
202
|
}
|
|
207
203
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
return waitAndSummarizeBuilds(startedBuilds, buildersByActorFullName, 'LOCAL BUILDS');
|
|
204
|
+
return runAndSummarizeBuilds(actorConfigs, 'LOCAL BUILDS', async (actorConfig, builder) => {
|
|
205
|
+
// Zipped and uploaded like `apify push` does, since inline SOURCE_FILES are capped at ~9 MB.
|
|
206
|
+
const zip = new AdmZip();
|
|
207
|
+
for (const { name, content } of await collectSourceFiles(actorConfig.actorFullName, actorConfig.folder)) {
|
|
208
|
+
zip.addFile(name, content);
|
|
209
|
+
}
|
|
210
|
+
const actorVersion: ActorVersion = {
|
|
211
|
+
versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
|
|
212
|
+
tarballUrl: await builder.uploadSourceZip(LOCAL_SOURCE_VERSION_NUMBER, zip.toBuffer()),
|
|
213
|
+
sourceType: ActorSourceType.Tarball,
|
|
214
|
+
};
|
|
215
|
+
return builder.createVersionAndBuild(LOCAL_SOURCE_VERSION_NUMBER, actorVersion, false);
|
|
216
|
+
});
|
|
222
217
|
};
|
package/bin/build.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import { ApifyClient } from 'apify-client';
|
|
3
|
-
|
|
4
|
-
import { ACTOR_SOURCE_TYPES } from '@apify/consts';
|
|
1
|
+
import type { ActorVersion, Build } from 'apify-client';
|
|
2
|
+
import { ActorSourceType, ApifyClient } from 'apify-client';
|
|
5
3
|
|
|
6
4
|
import type { ActorConfig, BuildData } from './types.js';
|
|
7
5
|
|
|
@@ -12,6 +10,11 @@ type BuildPrActorOptions = {
|
|
|
12
10
|
actorConfig: ActorConfig;
|
|
13
11
|
useDockerCache: boolean;
|
|
14
12
|
};
|
|
13
|
+
|
|
14
|
+
// Fixed version number used to build from local source files, since there is no real version to track.
|
|
15
|
+
export const LOCAL_SOURCE_VERSION_NUMBER = '0.98';
|
|
16
|
+
const DEFAULT_TEST_VERSION_NUMBER = '0.99';
|
|
17
|
+
|
|
15
18
|
export class ApifyBuilder {
|
|
16
19
|
private constructor(
|
|
17
20
|
private readonly apifyClient: ApifyClient,
|
|
@@ -57,12 +60,27 @@ export class ApifyBuilder {
|
|
|
57
60
|
return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag };
|
|
58
61
|
};
|
|
59
62
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
// Mirrors `apify push`: stores the zip in the Actor's `actor-<id>-source` KV store and returns a signed
|
|
64
|
+
// record URL the builder can download without a token.
|
|
65
|
+
uploadSourceZip = async (versionNumber: string, zip: Buffer): Promise<string> => {
|
|
66
|
+
const actorInfo = await this.apifyClient.actor(this.actorFullName).get();
|
|
67
|
+
if (!actorInfo) throw new Error(`No actor named '${this.actorFullName}' was found on the platform.`);
|
|
68
|
+
|
|
69
|
+
const store = await this.apifyClient.keyValueStores().getOrCreate(`actor-${actorInfo.id}-source`);
|
|
70
|
+
const storeClient = this.apifyClient.keyValueStore(store.id);
|
|
71
|
+
const key = `version-${versionNumber}.zip`;
|
|
72
|
+
await storeClient.setRecord({ key, value: zip as never, contentType: 'application/zip' });
|
|
73
|
+
|
|
74
|
+
const url = new URL(await storeClient.getRecordPublicUrl(key));
|
|
75
|
+
url.searchParams.set('disableRedirect', 'true');
|
|
76
|
+
return url.toString();
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
createVersionAndBuild = async (
|
|
80
|
+
versionNumber: string,
|
|
81
|
+
actorVersion: ActorVersion,
|
|
82
|
+
useCache: boolean,
|
|
83
|
+
): Promise<BuildData> => {
|
|
66
84
|
const actorClient = this.apifyClient.actor(this.actorFullName);
|
|
67
85
|
const actorInfo = await actorClient.get();
|
|
68
86
|
if (!actorInfo) {
|
|
@@ -73,17 +91,6 @@ export class ApifyBuilder {
|
|
|
73
91
|
);
|
|
74
92
|
}
|
|
75
93
|
|
|
76
|
-
// NOTE: I couldn't find this type, so I had to extract it :(
|
|
77
|
-
type ActorVersion = Parameters<ReturnType<typeof actorClient.version>['update']>[0];
|
|
78
|
-
const actorVersion: ActorVersion = {
|
|
79
|
-
buildTag,
|
|
80
|
-
versionNumber,
|
|
81
|
-
gitRepoUrl,
|
|
82
|
-
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
83
|
-
// @ts-ignore: coudn't find this type either :(
|
|
84
|
-
sourceType: ACTOR_SOURCE_TYPES.GIT_REPO,
|
|
85
|
-
};
|
|
86
|
-
|
|
87
94
|
// Prepare version
|
|
88
95
|
const versionExists = !actorInfo.versions.find((version) => version.versionNumber === versionNumber);
|
|
89
96
|
if (versionExists) {
|
|
@@ -95,45 +102,12 @@ export class ApifyBuilder {
|
|
|
95
102
|
}
|
|
96
103
|
|
|
97
104
|
// We also get back actId so the testing actor can both match by actor ID and name
|
|
98
|
-
const { id, actId, buildNumber } = await actorClient.build(versionNumber, { useCache
|
|
105
|
+
const { id, actId, buildNumber } = await actorClient.build(versionNumber, { useCache });
|
|
99
106
|
|
|
100
107
|
console.error(`[${this.actorFullName}]: ${id} (${buildNumber})`);
|
|
101
108
|
return { buildId: id, actorRawId: actId, buildNumber, actorFullName: this.actorFullName };
|
|
102
109
|
};
|
|
103
110
|
|
|
104
|
-
startActorBuildFromSourceFiles = async (sourceFiles: ActorVersionSourceFile[]): Promise<BuildData> => {
|
|
105
|
-
const ZIP_VERSION = '0.98';
|
|
106
|
-
const actorClient = this.apifyClient.actor(this.actorFullName);
|
|
107
|
-
const actorInfo = await actorClient.get();
|
|
108
|
-
if (!actorInfo) {
|
|
109
|
-
throw new Error(
|
|
110
|
-
`No actor named '${this.actorFullName}' was found on the platform. If this` +
|
|
111
|
-
' is unexpected, make sure the actor you are targeting is spelled the' +
|
|
112
|
-
' same as the folder in the repository.',
|
|
113
|
-
);
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
type ActorVersion = Parameters<ReturnType<typeof actorClient.version>['update']>[0];
|
|
117
|
-
const actorVersion: ActorVersion = {
|
|
118
|
-
versionNumber: ZIP_VERSION,
|
|
119
|
-
sourceFiles,
|
|
120
|
-
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
121
|
-
// @ts-ignore: couldn't find this type :(
|
|
122
|
-
sourceType: ACTOR_SOURCE_TYPES.SOURCE_FILES,
|
|
123
|
-
};
|
|
124
|
-
|
|
125
|
-
const versionExists = !actorInfo.versions.find((v) => v.versionNumber === ZIP_VERSION);
|
|
126
|
-
if (versionExists) {
|
|
127
|
-
await actorClient.versions().create(actorVersion);
|
|
128
|
-
} else {
|
|
129
|
-
await actorClient.version(ZIP_VERSION).update(actorVersion);
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
const { id, actId, buildNumber } = await actorClient.build(ZIP_VERSION, { useCache: false });
|
|
133
|
-
console.error(`[${this.actorFullName}]: ${id} (${buildNumber})`);
|
|
134
|
-
return { buildId: id, actorRawId: actId, buildNumber, actorFullName: this.actorFullName };
|
|
135
|
-
};
|
|
136
|
-
|
|
137
111
|
waitForBuildToFinish = async (buildId: string): Promise<Build> => {
|
|
138
112
|
const build = await this.apifyClient.build(buildId).waitForFinish();
|
|
139
113
|
const versionNumber = build.buildNumber;
|
|
@@ -281,6 +255,33 @@ export const waitAndSummarizeBuilds = async (
|
|
|
281
255
|
return startedBuilds;
|
|
282
256
|
};
|
|
283
257
|
|
|
258
|
+
export const runAndSummarizeBuilds = async (
|
|
259
|
+
actorConfigs: ActorConfig[],
|
|
260
|
+
label: string,
|
|
261
|
+
buildOneActor: (actorConfig: ActorConfig, builder: ApifyBuilder) => Promise<BuildData>,
|
|
262
|
+
): Promise<BuildData[]> => {
|
|
263
|
+
const buildersByActorFullName = new Map<string, ApifyBuilder>(
|
|
264
|
+
actorConfigs.map((actorConfig) => [actorConfig.actorFullName, ApifyBuilder.fromActorConfig(actorConfig)]),
|
|
265
|
+
);
|
|
266
|
+
console.error('=========================================');
|
|
267
|
+
console.error(`STARTED ${label}:`);
|
|
268
|
+
const startedBuilds = await Promise.all(
|
|
269
|
+
actorConfigs.map(async (actorConfig) =>
|
|
270
|
+
buildOneActor(actorConfig, buildersByActorFullName.get(actorConfig.actorFullName)!),
|
|
271
|
+
),
|
|
272
|
+
);
|
|
273
|
+
|
|
274
|
+
return waitAndSummarizeBuilds(startedBuilds, buildersByActorFullName, label);
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
// Placeholder BuildData for dry runs, since no real build was triggered.
|
|
278
|
+
export const dryRunBuildData = (actorFullName: string, versionNumber: string): BuildData => ({
|
|
279
|
+
buildId: 'dry-run',
|
|
280
|
+
actorRawId: 'dry-run',
|
|
281
|
+
buildNumber: versionNumber,
|
|
282
|
+
actorFullName,
|
|
283
|
+
});
|
|
284
|
+
|
|
284
285
|
type RunBuildsOptions = {
|
|
285
286
|
actorConfigs: ActorConfig[];
|
|
286
287
|
isLatest?: boolean;
|
|
@@ -297,7 +298,7 @@ export const runBuilds = async ({
|
|
|
297
298
|
isLatest = false,
|
|
298
299
|
dryRun,
|
|
299
300
|
useDockerCache,
|
|
300
|
-
}: RunBuildsOptions) => {
|
|
301
|
+
}: RunBuildsOptions): Promise<BuildData[]> => {
|
|
301
302
|
const buildConfigs: BuildPrActorOptions[] = [];
|
|
302
303
|
|
|
303
304
|
for (const actorConfig of actorConfigs) {
|
|
@@ -310,7 +311,7 @@ export const runBuilds = async ({
|
|
|
310
311
|
versionNumber = defaultVersionNumber;
|
|
311
312
|
buildTag = defaultBuildTag;
|
|
312
313
|
} else {
|
|
313
|
-
versionNumber =
|
|
314
|
+
versionNumber = DEFAULT_TEST_VERSION_NUMBER;
|
|
314
315
|
}
|
|
315
316
|
|
|
316
317
|
// Depending on if these are miniactors or standaloneActors
|
|
@@ -322,23 +323,34 @@ export const runBuilds = async ({
|
|
|
322
323
|
}
|
|
323
324
|
|
|
324
325
|
if (dryRun) {
|
|
325
|
-
|
|
326
|
+
console.error('[DRY RUN] Would build:');
|
|
327
|
+
for (const { actorConfig, versionNumber } of buildConfigs) {
|
|
328
|
+
console.error(` ${actorConfig.actorFullName} (${versionNumber})`);
|
|
329
|
+
}
|
|
330
|
+
return buildConfigs.map(({ actorConfig, versionNumber }) =>
|
|
331
|
+
dryRunBuildData(actorConfig.actorFullName, versionNumber),
|
|
332
|
+
);
|
|
326
333
|
}
|
|
327
334
|
|
|
328
|
-
const
|
|
329
|
-
|
|
330
|
-
);
|
|
331
|
-
console.error('=========================================');
|
|
332
|
-
console.error('STARTED BUILDS:');
|
|
333
|
-
const startedBuilds = await Promise.all(
|
|
334
|
-
buildConfigs.map(async (buildConfig) => {
|
|
335
|
-
const builder = buildersByActorFullName.get(buildConfig.actorConfig.actorFullName)!;
|
|
336
|
-
const buildData = await builder.startActorBuild(buildConfig);
|
|
337
|
-
return buildData;
|
|
338
|
-
}),
|
|
335
|
+
const buildConfigsByActorFullName = new Map(
|
|
336
|
+
buildConfigs.map((buildConfig) => [buildConfig.actorConfig.actorFullName, buildConfig]),
|
|
339
337
|
);
|
|
340
338
|
|
|
341
|
-
return
|
|
339
|
+
return runAndSummarizeBuilds(actorConfigs, 'BUILDS', async (actorConfig, builder) => {
|
|
340
|
+
const {
|
|
341
|
+
buildTag,
|
|
342
|
+
versionNumber,
|
|
343
|
+
gitRepoUrl,
|
|
344
|
+
useDockerCache: useCache,
|
|
345
|
+
} = buildConfigsByActorFullName.get(actorConfig.actorFullName)!;
|
|
346
|
+
const actorVersion: ActorVersion = {
|
|
347
|
+
buildTag,
|
|
348
|
+
versionNumber,
|
|
349
|
+
gitRepoUrl,
|
|
350
|
+
sourceType: ActorSourceType.GitRepo,
|
|
351
|
+
};
|
|
352
|
+
return builder.createVersionAndBuild(versionNumber, actorVersion, useCache);
|
|
353
|
+
});
|
|
342
354
|
};
|
|
343
355
|
|
|
344
356
|
export const deleteOldBuilds = async (actorConfigs: ActorConfig[]) => {
|
package/bin/main.ts
CHANGED
|
@@ -14,7 +14,8 @@ import { getPushData } from './github.js';
|
|
|
14
14
|
import { notifyToSlack } from './slack.js';
|
|
15
15
|
import { reportTestResults } from './test-report.js';
|
|
16
16
|
import type { Config } from './types.js';
|
|
17
|
-
import {
|
|
17
|
+
import { setCwd, spawnCommandInGhWorkspace } from './utils.js';
|
|
18
|
+
import { readConfigFile } from './utils/config/load-config.js';
|
|
18
19
|
|
|
19
20
|
/**
|
|
20
21
|
* Middlewares to be run before every command execution
|
package/bin/types.ts
CHANGED
|
@@ -87,17 +87,6 @@ export interface GithubCommit {
|
|
|
87
87
|
modified: string[];
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
-
export interface ActorConfigFileEntry {
|
|
91
|
-
folder: string;
|
|
92
|
-
actorFullName: string;
|
|
93
|
-
tokenEnvVar: string;
|
|
94
|
-
overrideActorContext?: string[];
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
export interface ActorConfigFile {
|
|
98
|
-
actors: ActorConfigFileEntry[];
|
|
99
|
-
}
|
|
100
|
-
|
|
101
90
|
export interface BuildData {
|
|
102
91
|
buildId: string;
|
|
103
92
|
actorRawId: string;
|