apify-test-tools 0.9.1-beta.10 → 0.9.1-beta.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/.github/workflows/public_push-build-latest.yaml +3 -1
  2. package/.github/workflows-package-version +1 -1
  3. package/CHANGELOG.md +2 -0
  4. package/CONTRIBUTING.md +2 -4
  5. package/README.md +146 -35
  6. package/bin/build-from-local.ts +2 -2
  7. package/bin/build.ts +114 -70
  8. package/bin/git.ts +140 -0
  9. package/bin/main.ts +46 -19
  10. package/bin/slack.ts +5 -2
  11. package/bin/types.ts +0 -73
  12. package/bin/utils/config/parser.ts +6 -5
  13. package/bin/utils/config/structures/base.ts +5 -1
  14. package/bin/utils/config/structures/grouped.ts +40 -0
  15. package/bin/utils/config/structures/legacy.ts +3 -1
  16. package/bin/utils.ts +2 -3
  17. package/dist/bin/build-from-local.js.map +1 -1
  18. package/dist/bin/build.d.ts +25 -4
  19. package/dist/bin/build.d.ts.map +1 -1
  20. package/dist/bin/build.js +76 -47
  21. package/dist/bin/build.js.map +1 -1
  22. package/dist/bin/git.d.ts +40 -0
  23. package/dist/bin/git.d.ts.map +1 -1
  24. package/dist/bin/git.js +118 -0
  25. package/dist/bin/git.js.map +1 -1
  26. package/dist/bin/main.d.ts.map +1 -1
  27. package/dist/bin/main.js +33 -13
  28. package/dist/bin/main.js.map +1 -1
  29. package/dist/bin/slack.d.ts.map +1 -1
  30. package/dist/bin/slack.js +4 -2
  31. package/dist/bin/slack.js.map +1 -1
  32. package/dist/bin/types.d.ts +0 -64
  33. package/dist/bin/types.d.ts.map +1 -1
  34. package/dist/bin/utils/config/parser.d.ts.map +1 -1
  35. package/dist/bin/utils/config/parser.js +6 -5
  36. package/dist/bin/utils/config/parser.js.map +1 -1
  37. package/dist/bin/utils/config/structures/base.d.ts +3 -2
  38. package/dist/bin/utils/config/structures/base.d.ts.map +1 -1
  39. package/dist/bin/utils/config/structures/base.js +1 -0
  40. package/dist/bin/utils/config/structures/base.js.map +1 -1
  41. package/dist/bin/utils/config/structures/grouped.d.ts +6 -0
  42. package/dist/bin/utils/config/structures/grouped.d.ts.map +1 -0
  43. package/dist/bin/utils/config/structures/grouped.js +29 -0
  44. package/dist/bin/utils/config/structures/grouped.js.map +1 -0
  45. package/dist/bin/utils/config/structures/legacy.d.ts +1 -1
  46. package/dist/bin/utils/config/structures/legacy.d.ts.map +1 -1
  47. package/dist/bin/utils/config/structures/legacy.js +1 -1
  48. package/dist/bin/utils/config/structures/legacy.js.map +1 -1
  49. package/dist/bin/utils.d.ts.map +1 -1
  50. package/dist/bin/utils.js +2 -3
  51. package/dist/bin/utils.js.map +1 -1
  52. package/dist/test/unit/bin/build.test.d.ts +2 -0
  53. package/dist/test/unit/bin/build.test.d.ts.map +1 -0
  54. package/dist/test/unit/bin/build.test.js +46 -0
  55. package/dist/test/unit/bin/build.test.js.map +1 -0
  56. package/dist/test/unit/bin/git.test.js +104 -1
  57. package/dist/test/unit/bin/git.test.js.map +1 -1
  58. package/dist/test/unit/bin/utils/config/load-config.test.js +1 -0
  59. package/dist/test/unit/bin/utils/config/load-config.test.js.map +1 -1
  60. package/dist/test/unit/bin/utils/config/parser.test.js +23 -5
  61. package/dist/test/unit/bin/utils/config/parser.test.js.map +1 -1
  62. package/dist/test/unit/bin/utils/config/structures/grouped.test.d.ts +2 -0
  63. package/dist/test/unit/bin/utils/config/structures/grouped.test.d.ts.map +1 -0
  64. package/dist/test/unit/bin/utils/config/structures/grouped.test.js +75 -0
  65. package/dist/test/unit/bin/utils/config/structures/grouped.test.js.map +1 -0
  66. package/dist/tsconfig.tsbuildinfo +1 -1
  67. package/package.json +2 -2
  68. package/test/unit/bin/build.test.ts +68 -0
  69. package/test/unit/bin/git.test.ts +122 -0
  70. package/test/unit/bin/utils/config/load-config.test.ts +1 -0
  71. package/test/unit/bin/utils/config/parser.test.ts +26 -5
  72. package/test/unit/bin/utils/config/structures/grouped.test.ts +90 -0
  73. package/bin/github.ts +0 -69
  74. package/dist/bin/github.d.ts +0 -11
  75. package/dist/bin/github.d.ts.map +0 -1
  76. package/dist/bin/github.js +0 -59
  77. package/dist/bin/github.js.map +0 -1
@@ -25,6 +25,8 @@ jobs:
25
25
  with:
26
26
  npm-token: ${{ secrets.NPM_TOKEN }}
27
27
 
28
+ # --base-commit is the last commit that is already released: the push's "before" commit.
29
+ # Everything after it up to HEAD gets released. See the "Releasing Actors" section of the README.
28
30
  # run-with-apify-tokens.mjs reads the token names from in apify-test-tools.config.json
29
31
  # and hands the build those secrets and nothing else, so the tokens stay in this one command instead of the whole job.
30
32
  # The library never sees secrets it doesn't need
@@ -34,7 +36,7 @@ jobs:
34
36
  SLACK_TOKEN_RELEASES_BOT: ${{ secrets.SLACK_TOKEN_RELEASES_BOT }}
35
37
  run: |
36
38
  node "${{ steps.setup.outputs.scripts-path }}/run-with-apify-tokens.mjs" \
37
- npx apify-test-tools release --push-event-path ${{ github.event_path }} --release-slack-channel "#delivery-public-actors" --report-slack-channel "${{ inputs.report-slack-channel || format('#notif-{0}', github.event.repository.name) }}" --use-docker-cache
39
+ npx apify-test-tools release --base-commit ${{ github.event.before }} --release-slack-channel "#delivery-public-actors" --report-slack-channel "${{ inputs.report-slack-channel || format('#notif-{0}', github.event.repository.name) }}" --use-docker-cache
38
40
 
39
41
  - name: Delete old builds
40
42
  env:
@@ -1 +1 @@
1
- 0.9.0
1
+ 0.10.0
package/CHANGELOG.md CHANGED
@@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file.
9
9
 
10
10
  - **config/modes:** Lay groundwork for multiple configuration structures ([#142](https://github.com/apify/apify-test-tools/pull/142)) ([7e18638](https://github.com/apify/apify-test-tools/commit/7e18638a8d3523154dbe819312c2b16b54c548ff)) by [@JuanGalilea](https://github.com/JuanGalilea)
11
11
  - **workflows:** Host the reusable GitHub workflows in this repo ([#120](https://github.com/apify/apify-test-tools/pull/120)) ([10f089d](https://github.com/apify/apify-test-tools/commit/10f089d85146ceb7cdae964cfa1fc2577df43a37)) by [@metalwarrior665](https://github.com/metalwarrior665)
12
+ - **mode:** Add grouped config mode ([#143](https://github.com/apify/apify-test-tools/pull/143)) ([9d0b93e](https://github.com/apify/apify-test-tools/commit/9d0b93e194a2cd810a31bcef4ca9457fd9ebfb9b)) by [@JuanGalilea](https://github.com/JuanGalilea)
12
13
 
13
14
  ### 🐛 Bug Fixes
14
15
 
@@ -18,6 +19,7 @@ All notable changes to this project will be documented in this file.
18
19
  ### 🚜 Refactor
19
20
 
20
21
  - **workflows:** [**breaking**] Rename the floating tag to workflows-v0 ([#146](https://github.com/apify/apify-test-tools/pull/146)) ([c48fb22](https://github.com/apify/apify-test-tools/commit/c48fb223c885c4e5de7e5f64d8c14f6d03fcc7de)) by [@metalwarrior665](https://github.com/metalwarrior665)
22
+ - **bin:** [**breaking**] Remove GitHub event dependency from release command ([#153](https://github.com/apify/apify-test-tools/pull/153)) ([e6953aa](https://github.com/apify/apify-test-tools/commit/e6953aa095d1a92638c6d72441e1b312d9f835c3)) by [@metalwarrior665](https://github.com/metalwarrior665), closes [#131](https://github.com/apify/apify-test-tools/issues/131)
21
23
 
22
24
 
23
25
  <!-- git-cliff-unreleased-end -->
package/CONTRIBUTING.md CHANGED
@@ -10,7 +10,6 @@ The package consists of three parts:
10
10
 
11
11
  - `bin/build.ts` actor building
12
12
  - `bin/git.ts` git wrappers for getting relevant git commits and changed files
13
- - `bin/github.ts` github push event parsing
14
13
  - `bin/main.ts` entrypoing
15
14
  - `bin/slack.ts` sending notifications to slack
16
15
  - `bin/test-report.ts` processing vitest's test reports
@@ -40,12 +39,11 @@ git clone git@github.com:apify-store/testing-repo-for-github-actions.git
40
39
 
41
40
  #### Working on the CLI
42
41
 
43
- To work on the library, you just need to define `GITHUB_WORKSPACE` to tell the cli where you repo is located:
42
+ To work on the library, point the cli to your repo with `--workspace` (it defaults to the current directory):
44
43
 
45
44
  ```sh
46
- export GITHUB_WORKSPACE=../path/to/testing-repo-for-github-actions # path to the repo
47
45
  npx tsx bin/main.ts --help
48
- npx tsx bin/main.ts get-commits --target-branch master --source-branch feat/testing-feature-branch
46
+ npx tsx bin/main.ts --workspace ../path/to/testing-repo-for-github-actions get-commits --target-branch master --source-branch feat/testing-feature-branch
49
47
  ```
50
48
 
51
49
  #### Working on the library
package/README.md CHANGED
@@ -12,37 +12,15 @@ npm i -D apify-test-tools
12
12
 
13
13
  - Requires `vitest` version `3.2.0` or later (uses [annotate](https://vitest.dev/guide/test-context.html#annotate))
14
14
  - Make sure `target` and `module` in your `tsconfig.json`'s `compilerOptions` are set to `ES2022`
15
+ - Every Actor must already have at least one build under its default build tag (usually `latest`), built
16
+ from a Git repository version whose URL points to this repository. Build it once manually on the
17
+ platform before CI takes over. See [Repository check](#repository-check).
15
18
 
16
19
  ### 2. Create the config file
17
20
 
18
- Every repo that uses `apify-test-tools` must have an `apify-test-tools.config.json` file at the root. This file tells the tool which actors live in the repo, how to identify them, and which token to use.
21
+ Every repo that uses `apify-test-tools` must have an `apify-test-tools.config.json` file at the root. It lists the actors in the repo, their folders, their full names on the platform, and which env var holds the token to use for each. A top-level `mode` field picks the file's layout; leave it out to use the default `"legacy"` layout.
19
22
 
20
- ```json
21
- {
22
- "actors": [
23
- {
24
- "folder": "actors/web-scraper",
25
- "actorFullName": "myteam/web-scraper",
26
- "tokenEnvVar": "APIFY_TOKEN_MYTEAM"
27
- },
28
- {
29
- "folder": "actors/email-sender",
30
- "actorFullName": "myteam/email-sender",
31
- "tokenEnvVar": "APIFY_TOKEN_MYTEAM",
32
- "overrideActorContext": ["actors/email-sender", "packages/shared"]
33
- }
34
- ]
35
- }
36
- ```
37
-
38
- Each entry has:
39
-
40
- | Field | Required | Description |
41
- | ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
42
- | `folder` | yes | Relative path from repo root to the actor's own project directory — the folder that directly contains `.actor/actor.json` (i.e. `<folder>/.actor/actor.json`), the actor's README/CHANGELOG, and its source. Use `"."` for a single-actor repo where `.actor/` is at the root. |
43
- | `actorFullName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. |
44
- | `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails. |
45
- | `overrideActorContext` | no | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. Entries must not be prefixes of one another (e.g. `["", "code"]` or `["actors", "actors/foo"]` are rejected). The actor's own `folder` is always part of its context — if none of the listed entries reach it, it's added automatically. |
23
+ See [Config file](#config-file) for the available modes and every field.
46
24
 
47
25
  ### 3. Set up actor folders
48
26
 
@@ -97,6 +75,82 @@ mkdir -p test/platform/core
97
75
 
98
76
  See the [GitHub workflows](#github-worklows) section below.
99
77
 
78
+ ## Config file
79
+
80
+ `apify-test-tools.config.json` lives at the repo root and tells the tool which actors live in the repo, how to identify them, and which token to use.
81
+
82
+ ### Actor fields
83
+
84
+ Every actor in the config is described by these fields, whatever the mode:
85
+
86
+ | Field | Required | Description |
87
+ | ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
88
+ | `folder` | yes | Relative path from repo root to the actor's own project directory — the folder that directly contains `.actor/actor.json` (i.e. `<folder>/.actor/actor.json`), the actor's README/CHANGELOG, and its source. Use `"."` for a single-actor repo where `.actor/` is at the root. |
89
+ | `actorFullName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. |
90
+ | `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. In [grouped mode](#grouped-mode) it's inherited from the group unless the actor sets its own. No fallback — if the env var is not set at build time, the build fails. |
91
+ | `overrideActorContext` | no | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. Entries must not be prefixes of one another (e.g. `["", "code"]` or `["actors", "actors/foo"]` are rejected). The actor's own `folder` is always part of its context — if none of the listed entries reach it, it's added automatically. |
92
+
93
+ ### Modes
94
+
95
+ The file's layout is picked by the top-level `mode` field. Each mode is a different way of writing the same thing: every mode resolves to a list of actors with the fields above, and the same rules apply to all of them — no two actors may share a `folder`, and no two entries may point at the same `actorFullName`. If `mode` is left out, [`"legacy"`](#legacy-mode) is used, so existing config files keep working unchanged.
96
+
97
+ ### Legacy mode
98
+
99
+ The default. A flat `actors` array where every entry carries all of its settings. Good for repos with a few actors, or where every actor has different settings.
100
+
101
+ ```json
102
+ {
103
+ "mode": "legacy",
104
+ "actors": [
105
+ {
106
+ "folder": "actors/web-scraper",
107
+ "actorFullName": "myteam/web-scraper",
108
+ "tokenEnvVar": "APIFY_TOKEN_MYTEAM"
109
+ },
110
+ {
111
+ "folder": "actors/email-sender",
112
+ "actorFullName": "myteam/email-sender",
113
+ "tokenEnvVar": "APIFY_TOKEN_MYTEAM",
114
+ "overrideActorContext": ["actors/email-sender", "packages/shared"]
115
+ }
116
+ ]
117
+ }
118
+ ```
119
+
120
+ `actors` must hold at least one entry. Every entry takes all the [actor fields](#actor-fields) directly, and `tokenEnvVar` is required on each one.
121
+
122
+ ### Grouped mode
123
+
124
+ Actors are organized into `groups`, and each group sets `tokenEnvVar` and `overrideActorContext` once for all of its actors. Good for repos where many actors share the same token or context.
125
+
126
+ ```json
127
+ {
128
+ "mode": "grouped",
129
+ "groups": {
130
+ "myteam": {
131
+ "tokenEnvVar": "APIFY_TOKEN_MYTEAM",
132
+ "overrideActorContext": ["packages/shared"],
133
+ "actors": [
134
+ { "folder": "actors/web-scraper", "actorFullName": "myteam/web-scraper" },
135
+ {
136
+ "folder": "actors/email-sender",
137
+ "actorFullName": "myteam/email-sender",
138
+ "overrideActorContext": ["actors/email-sender", "packages/mailer"]
139
+ }
140
+ ]
141
+ },
142
+ "partner": {
143
+ "tokenEnvVar": "APIFY_TOKEN_PARTNER",
144
+ "actors": [{ "folder": "actors/partner-crawler", "actorFullName": "partner/crawler" }]
145
+ }
146
+ }
147
+ }
148
+ ```
149
+
150
+ `groups` is an object with at least one group. The group key (`myteam`, `partner` above) is only a label for readability; it has no effect on the result, and it does not need to match any folder or username. Each group needs a `tokenEnvVar` and at least one actor, and can optionally set `overrideActorContext`.
151
+
152
+ An actor inside a group can set its own `tokenEnvVar` or `overrideActorContext`, and that value replaces the group's for that actor only. Values are replaced, not merged: in the example above, `myteam/email-sender` gets `["actors/email-sender", "packages/mailer"]`, not `packages/shared` as well.
153
+
100
154
  ## Github worklows
101
155
 
102
156
  The reusable workflows live in this repo, alongside the package they call. They are the
@@ -172,6 +226,10 @@ jobs:
172
226
  secrets: inherit
173
227
  ```
174
228
 
229
+ This workflow runs `apify-test-tools release --base-commit ${{ github.event.before }}`. See
230
+ [Releasing Actors](#releasing-actors-release) for how the released range is determined and what to
231
+ pass if you call the CLI from somewhere else.
232
+
175
233
  ### `claude-review.yaml`
176
234
 
177
235
  Optional. Reviews a PR against the shared guidelines when you add the trigger label, and again on
@@ -537,7 +595,7 @@ The main local flow is:
537
595
  4. Build Actors on Apify (with your new code)
538
596
  5. Run tests against those builds. You can change tests and run on the same builds.
539
597
 
540
- `cd` into the actor repository you want to work with (or use `--workspace`).
598
+ Run every command from the root of the actor repository you want to work with (or point `--workspace` at it).
541
599
 
542
600
  #### 4. Build affected Actors
543
601
 
@@ -547,7 +605,6 @@ Requires `APIFY_TOKEN_<USERNAME>` for all Apify users that own your Actors (e.g.
547
605
 
548
606
  ```bash
549
607
  APIFY_TOKEN_JOHN_DOE=<token> \
550
- GITHUB_WORKSPACE=. \
551
608
  npx apify-test-tools build \
552
609
  --target-branch origin/master \
553
610
  --source-branch origin/my-dummy-branch \
@@ -566,7 +623,6 @@ If you don't want to push a dummy branch just to test a change and wait for all
566
623
 
567
624
  ```bash
568
625
  APIFY_TOKEN_JOHN_DOE=<token> \
569
- GITHUB_WORKSPACE=. \
570
626
  npx apify-test-tools build-from-local --actors john.doe/my-actor
571
627
  ```
572
628
 
@@ -575,8 +631,7 @@ Pass a hardcoded actor name via `--actors` to build only that Actor (comma-separ
575
631
  ```bash
576
632
  # Build from local source and capture output
577
633
  BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
578
- GITHUB_WORKSPACE=. \
579
- npx apify-test-tools build-from-local --actors apify/my-actor)
634
+ npx apify-test-tools build-from-local --actors apify/my-actor)
580
635
  ```
581
636
 
582
637
  Since you already scoped the build to just the Actor(s) you care about, point vitest at a specific test file (or a `-t` name filter) instead of the whole `test/platform` directory — you get feedback on that one test without waiting for the full suite to run.
@@ -598,8 +653,7 @@ TESTER_APIFY_TOKEN=<token> \
598
653
  ```bash
599
654
  # Build and capture output
600
655
  BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
601
- GITHUB_WORKSPACE=. \
602
- npx apify-test-tools build \
656
+ npx apify-test-tools build \
603
657
  --target-branch origin/master \
604
658
  --source-branch origin/my-dummy-branch)
605
659
 
@@ -609,10 +663,67 @@ TESTER_APIFY_TOKEN=apify_api_yyy \
609
663
  npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform
610
664
  ```
611
665
 
666
+ ### Releasing Actors (`release`)
667
+
668
+ `release` builds the `latest` (default) version of every Actor changed since the last release and
669
+ posts release notes to Slack. It reads everything else from git: the branch from the checked-out
670
+ `HEAD`, the commits and changed files from the range `<base-commit>..HEAD`, and the repository from
671
+ the `origin` remote.
672
+
673
+ ```bash
674
+ npx apify-test-tools release --base-commit <sha> --release-slack-channel "#releases" --dry-run
675
+ ```
676
+
677
+ #### `--base-commit`
678
+
679
+ The last commit that is **already released**. Everything after it, up to `HEAD`, is released. It is
680
+ required: git alone can't tell which commits a push brought, and guessing wrong silently leaves Actors
681
+ on old code.
682
+
683
+ The changed files come from diffing `<base-commit>` and `HEAD` directly, so this works for every
684
+ merge strategy as long as the base commit is the branch tip from **before** the push:
685
+
686
+ | Merge strategy | What lands on the branch | What to pass |
687
+ | ------------------------------ | --------------------------------- | ---------------------------------------------------------------------- |
688
+ | Squash and merge | 1 new commit | the push's "before" commit (`HEAD~1` also works) |
689
+ | Merge commit | the PR's commits + 1 merge commit | the push's "before" commit (`HEAD^1`, the first parent, also works) |
690
+ | Rebase and merge / direct push | N new commits | the push's "before" commit (`HEAD~1` would miss the first N-1 commits) |
691
+
692
+ On GitHub, the push's "before" commit is `${{ github.event.before }}`, which the reusable workflow
693
+ passes for you. On other CIs use the equivalent, e.g. `$CI_COMMIT_BEFORE_SHA` on GitLab. The checkout
694
+ needs the full history (`fetch-depth: 0` in `actions/checkout`) so the base commit is available locally.
695
+
696
+ `release` fails instead of guessing when:
697
+
698
+ - the base commit is all zeros (GitHub sends this for a push that created the branch)
699
+ - the base commit isn't in the local history (shallow checkout) or isn't an ancestor of `HEAD`
700
+ (the branch was force-pushed)
701
+
702
+ In those cases, rerun `release` with a `--base-commit` you pick yourself: an ancestor of `HEAD` from
703
+ before the changes you want to release (for a shallow checkout, fetch the full history instead).
704
+ `--actors` only narrows which of the changed Actors get released; it never forces a build, so on its
705
+ own it doesn't get past these errors.
706
+ If `HEAD` is the base commit, there is nothing to release and the command exits successfully.
707
+
708
+ A failed release isn't retried by the next push: that push's base commit is the failed one's head, so
709
+ its changes are skipped. Re-run the failed workflow run, or release the Actors manually.
710
+
711
+ #### Repository check
712
+
713
+ `release` and `build` point each Actor's version at the `origin` remote. Because a build overwrites the
714
+ version's Git URL, building from a fork or a mirror would repoint the Actor. So, before building, the
715
+ remote is compared with the Git URL of the Actor's default version (the one its default build tag
716
+ points to), and the command fails if they differ or if the default version isn't built from a Git
717
+ repository. The URL form doesn't matter (`https://github.com/org/repo` equals
718
+ `git@github.com:org/repo.git`).
719
+
720
+ To build from a different repository on purpose, e.g. when moving an Actor, pass `--repo-url`. That
721
+ skips the check.
722
+
612
723
  #### Dev mode
613
724
 
614
725
  For development on `apify-test-tools` itself, use `tsx` directly:
615
726
 
616
727
  ```bash
617
- GITHUB_WORKSPACE=local-clone tsx bin/main.ts get-actor-configs
728
+ tsx bin/main.ts --workspace local-clone get-actor-configs
618
729
  ```
@@ -3,7 +3,7 @@ import os from 'node:os';
3
3
  import path from 'node:path';
4
4
 
5
5
  import AdmZip from 'adm-zip';
6
- import type { ActorVersion } from 'apify-client';
6
+ import type * as ApifyClientTypes from 'apify-client';
7
7
  import { ActorSourceType } from 'apify-client';
8
8
 
9
9
  import { dryRunBuildData, LOCAL_SOURCE_VERSION_NUMBER, runAndSummarizeBuilds } from './build.js';
@@ -207,7 +207,7 @@ export const runBuildsFromLocal = async ({
207
207
  for (const { name, content } of await collectSourceFiles(actorConfig.actorFullName, actorConfig.folder)) {
208
208
  zip.addFile(name, content);
209
209
  }
210
- const actorVersion: ActorVersion = {
210
+ const actorVersion: ApifyClientTypes.ActorVersion = {
211
211
  versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
212
212
  tarballUrl: await builder.uploadSourceZip(LOCAL_SOURCE_VERSION_NUMBER, zip.toBuffer()),
213
213
  sourceType: ActorSourceType.Tarball,
package/bin/build.ts CHANGED
@@ -1,6 +1,7 @@
1
- import type { ActorVersion, Build } from 'apify-client';
1
+ import type * as ApifyClientTypes from 'apify-client';
2
2
  import { ActorSourceType, ApifyClient } from 'apify-client';
3
3
 
4
+ import { normalizeRepoUrl } from './git.js';
4
5
  import type { ActorConfig, BuildData } from './types.js';
5
6
 
6
7
  type BuildPrActorOptions = {
@@ -8,6 +9,7 @@ type BuildPrActorOptions = {
8
9
  versionNumber: string;
9
10
  gitRepoUrl: string;
10
11
  actorConfig: ActorConfig;
12
+ actorInfo: ApifyClientTypes.Actor;
11
13
  useDockerCache: boolean;
12
14
  };
13
15
 
@@ -15,51 +17,89 @@ type BuildPrActorOptions = {
15
17
  export const LOCAL_SOURCE_VERSION_NUMBER = '0.98';
16
18
  const DEFAULT_TEST_VERSION_NUMBER = '0.99';
17
19
 
20
+ // Shown whenever an Actor isn't set up the way CI builds need it
21
+ const ACTOR_SETUP_REQUIREMENT =
22
+ 'Before using apify-test-tools, every Actor needs at least one build under its default build tag (usually "latest"), ' +
23
+ 'built from a Git repository version whose URL points to this repository. Build it once manually on the platform.';
24
+
25
+ /**
26
+ * Finds the Actor's default version: the one whose build the default build tag points to.
27
+ * Usually tagged 'latest' but not necessarily (can be e.g. 'version-0').
28
+ */
29
+ export const resolveDefaultVersion = (actorFullName: string, actorInfo: ApifyClientTypes.Actor) => {
30
+ const defaultBuildTag = actorInfo.defaultRunOptions.build;
31
+ console.error(`Default build tag for ${actorFullName} is ${defaultBuildTag}`);
32
+
33
+ // We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
34
+ if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
35
+ throw new Error(
36
+ `[${actorFullName}] Default build is a build number, not a tag. While this could work, ` +
37
+ `we want to have a default as tag so this is often an accidental misconfiguration from the dev`,
38
+ );
39
+ }
40
+ // I reported that buildNumber should probably not be optional
41
+ const defaultBuildNumber = actorInfo.taggedBuilds?.[defaultBuildTag]?.buildNumber;
42
+ if (!defaultBuildNumber) {
43
+ throw new Error(`[${actorFullName}] No build found for tag "${defaultBuildTag}". ${ACTOR_SETUP_REQUIREMENT}`);
44
+ }
45
+ const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1];
46
+ console.error(`Default version for ${actorFullName} is ${defaultVersionNumber}`);
47
+
48
+ const defaultVersion = actorInfo.versions.find((version) => version.versionNumber === defaultVersionNumber);
49
+
50
+ return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag, defaultVersion };
51
+ };
52
+
53
+ /**
54
+ * Guards against building an Actor from a different repository than it is published from, e.g. when the
55
+ * local `origin` remote is a fork or a mirror. Every build overwrites the version's Git URL, so building
56
+ * from the wrong remote would silently repoint the Actor. The default version is the source of truth
57
+ * since it is what users run. Skipped when the repo URL is passed explicitly, which is how you move an
58
+ * Actor to a new repository on purpose.
59
+ */
60
+ export const assertRepoUrlMatchesDefaultVersion = (
61
+ actorFullName: string,
62
+ defaultVersion: ApifyClientTypes.ActorVersion | undefined,
63
+ repoUrl: string,
64
+ ) => {
65
+ if (!defaultVersion) {
66
+ throw new Error(
67
+ `[${actorFullName}] The version of the default build no longer exists. ${ACTOR_SETUP_REQUIREMENT}`,
68
+ );
69
+ }
70
+ if (defaultVersion.sourceType !== ActorSourceType.GitRepo) {
71
+ throw new Error(
72
+ `[${actorFullName}] Default version ${defaultVersion.versionNumber} has source type "${defaultVersion.sourceType}", ` +
73
+ `not "${ActorSourceType.GitRepo}". ${ACTOR_SETUP_REQUIREMENT}`,
74
+ );
75
+ }
76
+ if (normalizeRepoUrl(defaultVersion.gitRepoUrl) !== normalizeRepoUrl(repoUrl)) {
77
+ throw new Error(
78
+ `[${actorFullName}] Repository mismatch: the git remote "origin" is ${repoUrl} but default version ` +
79
+ `${defaultVersion.versionNumber} is built from ${defaultVersion.gitRepoUrl}. Fix the remote or the Actor's source, ` +
80
+ `or pass --repo-url to build from a different repository on purpose.`,
81
+ );
82
+ }
83
+ };
84
+
18
85
  export class ApifyBuilder {
19
86
  private constructor(
20
87
  private readonly apifyClient: ApifyClient,
21
88
  private readonly actorFullName: string,
22
89
  ) {}
23
90
 
24
- // Usually 'latest' but not necessarily (can be e.g. 'version-0')
25
- getDefaultVersionAndTag = async (): Promise<{
26
- defaultBuildNumber: string;
27
- defaultVersionNumber: string;
28
- defaultBuildTag: string;
29
- }> => {
30
- const actorClient = this.apifyClient.actor(this.actorFullName);
31
- const actorInfo = await actorClient.get();
32
-
91
+ getActorInfo = async (): Promise<ApifyClientTypes.Actor> => {
92
+ const actorInfo = await this.apifyClient.actor(this.actorFullName).get();
33
93
  if (!actorInfo) {
34
94
  throw new Error(
35
95
  `[${this.actorFullName}] not found. It is not published or we are missing token to access it privately or its name is misspelled`,
36
96
  );
37
97
  }
38
-
39
- const defaultBuildTag = actorInfo.defaultRunOptions.build;
40
- console.error(`Default build tag for ${this.actorFullName} is ${defaultBuildTag}`);
41
-
42
- // We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
43
- if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
44
- throw new Error(
45
- `[${this.actorFullName}] Default build is a build number, not a tag. While this could work, ` +
46
- `we want to have a default as tag so this is often an accidental misconfiguration from the dev`,
47
- );
48
- }
49
- // I reported that buildNumber should probably not be optional
50
- if (!actorInfo.taggedBuilds?.[defaultBuildTag]?.buildNumber) {
51
- throw new Error(
52
- `[${this.actorFullName}] No build found for tag "${defaultBuildTag}". ` +
53
- `The first build must be triggered manually on the platform before CI can take over.`,
54
- );
55
- }
56
- const defaultBuildNumber = actorInfo.taggedBuilds![defaultBuildTag].buildNumber!;
57
- const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1];
58
- console.error(`Default version for ${this.actorFullName} is ${defaultVersionNumber}`);
59
-
60
- return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag };
98
+ return actorInfo;
61
99
  };
62
100
 
101
+ getDefaultVersionAndTag = async () => resolveDefaultVersion(this.actorFullName, await this.getActorInfo());
102
+
63
103
  // Mirrors `apify push`: stores the zip in the Actor's `actor-<id>-source` KV store and returns a signed
64
104
  // record URL the builder can download without a token.
65
105
  uploadSourceZip = async (versionNumber: string, zip: Buffer): Promise<string> => {
@@ -76,24 +116,19 @@ export class ApifyBuilder {
76
116
  return url.toString();
77
117
  };
78
118
 
119
+ // Pass actorInfo when the caller already fetched it, to save an API call
79
120
  createVersionAndBuild = async (
80
121
  versionNumber: string,
81
- actorVersion: ActorVersion,
122
+ actorVersion: ApifyClientTypes.ActorVersion,
82
123
  useCache: boolean,
124
+ actorInfo?: ApifyClientTypes.Actor,
83
125
  ): Promise<BuildData> => {
84
126
  const actorClient = this.apifyClient.actor(this.actorFullName);
85
- const actorInfo = await actorClient.get();
86
- if (!actorInfo) {
87
- throw new Error(
88
- `No actor named '${this.actorFullName}' was found on the platform. If this` +
89
- ' is unexpected, make sure the actor you are targeting is spelled the' +
90
- ' same as the folder in the repository.',
91
- );
92
- }
127
+ const { versions } = actorInfo ?? (await this.getActorInfo());
93
128
 
94
129
  // Prepare version
95
- const versionExists = !actorInfo.versions.find((version) => version.versionNumber === versionNumber);
96
- if (versionExists) {
130
+ const versionMissing = !versions.find((version) => version.versionNumber === versionNumber);
131
+ if (versionMissing) {
97
132
  // create new version
98
133
  await actorClient.versions().create(actorVersion);
99
134
  } else {
@@ -108,7 +143,7 @@ export class ApifyBuilder {
108
143
  return { buildId: id, actorRawId: actId, buildNumber, actorFullName: this.actorFullName };
109
144
  };
110
145
 
111
- waitForBuildToFinish = async (buildId: string): Promise<Build> => {
146
+ waitForBuildToFinish = async (buildId: string): Promise<ApifyClientTypes.Build> => {
112
147
  const build = await this.apifyClient.build(buildId).waitForFinish();
113
148
  const versionNumber = build.buildNumber;
114
149
  if (build.status === 'FAILED' || build.status === 'TIMED-OUT') {
@@ -168,7 +203,7 @@ export class ApifyBuilder {
168
203
  const DEFAULT_DAYS_BACK_PROD_VERSIONS = 30;
169
204
  const DEFAULT_DAYS_BACK_DEVEL = 7;
170
205
 
171
- const actorInfo = (await this.apifyClient.actor(this.actorFullName).get())!;
206
+ const actorInfo = await this.getActorInfo();
172
207
 
173
208
  // 'devel' used to be hardcoded for testing version 0.99, once we get rid of this tag everywhere, we can remove this code
174
209
  const taggedDevelBuildNumber: string | undefined = actorInfo.taggedBuilds!.devel?.buildNumber;
@@ -183,7 +218,7 @@ export class ApifyBuilder {
183
218
  const { items } = await this.apifyClient.actor(this.actorFullName).builds().list();
184
219
 
185
220
  // Deleting default build throws an error, so we skip it
186
- const { defaultBuildNumber, defaultBuildTag } = await this.getDefaultVersionAndTag();
221
+ const { defaultBuildNumber, defaultBuildTag } = resolveDefaultVersion(this.actorFullName, actorInfo);
187
222
 
188
223
  const daysAgoUnixProd = Date.now() - DEFAULT_DAYS_BACK_PROD_VERSIONS * 24 * 60 * 60 * 1000;
189
224
  const daysAgoUnixDevel = Date.now() - DEFAULT_DAYS_BACK_DEVEL * 24 * 60 * 60 * 1000;
@@ -286,6 +321,8 @@ type RunBuildsOptions = {
286
321
  actorConfigs: ActorConfig[];
287
322
  isLatest?: boolean;
288
323
  repoUrl: string;
324
+ // False when the repo URL was passed explicitly, see assertRepoUrlMatchesDefaultVersion
325
+ shouldVerifyRepoUrl: boolean;
289
326
  branch: string;
290
327
  dryRun: boolean;
291
328
  useDockerCache: boolean;
@@ -293,34 +330,40 @@ type RunBuildsOptions = {
293
330
 
294
331
  export const runBuilds = async ({
295
332
  repoUrl,
333
+ shouldVerifyRepoUrl,
296
334
  branch,
297
335
  actorConfigs,
298
336
  isLatest = false,
299
337
  dryRun,
300
338
  useDockerCache,
301
339
  }: RunBuildsOptions): Promise<BuildData[]> => {
302
- const buildConfigs: BuildPrActorOptions[] = [];
303
-
304
- for (const actorConfig of actorConfigs) {
305
- let versionNumber: string;
306
- let buildTag: string | undefined;
307
-
308
- if (isLatest) {
309
- const { defaultVersionNumber, defaultBuildTag } =
310
- await ApifyBuilder.fromActorConfig(actorConfig).getDefaultVersionAndTag();
311
- versionNumber = defaultVersionNumber;
312
- buildTag = defaultBuildTag;
313
- } else {
314
- versionNumber = DEFAULT_TEST_VERSION_NUMBER;
315
- }
340
+ // Fetched once per Actor and reused for the checks, the version numbers and the build itself
341
+ const buildConfigs: BuildPrActorOptions[] = await Promise.all(
342
+ actorConfigs.map(async (actorConfig) => {
343
+ const actorInfo = await ApifyBuilder.fromActorConfig(actorConfig).getActorInfo();
344
+ const { defaultVersionNumber, defaultBuildTag, defaultVersion } = resolveDefaultVersion(
345
+ actorConfig.actorFullName,
346
+ actorInfo,
347
+ );
348
+ if (shouldVerifyRepoUrl) {
349
+ assertRepoUrlMatchesDefaultVersion(actorConfig.actorFullName, defaultVersion, repoUrl);
350
+ }
316
351
 
317
- // Depending on if these are miniactors or standaloneActors
318
- let gitRepoUrl = `${repoUrl}#${branch}`;
319
- if (actorConfig.folder) {
320
- gitRepoUrl = `${gitRepoUrl}:${actorConfig.folder}`;
321
- }
322
- buildConfigs.push({ actorConfig, gitRepoUrl, versionNumber, buildTag, useDockerCache });
323
- }
352
+ // Depending on if these are miniactors or standaloneActors
353
+ let gitRepoUrl = `${repoUrl}#${branch}`;
354
+ if (actorConfig.folder) {
355
+ gitRepoUrl = `${gitRepoUrl}:${actorConfig.folder}`;
356
+ }
357
+ return {
358
+ actorConfig,
359
+ actorInfo,
360
+ gitRepoUrl,
361
+ versionNumber: isLatest ? defaultVersionNumber : DEFAULT_TEST_VERSION_NUMBER,
362
+ buildTag: isLatest ? defaultBuildTag : undefined,
363
+ useDockerCache,
364
+ };
365
+ }),
366
+ );
324
367
 
325
368
  if (dryRun) {
326
369
  console.error('[DRY RUN] Would build:');
@@ -338,18 +381,19 @@ export const runBuilds = async ({
338
381
 
339
382
  return runAndSummarizeBuilds(actorConfigs, 'BUILDS', async (actorConfig, builder) => {
340
383
  const {
384
+ actorInfo,
341
385
  buildTag,
342
386
  versionNumber,
343
387
  gitRepoUrl,
344
388
  useDockerCache: useCache,
345
389
  } = buildConfigsByActorFullName.get(actorConfig.actorFullName)!;
346
- const actorVersion: ActorVersion = {
390
+ const actorVersion: ApifyClientTypes.ActorVersion = {
347
391
  buildTag,
348
392
  versionNumber,
349
393
  gitRepoUrl,
350
394
  sourceType: ActorSourceType.GitRepo,
351
395
  };
352
- return builder.createVersionAndBuild(versionNumber, actorVersion, useCache);
396
+ return builder.createVersionAndBuild(versionNumber, actorVersion, useCache, actorInfo);
353
397
  });
354
398
  };
355
399