apify-test-tools 0.9.1-beta.10 → 0.9.1-beta.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/public_push-build-latest.yaml +3 -1
- package/.github/workflows-package-version +1 -1
- package/CHANGELOG.md +2 -0
- package/CONTRIBUTING.md +2 -4
- package/README.md +146 -35
- package/bin/build-from-local.ts +2 -2
- package/bin/build.ts +114 -70
- package/bin/git.ts +140 -0
- package/bin/main.ts +46 -19
- package/bin/slack.ts +5 -2
- package/bin/types.ts +0 -73
- package/bin/utils/config/parser.ts +6 -5
- package/bin/utils/config/structures/base.ts +5 -1
- package/bin/utils/config/structures/grouped.ts +40 -0
- package/bin/utils/config/structures/legacy.ts +3 -1
- package/bin/utils.ts +2 -3
- package/dist/bin/build-from-local.js.map +1 -1
- package/dist/bin/build.d.ts +25 -4
- package/dist/bin/build.d.ts.map +1 -1
- package/dist/bin/build.js +76 -47
- package/dist/bin/build.js.map +1 -1
- package/dist/bin/git.d.ts +40 -0
- package/dist/bin/git.d.ts.map +1 -1
- package/dist/bin/git.js +118 -0
- package/dist/bin/git.js.map +1 -1
- package/dist/bin/main.d.ts.map +1 -1
- package/dist/bin/main.js +33 -13
- package/dist/bin/main.js.map +1 -1
- package/dist/bin/slack.d.ts.map +1 -1
- package/dist/bin/slack.js +4 -2
- package/dist/bin/slack.js.map +1 -1
- package/dist/bin/types.d.ts +0 -64
- package/dist/bin/types.d.ts.map +1 -1
- package/dist/bin/utils/config/parser.d.ts.map +1 -1
- package/dist/bin/utils/config/parser.js +6 -5
- package/dist/bin/utils/config/parser.js.map +1 -1
- package/dist/bin/utils/config/structures/base.d.ts +3 -2
- package/dist/bin/utils/config/structures/base.d.ts.map +1 -1
- package/dist/bin/utils/config/structures/base.js +1 -0
- package/dist/bin/utils/config/structures/base.js.map +1 -1
- package/dist/bin/utils/config/structures/grouped.d.ts +6 -0
- package/dist/bin/utils/config/structures/grouped.d.ts.map +1 -0
- package/dist/bin/utils/config/structures/grouped.js +29 -0
- package/dist/bin/utils/config/structures/grouped.js.map +1 -0
- package/dist/bin/utils/config/structures/legacy.d.ts +1 -1
- package/dist/bin/utils/config/structures/legacy.d.ts.map +1 -1
- package/dist/bin/utils/config/structures/legacy.js +1 -1
- package/dist/bin/utils/config/structures/legacy.js.map +1 -1
- package/dist/bin/utils.d.ts.map +1 -1
- package/dist/bin/utils.js +2 -3
- package/dist/bin/utils.js.map +1 -1
- package/dist/test/unit/bin/build.test.d.ts +2 -0
- package/dist/test/unit/bin/build.test.d.ts.map +1 -0
- package/dist/test/unit/bin/build.test.js +46 -0
- package/dist/test/unit/bin/build.test.js.map +1 -0
- package/dist/test/unit/bin/git.test.js +104 -1
- package/dist/test/unit/bin/git.test.js.map +1 -1
- package/dist/test/unit/bin/utils/config/load-config.test.js +1 -0
- package/dist/test/unit/bin/utils/config/load-config.test.js.map +1 -1
- package/dist/test/unit/bin/utils/config/parser.test.js +23 -5
- package/dist/test/unit/bin/utils/config/parser.test.js.map +1 -1
- package/dist/test/unit/bin/utils/config/structures/grouped.test.d.ts +2 -0
- package/dist/test/unit/bin/utils/config/structures/grouped.test.d.ts.map +1 -0
- package/dist/test/unit/bin/utils/config/structures/grouped.test.js +75 -0
- package/dist/test/unit/bin/utils/config/structures/grouped.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
- package/test/unit/bin/build.test.ts +68 -0
- package/test/unit/bin/git.test.ts +122 -0
- package/test/unit/bin/utils/config/load-config.test.ts +1 -0
- package/test/unit/bin/utils/config/parser.test.ts +26 -5
- package/test/unit/bin/utils/config/structures/grouped.test.ts +90 -0
- package/bin/github.ts +0 -69
- package/dist/bin/github.d.ts +0 -11
- package/dist/bin/github.d.ts.map +0 -1
- package/dist/bin/github.js +0 -59
- package/dist/bin/github.js.map +0 -1
|
@@ -25,6 +25,8 @@ jobs:
|
|
|
25
25
|
with:
|
|
26
26
|
npm-token: ${{ secrets.NPM_TOKEN }}
|
|
27
27
|
|
|
28
|
+
# --base-commit is the last commit that is already released: the push's "before" commit.
|
|
29
|
+
# Everything after it up to HEAD gets released. See the "Releasing Actors" section of the README.
|
|
28
30
|
# run-with-apify-tokens.mjs reads the token names from in apify-test-tools.config.json
|
|
29
31
|
# and hands the build those secrets and nothing else, so the tokens stay in this one command instead of the whole job.
|
|
30
32
|
# The library never sees secrets it doesn't need
|
|
@@ -34,7 +36,7 @@ jobs:
|
|
|
34
36
|
SLACK_TOKEN_RELEASES_BOT: ${{ secrets.SLACK_TOKEN_RELEASES_BOT }}
|
|
35
37
|
run: |
|
|
36
38
|
node "${{ steps.setup.outputs.scripts-path }}/run-with-apify-tokens.mjs" \
|
|
37
|
-
npx apify-test-tools release --
|
|
39
|
+
npx apify-test-tools release --base-commit ${{ github.event.before }} --release-slack-channel "#delivery-public-actors" --report-slack-channel "${{ inputs.report-slack-channel || format('#notif-{0}', github.event.repository.name) }}" --use-docker-cache
|
|
38
40
|
|
|
39
41
|
- name: Delete old builds
|
|
40
42
|
env:
|
|
@@ -1 +1 @@
|
|
|
1
|
-
0.
|
|
1
|
+
0.10.0
|
package/CHANGELOG.md
CHANGED
|
@@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file.
|
|
|
9
9
|
|
|
10
10
|
- **config/modes:** Lay groundwork for multiple configuration structures ([#142](https://github.com/apify/apify-test-tools/pull/142)) ([7e18638](https://github.com/apify/apify-test-tools/commit/7e18638a8d3523154dbe819312c2b16b54c548ff)) by [@JuanGalilea](https://github.com/JuanGalilea)
|
|
11
11
|
- **workflows:** Host the reusable GitHub workflows in this repo ([#120](https://github.com/apify/apify-test-tools/pull/120)) ([10f089d](https://github.com/apify/apify-test-tools/commit/10f089d85146ceb7cdae964cfa1fc2577df43a37)) by [@metalwarrior665](https://github.com/metalwarrior665)
|
|
12
|
+
- **mode:** Add grouped config mode ([#143](https://github.com/apify/apify-test-tools/pull/143)) ([9d0b93e](https://github.com/apify/apify-test-tools/commit/9d0b93e194a2cd810a31bcef4ca9457fd9ebfb9b)) by [@JuanGalilea](https://github.com/JuanGalilea)
|
|
12
13
|
|
|
13
14
|
### 🐛 Bug Fixes
|
|
14
15
|
|
|
@@ -18,6 +19,7 @@ All notable changes to this project will be documented in this file.
|
|
|
18
19
|
### 🚜 Refactor
|
|
19
20
|
|
|
20
21
|
- **workflows:** [**breaking**] Rename the floating tag to workflows-v0 ([#146](https://github.com/apify/apify-test-tools/pull/146)) ([c48fb22](https://github.com/apify/apify-test-tools/commit/c48fb223c885c4e5de7e5f64d8c14f6d03fcc7de)) by [@metalwarrior665](https://github.com/metalwarrior665)
|
|
22
|
+
- **bin:** [**breaking**] Remove GitHub event dependency from release command ([#153](https://github.com/apify/apify-test-tools/pull/153)) ([e6953aa](https://github.com/apify/apify-test-tools/commit/e6953aa095d1a92638c6d72441e1b312d9f835c3)) by [@metalwarrior665](https://github.com/metalwarrior665), closes [#131](https://github.com/apify/apify-test-tools/issues/131)
|
|
21
23
|
|
|
22
24
|
|
|
23
25
|
<!-- git-cliff-unreleased-end -->
|
package/CONTRIBUTING.md
CHANGED
|
@@ -10,7 +10,6 @@ The package consists of three parts:
|
|
|
10
10
|
|
|
11
11
|
- `bin/build.ts` actor building
|
|
12
12
|
- `bin/git.ts` git wrappers for getting relevant git commits and changed files
|
|
13
|
-
- `bin/github.ts` github push event parsing
|
|
14
13
|
- `bin/main.ts` entrypoing
|
|
15
14
|
- `bin/slack.ts` sending notifications to slack
|
|
16
15
|
- `bin/test-report.ts` processing vitest's test reports
|
|
@@ -40,12 +39,11 @@ git clone git@github.com:apify-store/testing-repo-for-github-actions.git
|
|
|
40
39
|
|
|
41
40
|
#### Working on the CLI
|
|
42
41
|
|
|
43
|
-
To work on the library,
|
|
42
|
+
To work on the library, point the cli to your repo with `--workspace` (it defaults to the current directory):
|
|
44
43
|
|
|
45
44
|
```sh
|
|
46
|
-
export GITHUB_WORKSPACE=../path/to/testing-repo-for-github-actions # path to the repo
|
|
47
45
|
npx tsx bin/main.ts --help
|
|
48
|
-
npx tsx bin/main.ts get-commits --target-branch master --source-branch feat/testing-feature-branch
|
|
46
|
+
npx tsx bin/main.ts --workspace ../path/to/testing-repo-for-github-actions get-commits --target-branch master --source-branch feat/testing-feature-branch
|
|
49
47
|
```
|
|
50
48
|
|
|
51
49
|
#### Working on the library
|
package/README.md
CHANGED
|
@@ -12,37 +12,15 @@ npm i -D apify-test-tools
|
|
|
12
12
|
|
|
13
13
|
- Requires `vitest` version `3.2.0` or later (uses [annotate](https://vitest.dev/guide/test-context.html#annotate))
|
|
14
14
|
- Make sure `target` and `module` in your `tsconfig.json`'s `compilerOptions` are set to `ES2022`
|
|
15
|
+
- Every Actor must already have at least one build under its default build tag (usually `latest`), built
|
|
16
|
+
from a Git repository version whose URL points to this repository. Build it once manually on the
|
|
17
|
+
platform before CI takes over. See [Repository check](#repository-check).
|
|
15
18
|
|
|
16
19
|
### 2. Create the config file
|
|
17
20
|
|
|
18
|
-
Every repo that uses `apify-test-tools` must have an `apify-test-tools.config.json` file at the root.
|
|
21
|
+
Every repo that uses `apify-test-tools` must have an `apify-test-tools.config.json` file at the root. It lists the actors in the repo, their folders, their full names on the platform, and which env var holds the token to use for each. A top-level `mode` field picks the file's layout; leave it out to use the default `"legacy"` layout.
|
|
19
22
|
|
|
20
|
-
|
|
21
|
-
{
|
|
22
|
-
"actors": [
|
|
23
|
-
{
|
|
24
|
-
"folder": "actors/web-scraper",
|
|
25
|
-
"actorFullName": "myteam/web-scraper",
|
|
26
|
-
"tokenEnvVar": "APIFY_TOKEN_MYTEAM"
|
|
27
|
-
},
|
|
28
|
-
{
|
|
29
|
-
"folder": "actors/email-sender",
|
|
30
|
-
"actorFullName": "myteam/email-sender",
|
|
31
|
-
"tokenEnvVar": "APIFY_TOKEN_MYTEAM",
|
|
32
|
-
"overrideActorContext": ["actors/email-sender", "packages/shared"]
|
|
33
|
-
}
|
|
34
|
-
]
|
|
35
|
-
}
|
|
36
|
-
```
|
|
37
|
-
|
|
38
|
-
Each entry has:
|
|
39
|
-
|
|
40
|
-
| Field | Required | Description |
|
|
41
|
-
| ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
42
|
-
| `folder` | yes | Relative path from repo root to the actor's own project directory — the folder that directly contains `.actor/actor.json` (i.e. `<folder>/.actor/actor.json`), the actor's README/CHANGELOG, and its source. Use `"."` for a single-actor repo where `.actor/` is at the root. |
|
|
43
|
-
| `actorFullName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. |
|
|
44
|
-
| `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails. |
|
|
45
|
-
| `overrideActorContext` | no | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. Entries must not be prefixes of one another (e.g. `["", "code"]` or `["actors", "actors/foo"]` are rejected). The actor's own `folder` is always part of its context — if none of the listed entries reach it, it's added automatically. |
|
|
23
|
+
See [Config file](#config-file) for the available modes and every field.
|
|
46
24
|
|
|
47
25
|
### 3. Set up actor folders
|
|
48
26
|
|
|
@@ -97,6 +75,82 @@ mkdir -p test/platform/core
|
|
|
97
75
|
|
|
98
76
|
See the [GitHub workflows](#github-worklows) section below.
|
|
99
77
|
|
|
78
|
+
## Config file
|
|
79
|
+
|
|
80
|
+
`apify-test-tools.config.json` lives at the repo root and tells the tool which actors live in the repo, how to identify them, and which token to use.
|
|
81
|
+
|
|
82
|
+
### Actor fields
|
|
83
|
+
|
|
84
|
+
Every actor in the config is described by these fields, whatever the mode:
|
|
85
|
+
|
|
86
|
+
| Field | Required | Description |
|
|
87
|
+
| ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
88
|
+
| `folder` | yes | Relative path from repo root to the actor's own project directory — the folder that directly contains `.actor/actor.json` (i.e. `<folder>/.actor/actor.json`), the actor's README/CHANGELOG, and its source. Use `"."` for a single-actor repo where `.actor/` is at the root. |
|
|
89
|
+
| `actorFullName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. |
|
|
90
|
+
| `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. In [grouped mode](#grouped-mode) it's inherited from the group unless the actor sets its own. No fallback — if the env var is not set at build time, the build fails. |
|
|
91
|
+
| `overrideActorContext` | no | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. Entries must not be prefixes of one another (e.g. `["", "code"]` or `["actors", "actors/foo"]` are rejected). The actor's own `folder` is always part of its context — if none of the listed entries reach it, it's added automatically. |
|
|
92
|
+
|
|
93
|
+
### Modes
|
|
94
|
+
|
|
95
|
+
The file's layout is picked by the top-level `mode` field. Each mode is a different way of writing the same thing: every mode resolves to a list of actors with the fields above, and the same rules apply to all of them — no two actors may share a `folder`, and no two entries may point at the same `actorFullName`. If `mode` is left out, [`"legacy"`](#legacy-mode) is used, so existing config files keep working unchanged.
|
|
96
|
+
|
|
97
|
+
### Legacy mode
|
|
98
|
+
|
|
99
|
+
The default. A flat `actors` array where every entry carries all of its settings. Good for repos with a few actors, or where every actor has different settings.
|
|
100
|
+
|
|
101
|
+
```json
|
|
102
|
+
{
|
|
103
|
+
"mode": "legacy",
|
|
104
|
+
"actors": [
|
|
105
|
+
{
|
|
106
|
+
"folder": "actors/web-scraper",
|
|
107
|
+
"actorFullName": "myteam/web-scraper",
|
|
108
|
+
"tokenEnvVar": "APIFY_TOKEN_MYTEAM"
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"folder": "actors/email-sender",
|
|
112
|
+
"actorFullName": "myteam/email-sender",
|
|
113
|
+
"tokenEnvVar": "APIFY_TOKEN_MYTEAM",
|
|
114
|
+
"overrideActorContext": ["actors/email-sender", "packages/shared"]
|
|
115
|
+
}
|
|
116
|
+
]
|
|
117
|
+
}
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
`actors` must hold at least one entry. Every entry takes all the [actor fields](#actor-fields) directly, and `tokenEnvVar` is required on each one.
|
|
121
|
+
|
|
122
|
+
### Grouped mode
|
|
123
|
+
|
|
124
|
+
Actors are organized into `groups`, and each group sets `tokenEnvVar` and `overrideActorContext` once for all of its actors. Good for repos where many actors share the same token or context.
|
|
125
|
+
|
|
126
|
+
```json
|
|
127
|
+
{
|
|
128
|
+
"mode": "grouped",
|
|
129
|
+
"groups": {
|
|
130
|
+
"myteam": {
|
|
131
|
+
"tokenEnvVar": "APIFY_TOKEN_MYTEAM",
|
|
132
|
+
"overrideActorContext": ["packages/shared"],
|
|
133
|
+
"actors": [
|
|
134
|
+
{ "folder": "actors/web-scraper", "actorFullName": "myteam/web-scraper" },
|
|
135
|
+
{
|
|
136
|
+
"folder": "actors/email-sender",
|
|
137
|
+
"actorFullName": "myteam/email-sender",
|
|
138
|
+
"overrideActorContext": ["actors/email-sender", "packages/mailer"]
|
|
139
|
+
}
|
|
140
|
+
]
|
|
141
|
+
},
|
|
142
|
+
"partner": {
|
|
143
|
+
"tokenEnvVar": "APIFY_TOKEN_PARTNER",
|
|
144
|
+
"actors": [{ "folder": "actors/partner-crawler", "actorFullName": "partner/crawler" }]
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
`groups` is an object with at least one group. The group key (`myteam`, `partner` above) is only a label for readability; it has no effect on the result, and it does not need to match any folder or username. Each group needs a `tokenEnvVar` and at least one actor, and can optionally set `overrideActorContext`.
|
|
151
|
+
|
|
152
|
+
An actor inside a group can set its own `tokenEnvVar` or `overrideActorContext`, and that value replaces the group's for that actor only. Values are replaced, not merged: in the example above, `myteam/email-sender` gets `["actors/email-sender", "packages/mailer"]`, not `packages/shared` as well.
|
|
153
|
+
|
|
100
154
|
## Github worklows
|
|
101
155
|
|
|
102
156
|
The reusable workflows live in this repo, alongside the package they call. They are the
|
|
@@ -172,6 +226,10 @@ jobs:
|
|
|
172
226
|
secrets: inherit
|
|
173
227
|
```
|
|
174
228
|
|
|
229
|
+
This workflow runs `apify-test-tools release --base-commit ${{ github.event.before }}`. See
|
|
230
|
+
[Releasing Actors](#releasing-actors-release) for how the released range is determined and what to
|
|
231
|
+
pass if you call the CLI from somewhere else.
|
|
232
|
+
|
|
175
233
|
### `claude-review.yaml`
|
|
176
234
|
|
|
177
235
|
Optional. Reviews a PR against the shared guidelines when you add the trigger label, and again on
|
|
@@ -537,7 +595,7 @@ The main local flow is:
|
|
|
537
595
|
4. Build Actors on Apify (with your new code)
|
|
538
596
|
5. Run tests against those builds. You can change tests and run on the same builds.
|
|
539
597
|
|
|
540
|
-
|
|
598
|
+
Run every command from the root of the actor repository you want to work with (or point `--workspace` at it).
|
|
541
599
|
|
|
542
600
|
#### 4. Build affected Actors
|
|
543
601
|
|
|
@@ -547,7 +605,6 @@ Requires `APIFY_TOKEN_<USERNAME>` for all Apify users that own your Actors (e.g.
|
|
|
547
605
|
|
|
548
606
|
```bash
|
|
549
607
|
APIFY_TOKEN_JOHN_DOE=<token> \
|
|
550
|
-
GITHUB_WORKSPACE=. \
|
|
551
608
|
npx apify-test-tools build \
|
|
552
609
|
--target-branch origin/master \
|
|
553
610
|
--source-branch origin/my-dummy-branch \
|
|
@@ -566,7 +623,6 @@ If you don't want to push a dummy branch just to test a change and wait for all
|
|
|
566
623
|
|
|
567
624
|
```bash
|
|
568
625
|
APIFY_TOKEN_JOHN_DOE=<token> \
|
|
569
|
-
GITHUB_WORKSPACE=. \
|
|
570
626
|
npx apify-test-tools build-from-local --actors john.doe/my-actor
|
|
571
627
|
```
|
|
572
628
|
|
|
@@ -575,8 +631,7 @@ Pass a hardcoded actor name via `--actors` to build only that Actor (comma-separ
|
|
|
575
631
|
```bash
|
|
576
632
|
# Build from local source and capture output
|
|
577
633
|
BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
|
|
578
|
-
|
|
579
|
-
npx apify-test-tools build-from-local --actors apify/my-actor)
|
|
634
|
+
npx apify-test-tools build-from-local --actors apify/my-actor)
|
|
580
635
|
```
|
|
581
636
|
|
|
582
637
|
Since you already scoped the build to just the Actor(s) you care about, point vitest at a specific test file (or a `-t` name filter) instead of the whole `test/platform` directory — you get feedback on that one test without waiting for the full suite to run.
|
|
@@ -598,8 +653,7 @@ TESTER_APIFY_TOKEN=<token> \
|
|
|
598
653
|
```bash
|
|
599
654
|
# Build and capture output
|
|
600
655
|
BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
|
|
601
|
-
|
|
602
|
-
npx apify-test-tools build \
|
|
656
|
+
npx apify-test-tools build \
|
|
603
657
|
--target-branch origin/master \
|
|
604
658
|
--source-branch origin/my-dummy-branch)
|
|
605
659
|
|
|
@@ -609,10 +663,67 @@ TESTER_APIFY_TOKEN=apify_api_yyy \
|
|
|
609
663
|
npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform
|
|
610
664
|
```
|
|
611
665
|
|
|
666
|
+
### Releasing Actors (`release`)
|
|
667
|
+
|
|
668
|
+
`release` builds the `latest` (default) version of every Actor changed since the last release and
|
|
669
|
+
posts release notes to Slack. It reads everything else from git: the branch from the checked-out
|
|
670
|
+
`HEAD`, the commits and changed files from the range `<base-commit>..HEAD`, and the repository from
|
|
671
|
+
the `origin` remote.
|
|
672
|
+
|
|
673
|
+
```bash
|
|
674
|
+
npx apify-test-tools release --base-commit <sha> --release-slack-channel "#releases" --dry-run
|
|
675
|
+
```
|
|
676
|
+
|
|
677
|
+
#### `--base-commit`
|
|
678
|
+
|
|
679
|
+
The last commit that is **already released**. Everything after it, up to `HEAD`, is released. It is
|
|
680
|
+
required: git alone can't tell which commits a push brought, and guessing wrong silently leaves Actors
|
|
681
|
+
on old code.
|
|
682
|
+
|
|
683
|
+
The changed files come from diffing `<base-commit>` and `HEAD` directly, so this works for every
|
|
684
|
+
merge strategy as long as the base commit is the branch tip from **before** the push:
|
|
685
|
+
|
|
686
|
+
| Merge strategy | What lands on the branch | What to pass |
|
|
687
|
+
| ------------------------------ | --------------------------------- | ---------------------------------------------------------------------- |
|
|
688
|
+
| Squash and merge | 1 new commit | the push's "before" commit (`HEAD~1` also works) |
|
|
689
|
+
| Merge commit | the PR's commits + 1 merge commit | the push's "before" commit (`HEAD^1`, the first parent, also works) |
|
|
690
|
+
| Rebase and merge / direct push | N new commits | the push's "before" commit (`HEAD~1` would miss the first N-1 commits) |
|
|
691
|
+
|
|
692
|
+
On GitHub, the push's "before" commit is `${{ github.event.before }}`, which the reusable workflow
|
|
693
|
+
passes for you. On other CIs use the equivalent, e.g. `$CI_COMMIT_BEFORE_SHA` on GitLab. The checkout
|
|
694
|
+
needs the full history (`fetch-depth: 0` in `actions/checkout`) so the base commit is available locally.
|
|
695
|
+
|
|
696
|
+
`release` fails instead of guessing when:
|
|
697
|
+
|
|
698
|
+
- the base commit is all zeros (GitHub sends this for a push that created the branch)
|
|
699
|
+
- the base commit isn't in the local history (shallow checkout) or isn't an ancestor of `HEAD`
|
|
700
|
+
(the branch was force-pushed)
|
|
701
|
+
|
|
702
|
+
In those cases, rerun `release` with a `--base-commit` you pick yourself: an ancestor of `HEAD` from
|
|
703
|
+
before the changes you want to release (for a shallow checkout, fetch the full history instead).
|
|
704
|
+
`--actors` only narrows which of the changed Actors get released; it never forces a build, so on its
|
|
705
|
+
own it doesn't get past these errors.
|
|
706
|
+
If `HEAD` is the base commit, there is nothing to release and the command exits successfully.
|
|
707
|
+
|
|
708
|
+
A failed release isn't retried by the next push: that push's base commit is the failed one's head, so
|
|
709
|
+
its changes are skipped. Re-run the failed workflow run, or release the Actors manually.
|
|
710
|
+
|
|
711
|
+
#### Repository check
|
|
712
|
+
|
|
713
|
+
`release` and `build` point each Actor's version at the `origin` remote. Because a build overwrites the
|
|
714
|
+
version's Git URL, building from a fork or a mirror would repoint the Actor. So, before building, the
|
|
715
|
+
remote is compared with the Git URL of the Actor's default version (the one its default build tag
|
|
716
|
+
points to), and the command fails if they differ or if the default version isn't built from a Git
|
|
717
|
+
repository. The URL form doesn't matter (`https://github.com/org/repo` equals
|
|
718
|
+
`git@github.com:org/repo.git`).
|
|
719
|
+
|
|
720
|
+
To build from a different repository on purpose, e.g. when moving an Actor, pass `--repo-url`. That
|
|
721
|
+
skips the check.
|
|
722
|
+
|
|
612
723
|
#### Dev mode
|
|
613
724
|
|
|
614
725
|
For development on `apify-test-tools` itself, use `tsx` directly:
|
|
615
726
|
|
|
616
727
|
```bash
|
|
617
|
-
|
|
728
|
+
tsx bin/main.ts --workspace local-clone get-actor-configs
|
|
618
729
|
```
|
package/bin/build-from-local.ts
CHANGED
|
@@ -3,7 +3,7 @@ import os from 'node:os';
|
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
|
|
5
5
|
import AdmZip from 'adm-zip';
|
|
6
|
-
import type
|
|
6
|
+
import type * as ApifyClientTypes from 'apify-client';
|
|
7
7
|
import { ActorSourceType } from 'apify-client';
|
|
8
8
|
|
|
9
9
|
import { dryRunBuildData, LOCAL_SOURCE_VERSION_NUMBER, runAndSummarizeBuilds } from './build.js';
|
|
@@ -207,7 +207,7 @@ export const runBuildsFromLocal = async ({
|
|
|
207
207
|
for (const { name, content } of await collectSourceFiles(actorConfig.actorFullName, actorConfig.folder)) {
|
|
208
208
|
zip.addFile(name, content);
|
|
209
209
|
}
|
|
210
|
-
const actorVersion: ActorVersion = {
|
|
210
|
+
const actorVersion: ApifyClientTypes.ActorVersion = {
|
|
211
211
|
versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
|
|
212
212
|
tarballUrl: await builder.uploadSourceZip(LOCAL_SOURCE_VERSION_NUMBER, zip.toBuffer()),
|
|
213
213
|
sourceType: ActorSourceType.Tarball,
|
package/bin/build.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import type * as ApifyClientTypes from 'apify-client';
|
|
2
2
|
import { ActorSourceType, ApifyClient } from 'apify-client';
|
|
3
3
|
|
|
4
|
+
import { normalizeRepoUrl } from './git.js';
|
|
4
5
|
import type { ActorConfig, BuildData } from './types.js';
|
|
5
6
|
|
|
6
7
|
type BuildPrActorOptions = {
|
|
@@ -8,6 +9,7 @@ type BuildPrActorOptions = {
|
|
|
8
9
|
versionNumber: string;
|
|
9
10
|
gitRepoUrl: string;
|
|
10
11
|
actorConfig: ActorConfig;
|
|
12
|
+
actorInfo: ApifyClientTypes.Actor;
|
|
11
13
|
useDockerCache: boolean;
|
|
12
14
|
};
|
|
13
15
|
|
|
@@ -15,51 +17,89 @@ type BuildPrActorOptions = {
|
|
|
15
17
|
export const LOCAL_SOURCE_VERSION_NUMBER = '0.98';
|
|
16
18
|
const DEFAULT_TEST_VERSION_NUMBER = '0.99';
|
|
17
19
|
|
|
20
|
+
// Shown whenever an Actor isn't set up the way CI builds need it
|
|
21
|
+
const ACTOR_SETUP_REQUIREMENT =
|
|
22
|
+
'Before using apify-test-tools, every Actor needs at least one build under its default build tag (usually "latest"), ' +
|
|
23
|
+
'built from a Git repository version whose URL points to this repository. Build it once manually on the platform.';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Finds the Actor's default version: the one whose build the default build tag points to.
|
|
27
|
+
* Usually tagged 'latest' but not necessarily (can be e.g. 'version-0').
|
|
28
|
+
*/
|
|
29
|
+
export const resolveDefaultVersion = (actorFullName: string, actorInfo: ApifyClientTypes.Actor) => {
|
|
30
|
+
const defaultBuildTag = actorInfo.defaultRunOptions.build;
|
|
31
|
+
console.error(`Default build tag for ${actorFullName} is ${defaultBuildTag}`);
|
|
32
|
+
|
|
33
|
+
// We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
|
|
34
|
+
if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
|
|
35
|
+
throw new Error(
|
|
36
|
+
`[${actorFullName}] Default build is a build number, not a tag. While this could work, ` +
|
|
37
|
+
`we want to have a default as tag so this is often an accidental misconfiguration from the dev`,
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
// I reported that buildNumber should probably not be optional
|
|
41
|
+
const defaultBuildNumber = actorInfo.taggedBuilds?.[defaultBuildTag]?.buildNumber;
|
|
42
|
+
if (!defaultBuildNumber) {
|
|
43
|
+
throw new Error(`[${actorFullName}] No build found for tag "${defaultBuildTag}". ${ACTOR_SETUP_REQUIREMENT}`);
|
|
44
|
+
}
|
|
45
|
+
const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1];
|
|
46
|
+
console.error(`Default version for ${actorFullName} is ${defaultVersionNumber}`);
|
|
47
|
+
|
|
48
|
+
const defaultVersion = actorInfo.versions.find((version) => version.versionNumber === defaultVersionNumber);
|
|
49
|
+
|
|
50
|
+
return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag, defaultVersion };
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Guards against building an Actor from a different repository than it is published from, e.g. when the
|
|
55
|
+
* local `origin` remote is a fork or a mirror. Every build overwrites the version's Git URL, so building
|
|
56
|
+
* from the wrong remote would silently repoint the Actor. The default version is the source of truth
|
|
57
|
+
* since it is what users run. Skipped when the repo URL is passed explicitly, which is how you move an
|
|
58
|
+
* Actor to a new repository on purpose.
|
|
59
|
+
*/
|
|
60
|
+
export const assertRepoUrlMatchesDefaultVersion = (
|
|
61
|
+
actorFullName: string,
|
|
62
|
+
defaultVersion: ApifyClientTypes.ActorVersion | undefined,
|
|
63
|
+
repoUrl: string,
|
|
64
|
+
) => {
|
|
65
|
+
if (!defaultVersion) {
|
|
66
|
+
throw new Error(
|
|
67
|
+
`[${actorFullName}] The version of the default build no longer exists. ${ACTOR_SETUP_REQUIREMENT}`,
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
if (defaultVersion.sourceType !== ActorSourceType.GitRepo) {
|
|
71
|
+
throw new Error(
|
|
72
|
+
`[${actorFullName}] Default version ${defaultVersion.versionNumber} has source type "${defaultVersion.sourceType}", ` +
|
|
73
|
+
`not "${ActorSourceType.GitRepo}". ${ACTOR_SETUP_REQUIREMENT}`,
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
if (normalizeRepoUrl(defaultVersion.gitRepoUrl) !== normalizeRepoUrl(repoUrl)) {
|
|
77
|
+
throw new Error(
|
|
78
|
+
`[${actorFullName}] Repository mismatch: the git remote "origin" is ${repoUrl} but default version ` +
|
|
79
|
+
`${defaultVersion.versionNumber} is built from ${defaultVersion.gitRepoUrl}. Fix the remote or the Actor's source, ` +
|
|
80
|
+
`or pass --repo-url to build from a different repository on purpose.`,
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
|
|
18
85
|
export class ApifyBuilder {
|
|
19
86
|
private constructor(
|
|
20
87
|
private readonly apifyClient: ApifyClient,
|
|
21
88
|
private readonly actorFullName: string,
|
|
22
89
|
) {}
|
|
23
90
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
defaultBuildNumber: string;
|
|
27
|
-
defaultVersionNumber: string;
|
|
28
|
-
defaultBuildTag: string;
|
|
29
|
-
}> => {
|
|
30
|
-
const actorClient = this.apifyClient.actor(this.actorFullName);
|
|
31
|
-
const actorInfo = await actorClient.get();
|
|
32
|
-
|
|
91
|
+
getActorInfo = async (): Promise<ApifyClientTypes.Actor> => {
|
|
92
|
+
const actorInfo = await this.apifyClient.actor(this.actorFullName).get();
|
|
33
93
|
if (!actorInfo) {
|
|
34
94
|
throw new Error(
|
|
35
95
|
`[${this.actorFullName}] not found. It is not published or we are missing token to access it privately or its name is misspelled`,
|
|
36
96
|
);
|
|
37
97
|
}
|
|
38
|
-
|
|
39
|
-
const defaultBuildTag = actorInfo.defaultRunOptions.build;
|
|
40
|
-
console.error(`Default build tag for ${this.actorFullName} is ${defaultBuildTag}`);
|
|
41
|
-
|
|
42
|
-
// We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
|
|
43
|
-
if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
|
|
44
|
-
throw new Error(
|
|
45
|
-
`[${this.actorFullName}] Default build is a build number, not a tag. While this could work, ` +
|
|
46
|
-
`we want to have a default as tag so this is often an accidental misconfiguration from the dev`,
|
|
47
|
-
);
|
|
48
|
-
}
|
|
49
|
-
// I reported that buildNumber should probably not be optional
|
|
50
|
-
if (!actorInfo.taggedBuilds?.[defaultBuildTag]?.buildNumber) {
|
|
51
|
-
throw new Error(
|
|
52
|
-
`[${this.actorFullName}] No build found for tag "${defaultBuildTag}". ` +
|
|
53
|
-
`The first build must be triggered manually on the platform before CI can take over.`,
|
|
54
|
-
);
|
|
55
|
-
}
|
|
56
|
-
const defaultBuildNumber = actorInfo.taggedBuilds![defaultBuildTag].buildNumber!;
|
|
57
|
-
const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1];
|
|
58
|
-
console.error(`Default version for ${this.actorFullName} is ${defaultVersionNumber}`);
|
|
59
|
-
|
|
60
|
-
return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag };
|
|
98
|
+
return actorInfo;
|
|
61
99
|
};
|
|
62
100
|
|
|
101
|
+
getDefaultVersionAndTag = async () => resolveDefaultVersion(this.actorFullName, await this.getActorInfo());
|
|
102
|
+
|
|
63
103
|
// Mirrors `apify push`: stores the zip in the Actor's `actor-<id>-source` KV store and returns a signed
|
|
64
104
|
// record URL the builder can download without a token.
|
|
65
105
|
uploadSourceZip = async (versionNumber: string, zip: Buffer): Promise<string> => {
|
|
@@ -76,24 +116,19 @@ export class ApifyBuilder {
|
|
|
76
116
|
return url.toString();
|
|
77
117
|
};
|
|
78
118
|
|
|
119
|
+
// Pass actorInfo when the caller already fetched it, to save an API call
|
|
79
120
|
createVersionAndBuild = async (
|
|
80
121
|
versionNumber: string,
|
|
81
|
-
actorVersion: ActorVersion,
|
|
122
|
+
actorVersion: ApifyClientTypes.ActorVersion,
|
|
82
123
|
useCache: boolean,
|
|
124
|
+
actorInfo?: ApifyClientTypes.Actor,
|
|
83
125
|
): Promise<BuildData> => {
|
|
84
126
|
const actorClient = this.apifyClient.actor(this.actorFullName);
|
|
85
|
-
const
|
|
86
|
-
if (!actorInfo) {
|
|
87
|
-
throw new Error(
|
|
88
|
-
`No actor named '${this.actorFullName}' was found on the platform. If this` +
|
|
89
|
-
' is unexpected, make sure the actor you are targeting is spelled the' +
|
|
90
|
-
' same as the folder in the repository.',
|
|
91
|
-
);
|
|
92
|
-
}
|
|
127
|
+
const { versions } = actorInfo ?? (await this.getActorInfo());
|
|
93
128
|
|
|
94
129
|
// Prepare version
|
|
95
|
-
const
|
|
96
|
-
if (
|
|
130
|
+
const versionMissing = !versions.find((version) => version.versionNumber === versionNumber);
|
|
131
|
+
if (versionMissing) {
|
|
97
132
|
// create new version
|
|
98
133
|
await actorClient.versions().create(actorVersion);
|
|
99
134
|
} else {
|
|
@@ -108,7 +143,7 @@ export class ApifyBuilder {
|
|
|
108
143
|
return { buildId: id, actorRawId: actId, buildNumber, actorFullName: this.actorFullName };
|
|
109
144
|
};
|
|
110
145
|
|
|
111
|
-
waitForBuildToFinish = async (buildId: string): Promise<Build> => {
|
|
146
|
+
waitForBuildToFinish = async (buildId: string): Promise<ApifyClientTypes.Build> => {
|
|
112
147
|
const build = await this.apifyClient.build(buildId).waitForFinish();
|
|
113
148
|
const versionNumber = build.buildNumber;
|
|
114
149
|
if (build.status === 'FAILED' || build.status === 'TIMED-OUT') {
|
|
@@ -168,7 +203,7 @@ export class ApifyBuilder {
|
|
|
168
203
|
const DEFAULT_DAYS_BACK_PROD_VERSIONS = 30;
|
|
169
204
|
const DEFAULT_DAYS_BACK_DEVEL = 7;
|
|
170
205
|
|
|
171
|
-
const actorInfo =
|
|
206
|
+
const actorInfo = await this.getActorInfo();
|
|
172
207
|
|
|
173
208
|
// 'devel' used to be hardcoded for testing version 0.99, once we get rid of this tag everywhere, we can remove this code
|
|
174
209
|
const taggedDevelBuildNumber: string | undefined = actorInfo.taggedBuilds!.devel?.buildNumber;
|
|
@@ -183,7 +218,7 @@ export class ApifyBuilder {
|
|
|
183
218
|
const { items } = await this.apifyClient.actor(this.actorFullName).builds().list();
|
|
184
219
|
|
|
185
220
|
// Deleting default build throws an error, so we skip it
|
|
186
|
-
const { defaultBuildNumber, defaultBuildTag } =
|
|
221
|
+
const { defaultBuildNumber, defaultBuildTag } = resolveDefaultVersion(this.actorFullName, actorInfo);
|
|
187
222
|
|
|
188
223
|
const daysAgoUnixProd = Date.now() - DEFAULT_DAYS_BACK_PROD_VERSIONS * 24 * 60 * 60 * 1000;
|
|
189
224
|
const daysAgoUnixDevel = Date.now() - DEFAULT_DAYS_BACK_DEVEL * 24 * 60 * 60 * 1000;
|
|
@@ -286,6 +321,8 @@ type RunBuildsOptions = {
|
|
|
286
321
|
actorConfigs: ActorConfig[];
|
|
287
322
|
isLatest?: boolean;
|
|
288
323
|
repoUrl: string;
|
|
324
|
+
// False when the repo URL was passed explicitly, see assertRepoUrlMatchesDefaultVersion
|
|
325
|
+
shouldVerifyRepoUrl: boolean;
|
|
289
326
|
branch: string;
|
|
290
327
|
dryRun: boolean;
|
|
291
328
|
useDockerCache: boolean;
|
|
@@ -293,34 +330,40 @@ type RunBuildsOptions = {
|
|
|
293
330
|
|
|
294
331
|
export const runBuilds = async ({
|
|
295
332
|
repoUrl,
|
|
333
|
+
shouldVerifyRepoUrl,
|
|
296
334
|
branch,
|
|
297
335
|
actorConfigs,
|
|
298
336
|
isLatest = false,
|
|
299
337
|
dryRun,
|
|
300
338
|
useDockerCache,
|
|
301
339
|
}: RunBuildsOptions): Promise<BuildData[]> => {
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
} else {
|
|
314
|
-
versionNumber = DEFAULT_TEST_VERSION_NUMBER;
|
|
315
|
-
}
|
|
340
|
+
// Fetched once per Actor and reused for the checks, the version numbers and the build itself
|
|
341
|
+
const buildConfigs: BuildPrActorOptions[] = await Promise.all(
|
|
342
|
+
actorConfigs.map(async (actorConfig) => {
|
|
343
|
+
const actorInfo = await ApifyBuilder.fromActorConfig(actorConfig).getActorInfo();
|
|
344
|
+
const { defaultVersionNumber, defaultBuildTag, defaultVersion } = resolveDefaultVersion(
|
|
345
|
+
actorConfig.actorFullName,
|
|
346
|
+
actorInfo,
|
|
347
|
+
);
|
|
348
|
+
if (shouldVerifyRepoUrl) {
|
|
349
|
+
assertRepoUrlMatchesDefaultVersion(actorConfig.actorFullName, defaultVersion, repoUrl);
|
|
350
|
+
}
|
|
316
351
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
352
|
+
// Depending on if these are miniactors or standaloneActors
|
|
353
|
+
let gitRepoUrl = `${repoUrl}#${branch}`;
|
|
354
|
+
if (actorConfig.folder) {
|
|
355
|
+
gitRepoUrl = `${gitRepoUrl}:${actorConfig.folder}`;
|
|
356
|
+
}
|
|
357
|
+
return {
|
|
358
|
+
actorConfig,
|
|
359
|
+
actorInfo,
|
|
360
|
+
gitRepoUrl,
|
|
361
|
+
versionNumber: isLatest ? defaultVersionNumber : DEFAULT_TEST_VERSION_NUMBER,
|
|
362
|
+
buildTag: isLatest ? defaultBuildTag : undefined,
|
|
363
|
+
useDockerCache,
|
|
364
|
+
};
|
|
365
|
+
}),
|
|
366
|
+
);
|
|
324
367
|
|
|
325
368
|
if (dryRun) {
|
|
326
369
|
console.error('[DRY RUN] Would build:');
|
|
@@ -338,18 +381,19 @@ export const runBuilds = async ({
|
|
|
338
381
|
|
|
339
382
|
return runAndSummarizeBuilds(actorConfigs, 'BUILDS', async (actorConfig, builder) => {
|
|
340
383
|
const {
|
|
384
|
+
actorInfo,
|
|
341
385
|
buildTag,
|
|
342
386
|
versionNumber,
|
|
343
387
|
gitRepoUrl,
|
|
344
388
|
useDockerCache: useCache,
|
|
345
389
|
} = buildConfigsByActorFullName.get(actorConfig.actorFullName)!;
|
|
346
|
-
const actorVersion: ActorVersion = {
|
|
390
|
+
const actorVersion: ApifyClientTypes.ActorVersion = {
|
|
347
391
|
buildTag,
|
|
348
392
|
versionNumber,
|
|
349
393
|
gitRepoUrl,
|
|
350
394
|
sourceType: ActorSourceType.GitRepo,
|
|
351
395
|
};
|
|
352
|
-
return builder.createVersionAndBuild(versionNumber, actorVersion, useCache);
|
|
396
|
+
return builder.createVersionAndBuild(versionNumber, actorVersion, useCache, actorInfo);
|
|
353
397
|
});
|
|
354
398
|
};
|
|
355
399
|
|