apify-test-tools 0.9.1-beta.11 → 0.9.1-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/public_push-build-latest.yaml +3 -1
- package/.github/workflows-package-version +1 -1
- package/CHANGELOG.md +1 -0
- package/CONTRIBUTING.md +2 -4
- package/README.md +68 -8
- package/bin/build-from-local.ts +2 -2
- package/bin/build.ts +114 -70
- package/bin/git.ts +140 -0
- package/bin/main.ts +128 -91
- package/bin/slack.ts +5 -2
- package/bin/types.ts +0 -73
- package/bin/utils.ts +2 -3
- package/dist/bin/build-from-local.js.map +1 -1
- package/dist/bin/build.d.ts +25 -4
- package/dist/bin/build.d.ts.map +1 -1
- package/dist/bin/build.js +76 -47
- package/dist/bin/build.js.map +1 -1
- package/dist/bin/git.d.ts +40 -0
- package/dist/bin/git.d.ts.map +1 -1
- package/dist/bin/git.js +118 -0
- package/dist/bin/git.js.map +1 -1
- package/dist/bin/main.d.ts +1 -20
- package/dist/bin/main.d.ts.map +1 -1
- package/dist/bin/main.js +65 -62
- package/dist/bin/main.js.map +1 -1
- package/dist/bin/slack.d.ts.map +1 -1
- package/dist/bin/slack.js +4 -2
- package/dist/bin/slack.js.map +1 -1
- package/dist/bin/types.d.ts +0 -64
- package/dist/bin/types.d.ts.map +1 -1
- package/dist/bin/utils.d.ts.map +1 -1
- package/dist/bin/utils.js +2 -3
- package/dist/bin/utils.js.map +1 -1
- package/dist/test/unit/bin/build.test.d.ts +2 -0
- package/dist/test/unit/bin/build.test.d.ts.map +1 -0
- package/dist/test/unit/bin/build.test.js +46 -0
- package/dist/test/unit/bin/build.test.js.map +1 -0
- package/dist/test/unit/bin/git.test.js +104 -1
- package/dist/test/unit/bin/git.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
- package/test/unit/bin/build.test.ts +68 -0
- package/test/unit/bin/git.test.ts +122 -0
- package/bin/github.ts +0 -69
- package/dist/bin/github.d.ts +0 -11
- package/dist/bin/github.d.ts.map +0 -1
- package/dist/bin/github.js +0 -59
- package/dist/bin/github.js.map +0 -1
|
@@ -25,6 +25,8 @@ jobs:
|
|
|
25
25
|
with:
|
|
26
26
|
npm-token: ${{ secrets.NPM_TOKEN }}
|
|
27
27
|
|
|
28
|
+
# --base-commit is the last commit that is already released: the push's "before" commit.
|
|
29
|
+
# Everything after it up to HEAD gets released. See the "Releasing Actors" section of the README.
|
|
28
30
|
# run-with-apify-tokens.mjs reads the token names from in apify-test-tools.config.json
|
|
29
31
|
# and hands the build those secrets and nothing else, so the tokens stay in this one command instead of the whole job.
|
|
30
32
|
# The library never sees secrets it doesn't need
|
|
@@ -34,7 +36,7 @@ jobs:
|
|
|
34
36
|
SLACK_TOKEN_RELEASES_BOT: ${{ secrets.SLACK_TOKEN_RELEASES_BOT }}
|
|
35
37
|
run: |
|
|
36
38
|
node "${{ steps.setup.outputs.scripts-path }}/run-with-apify-tokens.mjs" \
|
|
37
|
-
npx apify-test-tools release --
|
|
39
|
+
npx apify-test-tools release --base-commit ${{ github.event.before }} --release-slack-channel "#delivery-public-actors" --report-slack-channel "${{ inputs.report-slack-channel || format('#notif-{0}', github.event.repository.name) }}" --use-docker-cache
|
|
38
40
|
|
|
39
41
|
- name: Delete old builds
|
|
40
42
|
env:
|
|
@@ -1 +1 @@
|
|
|
1
|
-
0.
|
|
1
|
+
0.10.0
|
package/CHANGELOG.md
CHANGED
|
@@ -19,6 +19,7 @@ All notable changes to this project will be documented in this file.
|
|
|
19
19
|
### 🚜 Refactor
|
|
20
20
|
|
|
21
21
|
- **workflows:** [**breaking**] Rename the floating tag to workflows-v0 ([#146](https://github.com/apify/apify-test-tools/pull/146)) ([c48fb22](https://github.com/apify/apify-test-tools/commit/c48fb223c885c4e5de7e5f64d8c14f6d03fcc7de)) by [@metalwarrior665](https://github.com/metalwarrior665)
|
|
22
|
+
- **bin:** [**breaking**] Remove GitHub event dependency from release command ([#153](https://github.com/apify/apify-test-tools/pull/153)) ([e6953aa](https://github.com/apify/apify-test-tools/commit/e6953aa095d1a92638c6d72441e1b312d9f835c3)) by [@metalwarrior665](https://github.com/metalwarrior665), closes [#131](https://github.com/apify/apify-test-tools/issues/131)
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
<!-- git-cliff-unreleased-end -->
|
package/CONTRIBUTING.md
CHANGED
|
@@ -10,7 +10,6 @@ The package consists of three parts:
|
|
|
10
10
|
|
|
11
11
|
- `bin/build.ts` actor building
|
|
12
12
|
- `bin/git.ts` git wrappers for getting relevant git commits and changed files
|
|
13
|
-
- `bin/github.ts` github push event parsing
|
|
14
13
|
- `bin/main.ts` entrypoing
|
|
15
14
|
- `bin/slack.ts` sending notifications to slack
|
|
16
15
|
- `bin/test-report.ts` processing vitest's test reports
|
|
@@ -40,12 +39,11 @@ git clone git@github.com:apify-store/testing-repo-for-github-actions.git
|
|
|
40
39
|
|
|
41
40
|
#### Working on the CLI
|
|
42
41
|
|
|
43
|
-
To work on the library,
|
|
42
|
+
To work on the library, point the cli to your repo with `--workspace` (it defaults to the current directory):
|
|
44
43
|
|
|
45
44
|
```sh
|
|
46
|
-
export GITHUB_WORKSPACE=../path/to/testing-repo-for-github-actions # path to the repo
|
|
47
45
|
npx tsx bin/main.ts --help
|
|
48
|
-
npx tsx bin/main.ts get-commits --target-branch master --source-branch feat/testing-feature-branch
|
|
46
|
+
npx tsx bin/main.ts --workspace ../path/to/testing-repo-for-github-actions get-commits --target-branch master --source-branch feat/testing-feature-branch
|
|
49
47
|
```
|
|
50
48
|
|
|
51
49
|
#### Working on the library
|
package/README.md
CHANGED
|
@@ -12,6 +12,9 @@ npm i -D apify-test-tools
|
|
|
12
12
|
|
|
13
13
|
- Requires `vitest` version `3.2.0` or later (uses [annotate](https://vitest.dev/guide/test-context.html#annotate))
|
|
14
14
|
- Make sure `target` and `module` in your `tsconfig.json`'s `compilerOptions` are set to `ES2022`
|
|
15
|
+
- Every Actor must already have at least one build under its default build tag (usually `latest`), built
|
|
16
|
+
from a Git repository version whose URL points to this repository. Build it once manually on the
|
|
17
|
+
platform before CI takes over. See [Repository check](#repository-check).
|
|
15
18
|
|
|
16
19
|
### 2. Create the config file
|
|
17
20
|
|
|
@@ -223,6 +226,10 @@ jobs:
|
|
|
223
226
|
secrets: inherit
|
|
224
227
|
```
|
|
225
228
|
|
|
229
|
+
This workflow runs `apify-test-tools release --base-commit ${{ github.event.before }}`. See
|
|
230
|
+
[Releasing Actors](#releasing-actors-release) for how the released range is determined and what to
|
|
231
|
+
pass if you call the CLI from somewhere else.
|
|
232
|
+
|
|
226
233
|
### `claude-review.yaml`
|
|
227
234
|
|
|
228
235
|
Optional. Reviews a PR against the shared guidelines when you add the trigger label, and again on
|
|
@@ -588,7 +595,7 @@ The main local flow is:
|
|
|
588
595
|
4. Build Actors on Apify (with your new code)
|
|
589
596
|
5. Run tests against those builds. You can change tests and run on the same builds.
|
|
590
597
|
|
|
591
|
-
|
|
598
|
+
Run every command from the root of the actor repository you want to work with (or point `--workspace` at it).
|
|
592
599
|
|
|
593
600
|
#### 4. Build affected Actors
|
|
594
601
|
|
|
@@ -598,7 +605,6 @@ Requires `APIFY_TOKEN_<USERNAME>` for all Apify users that own your Actors (e.g.
|
|
|
598
605
|
|
|
599
606
|
```bash
|
|
600
607
|
APIFY_TOKEN_JOHN_DOE=<token> \
|
|
601
|
-
GITHUB_WORKSPACE=. \
|
|
602
608
|
npx apify-test-tools build \
|
|
603
609
|
--target-branch origin/master \
|
|
604
610
|
--source-branch origin/my-dummy-branch \
|
|
@@ -617,7 +623,6 @@ If you don't want to push a dummy branch just to test a change and wait for all
|
|
|
617
623
|
|
|
618
624
|
```bash
|
|
619
625
|
APIFY_TOKEN_JOHN_DOE=<token> \
|
|
620
|
-
GITHUB_WORKSPACE=. \
|
|
621
626
|
npx apify-test-tools build-from-local --actors john.doe/my-actor
|
|
622
627
|
```
|
|
623
628
|
|
|
@@ -626,8 +631,7 @@ Pass a hardcoded actor name via `--actors` to build only that Actor (comma-separ
|
|
|
626
631
|
```bash
|
|
627
632
|
# Build from local source and capture output
|
|
628
633
|
BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
|
|
629
|
-
|
|
630
|
-
npx apify-test-tools build-from-local --actors apify/my-actor)
|
|
634
|
+
npx apify-test-tools build-from-local --actors apify/my-actor)
|
|
631
635
|
```
|
|
632
636
|
|
|
633
637
|
Since you already scoped the build to just the Actor(s) you care about, point vitest at a specific test file (or a `-t` name filter) instead of the whole `test/platform` directory — you get feedback on that one test without waiting for the full suite to run.
|
|
@@ -649,8 +653,7 @@ TESTER_APIFY_TOKEN=<token> \
|
|
|
649
653
|
```bash
|
|
650
654
|
# Build and capture output
|
|
651
655
|
BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
|
|
652
|
-
|
|
653
|
-
npx apify-test-tools build \
|
|
656
|
+
npx apify-test-tools build \
|
|
654
657
|
--target-branch origin/master \
|
|
655
658
|
--source-branch origin/my-dummy-branch)
|
|
656
659
|
|
|
@@ -660,10 +663,67 @@ TESTER_APIFY_TOKEN=apify_api_yyy \
|
|
|
660
663
|
npx vitest --run --maxConcurrency 20 --fileParallelism=true --maxWorkers 100 test/platform
|
|
661
664
|
```
|
|
662
665
|
|
|
666
|
+
### Releasing Actors (`release`)
|
|
667
|
+
|
|
668
|
+
`release` builds the `latest` (default) version of every Actor changed since the last release and
|
|
669
|
+
posts release notes to Slack. It reads everything else from git: the branch from the checked-out
|
|
670
|
+
`HEAD`, the commits and changed files from the range `<base-commit>..HEAD`, and the repository from
|
|
671
|
+
the `origin` remote.
|
|
672
|
+
|
|
673
|
+
```bash
|
|
674
|
+
npx apify-test-tools release --base-commit <sha> --release-slack-channel "#releases" --dry-run
|
|
675
|
+
```
|
|
676
|
+
|
|
677
|
+
#### `--base-commit`
|
|
678
|
+
|
|
679
|
+
The last commit that is **already released**. Everything after it, up to `HEAD`, is released. It is
|
|
680
|
+
required: git alone can't tell which commits a push brought, and guessing wrong silently leaves Actors
|
|
681
|
+
on old code.
|
|
682
|
+
|
|
683
|
+
The changed files come from diffing `<base-commit>` and `HEAD` directly, so this works for every
|
|
684
|
+
merge strategy as long as the base commit is the branch tip from **before** the push:
|
|
685
|
+
|
|
686
|
+
| Merge strategy | What lands on the branch | What to pass |
|
|
687
|
+
| ------------------------------ | --------------------------------- | ---------------------------------------------------------------------- |
|
|
688
|
+
| Squash and merge | 1 new commit | the push's "before" commit (`HEAD~1` also works) |
|
|
689
|
+
| Merge commit | the PR's commits + 1 merge commit | the push's "before" commit (`HEAD^1`, the first parent, also works) |
|
|
690
|
+
| Rebase and merge / direct push | N new commits | the push's "before" commit (`HEAD~1` would miss the first N-1 commits) |
|
|
691
|
+
|
|
692
|
+
On GitHub, the push's "before" commit is `${{ github.event.before }}`, which the reusable workflow
|
|
693
|
+
passes for you. On other CIs use the equivalent, e.g. `$CI_COMMIT_BEFORE_SHA` on GitLab. The checkout
|
|
694
|
+
needs the full history (`fetch-depth: 0` in `actions/checkout`) so the base commit is available locally.
|
|
695
|
+
|
|
696
|
+
`release` fails instead of guessing when:
|
|
697
|
+
|
|
698
|
+
- the base commit is all zeros (GitHub sends this for a push that created the branch)
|
|
699
|
+
- the base commit isn't in the local history (shallow checkout) or isn't an ancestor of `HEAD`
|
|
700
|
+
(the branch was force-pushed)
|
|
701
|
+
|
|
702
|
+
In those cases, rerun `release` with a `--base-commit` you pick yourself: an ancestor of `HEAD` from
|
|
703
|
+
before the changes you want to release (for a shallow checkout, fetch the full history instead).
|
|
704
|
+
`--actors` only narrows which of the changed Actors get released; it never forces a build, so on its
|
|
705
|
+
own it doesn't get past these errors.
|
|
706
|
+
If `HEAD` is the base commit, there is nothing to release and the command exits successfully.
|
|
707
|
+
|
|
708
|
+
A failed release isn't retried by the next push: that push's base commit is the failed one's head, so
|
|
709
|
+
its changes are skipped. Re-run the failed workflow run, or release the Actors manually.
|
|
710
|
+
|
|
711
|
+
#### Repository check
|
|
712
|
+
|
|
713
|
+
`release` and `build` point each Actor's version at the `origin` remote. Because a build overwrites the
|
|
714
|
+
version's Git URL, building from a fork or a mirror would repoint the Actor. So, before building, the
|
|
715
|
+
remote is compared with the Git URL of the Actor's default version (the one its default build tag
|
|
716
|
+
points to), and the command fails if they differ or if the default version isn't built from a Git
|
|
717
|
+
repository. The URL form doesn't matter (`https://github.com/org/repo` equals
|
|
718
|
+
`git@github.com:org/repo.git`).
|
|
719
|
+
|
|
720
|
+
To build from a different repository on purpose, e.g. when moving an Actor, pass `--repo-url`. That
|
|
721
|
+
skips the check.
|
|
722
|
+
|
|
663
723
|
#### Dev mode
|
|
664
724
|
|
|
665
725
|
For development on `apify-test-tools` itself, use `tsx` directly:
|
|
666
726
|
|
|
667
727
|
```bash
|
|
668
|
-
|
|
728
|
+
tsx bin/main.ts --workspace local-clone get-actor-configs
|
|
669
729
|
```
|
package/bin/build-from-local.ts
CHANGED
|
@@ -3,7 +3,7 @@ import os from 'node:os';
|
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
|
|
5
5
|
import AdmZip from 'adm-zip';
|
|
6
|
-
import type
|
|
6
|
+
import type * as ApifyClientTypes from 'apify-client';
|
|
7
7
|
import { ActorSourceType } from 'apify-client';
|
|
8
8
|
|
|
9
9
|
import { dryRunBuildData, LOCAL_SOURCE_VERSION_NUMBER, runAndSummarizeBuilds } from './build.js';
|
|
@@ -207,7 +207,7 @@ export const runBuildsFromLocal = async ({
|
|
|
207
207
|
for (const { name, content } of await collectSourceFiles(actorConfig.actorFullName, actorConfig.folder)) {
|
|
208
208
|
zip.addFile(name, content);
|
|
209
209
|
}
|
|
210
|
-
const actorVersion: ActorVersion = {
|
|
210
|
+
const actorVersion: ApifyClientTypes.ActorVersion = {
|
|
211
211
|
versionNumber: LOCAL_SOURCE_VERSION_NUMBER,
|
|
212
212
|
tarballUrl: await builder.uploadSourceZip(LOCAL_SOURCE_VERSION_NUMBER, zip.toBuffer()),
|
|
213
213
|
sourceType: ActorSourceType.Tarball,
|
package/bin/build.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import type * as ApifyClientTypes from 'apify-client';
|
|
2
2
|
import { ActorSourceType, ApifyClient } from 'apify-client';
|
|
3
3
|
|
|
4
|
+
import { normalizeRepoUrl } from './git.js';
|
|
4
5
|
import type { ActorConfig, BuildData } from './types.js';
|
|
5
6
|
|
|
6
7
|
type BuildPrActorOptions = {
|
|
@@ -8,6 +9,7 @@ type BuildPrActorOptions = {
|
|
|
8
9
|
versionNumber: string;
|
|
9
10
|
gitRepoUrl: string;
|
|
10
11
|
actorConfig: ActorConfig;
|
|
12
|
+
actorInfo: ApifyClientTypes.Actor;
|
|
11
13
|
useDockerCache: boolean;
|
|
12
14
|
};
|
|
13
15
|
|
|
@@ -15,51 +17,89 @@ type BuildPrActorOptions = {
|
|
|
15
17
|
export const LOCAL_SOURCE_VERSION_NUMBER = '0.98';
|
|
16
18
|
const DEFAULT_TEST_VERSION_NUMBER = '0.99';
|
|
17
19
|
|
|
20
|
+
// Shown whenever an Actor isn't set up the way CI builds need it
|
|
21
|
+
const ACTOR_SETUP_REQUIREMENT =
|
|
22
|
+
'Before using apify-test-tools, every Actor needs at least one build under its default build tag (usually "latest"), ' +
|
|
23
|
+
'built from a Git repository version whose URL points to this repository. Build it once manually on the platform.';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Finds the Actor's default version: the one whose build the default build tag points to.
|
|
27
|
+
* Usually tagged 'latest' but not necessarily (can be e.g. 'version-0').
|
|
28
|
+
*/
|
|
29
|
+
export const resolveDefaultVersion = (actorFullName: string, actorInfo: ApifyClientTypes.Actor) => {
|
|
30
|
+
const defaultBuildTag = actorInfo.defaultRunOptions.build;
|
|
31
|
+
console.error(`Default build tag for ${actorFullName} is ${defaultBuildTag}`);
|
|
32
|
+
|
|
33
|
+
// We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
|
|
34
|
+
if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
|
|
35
|
+
throw new Error(
|
|
36
|
+
`[${actorFullName}] Default build is a build number, not a tag. While this could work, ` +
|
|
37
|
+
`we want to have a default as tag so this is often an accidental misconfiguration from the dev`,
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
// I reported that buildNumber should probably not be optional
|
|
41
|
+
const defaultBuildNumber = actorInfo.taggedBuilds?.[defaultBuildTag]?.buildNumber;
|
|
42
|
+
if (!defaultBuildNumber) {
|
|
43
|
+
throw new Error(`[${actorFullName}] No build found for tag "${defaultBuildTag}". ${ACTOR_SETUP_REQUIREMENT}`);
|
|
44
|
+
}
|
|
45
|
+
const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1];
|
|
46
|
+
console.error(`Default version for ${actorFullName} is ${defaultVersionNumber}`);
|
|
47
|
+
|
|
48
|
+
const defaultVersion = actorInfo.versions.find((version) => version.versionNumber === defaultVersionNumber);
|
|
49
|
+
|
|
50
|
+
return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag, defaultVersion };
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Guards against building an Actor from a different repository than it is published from, e.g. when the
|
|
55
|
+
* local `origin` remote is a fork or a mirror. Every build overwrites the version's Git URL, so building
|
|
56
|
+
* from the wrong remote would silently repoint the Actor. The default version is the source of truth
|
|
57
|
+
* since it is what users run. Skipped when the repo URL is passed explicitly, which is how you move an
|
|
58
|
+
* Actor to a new repository on purpose.
|
|
59
|
+
*/
|
|
60
|
+
export const assertRepoUrlMatchesDefaultVersion = (
|
|
61
|
+
actorFullName: string,
|
|
62
|
+
defaultVersion: ApifyClientTypes.ActorVersion | undefined,
|
|
63
|
+
repoUrl: string,
|
|
64
|
+
) => {
|
|
65
|
+
if (!defaultVersion) {
|
|
66
|
+
throw new Error(
|
|
67
|
+
`[${actorFullName}] The version of the default build no longer exists. ${ACTOR_SETUP_REQUIREMENT}`,
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
if (defaultVersion.sourceType !== ActorSourceType.GitRepo) {
|
|
71
|
+
throw new Error(
|
|
72
|
+
`[${actorFullName}] Default version ${defaultVersion.versionNumber} has source type "${defaultVersion.sourceType}", ` +
|
|
73
|
+
`not "${ActorSourceType.GitRepo}". ${ACTOR_SETUP_REQUIREMENT}`,
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
if (normalizeRepoUrl(defaultVersion.gitRepoUrl) !== normalizeRepoUrl(repoUrl)) {
|
|
77
|
+
throw new Error(
|
|
78
|
+
`[${actorFullName}] Repository mismatch: the git remote "origin" is ${repoUrl} but default version ` +
|
|
79
|
+
`${defaultVersion.versionNumber} is built from ${defaultVersion.gitRepoUrl}. Fix the remote or the Actor's source, ` +
|
|
80
|
+
`or pass --repo-url to build from a different repository on purpose.`,
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
|
|
18
85
|
export class ApifyBuilder {
|
|
19
86
|
private constructor(
|
|
20
87
|
private readonly apifyClient: ApifyClient,
|
|
21
88
|
private readonly actorFullName: string,
|
|
22
89
|
) {}
|
|
23
90
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
defaultBuildNumber: string;
|
|
27
|
-
defaultVersionNumber: string;
|
|
28
|
-
defaultBuildTag: string;
|
|
29
|
-
}> => {
|
|
30
|
-
const actorClient = this.apifyClient.actor(this.actorFullName);
|
|
31
|
-
const actorInfo = await actorClient.get();
|
|
32
|
-
|
|
91
|
+
getActorInfo = async (): Promise<ApifyClientTypes.Actor> => {
|
|
92
|
+
const actorInfo = await this.apifyClient.actor(this.actorFullName).get();
|
|
33
93
|
if (!actorInfo) {
|
|
34
94
|
throw new Error(
|
|
35
95
|
`[${this.actorFullName}] not found. It is not published or we are missing token to access it privately or its name is misspelled`,
|
|
36
96
|
);
|
|
37
97
|
}
|
|
38
|
-
|
|
39
|
-
const defaultBuildTag = actorInfo.defaultRunOptions.build;
|
|
40
|
-
console.error(`Default build tag for ${this.actorFullName} is ${defaultBuildTag}`);
|
|
41
|
-
|
|
42
|
-
// We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
|
|
43
|
-
if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
|
|
44
|
-
throw new Error(
|
|
45
|
-
`[${this.actorFullName}] Default build is a build number, not a tag. While this could work, ` +
|
|
46
|
-
`we want to have a default as tag so this is often an accidental misconfiguration from the dev`,
|
|
47
|
-
);
|
|
48
|
-
}
|
|
49
|
-
// I reported that buildNumber should probably not be optional
|
|
50
|
-
if (!actorInfo.taggedBuilds?.[defaultBuildTag]?.buildNumber) {
|
|
51
|
-
throw new Error(
|
|
52
|
-
`[${this.actorFullName}] No build found for tag "${defaultBuildTag}". ` +
|
|
53
|
-
`The first build must be triggered manually on the platform before CI can take over.`,
|
|
54
|
-
);
|
|
55
|
-
}
|
|
56
|
-
const defaultBuildNumber = actorInfo.taggedBuilds![defaultBuildTag].buildNumber!;
|
|
57
|
-
const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1];
|
|
58
|
-
console.error(`Default version for ${this.actorFullName} is ${defaultVersionNumber}`);
|
|
59
|
-
|
|
60
|
-
return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag };
|
|
98
|
+
return actorInfo;
|
|
61
99
|
};
|
|
62
100
|
|
|
101
|
+
getDefaultVersionAndTag = async () => resolveDefaultVersion(this.actorFullName, await this.getActorInfo());
|
|
102
|
+
|
|
63
103
|
// Mirrors `apify push`: stores the zip in the Actor's `actor-<id>-source` KV store and returns a signed
|
|
64
104
|
// record URL the builder can download without a token.
|
|
65
105
|
uploadSourceZip = async (versionNumber: string, zip: Buffer): Promise<string> => {
|
|
@@ -76,24 +116,19 @@ export class ApifyBuilder {
|
|
|
76
116
|
return url.toString();
|
|
77
117
|
};
|
|
78
118
|
|
|
119
|
+
// Pass actorInfo when the caller already fetched it, to save an API call
|
|
79
120
|
createVersionAndBuild = async (
|
|
80
121
|
versionNumber: string,
|
|
81
|
-
actorVersion: ActorVersion,
|
|
122
|
+
actorVersion: ApifyClientTypes.ActorVersion,
|
|
82
123
|
useCache: boolean,
|
|
124
|
+
actorInfo?: ApifyClientTypes.Actor,
|
|
83
125
|
): Promise<BuildData> => {
|
|
84
126
|
const actorClient = this.apifyClient.actor(this.actorFullName);
|
|
85
|
-
const
|
|
86
|
-
if (!actorInfo) {
|
|
87
|
-
throw new Error(
|
|
88
|
-
`No actor named '${this.actorFullName}' was found on the platform. If this` +
|
|
89
|
-
' is unexpected, make sure the actor you are targeting is spelled the' +
|
|
90
|
-
' same as the folder in the repository.',
|
|
91
|
-
);
|
|
92
|
-
}
|
|
127
|
+
const { versions } = actorInfo ?? (await this.getActorInfo());
|
|
93
128
|
|
|
94
129
|
// Prepare version
|
|
95
|
-
const
|
|
96
|
-
if (
|
|
130
|
+
const versionMissing = !versions.find((version) => version.versionNumber === versionNumber);
|
|
131
|
+
if (versionMissing) {
|
|
97
132
|
// create new version
|
|
98
133
|
await actorClient.versions().create(actorVersion);
|
|
99
134
|
} else {
|
|
@@ -108,7 +143,7 @@ export class ApifyBuilder {
|
|
|
108
143
|
return { buildId: id, actorRawId: actId, buildNumber, actorFullName: this.actorFullName };
|
|
109
144
|
};
|
|
110
145
|
|
|
111
|
-
waitForBuildToFinish = async (buildId: string): Promise<Build> => {
|
|
146
|
+
waitForBuildToFinish = async (buildId: string): Promise<ApifyClientTypes.Build> => {
|
|
112
147
|
const build = await this.apifyClient.build(buildId).waitForFinish();
|
|
113
148
|
const versionNumber = build.buildNumber;
|
|
114
149
|
if (build.status === 'FAILED' || build.status === 'TIMED-OUT') {
|
|
@@ -168,7 +203,7 @@ export class ApifyBuilder {
|
|
|
168
203
|
const DEFAULT_DAYS_BACK_PROD_VERSIONS = 30;
|
|
169
204
|
const DEFAULT_DAYS_BACK_DEVEL = 7;
|
|
170
205
|
|
|
171
|
-
const actorInfo =
|
|
206
|
+
const actorInfo = await this.getActorInfo();
|
|
172
207
|
|
|
173
208
|
// 'devel' used to be hardcoded for testing version 0.99, once we get rid of this tag everywhere, we can remove this code
|
|
174
209
|
const taggedDevelBuildNumber: string | undefined = actorInfo.taggedBuilds!.devel?.buildNumber;
|
|
@@ -183,7 +218,7 @@ export class ApifyBuilder {
|
|
|
183
218
|
const { items } = await this.apifyClient.actor(this.actorFullName).builds().list();
|
|
184
219
|
|
|
185
220
|
// Deleting default build throws an error, so we skip it
|
|
186
|
-
const { defaultBuildNumber, defaultBuildTag } =
|
|
221
|
+
const { defaultBuildNumber, defaultBuildTag } = resolveDefaultVersion(this.actorFullName, actorInfo);
|
|
187
222
|
|
|
188
223
|
const daysAgoUnixProd = Date.now() - DEFAULT_DAYS_BACK_PROD_VERSIONS * 24 * 60 * 60 * 1000;
|
|
189
224
|
const daysAgoUnixDevel = Date.now() - DEFAULT_DAYS_BACK_DEVEL * 24 * 60 * 60 * 1000;
|
|
@@ -286,6 +321,8 @@ type RunBuildsOptions = {
|
|
|
286
321
|
actorConfigs: ActorConfig[];
|
|
287
322
|
isLatest?: boolean;
|
|
288
323
|
repoUrl: string;
|
|
324
|
+
// False when the repo URL was passed explicitly, see assertRepoUrlMatchesDefaultVersion
|
|
325
|
+
shouldVerifyRepoUrl: boolean;
|
|
289
326
|
branch: string;
|
|
290
327
|
dryRun: boolean;
|
|
291
328
|
useDockerCache: boolean;
|
|
@@ -293,34 +330,40 @@ type RunBuildsOptions = {
|
|
|
293
330
|
|
|
294
331
|
export const runBuilds = async ({
|
|
295
332
|
repoUrl,
|
|
333
|
+
shouldVerifyRepoUrl,
|
|
296
334
|
branch,
|
|
297
335
|
actorConfigs,
|
|
298
336
|
isLatest = false,
|
|
299
337
|
dryRun,
|
|
300
338
|
useDockerCache,
|
|
301
339
|
}: RunBuildsOptions): Promise<BuildData[]> => {
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
} else {
|
|
314
|
-
versionNumber = DEFAULT_TEST_VERSION_NUMBER;
|
|
315
|
-
}
|
|
340
|
+
// Fetched once per Actor and reused for the checks, the version numbers and the build itself
|
|
341
|
+
const buildConfigs: BuildPrActorOptions[] = await Promise.all(
|
|
342
|
+
actorConfigs.map(async (actorConfig) => {
|
|
343
|
+
const actorInfo = await ApifyBuilder.fromActorConfig(actorConfig).getActorInfo();
|
|
344
|
+
const { defaultVersionNumber, defaultBuildTag, defaultVersion } = resolveDefaultVersion(
|
|
345
|
+
actorConfig.actorFullName,
|
|
346
|
+
actorInfo,
|
|
347
|
+
);
|
|
348
|
+
if (shouldVerifyRepoUrl) {
|
|
349
|
+
assertRepoUrlMatchesDefaultVersion(actorConfig.actorFullName, defaultVersion, repoUrl);
|
|
350
|
+
}
|
|
316
351
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
352
|
+
// Depending on if these are miniactors or standaloneActors
|
|
353
|
+
let gitRepoUrl = `${repoUrl}#${branch}`;
|
|
354
|
+
if (actorConfig.folder) {
|
|
355
|
+
gitRepoUrl = `${gitRepoUrl}:${actorConfig.folder}`;
|
|
356
|
+
}
|
|
357
|
+
return {
|
|
358
|
+
actorConfig,
|
|
359
|
+
actorInfo,
|
|
360
|
+
gitRepoUrl,
|
|
361
|
+
versionNumber: isLatest ? defaultVersionNumber : DEFAULT_TEST_VERSION_NUMBER,
|
|
362
|
+
buildTag: isLatest ? defaultBuildTag : undefined,
|
|
363
|
+
useDockerCache,
|
|
364
|
+
};
|
|
365
|
+
}),
|
|
366
|
+
);
|
|
324
367
|
|
|
325
368
|
if (dryRun) {
|
|
326
369
|
console.error('[DRY RUN] Would build:');
|
|
@@ -338,18 +381,19 @@ export const runBuilds = async ({
|
|
|
338
381
|
|
|
339
382
|
return runAndSummarizeBuilds(actorConfigs, 'BUILDS', async (actorConfig, builder) => {
|
|
340
383
|
const {
|
|
384
|
+
actorInfo,
|
|
341
385
|
buildTag,
|
|
342
386
|
versionNumber,
|
|
343
387
|
gitRepoUrl,
|
|
344
388
|
useDockerCache: useCache,
|
|
345
389
|
} = buildConfigsByActorFullName.get(actorConfig.actorFullName)!;
|
|
346
|
-
const actorVersion: ActorVersion = {
|
|
390
|
+
const actorVersion: ApifyClientTypes.ActorVersion = {
|
|
347
391
|
buildTag,
|
|
348
392
|
versionNumber,
|
|
349
393
|
gitRepoUrl,
|
|
350
394
|
sourceType: ActorSourceType.GitRepo,
|
|
351
395
|
};
|
|
352
|
-
return builder.createVersionAndBuild(versionNumber, actorVersion, useCache);
|
|
396
|
+
return builder.createVersionAndBuild(versionNumber, actorVersion, useCache, actorInfo);
|
|
353
397
|
});
|
|
354
398
|
};
|
|
355
399
|
|
package/bin/git.ts
CHANGED
|
@@ -145,3 +145,143 @@ export const parseCommit = (commitString: string): Commit => {
|
|
|
145
145
|
message,
|
|
146
146
|
};
|
|
147
147
|
};
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Returns the currently checked-out branch. Release builds point the Actor version at this branch,
|
|
151
|
+
* so a detached HEAD (no branch to point at) is an error rather than a guess.
|
|
152
|
+
*/
|
|
153
|
+
export const getCurrentBranch = (): string => {
|
|
154
|
+
const branch = spawnCommandInGhWorkspace('git rev-parse --abbrev-ref HEAD');
|
|
155
|
+
if (branch === 'HEAD') {
|
|
156
|
+
throw new Error(
|
|
157
|
+
'Cannot determine the branch to release: HEAD is detached. Check out the branch you want to release.',
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
return branch;
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Reads the repository URL from the `origin` remote, rewritten to the SSH form the Apify platform
|
|
165
|
+
* uses for Git repo sources, e.g. git@github.com:apify-store/google-maps
|
|
166
|
+
*/
|
|
167
|
+
const getOriginRepoUrl = (): string => {
|
|
168
|
+
return spawnCommandInGhWorkspace('git remote get-url origin').replace(
|
|
169
|
+
/^https:\/\/github\.com\//,
|
|
170
|
+
'git@github.com:',
|
|
171
|
+
);
|
|
172
|
+
};
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Picks the repo URL to build from. By default it's the `origin` remote, and runBuilds then checks it against
|
|
176
|
+
* each Actor's default version, so a fork or mirror remote can't repoint a published Actor. An explicit
|
|
177
|
+
* --repo-url skips that check: passing it is how you move an Actor to another repository on purpose.
|
|
178
|
+
*/
|
|
179
|
+
export const resolveRepoUrl = (explicitRepoUrl: string | undefined) => ({
|
|
180
|
+
repoUrl: explicitRepoUrl ?? getOriginRepoUrl(),
|
|
181
|
+
shouldVerifyRepoUrl: explicitRepoUrl === undefined,
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Makes repo URLs comparable regardless of their form. All of these normalize to `github.com/org/repo`:
|
|
186
|
+
* - git@github.com:org/repo.git
|
|
187
|
+
* - https://github.com/org/repo
|
|
188
|
+
* - ssh://git@github.com/org/repo#master:actors/foo (Actor versions carry a `#branch:folder` fragment)
|
|
189
|
+
*/
|
|
190
|
+
export const normalizeRepoUrl = (repoUrl: string): string => {
|
|
191
|
+
return repoUrl
|
|
192
|
+
.trim()
|
|
193
|
+
.split('#')[0]
|
|
194
|
+
.replace(/^[a-z+]+:\/\//i, '')
|
|
195
|
+
.replace(/^[^@/]+@/, '')
|
|
196
|
+
.replace(':', '/')
|
|
197
|
+
.replace(/\/+$/, '')
|
|
198
|
+
.replace(/\.git$/, '')
|
|
199
|
+
.toLowerCase();
|
|
200
|
+
};
|
|
201
|
+
|
|
202
|
+
/** Repository name from its URL, e.g. `google-maps` for git@github.com:apify-store/google-maps.git */
|
|
203
|
+
export const getRepoName = (repoUrl: string): string => {
|
|
204
|
+
return normalizeRepoUrl(repoUrl).split('/').pop()!;
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
// Sent by e.g. GitHub as the "before" commit of a push that created the branch
|
|
208
|
+
const ZERO_SHA_REGEX = /^0{40}$/;
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Validates the base commit of a release: the last commit whose changes are already released.
|
|
212
|
+
* Unlike the PR path (getCommits), there is no lenient fallback. Falling back to "everything"
|
|
213
|
+
* would rebuild the latest build of every Actor and post it to Slack, so every problem is an error.
|
|
214
|
+
*/
|
|
215
|
+
export const resolveReleaseBaseCommit = (baseCommit: string): string => {
|
|
216
|
+
const sha = parseBaseCommit(baseCommit);
|
|
217
|
+
if (!sha) {
|
|
218
|
+
throw new Error('--base-commit is required for release. See the README section "Releasing Actors".');
|
|
219
|
+
}
|
|
220
|
+
if (ZERO_SHA_REGEX.test(sha)) {
|
|
221
|
+
throw new Error(
|
|
222
|
+
`Base commit is ${sha}, which means the branch was just created and there is no previous release to diff against. ` +
|
|
223
|
+
`Rerun with --base-commit set to the last commit before the changes you want to release.`,
|
|
224
|
+
);
|
|
225
|
+
}
|
|
226
|
+
// --quiet makes rev-parse print nothing (instead of an error) when the commit is missing
|
|
227
|
+
if (!spawnCommandInGhWorkspace(`git rev-parse --verify --quiet "${sha}^{commit}"`)) {
|
|
228
|
+
throw new Error(
|
|
229
|
+
`Base commit ${sha} is not in the local git history. Either the checkout is shallow ` +
|
|
230
|
+
`(fetch the full history, e.g. fetch-depth: 0 in actions/checkout) or the branch was force-pushed.`,
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
// git merge-base A B outputs the common ancestor. If that equals A, then A is an ancestor of B.
|
|
234
|
+
if (spawnCommandInGhWorkspace(`git merge-base ${sha} HEAD`) !== sha) {
|
|
235
|
+
throw new Error(
|
|
236
|
+
`Base commit ${sha} is not an ancestor of HEAD, most likely because the branch was force-pushed. ` +
|
|
237
|
+
`The changed files cannot be determined reliably. Rerun with --base-commit set to an ancestor of HEAD ` +
|
|
238
|
+
`from before the changes you want to release.`,
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
return sha;
|
|
242
|
+
};
|
|
243
|
+
|
|
244
|
+
const CHANGELOG_PATH = 'CHANGELOG.md';
|
|
245
|
+
|
|
246
|
+
/** Returns the lines added to the root CHANGELOG.md between baseSha and HEAD, or null if it didn't change. */
|
|
247
|
+
const getChangelogAdditions = (baseSha: string, changedFiles: string[]): string | null => {
|
|
248
|
+
if (!changedFiles.includes(CHANGELOG_PATH)) {
|
|
249
|
+
return null;
|
|
250
|
+
}
|
|
251
|
+
const diff = spawnCommandInGhWorkspace('git', ['diff', baseSha, 'HEAD', '--', CHANGELOG_PATH]);
|
|
252
|
+
|
|
253
|
+
const added: string[] = [];
|
|
254
|
+
let startedChangelog = false;
|
|
255
|
+
for (const line of diff.split('\n')) {
|
|
256
|
+
// The diff is already limited to the changelog but better to double check
|
|
257
|
+
if (line.startsWith('+++') && line.toLowerCase().includes(CHANGELOG_PATH.toLowerCase())) {
|
|
258
|
+
startedChangelog = true;
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
if (startedChangelog) {
|
|
262
|
+
if (line.startsWith('diff')) {
|
|
263
|
+
break;
|
|
264
|
+
}
|
|
265
|
+
if (line.startsWith('+')) {
|
|
266
|
+
added.push(line.slice(1).trim());
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return added.join('\n').trim();
|
|
271
|
+
};
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Everything that changed since the last release (baseSha, exclusive) up to HEAD, or null when there
|
|
275
|
+
* are no new commits. Changed files come from diffing the range directly instead of from the commit
|
|
276
|
+
* list: with a merge commit, the oldest commit of the merged branch can be older than baseSha, and
|
|
277
|
+
* diffing from its parent would pull in already-released changes.
|
|
278
|
+
*/
|
|
279
|
+
export const getReleaseChanges = (baseSha: string) => {
|
|
280
|
+
if (spawnCommandInGhWorkspace('git rev-parse HEAD') === baseSha) {
|
|
281
|
+
return null;
|
|
282
|
+
}
|
|
283
|
+
const commits = fetchAllBranchCommits('HEAD', baseSha);
|
|
284
|
+
const changedFiles = spawnCommandInGhWorkspace(`git diff --name-only ${baseSha} HEAD`).split('\n').filter(Boolean);
|
|
285
|
+
const changelog = getChangelogAdditions(baseSha, changedFiles);
|
|
286
|
+
return { commits, changedFiles, changelog };
|
|
287
|
+
};
|