apify-test-tools 0.8.6 → 0.9.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -13
- package/bin/build-from-local.ts +35 -34
- package/bin/build.ts +55 -59
- package/bin/diff-changes.ts +166 -125
- package/bin/dockerignore.ts +50 -0
- package/bin/main.ts +7 -7
- package/bin/path-utils.ts +13 -0
- package/bin/test-report.ts +5 -5
- package/bin/types.ts +17 -5
- package/bin/utils.ts +124 -95
- package/dist/bin/build-from-local.d.ts +2 -2
- package/dist/bin/build-from-local.d.ts.map +1 -1
- package/dist/bin/build-from-local.js +33 -28
- package/dist/bin/build-from-local.js.map +1 -1
- package/dist/bin/build.d.ts +5 -8
- package/dist/bin/build.d.ts.map +1 -1
- package/dist/bin/build.js +49 -56
- package/dist/bin/build.js.map +1 -1
- package/dist/bin/diff-changes.d.ts +0 -6
- package/dist/bin/diff-changes.d.ts.map +1 -1
- package/dist/bin/diff-changes.js +129 -89
- package/dist/bin/diff-changes.js.map +1 -1
- package/dist/bin/dockerignore.d.ts +17 -0
- package/dist/bin/dockerignore.d.ts.map +1 -0
- package/dist/bin/dockerignore.js +41 -0
- package/dist/bin/dockerignore.js.map +1 -0
- package/dist/bin/main.js +7 -7
- package/dist/bin/main.js.map +1 -1
- package/dist/bin/path-utils.d.ts +4 -0
- package/dist/bin/path-utils.d.ts.map +1 -0
- package/dist/bin/path-utils.js +9 -0
- package/dist/bin/path-utils.js.map +1 -0
- package/dist/bin/test-report.js +3 -3
- package/dist/bin/test-report.js.map +1 -1
- package/dist/bin/types.d.ts +15 -4
- package/dist/bin/types.d.ts.map +1 -1
- package/dist/bin/utils.d.ts +2 -19
- package/dist/bin/utils.d.ts.map +1 -1
- package/dist/bin/utils.js +82 -81
- package/dist/bin/utils.js.map +1 -1
- package/dist/lib/lib.d.ts +10 -4
- package/dist/lib/lib.d.ts.map +1 -1
- package/dist/lib/lib.js +28 -22
- package/dist/lib/lib.js.map +1 -1
- package/dist/lib/types.d.ts +2 -2
- package/dist/lib/types.d.ts.map +1 -1
- package/dist/lib/utils.d.ts +1 -1
- package/dist/lib/utils.d.ts.map +1 -1
- package/dist/lib/utils.js +4 -4
- package/dist/lib/utils.js.map +1 -1
- package/dist/test/unit/bin/build-from-local.test.js +55 -86
- package/dist/test/unit/bin/build-from-local.test.js.map +1 -1
- package/dist/test/unit/bin/diff-changes.test.js +358 -36
- package/dist/test/unit/bin/diff-changes.test.js.map +1 -1
- package/dist/test/unit/bin/dockerignore.test.d.ts +2 -0
- package/dist/test/unit/bin/dockerignore.test.d.ts.map +1 -0
- package/dist/test/unit/bin/dockerignore.test.js +102 -0
- package/dist/test/unit/bin/dockerignore.test.js.map +1 -0
- package/dist/test/unit/bin/path-utils.test.d.ts +2 -0
- package/dist/test/unit/bin/path-utils.test.d.ts.map +1 -0
- package/dist/test/unit/bin/path-utils.test.js +14 -0
- package/dist/test/unit/bin/path-utils.test.js.map +1 -0
- package/dist/test/unit/bin/utils.test.d.ts +2 -0
- package/dist/test/unit/bin/utils.test.d.ts.map +1 -0
- package/dist/test/unit/bin/utils.test.js +292 -0
- package/dist/test/unit/bin/utils.test.js.map +1 -0
- package/dist/test/unit/should-built-and-test.test.js +83 -32
- package/dist/test/unit/should-built-and-test.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/lib/lib.ts +28 -22
- package/lib/types.ts +2 -2
- package/lib/utils.ts +4 -4
- package/package.json +62 -61
- package/test/unit/bin/build-from-local.test.ts +62 -121
- package/test/unit/bin/diff-changes.test.ts +397 -41
- package/test/unit/bin/dockerignore.test.ts +123 -0
- package/test/unit/bin/path-utils.test.ts +17 -0
- package/test/unit/bin/utils.test.ts +342 -0
- package/test/unit/should-built-and-test.test.ts +89 -32
package/README.md
CHANGED
|
@@ -4,29 +4,99 @@
|
|
|
4
4
|
|
|
5
5
|
## Getting Started
|
|
6
6
|
|
|
7
|
-
1. Install the package
|
|
8
|
-
- because it uses [annotate](https://vitest.dev/guide/test-context.html#annotate), `vitest` version to be at least `3.2.0`
|
|
9
|
-
- make sure that `target` and `module` in your `tsconfig.json`'s `compilerOptions` are set to `ES2022`
|
|
10
|
-
2. create test directories: `mkdir -p test/platform/core`
|
|
11
|
-
- core (hourly) tests should go to `test/platform/core`
|
|
12
|
-
- daily tests should go to `test/platform`
|
|
13
|
-
3. setup github worklows TODO
|
|
7
|
+
### 1. Install the package
|
|
14
8
|
|
|
15
|
-
|
|
9
|
+
```bash
|
|
10
|
+
npm i -D apify-test-tools
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
- Requires `vitest` version `3.2.0` or later (uses [annotate](https://vitest.dev/guide/test-context.html#annotate))
|
|
14
|
+
- Make sure `target` and `module` in your `tsconfig.json`'s `compilerOptions` are set to `ES2022`
|
|
15
|
+
|
|
16
|
+
### 2. Create the config file
|
|
17
|
+
|
|
18
|
+
Every repo that uses `apify-test-tools` must have an `apify-test-tools.config.json` file at the root. This file tells the tool which actors live in the repo, how to identify them, and which token to use.
|
|
19
|
+
|
|
20
|
+
```json
|
|
21
|
+
{
|
|
22
|
+
"actors": [
|
|
23
|
+
{
|
|
24
|
+
"folder": "actors/web-scraper",
|
|
25
|
+
"actorFullName": "myteam/web-scraper",
|
|
26
|
+
"tokenEnvVar": "APIFY_TOKEN_MYTEAM"
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"folder": "actors/email-sender",
|
|
30
|
+
"actorFullName": "myteam/email-sender",
|
|
31
|
+
"tokenEnvVar": "APIFY_TOKEN_MYTEAM",
|
|
32
|
+
"overrideActorContext": ["actors/email-sender", "packages/shared"]
|
|
33
|
+
}
|
|
34
|
+
]
|
|
35
|
+
}
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Each entry has:
|
|
39
|
+
|
|
40
|
+
| Field | Required | Description |
|
|
41
|
+
| ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
42
|
+
| `folder` | yes | Relative path from repo root to the actor's own project directory — the folder that directly contains `.actor/actor.json` (i.e. `<folder>/.actor/actor.json`), the actor's README/CHANGELOG, and its source. Use `"."` for a single-actor repo where `.actor/` is at the root. |
|
|
43
|
+
| `actorFullName` | yes | Full actor identifier in `owner/name` format (e.g. `"apify/web-scraper"`). This is the source of truth for the actor name — the `name` field in `actor.json` is not used. |
|
|
44
|
+
| `tokenEnvVar` | yes | Name of the environment variable holding the Apify API token for this actor. No fallback — if the env var is not set at build time, the build fails. |
|
|
45
|
+
| `overrideActorContext` | no | Array of paths (relative to repo root) that define which files are relevant to this actor. When set, replaces the `dockerContextDir` from `actor.json` for change detection. Useful when an actor depends on shared packages outside its Docker build context. Entries must not be prefixes of one another (e.g. `["", "code"]` or `["actors", "actors/foo"]` are rejected), and the list must include a path that reaches the actor's own `folder` — otherwise the actor could never be detected as changed. |
|
|
46
|
+
|
|
47
|
+
### 3. Set up actor folders
|
|
48
|
+
|
|
49
|
+
Each actor in the config must have a `.actor/actor.json` file. The `dockerContextDir` field in `actor.json` defines the build context boundary — this is what the tool uses to determine which files can affect the actor's build.
|
|
16
50
|
|
|
17
51
|
```
|
|
18
|
-
|
|
52
|
+
my-repo
|
|
53
|
+
├── apify-test-tools.config.json
|
|
19
54
|
├── actors
|
|
20
|
-
|
|
55
|
+
│ ├── web-scraper
|
|
56
|
+
│ │ ├── .actor
|
|
57
|
+
│ │ │ └── actor.json
|
|
58
|
+
│ │ └── src/
|
|
59
|
+
│ └── email-sender
|
|
60
|
+
│ ├── .actor
|
|
61
|
+
│ │ └── actor.json
|
|
62
|
+
│ └── src/
|
|
21
63
|
└── test
|
|
22
64
|
├── unit
|
|
23
65
|
└── platform
|
|
24
|
-
├── core
|
|
66
|
+
├── core <- Core (hourly) tests
|
|
25
67
|
│ └── core.test.ts
|
|
26
|
-
├── some.test.ts
|
|
68
|
+
├── some.test.ts <- Daily tests can be anywhere inside platform/
|
|
27
69
|
└── some-other.test.ts
|
|
28
70
|
```
|
|
29
71
|
|
|
72
|
+
For a single-actor repo, set `"folder": "."` in the config and place `.actor/actor.json` at the repo root.
|
|
73
|
+
|
|
74
|
+
### Change detection
|
|
75
|
+
|
|
76
|
+
When a PR is opened or code is pushed, the tool determines which actors need to be built and tested based on the changed files. For each changed file, for each actor:
|
|
77
|
+
|
|
78
|
+
1. **Sibling exclusion** — files inside another actor's `folder` are excluded first. This prevents an actor with broad context from being triggered by changes that belong to a sibling actor.
|
|
79
|
+
2. **CHANGELOG classification** — a `CHANGELOG.md` file is always `cosmetic` (only triggers a release build, not tests), for every actor, regardless of context or folder. (See [issue #106](https://github.com/apify/apify-test-tools/issues/106).)
|
|
80
|
+
3. **Context matching** — the file must fall within one of the actor's context paths (`dockerContextDir` from `actor.json` by default, or `overrideActorContext` from config if set). Files outside every context path are skipped.
|
|
81
|
+
4. **Hardcoded ignore list, context-aware** — the file path is first "hoisted" relative to the context path it matched (e.g. a standalone actor's own `.eslintrc` is checked as just `.eslintrc`, not the full repo-root-relative path), then checked against repo-level dev file patterns (`.vscode/`, `.gitignore`, `.husky/`, `.eslintrc`, `eslint.config.mjs`, `.prettierrc`, `.editorconfig`). There's no hardcoded special-casing for legacy `code/`/`shared/` layouts — repos that need those directories treated as top-level must list them explicitly in `overrideActorContext`.
|
|
82
|
+
5. **`.dockerignore` filtering** — if a `.dockerignore` exists at the root of the actor's `dockerContextDir`, matching files are ignored. Patterns are resolved relative to `dockerContextDir`, matching Docker's own behavior.
|
|
83
|
+
6. **README classification** — a `README.md` file is `cosmetic` (only triggers a release build, not tests) if it lives inside the actor's own `folder`; otherwise it's ignored entirely, since it isn't documentation for this actor.
|
|
84
|
+
7. **Cosmetic JSON classification** — `.json` files inside the actor's own `.actor/` directory with only cosmetic schema changes (whitespace, key ordering) only trigger a release build.
|
|
85
|
+
8. **Functional** — everything else triggers both build and tests.
|
|
86
|
+
|
|
87
|
+
### 4. Create test directories
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
mkdir -p test/platform/core
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
- Core (hourly) tests go in `test/platform/core`
|
|
94
|
+
- Daily tests go anywhere in `test/platform`
|
|
95
|
+
|
|
96
|
+
### 5. Set up GitHub workflows
|
|
97
|
+
|
|
98
|
+
See the [GitHub workflows](#github-worklows) section below.
|
|
99
|
+
|
|
30
100
|
## Github worklows
|
|
31
101
|
|
|
32
102
|
There should be 4 GH workflow files in `.github/workflows`.
|
|
@@ -343,7 +413,7 @@ GITHUB_WORKSPACE=. \
|
|
|
343
413
|
Remove `--dry-run` to actually trigger builds and update the branch names/ The command outputs a JSON array of build objects to stdout:
|
|
344
414
|
|
|
345
415
|
```json
|
|
346
|
-
[{ "buildId": "...", "
|
|
416
|
+
[{ "buildId": "...", "actorRawId": "...", "buildNumber": "...", "actorFullName": "john.doe/my-actor" }]
|
|
347
417
|
```
|
|
348
418
|
|
|
349
419
|
#### Build from local source (no push needed)
|
package/bin/build-from-local.ts
CHANGED
|
@@ -5,14 +5,10 @@ import path from 'node:path';
|
|
|
5
5
|
import type { ActorVersionSourceFile } from 'apify-client';
|
|
6
6
|
|
|
7
7
|
import { ApifyBuilder, waitAndSummarizeBuilds } from './build.js';
|
|
8
|
+
import { buildDockerIgnoreMatcher } from './dockerignore.js';
|
|
9
|
+
import { isPathWithinScope } from './path-utils.js';
|
|
8
10
|
import type { ActorConfig, BuildData } from './types.js';
|
|
9
|
-
import {
|
|
10
|
-
getDockerignoredPaths,
|
|
11
|
-
getGitignoredPaths,
|
|
12
|
-
isOutsideDir,
|
|
13
|
-
listRepoFilePaths,
|
|
14
|
-
toActorVersionSourceFile,
|
|
15
|
-
} from './utils.js';
|
|
11
|
+
import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, toActorVersionSourceFile } from './utils.js';
|
|
16
12
|
|
|
17
13
|
// JUST IN CASE. File patterns that commonly hold credentials — never ship these into a build, regardless
|
|
18
14
|
// of sourceType or of whether the repo's .gitignore happens to list them. Everything else that should be
|
|
@@ -34,11 +30,13 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P
|
|
|
34
30
|
const contextAbsDir = rawContextDir ? path.resolve(absActorDir, '.actor', rawContextDir) : undefined;
|
|
35
31
|
const isMonorepoActor = !!contextAbsDir && isOutsideDir(contextAbsDir, absActorDir);
|
|
36
32
|
|
|
37
|
-
const
|
|
38
|
-
const keptFilePaths = collectNonIgnoredFiles(
|
|
33
|
+
const dockerContextDirAbs = isMonorepoActor ? contextAbsDir! : absActorDir;
|
|
34
|
+
const keptFilePaths = collectNonIgnoredFiles(dockerContextDirAbs, repoRoot);
|
|
39
35
|
|
|
40
36
|
if (!isMonorepoActor) {
|
|
41
|
-
return Promise.all(
|
|
37
|
+
return Promise.all(
|
|
38
|
+
keptFilePaths.map(async (filePath) => toActorVersionSourceFile(filePath, dockerContextDirAbs)),
|
|
39
|
+
);
|
|
42
40
|
}
|
|
43
41
|
|
|
44
42
|
const { tempDir, filePaths } = await flattenMonorepoContext(
|
|
@@ -47,7 +45,6 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P
|
|
|
47
45
|
contextAbsDir!,
|
|
48
46
|
actorJson,
|
|
49
47
|
keptFilePaths,
|
|
50
|
-
repoRoot,
|
|
51
48
|
);
|
|
52
49
|
try {
|
|
53
50
|
return await Promise.all(filePaths.map(async (filePath) => toActorVersionSourceFile(filePath, tempDir)));
|
|
@@ -61,21 +58,21 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P
|
|
|
61
58
|
// manual directory walk — nested .gitignore files, `.git/info/exclude`, and global excludes are
|
|
62
59
|
// all honored since this delegates to git itself instead of re-implementing gitignore matching,
|
|
63
60
|
// and .git/ is never walked because git never lists its own internals here. `.dockerignore` at
|
|
64
|
-
// `
|
|
65
|
-
// reach a real Docker build either. `.actor/` (the Actor specification folder) is always kept
|
|
61
|
+
// `dockerContextDir` (the Docker build context) is honored the same way, since those files would
|
|
62
|
+
// never reach a real Docker build either. `.actor/` (the Actor specification folder) is always kept
|
|
66
63
|
// regardless of .gitignore/.dockerignore, matching Apify CLI's own behavior. Files matching the
|
|
67
64
|
// hardcoded secret-pattern backstop (keys, certs, .env variants) are dropped unconditionally,
|
|
68
65
|
// .actor/ included, since those should never ship regardless of what the ignore files say.
|
|
69
|
-
export const collectNonIgnoredFiles = (
|
|
70
|
-
const relativePaths = listRepoFilePaths(repoRoot,
|
|
66
|
+
export const collectNonIgnoredFiles = (dockerContextDir: string, repoRoot: string): string[] => {
|
|
67
|
+
const relativePaths = listRepoFilePaths(repoRoot, dockerContextDir);
|
|
71
68
|
const ignoredPaths = getGitignoredPaths(relativePaths);
|
|
72
69
|
const rootRelativePaths = new Map(
|
|
73
70
|
relativePaths.map((relPath) => [
|
|
74
71
|
relPath,
|
|
75
|
-
path.relative(
|
|
72
|
+
path.relative(dockerContextDir, path.join(repoRoot, relPath)).split(path.sep).join('/'),
|
|
76
73
|
]),
|
|
77
74
|
);
|
|
78
|
-
const
|
|
75
|
+
const isDockerIgnored = buildDockerIgnoreMatcher(dockerContextDir);
|
|
79
76
|
|
|
80
77
|
return relativePaths
|
|
81
78
|
.filter((relPath) => {
|
|
@@ -83,7 +80,7 @@ export const collectNonIgnoredFiles = (rootDir: string, repoRoot: string): strin
|
|
|
83
80
|
const isUnderActorDir = relPath.split('/').includes('.actor');
|
|
84
81
|
if (isUnderActorDir) return true;
|
|
85
82
|
if (ignoredPaths.has(relPath)) return false;
|
|
86
|
-
return !
|
|
83
|
+
return !isDockerIgnored(rootRelativePaths.get(relPath)!);
|
|
87
84
|
})
|
|
88
85
|
.map((relPath) => path.join(repoRoot, relPath));
|
|
89
86
|
};
|
|
@@ -92,8 +89,8 @@ export const collectNonIgnoredFiles = (rootDir: string, repoRoot: string): strin
|
|
|
92
89
|
// collect the actor directory of a monorepo actor — the platform would reject any path
|
|
93
90
|
// escaping it. Fix: create a temporary "flattened" directory where:
|
|
94
91
|
// - the Docker context's non-ignored files (repo root) are copied to the temp dir root
|
|
95
|
-
// - the actor's .actor/ directory
|
|
96
|
-
//
|
|
92
|
+
// - the actor's own .actor/ directory (already present within those same non-ignored files)
|
|
93
|
+
// is overlaid at the temp dir root instead of its original nested position
|
|
97
94
|
// - actor.json path fields are rewritten to be relative to the new location
|
|
98
95
|
//
|
|
99
96
|
// Result: the collected root IS the Docker context, .actor/ is at that root, and
|
|
@@ -105,7 +102,6 @@ export const flattenMonorepoContext = async (
|
|
|
105
102
|
contextAbsDir: string,
|
|
106
103
|
actorJson: Record<string, unknown>,
|
|
107
104
|
keptContextFiles: string[],
|
|
108
|
-
repoRoot: string,
|
|
109
105
|
): Promise<{ tempDir: string; filePaths: string[] }> => {
|
|
110
106
|
console.error(`[${actorName}]: monorepo actor detected — flattening from Docker context`);
|
|
111
107
|
|
|
@@ -124,11 +120,13 @@ export const flattenMonorepoContext = async (
|
|
|
124
120
|
}),
|
|
125
121
|
);
|
|
126
122
|
|
|
127
|
-
// Step 2: overlay the actor's .actor/ directory at the temp dir root.
|
|
128
|
-
//
|
|
129
|
-
//
|
|
123
|
+
// Step 2: overlay the actor's own .actor/ directory at the temp dir root. Its files are already
|
|
124
|
+
// present in keptContextFiles (collectNonIgnoredFiles keeps .actor/ paths unconditionally — see
|
|
125
|
+
// step 1 above) — pick out this actor's own subset (ignoring any sibling actors' .actor/ folders
|
|
126
|
+
// that might also appear in the broader context) and hoist each to its position relative to
|
|
127
|
+
// .actor/ itself, so it lands under tempDir/.actor/ instead of its original nested location.
|
|
130
128
|
const actorMetaDir = path.join(absActorDir, '.actor');
|
|
131
|
-
const keptActorFiles =
|
|
129
|
+
const keptActorFiles = keptContextFiles.filter((absFilePath) => isPathWithinScope(absFilePath, actorMetaDir));
|
|
132
130
|
await Promise.all(
|
|
133
131
|
keptActorFiles.map(async (absFilePath) => {
|
|
134
132
|
const relPath = path.relative(actorMetaDir, absFilePath);
|
|
@@ -196,26 +194,29 @@ export const runBuildsFromLocal = async ({
|
|
|
196
194
|
}): Promise<BuildData[]> => {
|
|
197
195
|
if (dryRun) {
|
|
198
196
|
console.error('[DRY RUN] Would build from local source:');
|
|
199
|
-
for (const {
|
|
200
|
-
console.error(` ${
|
|
197
|
+
for (const { actorFullName, folder } of actorConfigs) {
|
|
198
|
+
console.error(` ${actorFullName} (${folder})`);
|
|
201
199
|
}
|
|
202
|
-
return actorConfigs.map(({
|
|
200
|
+
return actorConfigs.map(({ actorFullName }) => ({
|
|
203
201
|
buildId: 'dry-run',
|
|
204
|
-
|
|
202
|
+
actorRawId: 'dry-run',
|
|
205
203
|
buildNumber: '0.98.0',
|
|
206
|
-
|
|
204
|
+
actorFullName,
|
|
207
205
|
}));
|
|
208
206
|
}
|
|
209
207
|
|
|
210
208
|
console.error('=========================================');
|
|
211
209
|
console.error('STARTED LOCAL BUILDS:');
|
|
210
|
+
const buildersByActorFullName = new Map<string, ApifyBuilder>(
|
|
211
|
+
actorConfigs.map((actorConfig) => [actorConfig.actorFullName, ApifyBuilder.fromActorConfig(actorConfig)]),
|
|
212
|
+
);
|
|
212
213
|
const startedBuilds = await Promise.all(
|
|
213
|
-
actorConfigs.map(async ({
|
|
214
|
-
const builder =
|
|
215
|
-
const sourceFiles = await collectSourceFiles(
|
|
214
|
+
actorConfigs.map(async ({ actorFullName, folder }) => {
|
|
215
|
+
const builder = buildersByActorFullName.get(actorFullName)!;
|
|
216
|
+
const sourceFiles = await collectSourceFiles(actorFullName, folder);
|
|
216
217
|
return builder.startActorBuildFromSourceFiles(sourceFiles);
|
|
217
218
|
}),
|
|
218
219
|
);
|
|
219
220
|
|
|
220
|
-
return waitAndSummarizeBuilds(startedBuilds, 'LOCAL BUILDS');
|
|
221
|
+
return waitAndSummarizeBuilds(startedBuilds, buildersByActorFullName, 'LOCAL BUILDS');
|
|
221
222
|
};
|
package/bin/build.ts
CHANGED
|
@@ -9,13 +9,13 @@ type BuildPrActorOptions = {
|
|
|
9
9
|
buildTag?: string;
|
|
10
10
|
versionNumber: string;
|
|
11
11
|
gitRepoUrl: string;
|
|
12
|
-
|
|
12
|
+
actorConfig: ActorConfig;
|
|
13
13
|
useDockerCache: boolean;
|
|
14
14
|
};
|
|
15
15
|
export class ApifyBuilder {
|
|
16
16
|
private constructor(
|
|
17
17
|
private readonly apifyClient: ApifyClient,
|
|
18
|
-
private readonly
|
|
18
|
+
private readonly actorFullName: string,
|
|
19
19
|
) {}
|
|
20
20
|
|
|
21
21
|
// Usually 'latest' but not necessarily (can be e.g. 'version-0')
|
|
@@ -24,35 +24,35 @@ export class ApifyBuilder {
|
|
|
24
24
|
defaultVersionNumber: string;
|
|
25
25
|
defaultBuildTag: string;
|
|
26
26
|
}> => {
|
|
27
|
-
const actorClient = this.apifyClient.actor(this.
|
|
27
|
+
const actorClient = this.apifyClient.actor(this.actorFullName);
|
|
28
28
|
const actorInfo = await actorClient.get();
|
|
29
29
|
|
|
30
30
|
if (!actorInfo) {
|
|
31
31
|
throw new Error(
|
|
32
|
-
`[${this.
|
|
32
|
+
`[${this.actorFullName}] not found. It is not published or we are missing token to access it privately or its name is misspelled`,
|
|
33
33
|
);
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
const defaultBuildTag = actorInfo.defaultRunOptions.build;
|
|
37
|
-
console.error(`Default build tag for ${this.
|
|
37
|
+
console.error(`Default build tag for ${this.actorFullName} is ${defaultBuildTag}`);
|
|
38
38
|
|
|
39
39
|
// We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
|
|
40
40
|
if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
|
|
41
41
|
throw new Error(
|
|
42
|
-
`[${this.
|
|
42
|
+
`[${this.actorFullName}] Default build is a build number, not a tag. While this could work, ` +
|
|
43
43
|
`we want to have a default as tag so this is often an accidental misconfiguration from the dev`,
|
|
44
44
|
);
|
|
45
45
|
}
|
|
46
46
|
// I reported that buildNumber should probably not be optional
|
|
47
47
|
if (!actorInfo.taggedBuilds?.[defaultBuildTag]?.buildNumber) {
|
|
48
48
|
throw new Error(
|
|
49
|
-
`[${this.
|
|
49
|
+
`[${this.actorFullName}] No build found for tag "${defaultBuildTag}". ` +
|
|
50
50
|
`The first build must be triggered manually on the platform before CI can take over.`,
|
|
51
51
|
);
|
|
52
52
|
}
|
|
53
53
|
const defaultBuildNumber = actorInfo.taggedBuilds![defaultBuildTag].buildNumber!;
|
|
54
54
|
const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1];
|
|
55
|
-
console.error(`Default version for ${this.
|
|
55
|
+
console.error(`Default version for ${this.actorFullName} is ${defaultVersionNumber}`);
|
|
56
56
|
|
|
57
57
|
return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag };
|
|
58
58
|
};
|
|
@@ -63,11 +63,11 @@ export class ApifyBuilder {
|
|
|
63
63
|
gitRepoUrl,
|
|
64
64
|
useDockerCache,
|
|
65
65
|
}: BuildPrActorOptions): Promise<BuildData> => {
|
|
66
|
-
const actorClient = this.apifyClient.actor(this.
|
|
66
|
+
const actorClient = this.apifyClient.actor(this.actorFullName);
|
|
67
67
|
const actorInfo = await actorClient.get();
|
|
68
68
|
if (!actorInfo) {
|
|
69
69
|
throw new Error(
|
|
70
|
-
`No actor named '${this.
|
|
70
|
+
`No actor named '${this.actorFullName}' was found on the platform. If this` +
|
|
71
71
|
' is unexpected, make sure the actor you are targeting is spelled the' +
|
|
72
72
|
' same as the folder in the repository.',
|
|
73
73
|
);
|
|
@@ -97,17 +97,17 @@ export class ApifyBuilder {
|
|
|
97
97
|
// We also get back actId so the testing actor can both match by actor ID and name
|
|
98
98
|
const { id, actId, buildNumber } = await actorClient.build(versionNumber, { useCache: useDockerCache });
|
|
99
99
|
|
|
100
|
-
console.error(`[${this.
|
|
101
|
-
return { buildId: id,
|
|
100
|
+
console.error(`[${this.actorFullName}]: ${id} (${buildNumber})`);
|
|
101
|
+
return { buildId: id, actorRawId: actId, buildNumber, actorFullName: this.actorFullName };
|
|
102
102
|
};
|
|
103
103
|
|
|
104
104
|
startActorBuildFromSourceFiles = async (sourceFiles: ActorVersionSourceFile[]): Promise<BuildData> => {
|
|
105
105
|
const ZIP_VERSION = '0.98';
|
|
106
|
-
const actorClient = this.apifyClient.actor(this.
|
|
106
|
+
const actorClient = this.apifyClient.actor(this.actorFullName);
|
|
107
107
|
const actorInfo = await actorClient.get();
|
|
108
108
|
if (!actorInfo) {
|
|
109
109
|
throw new Error(
|
|
110
|
-
`No actor named '${this.
|
|
110
|
+
`No actor named '${this.actorFullName}' was found on the platform. If this` +
|
|
111
111
|
' is unexpected, make sure the actor you are targeting is spelled the' +
|
|
112
112
|
' same as the folder in the repository.',
|
|
113
113
|
);
|
|
@@ -130,49 +130,39 @@ export class ApifyBuilder {
|
|
|
130
130
|
}
|
|
131
131
|
|
|
132
132
|
const { id, actId, buildNumber } = await actorClient.build(ZIP_VERSION, { useCache: false });
|
|
133
|
-
console.error(`[${this.
|
|
134
|
-
return { buildId: id,
|
|
133
|
+
console.error(`[${this.actorFullName}]: ${id} (${buildNumber})`);
|
|
134
|
+
return { buildId: id, actorRawId: actId, buildNumber, actorFullName: this.actorFullName };
|
|
135
135
|
};
|
|
136
136
|
|
|
137
|
-
waitForBuildToFinish = async (buildId: string
|
|
137
|
+
waitForBuildToFinish = async (buildId: string): Promise<Build> => {
|
|
138
138
|
const build = await this.apifyClient.build(buildId).waitForFinish();
|
|
139
139
|
const versionNumber = build.buildNumber;
|
|
140
140
|
if (build.status === 'FAILED' || build.status === 'TIMED-OUT') {
|
|
141
|
-
console.error(`[${this.
|
|
141
|
+
console.error(`[${this.actorFullName}]: ${versionNumber}`);
|
|
142
142
|
try {
|
|
143
143
|
const log = await this.apifyClient.build(buildId).log().get();
|
|
144
144
|
const logTail = log?.split('\n').slice(-40).join('\n');
|
|
145
145
|
console.error(`\n--- BUILD LOG (last 40 lines) ---\n${logTail}\n---`);
|
|
146
146
|
} catch (err) {
|
|
147
|
-
console.error(`[${this.
|
|
147
|
+
console.error(`[${this.actorFullName}]: Failed to fetch build log: ${err}`);
|
|
148
148
|
}
|
|
149
149
|
throw new Error(
|
|
150
|
-
`[BUILD][${
|
|
150
|
+
`[BUILD][${this.actorFullName}]: Build ${buildId} (${versionNumber}) failed. ` +
|
|
151
151
|
`Not continuing with other builds and tests.`,
|
|
152
152
|
);
|
|
153
153
|
}
|
|
154
|
-
console.error(`[${this.
|
|
154
|
+
console.error(`[${this.actorFullName}]: ${versionNumber}`);
|
|
155
155
|
return build;
|
|
156
156
|
};
|
|
157
157
|
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
static fromActorName = (actorName: string): ApifyBuilder => {
|
|
162
|
-
const username = actorName.split('/')[0];
|
|
163
|
-
// GitHib secrets only allow word characters (alphanum + underscore)
|
|
164
|
-
const usernameInGitHubSecretsFormat = username.replaceAll(/\W/g, '_').toUpperCase();
|
|
165
|
-
const usernameEnvVar = `APIFY_TOKEN_${usernameInGitHubSecretsFormat}`;
|
|
166
|
-
const token = process.env[usernameEnvVar];
|
|
158
|
+
static fromActorConfig = (actorConfig: ActorConfig): ApifyBuilder => {
|
|
159
|
+
const { actorFullName, tokenEnvVar } = actorConfig;
|
|
160
|
+
const token = process.env[tokenEnvVar];
|
|
167
161
|
if (!token) {
|
|
168
|
-
throw new Error(
|
|
169
|
-
`Cannot find Apify API token for username: ${username}. ` +
|
|
170
|
-
`Have you set secret env var to this GitHub repo with key: ${usernameEnvVar}?`,
|
|
171
|
-
);
|
|
162
|
+
throw new Error(`Env var ${tokenEnvVar} is not set (needed for actor "${actorFullName}").`);
|
|
172
163
|
}
|
|
173
164
|
const apifyClient = new ApifyClient({ token });
|
|
174
|
-
|
|
175
|
-
return builder;
|
|
165
|
+
return new ApifyBuilder(apifyClient, actorFullName);
|
|
176
166
|
};
|
|
177
167
|
|
|
178
168
|
/**
|
|
@@ -204,7 +194,7 @@ export class ApifyBuilder {
|
|
|
204
194
|
const DEFAULT_DAYS_BACK_PROD_VERSIONS = 30;
|
|
205
195
|
const DEFAULT_DAYS_BACK_DEVEL = 7;
|
|
206
196
|
|
|
207
|
-
const actorInfo = (await this.apifyClient.actor(this.
|
|
197
|
+
const actorInfo = (await this.apifyClient.actor(this.actorFullName).get())!;
|
|
208
198
|
|
|
209
199
|
// 'devel' used to be hardcoded for testing version 0.99, once we get rid of this tag everywhere, we can remove this code
|
|
210
200
|
const taggedDevelBuildNumber: string | undefined = actorInfo.taggedBuilds!.devel?.buildNumber;
|
|
@@ -216,12 +206,10 @@ export class ApifyBuilder {
|
|
|
216
206
|
tag,
|
|
217
207
|
}));
|
|
218
208
|
|
|
219
|
-
const { items } = await this.apifyClient.actor(this.
|
|
209
|
+
const { items } = await this.apifyClient.actor(this.actorFullName).builds().list();
|
|
220
210
|
|
|
221
211
|
// Deleting default build throws an error, so we skip it
|
|
222
|
-
const { defaultBuildNumber, defaultBuildTag } = await
|
|
223
|
-
this.actorName,
|
|
224
|
-
).getDefaultVersionAndTag();
|
|
212
|
+
const { defaultBuildNumber, defaultBuildTag } = await this.getDefaultVersionAndTag();
|
|
225
213
|
|
|
226
214
|
const daysAgoUnixProd = Date.now() - DEFAULT_DAYS_BACK_PROD_VERSIONS * 24 * 60 * 60 * 1000;
|
|
227
215
|
const daysAgoUnixDevel = Date.now() - DEFAULT_DAYS_BACK_DEVEL * 24 * 60 * 60 * 1000;
|
|
@@ -231,7 +219,7 @@ export class ApifyBuilder {
|
|
|
231
219
|
const buildsToDelete = (items as CorrectBuildColletionItem[]).filter((build) => {
|
|
232
220
|
if (build.buildNumber === defaultBuildNumber) {
|
|
233
221
|
console.error(
|
|
234
|
-
`[DELETE OLD BUILDS][${this.
|
|
222
|
+
`[DELETE OLD BUILDS][${this.actorFullName}]: Skipping default build ${defaultBuildNumber} (${defaultBuildTag}). ` +
|
|
235
223
|
`We never delete default builds`,
|
|
236
224
|
);
|
|
237
225
|
return false;
|
|
@@ -242,7 +230,7 @@ export class ApifyBuilder {
|
|
|
242
230
|
);
|
|
243
231
|
if (protectedTagFound) {
|
|
244
232
|
console.error(
|
|
245
|
-
`[DELETE OLD BUILDS][${this.
|
|
233
|
+
`[DELETE OLD BUILDS][${this.actorFullName}]: Skipping protected build ${protectedTagFound.buildNumber} (${protectedTagFound.tag}).`,
|
|
246
234
|
);
|
|
247
235
|
return false;
|
|
248
236
|
}
|
|
@@ -251,7 +239,7 @@ export class ApifyBuilder {
|
|
|
251
239
|
const shouldDeleteDevelBuild = build.startedAt.getTime() < daysAgoUnixDevel;
|
|
252
240
|
if (shouldDeleteDevelBuild) {
|
|
253
241
|
console.error(
|
|
254
|
-
`[DELETE OLD BUILDS][${this.
|
|
242
|
+
`[DELETE OLD BUILDS][${this.actorFullName}]: Removing olf devel build ${taggedDevelBuildNumber}.`,
|
|
255
243
|
);
|
|
256
244
|
}
|
|
257
245
|
return shouldDeleteDevelBuild;
|
|
@@ -260,7 +248,7 @@ export class ApifyBuilder {
|
|
|
260
248
|
});
|
|
261
249
|
|
|
262
250
|
console.error(
|
|
263
|
-
`[DELETE OLD BUILDS][${this.
|
|
251
|
+
`[DELETE OLD BUILDS][${this.actorFullName}]: Deleting ${buildsToDelete.length} old builds that are non-default and ` +
|
|
264
252
|
`older than 30 days from total ${items.length}`,
|
|
265
253
|
);
|
|
266
254
|
for (const build of buildsToDelete) {
|
|
@@ -269,20 +257,24 @@ export class ApifyBuilder {
|
|
|
269
257
|
}
|
|
270
258
|
}
|
|
271
259
|
|
|
272
|
-
export const waitAndSummarizeBuilds = async (
|
|
260
|
+
export const waitAndSummarizeBuilds = async (
|
|
261
|
+
startedBuilds: BuildData[],
|
|
262
|
+
buildersMap: Map<string, ApifyBuilder>,
|
|
263
|
+
label: string,
|
|
264
|
+
): Promise<BuildData[]> => {
|
|
273
265
|
console.error('=========================================');
|
|
274
266
|
console.error(`FINISHED ${label}:`);
|
|
275
267
|
await Promise.all(
|
|
276
268
|
startedBuilds.map(async (buildData) => {
|
|
277
|
-
const builder =
|
|
278
|
-
await builder.waitForBuildToFinish(buildData.buildId
|
|
269
|
+
const builder = buildersMap.get(buildData.actorFullName)!;
|
|
270
|
+
await builder.waitForBuildToFinish(buildData.buildId);
|
|
279
271
|
}),
|
|
280
272
|
);
|
|
281
273
|
|
|
282
274
|
console.error('=========================================');
|
|
283
275
|
console.error('SUMMARY:');
|
|
284
|
-
for (const buildData of startedBuilds.sort((a, b) => a.
|
|
285
|
-
console.error(`[${buildData.
|
|
276
|
+
for (const buildData of startedBuilds.sort((a, b) => a.actorFullName.localeCompare(b.actorFullName))) {
|
|
277
|
+
console.error(`[${buildData.actorFullName}]: ${buildData.buildNumber}`);
|
|
286
278
|
}
|
|
287
279
|
console.error('=========================================');
|
|
288
280
|
|
|
@@ -308,13 +300,13 @@ export const runBuilds = async ({
|
|
|
308
300
|
}: RunBuildsOptions) => {
|
|
309
301
|
const buildConfigs: BuildPrActorOptions[] = [];
|
|
310
302
|
|
|
311
|
-
for (const
|
|
303
|
+
for (const actorConfig of actorConfigs) {
|
|
312
304
|
let versionNumber: string;
|
|
313
305
|
let buildTag: string | undefined;
|
|
314
306
|
|
|
315
307
|
if (isLatest) {
|
|
316
308
|
const { defaultVersionNumber, defaultBuildTag } =
|
|
317
|
-
await ApifyBuilder.
|
|
309
|
+
await ApifyBuilder.fromActorConfig(actorConfig).getDefaultVersionAndTag();
|
|
318
310
|
versionNumber = defaultVersionNumber;
|
|
319
311
|
buildTag = defaultBuildTag;
|
|
320
312
|
} else {
|
|
@@ -323,30 +315,34 @@ export const runBuilds = async ({
|
|
|
323
315
|
|
|
324
316
|
// Depending on if these are miniactors or standaloneActors
|
|
325
317
|
let gitRepoUrl = `${repoUrl}#${branch}`;
|
|
326
|
-
if (folder) {
|
|
327
|
-
gitRepoUrl = `${gitRepoUrl}:${folder}`;
|
|
318
|
+
if (actorConfig.folder) {
|
|
319
|
+
gitRepoUrl = `${gitRepoUrl}:${actorConfig.folder}`;
|
|
328
320
|
}
|
|
329
|
-
buildConfigs.push({
|
|
321
|
+
buildConfigs.push({ actorConfig, gitRepoUrl, versionNumber, buildTag, useDockerCache });
|
|
330
322
|
}
|
|
331
323
|
|
|
332
324
|
if (dryRun) {
|
|
333
325
|
return buildConfigs;
|
|
334
326
|
}
|
|
327
|
+
|
|
328
|
+
const buildersByActorFullName = new Map<string, ApifyBuilder>(
|
|
329
|
+
actorConfigs.map((actorConfig) => [actorConfig.actorFullName, ApifyBuilder.fromActorConfig(actorConfig)]),
|
|
330
|
+
);
|
|
335
331
|
console.error('=========================================');
|
|
336
332
|
console.error('STARTED BUILDS:');
|
|
337
333
|
const startedBuilds = await Promise.all(
|
|
338
334
|
buildConfigs.map(async (buildConfig) => {
|
|
339
|
-
const builder =
|
|
335
|
+
const builder = buildersByActorFullName.get(buildConfig.actorConfig.actorFullName)!;
|
|
340
336
|
const buildData = await builder.startActorBuild(buildConfig);
|
|
341
337
|
return buildData;
|
|
342
338
|
}),
|
|
343
339
|
);
|
|
344
340
|
|
|
345
|
-
return waitAndSummarizeBuilds(startedBuilds, 'BUILDS');
|
|
341
|
+
return waitAndSummarizeBuilds(startedBuilds, buildersByActorFullName, 'BUILDS');
|
|
346
342
|
};
|
|
347
343
|
|
|
348
344
|
export const deleteOldBuilds = async (actorConfigs: ActorConfig[]) => {
|
|
349
|
-
for (const
|
|
350
|
-
await ApifyBuilder.
|
|
345
|
+
for (const actorConfig of actorConfigs) {
|
|
346
|
+
await ApifyBuilder.fromActorConfig(actorConfig).deleteOldBuilds();
|
|
351
347
|
}
|
|
352
348
|
};
|