apify-test-tools 0.8.3-beta.0 → 0.8.5-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/scripts/before-beta-release.js +3 -1
- package/.github/workflows/manual_publish_to_npm.yaml +3 -1
- package/CHANGELOG.md +12 -2
- package/README.md +21 -0
- package/bin/build-from-local.ts +204 -0
- package/bin/build.ts +67 -21
- package/bin/main.ts +26 -0
- package/bin/utils.ts +64 -0
- package/dist/bin/build-from-local.d.ts +14 -0
- package/dist/bin/build-from-local.d.ts.map +1 -0
- package/dist/bin/build-from-local.js +153 -0
- package/dist/bin/build-from-local.js.map +1 -0
- package/dist/bin/build.d.ts +25 -1
- package/dist/bin/build.d.ts.map +1 -1
- package/dist/bin/build.js +54 -17
- package/dist/bin/build.js.map +1 -1
- package/dist/bin/main.js +20 -0
- package/dist/bin/main.js.map +1 -1
- package/dist/bin/utils.d.ts +18 -0
- package/dist/bin/utils.d.ts.map +1 -1
- package/dist/bin/utils.js +51 -0
- package/dist/bin/utils.js.map +1 -1
- package/dist/test/unit/bin/build-from-local.test.d.ts +2 -0
- package/dist/test/unit/bin/build-from-local.test.d.ts.map +1 -0
- package/dist/test/unit/bin/build-from-local.test.js +152 -0
- package/dist/test/unit/bin/build-from-local.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/test/unit/bin/build-from-local.test.ts +233 -0
|
@@ -19,7 +19,9 @@ writeFileSync(PKG_JSON_PATH, `${JSON.stringify(pkgJson, null, 2)}\n`);
|
|
|
19
19
|
|
|
20
20
|
function addBetaSuffixToVersion(version) {
|
|
21
21
|
const versionString = execSync(`npm show ${PACKAGE_NAME} versions --json`, { encoding: 'utf8' });
|
|
22
|
-
|
|
22
|
+
// Normalize npm's output: npm 12 wraps the versions in an extra array ([[...]]),
|
|
23
|
+
// and npm returns a bare string for packages with a single published version.
|
|
24
|
+
const versions = [JSON.parse(versionString)].flat(Infinity).filter((v) => typeof v === 'string');
|
|
23
25
|
|
|
24
26
|
if (versions.some((v) => v === version)) {
|
|
25
27
|
console.error(
|
|
@@ -37,7 +37,9 @@ jobs:
|
|
|
37
37
|
cache: 'npm'
|
|
38
38
|
cache-dependency-path: 'package-lock.json'
|
|
39
39
|
- name: Update npm
|
|
40
|
-
|
|
40
|
+
# Pin to npm 11: npm 12.0.0 ships a broken `libnpmpublish` that requires the
|
|
41
|
+
# top-level `sigstore` module without bundling it, breaking `npm publish --provenance`.
|
|
42
|
+
run: npm install -g npm@11
|
|
41
43
|
- name: Install dependencies
|
|
42
44
|
run: npm ci
|
|
43
45
|
- name: Bump pre-release version
|
package/CHANGELOG.md
CHANGED
|
@@ -3,14 +3,24 @@
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
5
|
<!-- git-cliff-unreleased-start -->
|
|
6
|
-
## 0.8.
|
|
6
|
+
## 0.8.5 - **not yet released**
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
<!-- git-cliff-unreleased-end -->
|
|
10
|
+
## [0.8.4](https://github.com/apify/apify-test-tools/releases/tag/v0.8.4) (2026-07-10)
|
|
11
|
+
|
|
12
|
+
### 🚀 Features
|
|
13
|
+
|
|
14
|
+
- Build from local source, pushed as zip ([#100](https://github.com/apify/apify-test-tools/pull/100)) ([6058a39](https://github.com/apify/apify-test-tools/commit/6058a395faf006ddb6ee155f850e3b8fa9e69c94)) by [@gytelio](https://github.com/gytelio), closes [#93](https://github.com/apify/apify-test-tools/issues/93), [#94](https://github.com/apify/apify-test-tools/issues/94)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
## [0.8.3](https://github.com/apify/apify-test-tools/releases/tag/v0.8.3) (2026-07-01)
|
|
7
18
|
|
|
8
19
|
### 🐛 Bug Fixes
|
|
9
20
|
|
|
10
21
|
- Add error explaining to build manually ([#99](https://github.com/apify/apify-test-tools/pull/99)) ([20eb7f5](https://github.com/apify/apify-test-tools/commit/20eb7f5e0493ea77e8d93f94c350d425a43cd570)) by [@gytelio](https://github.com/gytelio), closes [#98](https://github.com/apify/apify-test-tools/issues/98)
|
|
11
22
|
|
|
12
23
|
|
|
13
|
-
<!-- git-cliff-unreleased-end -->
|
|
14
24
|
## [0.8.2](https://github.com/apify/apify-test-tools/releases/tag/v0.8.2) (2026-06-25)
|
|
15
25
|
|
|
16
26
|
### Change
|
package/README.md
CHANGED
|
@@ -346,6 +346,27 @@ Remove `--dry-run` to actually trigger builds and update the branch names/ The c
|
|
|
346
346
|
[{ "buildId": "...", "actorId": "...", "buildNumber": "...", "actorName": "john.doe/my-actor" }]
|
|
347
347
|
```
|
|
348
348
|
|
|
349
|
+
#### Build from local source (no push needed)
|
|
350
|
+
|
|
351
|
+
If you don't want to push a dummy branch just to test a change and wait for all the tests to finish, `build-from-local` builds Actors directly from your local files (zipped and uploaded as `SOURCE_FILES`), skipping steps 1-4 above.
|
|
352
|
+
|
|
353
|
+
```bash
|
|
354
|
+
APIFY_TOKEN_JOHN_DOE=<token> \
|
|
355
|
+
GITHUB_WORKSPACE=. \
|
|
356
|
+
npx apify-test-tools build-from-local --actors john.doe/my-actor
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
Pass a hardcoded actor name via `--actors` to build only that Actor (comma-separate multiple names). Omit `--actors` to build all Actors in the repo, or add `--dry-run` to preview without building. It outputs the same JSON build array as `build`, so you run tests against it the same way as in step 5 below:
|
|
360
|
+
|
|
361
|
+
```bash
|
|
362
|
+
# Build from local source and capture output
|
|
363
|
+
BUILDS=$(APIFY_TOKEN_JOHN_DOE=apify_api_xxx \
|
|
364
|
+
GITHUB_WORKSPACE=. \
|
|
365
|
+
npx apify-test-tools build-from-local --actors apify/my-actor)
|
|
366
|
+
```
|
|
367
|
+
|
|
368
|
+
Since you already scoped the build to just the Actor(s) you care about, point vitest at a specific test file (or a `-t` name filter) instead of the whole `test/platform` directory — you get feedback on that one test without waiting for the full suite to run.
|
|
369
|
+
|
|
349
370
|
#### 5. Run tests against the builds
|
|
350
371
|
|
|
351
372
|
Pass the build output as `ACTOR_BUILDS` and provide `TESTER_APIFY_TOKEN`. The token can point to your own account (if you have enough memory) or you can use the testing account (xRGg9iAfJSymqartk).
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
|
|
5
|
+
import type { ActorVersionSourceFile } from 'apify-client';
|
|
6
|
+
|
|
7
|
+
import { ApifyBuilder, waitAndSummarizeBuilds } from './build.js';
|
|
8
|
+
import type { ActorConfig, BuildData } from './types.js';
|
|
9
|
+
import { getGitignoredPaths, isOutsideDir, listRepoFilePaths, toActorVersionSourceFile } from './utils.js';
|
|
10
|
+
|
|
11
|
+
// JUST IN CASE. File patterns that commonly hold credentials — never ship these into a build, regardless
|
|
12
|
+
// of sourceType or of whether the repo's .gitignore happens to list them. Everything else that should be
|
|
13
|
+
// excluded (build output, local overrides, project-specific secret files, ...) is expected to already be
|
|
14
|
+
// in the repo's .gitignore — see collectNonIgnoredFiles.
|
|
15
|
+
const SKIP_FILE_PATTERNS = [/^\.env(\..+)?$/, /\.pem$/, /\.key$/, /\.pfx$/, /\.p12$/];
|
|
16
|
+
const isSecretFile = (fileName: string): boolean => SKIP_FILE_PATTERNS.some((pattern) => pattern.test(fileName));
|
|
17
|
+
|
|
18
|
+
export const collectSourceFiles = async (actorName: string, actorDir: string): Promise<ActorVersionSourceFile[]> => {
|
|
19
|
+
const repoRoot = process.cwd();
|
|
20
|
+
const absActorDir = path.resolve(actorDir);
|
|
21
|
+
|
|
22
|
+
// Read actor.json to check if this is a monorepo actor with an external dockerContextDir.
|
|
23
|
+
// Monorepo actors point their dockerContextDir to a parent directory (e.g. "../../.."),
|
|
24
|
+
// which means the Docker build context is the repo root, not the actor directory itself.
|
|
25
|
+
const actorJsonPath = path.join(absActorDir, '.actor', 'actor.json');
|
|
26
|
+
const actorJson = JSON.parse(await fs.readFile(actorJsonPath, 'utf8')) as Record<string, unknown>;
|
|
27
|
+
const rawContextDir = actorJson.dockerContextDir as string | undefined;
|
|
28
|
+
const contextAbsDir = rawContextDir ? path.resolve(absActorDir, '.actor', rawContextDir) : undefined;
|
|
29
|
+
const isMonorepoActor = !!contextAbsDir && isOutsideDir(contextAbsDir, absActorDir);
|
|
30
|
+
|
|
31
|
+
const collectRootDir = isMonorepoActor ? contextAbsDir! : absActorDir;
|
|
32
|
+
const keptFilePaths = collectNonIgnoredFiles(collectRootDir, repoRoot);
|
|
33
|
+
|
|
34
|
+
if (!isMonorepoActor) {
|
|
35
|
+
return Promise.all(keptFilePaths.map(async (filePath) => toActorVersionSourceFile(filePath, collectRootDir)));
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const { tempDir, filePaths } = await flattenMonorepoContext(
|
|
39
|
+
actorName,
|
|
40
|
+
absActorDir,
|
|
41
|
+
contextAbsDir!,
|
|
42
|
+
actorJson,
|
|
43
|
+
keptFilePaths,
|
|
44
|
+
repoRoot,
|
|
45
|
+
);
|
|
46
|
+
try {
|
|
47
|
+
return await Promise.all(filePaths.map(async (filePath) => toActorVersionSourceFile(filePath, tempDir)));
|
|
48
|
+
} finally {
|
|
49
|
+
// Only the flattened copy is temporary — never delete the actor's own directory.
|
|
50
|
+
await fs.rm(tempDir, { recursive: true, force: true });
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
// Candidates come from `git ls-files` (tracked + untracked, gitignored included) rather than a
|
|
55
|
+
// manual directory walk — nested .gitignore files, `.git/info/exclude`, and global excludes are
|
|
56
|
+
// all honored since this delegates to git itself instead of re-implementing gitignore matching,
|
|
57
|
+
// and .git/ is never walked because git never lists its own internals here. `.actor/` (the Actor
|
|
58
|
+
// specification folder) is always kept regardless of .gitignore, matching Apify CLI's own behavior.
|
|
59
|
+
// Files matching the hardcoded secret-pattern backstop (keys, certs, .env variants) are dropped
|
|
60
|
+
// unconditionally, .actor/ included, since those should never ship regardless of what .gitignore says.
|
|
61
|
+
export const collectNonIgnoredFiles = (rootDir: string, repoRoot: string): string[] => {
|
|
62
|
+
const relativePaths = listRepoFilePaths(repoRoot, rootDir);
|
|
63
|
+
const ignoredPaths = getGitignoredPaths(relativePaths);
|
|
64
|
+
|
|
65
|
+
return relativePaths
|
|
66
|
+
.filter((relPath) => {
|
|
67
|
+
if (isSecretFile(path.basename(relPath))) return false;
|
|
68
|
+
const isUnderActorDir = relPath.split('/').includes('.actor');
|
|
69
|
+
return isUnderActorDir || !ignoredPaths.has(relPath);
|
|
70
|
+
})
|
|
71
|
+
.map((relPath) => path.join(repoRoot, relPath));
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
// SOURCE_FILES always treats the collected root as the actor root, so we cannot simply
|
|
75
|
+
// collect the actor directory of a monorepo actor — the platform would reject any path
|
|
76
|
+
// escaping it. Fix: create a temporary "flattened" directory where:
|
|
77
|
+
// - the Docker context's non-ignored files (repo root) are copied to the temp dir root
|
|
78
|
+
// - the actor's .actor/ directory is overlaid at the temp dir root (through the same
|
|
79
|
+
// gitignore/secret-pattern filter as the rest of the context — see collectNonIgnoredFiles)
|
|
80
|
+
// - actor.json path fields are rewritten to be relative to the new location
|
|
81
|
+
//
|
|
82
|
+
// Result: the collected root IS the Docker context, .actor/ is at that root, and
|
|
83
|
+
// all relative paths (dockerfile, dockerContextDir, changelog) are exactly one
|
|
84
|
+
// level up ("..") instead of three ("../../..").
|
|
85
|
+
export const flattenMonorepoContext = async (
|
|
86
|
+
actorName: string,
|
|
87
|
+
absActorDir: string,
|
|
88
|
+
contextAbsDir: string,
|
|
89
|
+
actorJson: Record<string, unknown>,
|
|
90
|
+
keptContextFiles: string[],
|
|
91
|
+
repoRoot: string,
|
|
92
|
+
): Promise<{ tempDir: string; filePaths: string[] }> => {
|
|
93
|
+
console.error(`[${actorName}]: monorepo actor detected — flattening from Docker context`);
|
|
94
|
+
|
|
95
|
+
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), `apify-build-${actorName.replace('/', '_')}-`));
|
|
96
|
+
const filePaths: string[] = [];
|
|
97
|
+
|
|
98
|
+
// Step 1: copy only the files that survived gitignore/secret filtering, preserving their
|
|
99
|
+
// position relative to the Docker context root.
|
|
100
|
+
await Promise.all(
|
|
101
|
+
keptContextFiles.map(async (absFilePath) => {
|
|
102
|
+
const relPath = path.relative(contextAbsDir, absFilePath);
|
|
103
|
+
const destPath = path.join(tempDir, relPath);
|
|
104
|
+
await fs.mkdir(path.dirname(destPath), { recursive: true });
|
|
105
|
+
await fs.copyFile(absFilePath, destPath);
|
|
106
|
+
filePaths.push(destPath);
|
|
107
|
+
}),
|
|
108
|
+
);
|
|
109
|
+
|
|
110
|
+
// Step 2: overlay the actor's .actor/ directory at the temp dir root. collectNonIgnoredFiles
|
|
111
|
+
// always keeps .actor/ paths regardless of .gitignore, but still drops the hardcoded secret
|
|
112
|
+
// patterns — so this isn't a raw copy, a stray secret file living inside .actor/ is still dropped.
|
|
113
|
+
const actorMetaDir = path.join(absActorDir, '.actor');
|
|
114
|
+
const keptActorFiles = collectNonIgnoredFiles(actorMetaDir, repoRoot);
|
|
115
|
+
await Promise.all(
|
|
116
|
+
keptActorFiles.map(async (absFilePath) => {
|
|
117
|
+
const relPath = path.relative(actorMetaDir, absFilePath);
|
|
118
|
+
const destPath = path.join(tempDir, '.actor', relPath);
|
|
119
|
+
await fs.mkdir(path.dirname(destPath), { recursive: true });
|
|
120
|
+
await fs.copyFile(absFilePath, destPath);
|
|
121
|
+
filePaths.push(destPath);
|
|
122
|
+
}),
|
|
123
|
+
);
|
|
124
|
+
|
|
125
|
+
// Step 3: rewrite actor.json path fields so they resolve correctly from the new location.
|
|
126
|
+
// This overwrites the actor.json already copied in step 2 in place, so its path is already
|
|
127
|
+
// accounted for in filePaths — no need to add it again.
|
|
128
|
+
await rewriteActorJsonPaths(absActorDir, contextAbsDir, tempDir, actorJson);
|
|
129
|
+
|
|
130
|
+
return { tempDir, filePaths };
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
// Rewrites actor.json path fields so they resolve correctly from the new .actor/ location
|
|
134
|
+
// (one level below the root) instead of the original three-levels-deep location.
|
|
135
|
+
//
|
|
136
|
+
// Algorithm for each path field:
|
|
137
|
+
// 1. Resolve the original value to an absolute path on disk.
|
|
138
|
+
// 2. Compute its position relative to the Docker context root (e.g. repo root).
|
|
139
|
+
// That relative position is exactly where the file landed inside tempDir,
|
|
140
|
+
// because we copied contextAbsDir → tempDir in flattenMonorepoContext's step 1.
|
|
141
|
+
// 3. Build the new path from newActorDir to that file in tempDir.
|
|
142
|
+
//
|
|
143
|
+
// Local paths (e.g. "./dataset_schema.json") point inside .actor/ and are left
|
|
144
|
+
// unchanged — .actor/ was copied intact so those paths still resolve correctly.
|
|
145
|
+
export const rewriteActorJsonPaths = async (
|
|
146
|
+
absActorDir: string,
|
|
147
|
+
contextAbsDir: string,
|
|
148
|
+
tempDir: string,
|
|
149
|
+
actorJson: Record<string, unknown>,
|
|
150
|
+
): Promise<void> => {
|
|
151
|
+
const originalActorDir = path.join(absActorDir, '.actor');
|
|
152
|
+
const newActorDir = path.join(tempDir, '.actor');
|
|
153
|
+
const pathFields = ['dockerfile', 'dockerContextDir', 'changelog', 'readme'] as const;
|
|
154
|
+
const rewritten = { ...actorJson };
|
|
155
|
+
for (const field of pathFields) {
|
|
156
|
+
const value = rewritten[field];
|
|
157
|
+
if (typeof value !== 'string') continue;
|
|
158
|
+
|
|
159
|
+
const absPath = path.resolve(originalActorDir, value);
|
|
160
|
+
|
|
161
|
+
// Skip paths that stay inside .actor/ — they don't need rewriting.
|
|
162
|
+
if (!isOutsideDir(absPath, originalActorDir)) continue;
|
|
163
|
+
|
|
164
|
+
// Where does this file live inside the Docker context? That's also where
|
|
165
|
+
// it lives inside tempDir after the copy in flattenMonorepoContext's step 1.
|
|
166
|
+
const relativeToContext = path.relative(contextAbsDir, absPath);
|
|
167
|
+
const newAbsPath = path.join(tempDir, relativeToContext);
|
|
168
|
+
rewritten[field] = path.relative(newActorDir, newAbsPath);
|
|
169
|
+
}
|
|
170
|
+
await fs.writeFile(path.join(newActorDir, 'actor.json'), JSON.stringify(rewritten, null, 4));
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
export const runBuildsFromLocal = async ({
|
|
174
|
+
actorConfigs,
|
|
175
|
+
dryRun,
|
|
176
|
+
}: {
|
|
177
|
+
actorConfigs: ActorConfig[];
|
|
178
|
+
dryRun: boolean;
|
|
179
|
+
}): Promise<BuildData[]> => {
|
|
180
|
+
if (dryRun) {
|
|
181
|
+
console.error('[DRY RUN] Would build from local source:');
|
|
182
|
+
for (const { actorName, folder } of actorConfigs) {
|
|
183
|
+
console.error(` ${actorName} (${folder})`);
|
|
184
|
+
}
|
|
185
|
+
return actorConfigs.map(({ actorName }) => ({
|
|
186
|
+
buildId: 'dry-run',
|
|
187
|
+
actorId: 'dry-run',
|
|
188
|
+
buildNumber: '0.98.0',
|
|
189
|
+
actorName,
|
|
190
|
+
}));
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
console.error('=========================================');
|
|
194
|
+
console.error('STARTED LOCAL BUILDS:');
|
|
195
|
+
const startedBuilds = await Promise.all(
|
|
196
|
+
actorConfigs.map(async ({ actorName, folder }) => {
|
|
197
|
+
const builder = ApifyBuilder.fromActorName(actorName);
|
|
198
|
+
const sourceFiles = await collectSourceFiles(actorName, folder);
|
|
199
|
+
return builder.startActorBuildFromSourceFiles(sourceFiles);
|
|
200
|
+
}),
|
|
201
|
+
);
|
|
202
|
+
|
|
203
|
+
return waitAndSummarizeBuilds(startedBuilds, 'LOCAL BUILDS');
|
|
204
|
+
};
|
package/bin/build.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Build } from 'apify-client';
|
|
1
|
+
import type { ActorVersionSourceFile, Build } from 'apify-client';
|
|
2
2
|
import { ApifyClient } from 'apify-client';
|
|
3
3
|
|
|
4
4
|
import { ACTOR_SOURCE_TYPES } from '@apify/consts';
|
|
@@ -12,7 +12,7 @@ type BuildPrActorOptions = {
|
|
|
12
12
|
actorName: string;
|
|
13
13
|
useDockerCache: boolean;
|
|
14
14
|
};
|
|
15
|
-
class ApifyBuilder {
|
|
15
|
+
export class ApifyBuilder {
|
|
16
16
|
private constructor(
|
|
17
17
|
private readonly apifyClient: ApifyClient,
|
|
18
18
|
private readonly actorName: string,
|
|
@@ -101,15 +101,55 @@ class ApifyBuilder {
|
|
|
101
101
|
return { buildId: id, actorId: actId, buildNumber, actorName: this.actorName };
|
|
102
102
|
};
|
|
103
103
|
|
|
104
|
+
startActorBuildFromSourceFiles = async (sourceFiles: ActorVersionSourceFile[]): Promise<BuildData> => {
|
|
105
|
+
const ZIP_VERSION = '0.98';
|
|
106
|
+
const actorClient = this.apifyClient.actor(this.actorName);
|
|
107
|
+
const actorInfo = await actorClient.get();
|
|
108
|
+
if (!actorInfo) {
|
|
109
|
+
throw new Error(
|
|
110
|
+
`No actor named '${this.actorName}' was found on the platform. If this` +
|
|
111
|
+
' is unexpected, make sure the actor you are targeting is spelled the' +
|
|
112
|
+
' same as the folder in the repository.',
|
|
113
|
+
);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
type ActorVersion = Parameters<ReturnType<typeof actorClient.version>['update']>[0];
|
|
117
|
+
const actorVersion: ActorVersion = {
|
|
118
|
+
versionNumber: ZIP_VERSION,
|
|
119
|
+
sourceFiles,
|
|
120
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
121
|
+
// @ts-ignore: couldn't find this type :(
|
|
122
|
+
sourceType: ACTOR_SOURCE_TYPES.SOURCE_FILES,
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
const versionExists = !actorInfo.versions.find((v) => v.versionNumber === ZIP_VERSION);
|
|
126
|
+
if (versionExists) {
|
|
127
|
+
await actorClient.versions().create(actorVersion);
|
|
128
|
+
} else {
|
|
129
|
+
await actorClient.version(ZIP_VERSION).update(actorVersion);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const { id, actId, buildNumber } = await actorClient.build(ZIP_VERSION, { useCache: false });
|
|
133
|
+
console.error(`[${this.actorName}]: ${id} (${buildNumber})`);
|
|
134
|
+
return { buildId: id, actorId: actId, buildNumber, actorName: this.actorName };
|
|
135
|
+
};
|
|
136
|
+
|
|
104
137
|
waitForBuildToFinish = async (buildId: string, actorName: string): Promise<Build> => {
|
|
105
138
|
const build = await this.apifyClient.build(buildId).waitForFinish();
|
|
106
139
|
const versionNumber = build.buildNumber;
|
|
107
140
|
if (build.status === 'FAILED' || build.status === 'TIMED-OUT') {
|
|
108
|
-
const message =
|
|
109
|
-
`[BUILD][${actorName}]: Build ${buildId} (${versionNumber}) failed. ` +
|
|
110
|
-
`Not continuing with other builds and tests.`;
|
|
111
141
|
console.error(`[${this.actorName}]: ${versionNumber}`);
|
|
112
|
-
|
|
142
|
+
try {
|
|
143
|
+
const log = await this.apifyClient.build(buildId).log().get();
|
|
144
|
+
const logTail = log?.split('\n').slice(-40).join('\n');
|
|
145
|
+
console.error(`\n--- BUILD LOG (last 40 lines) ---\n${logTail}\n---`);
|
|
146
|
+
} catch (err) {
|
|
147
|
+
console.error(`[${this.actorName}]: Failed to fetch build log: ${err}`);
|
|
148
|
+
}
|
|
149
|
+
throw new Error(
|
|
150
|
+
`[BUILD][${actorName}]: Build ${buildId} (${versionNumber}) failed. ` +
|
|
151
|
+
`Not continuing with other builds and tests.`,
|
|
152
|
+
);
|
|
113
153
|
}
|
|
114
154
|
console.error(`[${this.actorName}]: ${versionNumber}`);
|
|
115
155
|
return build;
|
|
@@ -229,6 +269,26 @@ class ApifyBuilder {
|
|
|
229
269
|
}
|
|
230
270
|
}
|
|
231
271
|
|
|
272
|
+
export const waitAndSummarizeBuilds = async (startedBuilds: BuildData[], label: string): Promise<BuildData[]> => {
|
|
273
|
+
console.error('=========================================');
|
|
274
|
+
console.error(`FINISHED ${label}:`);
|
|
275
|
+
await Promise.all(
|
|
276
|
+
startedBuilds.map(async (buildData) => {
|
|
277
|
+
const builder = ApifyBuilder.fromActorName(buildData.actorName);
|
|
278
|
+
await builder.waitForBuildToFinish(buildData.buildId, buildData.actorName);
|
|
279
|
+
}),
|
|
280
|
+
);
|
|
281
|
+
|
|
282
|
+
console.error('=========================================');
|
|
283
|
+
console.error('SUMMARY:');
|
|
284
|
+
for (const buildData of startedBuilds.sort((a, b) => a.actorName.localeCompare(b.actorName))) {
|
|
285
|
+
console.error(`[${buildData.actorName}]: ${buildData.buildNumber}`);
|
|
286
|
+
}
|
|
287
|
+
console.error('=========================================');
|
|
288
|
+
|
|
289
|
+
return startedBuilds;
|
|
290
|
+
};
|
|
291
|
+
|
|
232
292
|
type RunBuildsOptions = {
|
|
233
293
|
actorConfigs: ActorConfig[];
|
|
234
294
|
isLatest?: boolean;
|
|
@@ -281,22 +341,8 @@ export const runBuilds = async ({
|
|
|
281
341
|
return buildData;
|
|
282
342
|
}),
|
|
283
343
|
);
|
|
284
|
-
console.error('=========================================');
|
|
285
|
-
console.error('FINISHED BUILDS:');
|
|
286
|
-
await Promise.all(
|
|
287
|
-
startedBuilds.map(async (buildData) => {
|
|
288
|
-
const builder = ApifyBuilder.fromActorName(buildData.actorName);
|
|
289
|
-
await builder.waitForBuildToFinish(buildData.buildId, buildData.actorName);
|
|
290
|
-
}),
|
|
291
|
-
);
|
|
292
|
-
console.error('=========================================');
|
|
293
|
-
console.error('SUMMARY:');
|
|
294
|
-
for (const buildData of startedBuilds.sort((a, b) => a.actorName.localeCompare(b.actorName))) {
|
|
295
|
-
console.error(`[${buildData.actorName}]: ${buildData.buildNumber} `);
|
|
296
|
-
}
|
|
297
|
-
console.error('=========================================');
|
|
298
344
|
|
|
299
|
-
return startedBuilds;
|
|
345
|
+
return waitAndSummarizeBuilds(startedBuilds, 'BUILDS');
|
|
300
346
|
};
|
|
301
347
|
|
|
302
348
|
export const deleteOldBuilds = async (actorConfigs: ActorConfig[]) => {
|
package/bin/main.ts
CHANGED
|
@@ -7,6 +7,7 @@ import yargs, { type Argv } from 'yargs';
|
|
|
7
7
|
import { hideBin } from 'yargs/helpers';
|
|
8
8
|
|
|
9
9
|
import { deleteOldBuilds, runBuilds } from './build.js';
|
|
10
|
+
import { runBuildsFromLocal } from './build-from-local.js';
|
|
10
11
|
import { getChangedActors } from './diff-changes.js';
|
|
11
12
|
import { getBranchOnlyChangedFiles, getChangedFiles, getCommits, hasMergeFromTarget } from './git.js';
|
|
12
13
|
import { getPushData } from './github.js';
|
|
@@ -202,6 +203,31 @@ await yargs()
|
|
|
202
203
|
});
|
|
203
204
|
},
|
|
204
205
|
)
|
|
206
|
+
.command(
|
|
207
|
+
'build-from-local',
|
|
208
|
+
'',
|
|
209
|
+
(args) =>
|
|
210
|
+
args
|
|
211
|
+
.option('actors', {
|
|
212
|
+
type: 'string',
|
|
213
|
+
description:
|
|
214
|
+
'Comma-separated actor names (owner/name) to build. Defaults to all actors in the repo.',
|
|
215
|
+
})
|
|
216
|
+
.option('dry-run', { type: 'boolean', default: false }),
|
|
217
|
+
async ({ actors, dryRun }) => {
|
|
218
|
+
const allActorConfigs = await getRepoActors();
|
|
219
|
+
const actorConfigs = actors
|
|
220
|
+
? actors.split(',').map((name) => {
|
|
221
|
+
const trimmed = name.trim();
|
|
222
|
+
const config = allActorConfigs.find((c) => c.actorName === trimmed);
|
|
223
|
+
if (!config) throw new Error(`Actor "${trimmed}" not found in repo`);
|
|
224
|
+
return config;
|
|
225
|
+
})
|
|
226
|
+
: allActorConfigs;
|
|
227
|
+
const builds = await runBuildsFromLocal({ actorConfigs, dryRun });
|
|
228
|
+
console.log(JSON.stringify(builds));
|
|
229
|
+
},
|
|
230
|
+
)
|
|
205
231
|
.command(
|
|
206
232
|
'delete-old-builds',
|
|
207
233
|
'',
|
package/bin/utils.ts
CHANGED
|
@@ -1,8 +1,72 @@
|
|
|
1
1
|
import { spawnSync } from 'node:child_process';
|
|
2
2
|
import fs from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
|
|
5
|
+
import type { ActorVersionSourceFile } from 'apify-client';
|
|
6
|
+
|
|
7
|
+
import { SOURCE_FILE_FORMATS } from '@apify/consts';
|
|
3
8
|
|
|
4
9
|
import type { ActorConfig } from './types.js';
|
|
5
10
|
|
|
11
|
+
// Returns true when `childPath` is not inside `parentPath`.
|
|
12
|
+
// Used to detect monorepo actors whose dockerContextDir escapes the actor directory.
|
|
13
|
+
export const isOutsideDir = (childPath: string, parentPath: string): boolean =>
|
|
14
|
+
path.relative(parentPath, childPath).startsWith('..');
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Lists every file under `subDir` (paths relative to `repoRoot`) that's either tracked by git or
|
|
18
|
+
* present but untracked in the working tree — deliberately omitting `--exclude-standard`, so
|
|
19
|
+
* gitignored files are included too. Callers combine this with getGitignoredPaths to decide what
|
|
20
|
+
* to keep, e.g. because .actor/ must survive even if .gitignore would otherwise exclude it.
|
|
21
|
+
* This also means .git/ itself is never walked, since git never lists its own internals here.
|
|
22
|
+
*/
|
|
23
|
+
export const listRepoFilePaths = (repoRoot: string, subDir: string): string[] => {
|
|
24
|
+
const relSubDir = path.relative(repoRoot, subDir).split(path.sep).join('/') || '.';
|
|
25
|
+
const result = spawnSync('git', ['ls-files', '--cached', '--others', '-z', '--', relSubDir], {
|
|
26
|
+
cwd: repoRoot,
|
|
27
|
+
maxBuffer: 100 * 1024 * 1024,
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
if (result.status !== 0) {
|
|
31
|
+
throw new Error(`[Command failed]: git ls-files\n${result.stderr.toString()}`);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
return result.stdout.toString().split('\0').filter(Boolean);
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Given paths relative to the repo root, returns the subset that `git` would exclude because of
|
|
39
|
+
* .gitignore rules (including nested .gitignore files, `.git/info/exclude`, and global excludes —
|
|
40
|
+
* anything `git` itself respects). Delegating to `git check-ignore` avoids re-implementing gitignore
|
|
41
|
+
* pattern matching.
|
|
42
|
+
*/
|
|
43
|
+
export const getGitignoredPaths = (relativePaths: string[]): Set<string> => {
|
|
44
|
+
if (relativePaths.length === 0) return new Set();
|
|
45
|
+
|
|
46
|
+
const result = spawnSync('git', ['check-ignore', '--stdin'], {
|
|
47
|
+
input: relativePaths.join('\n'),
|
|
48
|
+
maxBuffer: 100 * 1024 * 1024,
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
// Exit code 1 means none of the given paths are ignored - not an error. Anything else
|
|
52
|
+
// (e.g. 128 for "not a git repository") is a real failure.
|
|
53
|
+
if (result.status !== 0 && result.status !== 1) {
|
|
54
|
+
throw new Error(`[Command failed]: git check-ignore\n${result.stderr.toString()}`);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
return new Set(result.stdout.toString().split('\n').filter(Boolean));
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
const isBinary = (buffer: Buffer): boolean => buffer.includes(0);
|
|
61
|
+
|
|
62
|
+
export const toActorVersionSourceFile = async (absPath: string, rootDir: string): Promise<ActorVersionSourceFile> => {
|
|
63
|
+
const buffer = await fs.readFile(absPath);
|
|
64
|
+
const name = path.relative(rootDir, absPath).split(path.sep).join('/');
|
|
65
|
+
return isBinary(buffer)
|
|
66
|
+
? { name, format: SOURCE_FILE_FORMATS.BASE64, content: buffer.toString('base64') }
|
|
67
|
+
: { name, format: SOURCE_FILE_FORMATS.TEXT, content: buffer.toString('utf8') };
|
|
68
|
+
};
|
|
69
|
+
|
|
6
70
|
export const spawnCommandInGhWorkspace = (command: string, args: string[] = []) => {
|
|
7
71
|
console.error(command, args.join(' '));
|
|
8
72
|
const commandResult = spawnSync(command, args, { shell: true, maxBuffer: 100 * 1024 * 1024 });
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { ActorVersionSourceFile } from 'apify-client';
|
|
2
|
+
import type { ActorConfig, BuildData } from './types.js';
|
|
3
|
+
export declare const collectSourceFiles: (actorName: string, actorDir: string) => Promise<ActorVersionSourceFile[]>;
|
|
4
|
+
export declare const collectNonIgnoredFiles: (rootDir: string, repoRoot: string) => string[];
|
|
5
|
+
export declare const flattenMonorepoContext: (actorName: string, absActorDir: string, contextAbsDir: string, actorJson: Record<string, unknown>, keptContextFiles: string[], repoRoot: string) => Promise<{
|
|
6
|
+
tempDir: string;
|
|
7
|
+
filePaths: string[];
|
|
8
|
+
}>;
|
|
9
|
+
export declare const rewriteActorJsonPaths: (absActorDir: string, contextAbsDir: string, tempDir: string, actorJson: Record<string, unknown>) => Promise<void>;
|
|
10
|
+
export declare const runBuildsFromLocal: ({ actorConfigs, dryRun, }: {
|
|
11
|
+
actorConfigs: ActorConfig[];
|
|
12
|
+
dryRun: boolean;
|
|
13
|
+
}) => Promise<BuildData[]>;
|
|
14
|
+
//# sourceMappingURL=build-from-local.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build-from-local.d.ts","sourceRoot":"","sources":["../../bin/build-from-local.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AAG3D,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAUzD,eAAO,MAAM,kBAAkB,GAAU,WAAW,MAAM,EAAE,UAAU,MAAM,KAAG,OAAO,CAAC,sBAAsB,EAAE,CAkC9G,CAAC;AASF,eAAO,MAAM,sBAAsB,GAAI,SAAS,MAAM,EAAE,UAAU,MAAM,KAAG,MAAM,EAWhF,CAAC;AAaF,eAAO,MAAM,sBAAsB,GAC/B,WAAW,MAAM,EACjB,aAAa,MAAM,EACnB,eAAe,MAAM,EACrB,WAAW,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClC,kBAAkB,MAAM,EAAE,EAC1B,UAAU,MAAM,KACjB,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,EAAE,CAAA;CAAE,CAuClD,CAAC;AAcF,eAAO,MAAM,qBAAqB,GAC9B,aAAa,MAAM,EACnB,eAAe,MAAM,EACrB,SAAS,MAAM,EACf,WAAW,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KACnC,OAAO,CAAC,IAAI,CAqBd,CAAC;AAEF,eAAO,MAAM,kBAAkB,GAAU,2BAGtC;IACC,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,MAAM,EAAE,OAAO,CAAC;CACnB,KAAG,OAAO,CAAC,SAAS,EAAE,CAyBtB,CAAC"}
|