apify-test-tools 0.10.1-beta.2 → 0.10.1-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/actions/checkout-restore-dependencies/action.yaml +4 -3
- package/.github/workflows/public_platform-tests-claude-investigate-and-fix.yaml +7 -2
- package/.github/workflows/public_platform-tests.yaml +0 -1
- package/.github/workflows-package-version +1 -1
- package/CHANGELOG.md +6 -0
- package/CONTRIBUTING.md +13 -3
- package/README.md +2 -2
- package/bin/build-from-local.ts +8 -7
- package/bin/build.ts +22 -21
- package/bin/diff-changes.ts +11 -10
- package/bin/diff-json-schema.ts +14 -12
- package/bin/git.ts +89 -47
- package/bin/logger.ts +23 -0
- package/bin/main.ts +44 -38
- package/bin/output.ts +6 -0
- package/bin/slack.ts +8 -7
- package/bin/test-report.ts +22 -21
- package/bin/types.ts +0 -1
- package/bin/utils/config/actor-json.ts +84 -0
- package/bin/utils/config/load-config.ts +46 -46
- package/bin/utils/config/parser.ts +33 -28
- package/bin/utils/config/structures/base.ts +7 -0
- package/bin/utils/json-file.ts +11 -14
- package/bin/utils/path/repo-relative.ts +156 -0
- package/bin/utils/path/schema.ts +18 -0
- package/bin/utils.ts +63 -44
- package/dist/bin/build-from-local.d.ts +1 -1
- package/dist/bin/build-from-local.d.ts.map +1 -1
- package/dist/bin/build-from-local.js +8 -7
- package/dist/bin/build-from-local.js.map +1 -1
- package/dist/bin/build.d.ts.map +1 -1
- package/dist/bin/build.js +22 -21
- package/dist/bin/build.js.map +1 -1
- package/dist/bin/diff-changes.d.ts +1 -1
- package/dist/bin/diff-changes.d.ts.map +1 -1
- package/dist/bin/diff-changes.js +9 -8
- package/dist/bin/diff-changes.js.map +1 -1
- package/dist/bin/diff-json-schema.d.ts +1 -1
- package/dist/bin/diff-json-schema.d.ts.map +1 -1
- package/dist/bin/diff-json-schema.js +13 -12
- package/dist/bin/diff-json-schema.js.map +1 -1
- package/dist/bin/git.d.ts +10 -10
- package/dist/bin/git.d.ts.map +1 -1
- package/dist/bin/git.js +78 -37
- package/dist/bin/git.js.map +1 -1
- package/dist/bin/logger.d.ts +11 -0
- package/dist/bin/logger.d.ts.map +1 -0
- package/dist/bin/logger.js +20 -0
- package/dist/bin/logger.js.map +1 -0
- package/dist/bin/main.d.ts +1 -1
- package/dist/bin/main.js +44 -38
- package/dist/bin/main.js.map +1 -1
- package/dist/bin/output.d.ts +3 -0
- package/dist/bin/output.d.ts.map +1 -0
- package/dist/bin/output.js +6 -0
- package/dist/bin/output.js.map +1 -0
- package/dist/bin/slack.d.ts.map +1 -1
- package/dist/bin/slack.js +8 -7
- package/dist/bin/slack.js.map +1 -1
- package/dist/bin/test-report.d.ts.map +1 -1
- package/dist/bin/test-report.js +22 -21
- package/dist/bin/test-report.js.map +1 -1
- package/dist/bin/types.d.ts +0 -1
- package/dist/bin/types.d.ts.map +1 -1
- package/dist/bin/utils/config/actor-json.d.ts +14 -0
- package/dist/bin/utils/config/actor-json.d.ts.map +1 -0
- package/dist/bin/utils/config/actor-json.js +60 -0
- package/dist/bin/utils/config/actor-json.js.map +1 -0
- package/dist/bin/utils/config/load-config.d.ts +12 -2
- package/dist/bin/utils/config/load-config.d.ts.map +1 -1
- package/dist/bin/utils/config/load-config.js +32 -31
- package/dist/bin/utils/config/load-config.js.map +1 -1
- package/dist/bin/utils/config/parser.d.ts +4 -12
- package/dist/bin/utils/config/parser.d.ts.map +1 -1
- package/dist/bin/utils/config/parser.js +24 -23
- package/dist/bin/utils/config/parser.js.map +1 -1
- package/dist/bin/utils/config/structures/base.d.ts +5 -0
- package/dist/bin/utils/config/structures/base.d.ts.map +1 -1
- package/dist/bin/utils/config/structures/base.js.map +1 -1
- package/dist/bin/utils/json-file.d.ts +3 -2
- package/dist/bin/utils/json-file.d.ts.map +1 -1
- package/dist/bin/utils/json-file.js +8 -13
- package/dist/bin/utils/json-file.js.map +1 -1
- package/dist/bin/utils/path/repo-relative.d.ts +35 -0
- package/dist/bin/utils/path/repo-relative.d.ts.map +1 -0
- package/dist/bin/utils/path/repo-relative.js +137 -0
- package/dist/bin/utils/path/repo-relative.js.map +1 -0
- package/dist/bin/utils/path/schema.d.ts +7 -0
- package/dist/bin/utils/path/schema.d.ts.map +1 -0
- package/dist/bin/utils/path/schema.js +16 -0
- package/dist/bin/utils/path/schema.js.map +1 -0
- package/dist/bin/utils.d.ts +14 -6
- package/dist/bin/utils.d.ts.map +1 -1
- package/dist/bin/utils.js +52 -36
- package/dist/bin/utils.js.map +1 -1
- package/dist/test/unit/bin/build-from-local.test.js +4 -40
- package/dist/test/unit/bin/build-from-local.test.js.map +1 -1
- package/dist/test/unit/bin/diff-changes.test.js +97 -96
- package/dist/test/unit/bin/diff-changes.test.js.map +1 -1
- package/dist/test/unit/bin/diff.test.js +52 -48
- package/dist/test/unit/bin/diff.test.js.map +1 -1
- package/dist/test/unit/bin/git-command.test.d.ts +2 -0
- package/dist/test/unit/bin/git-command.test.d.ts.map +1 -0
- package/dist/test/unit/bin/git-command.test.js +54 -0
- package/dist/test/unit/bin/git-command.test.js.map +1 -0
- package/dist/test/unit/bin/git.test.js +211 -94
- package/dist/test/unit/bin/git.test.js.map +1 -1
- package/dist/test/unit/bin/logger.test.d.ts +2 -0
- package/dist/test/unit/bin/logger.test.d.ts.map +1 -0
- package/dist/test/unit/bin/logger.test.js +44 -0
- package/dist/test/unit/bin/logger.test.js.map +1 -0
- package/dist/test/unit/bin/logging.test.d.ts +2 -0
- package/dist/test/unit/bin/logging.test.d.ts.map +1 -0
- package/dist/test/unit/bin/logging.test.js +43 -0
- package/dist/test/unit/bin/logging.test.js.map +1 -0
- package/dist/test/unit/bin/utils/config/load-config.test.js +47 -25
- package/dist/test/unit/bin/utils/config/load-config.test.js.map +1 -1
- package/dist/test/unit/bin/utils/config/parser.test.js +64 -19
- package/dist/test/unit/bin/utils/config/parser.test.js.map +1 -1
- package/dist/test/unit/bin/utils/files.test.js +16 -27
- package/dist/test/unit/bin/utils/files.test.js.map +1 -1
- package/dist/test/unit/bin/utils/path/repo-relative.test.d.ts +2 -0
- package/dist/test/unit/bin/utils/path/repo-relative.test.d.ts.map +1 -0
- package/dist/test/unit/bin/utils/path/repo-relative.test.js +196 -0
- package/dist/test/unit/bin/utils/path/repo-relative.test.js.map +1 -0
- package/dist/test/unit/bin/utils/path/schema.test.d.ts +2 -0
- package/dist/test/unit/bin/utils/path/schema.test.d.ts.map +1 -0
- package/dist/test/unit/bin/utils/path/schema.test.js +52 -0
- package/dist/test/unit/bin/utils/path/schema.test.js.map +1 -0
- package/dist/test/unit/should-built-and-test.test.js +40 -38
- package/dist/test/unit/should-built-and-test.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/eslint.config.mjs +9 -1
- package/package.json +5 -5
- package/test/unit/bin/build-from-local.test.ts +4 -54
- package/test/unit/bin/diff-changes.test.ts +97 -96
- package/test/unit/bin/diff.test.ts +53 -48
- package/test/unit/bin/git-command.test.ts +71 -0
- package/test/unit/bin/git.test.ts +246 -91
- package/test/unit/bin/logger.test.ts +55 -0
- package/test/unit/bin/logging.test.ts +52 -0
- package/test/unit/bin/utils/config/load-config.test.ts +63 -26
- package/test/unit/bin/utils/config/parser.test.ts +81 -27
- package/test/unit/bin/utils/files.test.ts +19 -33
- package/test/unit/bin/utils/path/repo-relative.test.ts +250 -0
- package/test/unit/bin/utils/path/schema.test.ts +65 -0
- package/test/unit/should-built-and-test.test.ts +40 -38
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import z from 'zod';
|
|
2
|
+
|
|
3
|
+
import { safeReadJsonObjectFile } from '../json-file.js';
|
|
4
|
+
import { ExistingFile, type RelativeDir, type RelativeFile } from '../path/repo-relative.js';
|
|
5
|
+
import type { ValidatedActorConfig } from './structures/base.js';
|
|
6
|
+
|
|
7
|
+
const inlineOrPath = z.union([z.string(), z.record(z.string(), z.unknown())]).nullish();
|
|
8
|
+
|
|
9
|
+
const ActorJsonFieldsSchema = z.object({
|
|
10
|
+
dockerContextDir: z.string().default('..'),
|
|
11
|
+
dockerfile: z.string().optional(),
|
|
12
|
+
readme: z.string().nullish(),
|
|
13
|
+
changelog: z.string().nullish(),
|
|
14
|
+
input: inlineOrPath,
|
|
15
|
+
inputSchema: inlineOrPath,
|
|
16
|
+
output: inlineOrPath,
|
|
17
|
+
outputSchema: inlineOrPath,
|
|
18
|
+
webServerSchema: inlineOrPath,
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
export interface ActorJsonPaths {
|
|
22
|
+
file: ExistingFile;
|
|
23
|
+
dockerContextDir: RelativeDir;
|
|
24
|
+
dockerfile?: RelativeFile;
|
|
25
|
+
readme?: RelativeFile;
|
|
26
|
+
changelog?: RelativeFile;
|
|
27
|
+
inputSchema?: RelativeFile;
|
|
28
|
+
outputSchema?: RelativeFile;
|
|
29
|
+
webServerSchema?: RelativeFile;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const resolveFile = (actorJsonFile: ExistingFile, field: string, value: unknown): RelativeFile | undefined => {
|
|
33
|
+
if (typeof value !== 'string') return undefined;
|
|
34
|
+
try {
|
|
35
|
+
return actorJsonFile.joinFile(value);
|
|
36
|
+
} catch (error) {
|
|
37
|
+
throw new Error(`Invalid "${field}" in "${actorJsonFile}": ${(error as Error).message}`);
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
export function readActorJson(config: ValidatedActorConfig): ActorJsonPaths {
|
|
42
|
+
const actorJsonPath = config.folder.joinFile('.actor/actor.json');
|
|
43
|
+
let actorJsonFile: ExistingFile;
|
|
44
|
+
try {
|
|
45
|
+
actorJsonFile = ExistingFile.initialize(actorJsonPath);
|
|
46
|
+
} catch {
|
|
47
|
+
throw new Error(
|
|
48
|
+
`Cannot read "${actorJsonPath}". Every actor entry must have a corresponding .actor/actor.json file.`,
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const json = safeReadJsonObjectFile(actorJsonFile);
|
|
53
|
+
if (!json.success) throw new Error(`Cannot read "${actorJsonFile}". ${json.failure.message}`);
|
|
54
|
+
|
|
55
|
+
const parsed = ActorJsonFieldsSchema.safeParse(json.contents);
|
|
56
|
+
if (!parsed.success) throw new Error(`Invalid "${actorJsonFile}": ${z.prettifyError(parsed.error)}`);
|
|
57
|
+
|
|
58
|
+
const fields = parsed.data;
|
|
59
|
+
let dockerContextDir: RelativeDir;
|
|
60
|
+
try {
|
|
61
|
+
dockerContextDir = actorJsonFile.joinDir(fields.dockerContextDir);
|
|
62
|
+
} catch (error) {
|
|
63
|
+
throw new Error(`Invalid "dockerContextDir" in "${actorJsonFile}": ${(error as Error).message}`);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return {
|
|
67
|
+
file: actorJsonFile,
|
|
68
|
+
dockerContextDir,
|
|
69
|
+
dockerfile: resolveFile(actorJsonFile, 'dockerfile', fields.dockerfile),
|
|
70
|
+
readme: resolveFile(actorJsonFile, 'readme', fields.readme),
|
|
71
|
+
changelog: resolveFile(actorJsonFile, 'changelog', fields.changelog),
|
|
72
|
+
inputSchema: resolveFile(
|
|
73
|
+
actorJsonFile,
|
|
74
|
+
fields.input != null ? 'input' : 'inputSchema',
|
|
75
|
+
fields.input ?? fields.inputSchema,
|
|
76
|
+
),
|
|
77
|
+
outputSchema: resolveFile(
|
|
78
|
+
actorJsonFile,
|
|
79
|
+
fields.output != null ? 'output' : 'outputSchema',
|
|
80
|
+
fields.output ?? fields.outputSchema,
|
|
81
|
+
),
|
|
82
|
+
webServerSchema: resolveFile(actorJsonFile, 'webServerSchema', fields.webServerSchema),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
@@ -1,30 +1,26 @@
|
|
|
1
|
-
import path from 'node:path';
|
|
2
|
-
|
|
3
1
|
import { selectActors } from '../../actor-filtering.js';
|
|
4
|
-
import { isPathWithinScope } from '../../path-utils.js';
|
|
5
2
|
import type { ActorConfig } from '../../types.js';
|
|
6
3
|
import { safeReadJsonObjectFile } from '../json-file.js';
|
|
4
|
+
import { type ExistingDir, ExistingFile, type RelativeDir } from '../path/repo-relative.js';
|
|
5
|
+
import { type ActorJsonPaths, readActorJson } from './actor-json.js';
|
|
7
6
|
import { parseConfigFile } from './parser.js';
|
|
8
|
-
import type {
|
|
7
|
+
import type { ValidatedActorConfig } from './structures/base.js';
|
|
9
8
|
|
|
10
9
|
export const CONFIG_FILE_NAME = 'apify-test-tools.config.json';
|
|
11
10
|
|
|
12
11
|
// Reading the config happens in three stages, each one swappable on its own:
|
|
13
12
|
//
|
|
14
13
|
// 1. file -> plain object (safeReadJsonObjectFile)
|
|
15
|
-
// 2. plain object ->
|
|
14
|
+
// 2. plain object -> ValidatedActorConfig[] (parseConfigFile — picks a strategy, validates
|
|
16
15
|
// and normalizes; see ./parser.ts)
|
|
17
|
-
// 3.
|
|
16
|
+
// 3. ValidatedActorConfig -> LoadedActorConfig (loadActorConfig — reads .actor/actor.json)
|
|
18
17
|
|
|
19
18
|
// #region utils
|
|
20
19
|
|
|
21
|
-
const findOverlappingContextPaths = (contextPaths:
|
|
20
|
+
const findOverlappingContextPaths = (contextPaths: RelativeDir[]): [RelativeDir, RelativeDir] | undefined => {
|
|
22
21
|
for (let i = 0; i < contextPaths.length; i++) {
|
|
23
22
|
for (let j = i + 1; j < contextPaths.length; j++) {
|
|
24
|
-
if (
|
|
25
|
-
isPathWithinScope(contextPaths[i], contextPaths[j]) ||
|
|
26
|
-
isPathWithinScope(contextPaths[j], contextPaths[i])
|
|
27
|
-
) {
|
|
23
|
+
if (contextPaths[i].contains(contextPaths[j]) || contextPaths[j].contains(contextPaths[i])) {
|
|
28
24
|
return [contextPaths[i], contextPaths[j]];
|
|
29
25
|
}
|
|
30
26
|
}
|
|
@@ -32,9 +28,17 @@ const findOverlappingContextPaths = (contextPaths: string[]): [string, string] |
|
|
|
32
28
|
return undefined;
|
|
33
29
|
};
|
|
34
30
|
|
|
35
|
-
//
|
|
36
|
-
|
|
37
|
-
|
|
31
|
+
// Downstream ActorConfig consumers still use "" for the repo root.
|
|
32
|
+
const legacyPath = (dir: RelativeDir): string => (dir.path === '.' ? '' : dir.path);
|
|
33
|
+
|
|
34
|
+
export interface LoadedActorConfig {
|
|
35
|
+
actorFullName: string;
|
|
36
|
+
folder: ExistingDir;
|
|
37
|
+
tokenEnvVar: string;
|
|
38
|
+
actorJson: ActorJsonPaths;
|
|
39
|
+
dockerContextDir: RelativeDir;
|
|
40
|
+
contextPaths: RelativeDir[];
|
|
41
|
+
}
|
|
38
42
|
|
|
39
43
|
// #endregion
|
|
40
44
|
|
|
@@ -42,7 +46,16 @@ const displayFolder = (folder: string): string => folder || '.';
|
|
|
42
46
|
|
|
43
47
|
/** Stage 1 — the config file as a plain object, or a failure phrased for whoever wrote it. */
|
|
44
48
|
const readConfigFileContents = async (): Promise<Record<string, unknown>> => {
|
|
45
|
-
|
|
49
|
+
let configFile: ExistingFile;
|
|
50
|
+
try {
|
|
51
|
+
configFile = new ExistingFile(CONFIG_FILE_NAME);
|
|
52
|
+
} catch {
|
|
53
|
+
throw new Error(
|
|
54
|
+
`Config file "${CONFIG_FILE_NAME}" not found in the current directory. ` +
|
|
55
|
+
`Please create one with the required actor entries.`,
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
const file = safeReadJsonObjectFile(configFile);
|
|
46
59
|
if (file.success) {
|
|
47
60
|
return file.contents;
|
|
48
61
|
}
|
|
@@ -66,57 +79,44 @@ const readConfigFileContents = async (): Promise<Record<string, unknown>> => {
|
|
|
66
79
|
* Stage 3 — resolves one normalized entry against the repo, reading the actor's `.actor/actor.json`
|
|
67
80
|
* for its `dockerContextDir` and deciding which paths count as the actor's context.
|
|
68
81
|
*/
|
|
69
|
-
export const loadActorConfig = (entry:
|
|
82
|
+
export const loadActorConfig = (entry: ValidatedActorConfig): LoadedActorConfig => {
|
|
70
83
|
const { folder } = entry;
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
const
|
|
74
|
-
if (!actorJson.success) {
|
|
75
|
-
throw new Error(
|
|
76
|
-
`Cannot read "${actorJsonPath}". Every actor entry in "${CONFIG_FILE_NAME}" ` +
|
|
77
|
-
`must have a corresponding .actor/actor.json file.`,
|
|
78
|
-
);
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
const actorDotDir = folder ? `${folder}/.actor` : '.actor';
|
|
82
|
-
const rawDockerContextDir =
|
|
83
|
-
typeof actorJson.contents.dockerContextDir === 'string' ? actorJson.contents.dockerContextDir : '..';
|
|
84
|
-
const resolved = path.resolve(process.cwd(), actorDotDir, rawDockerContextDir);
|
|
85
|
-
const dockerContextDir = path.relative(process.cwd(), resolved);
|
|
86
|
-
|
|
87
|
-
if (dockerContextDir.startsWith('..')) {
|
|
88
|
-
throw new Error(
|
|
89
|
-
`"dockerContextDir" for folder "${displayFolder(folder)}" resolves outside the repository root. ` +
|
|
90
|
-
`Resolved path: "${dockerContextDir}".`,
|
|
91
|
-
);
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
const normalizedDockerContextDir = dockerContextDir === '.' ? '' : dockerContextDir;
|
|
95
|
-
const contextPaths = [...(entry.overrideActorContext ?? [normalizedDockerContextDir])];
|
|
84
|
+
const actorJson = readActorJson(entry);
|
|
85
|
+
const { dockerContextDir } = actorJson;
|
|
86
|
+
const contextPaths = [...(entry.overrideActorContext ?? [dockerContextDir])];
|
|
96
87
|
|
|
97
88
|
// The actor's own folder is always part of its context. When an explicit "overrideActorContext"
|
|
98
89
|
// doesn't already cover it, add it automatically instead of failing the workflow.
|
|
99
|
-
if (!contextPaths.some((contextPath) =>
|
|
90
|
+
if (!contextPaths.some((contextPath) => contextPath.contains(folder))) {
|
|
100
91
|
contextPaths.push(folder);
|
|
101
92
|
}
|
|
102
93
|
|
|
103
94
|
const overlap = findOverlappingContextPaths(contextPaths);
|
|
104
95
|
if (overlap) {
|
|
105
96
|
throw new Error(
|
|
106
|
-
`Invalid context paths for folder "${
|
|
97
|
+
`Invalid context paths for folder "${folder}" in "${CONFIG_FILE_NAME}": ` +
|
|
107
98
|
`"${overlap[0]}" and "${overlap[1]}" overlap. Context paths must not be prefixes of one another.`,
|
|
108
99
|
);
|
|
109
100
|
}
|
|
110
101
|
|
|
111
102
|
return {
|
|
112
103
|
actorFullName: entry.actorFullName,
|
|
113
|
-
folder,
|
|
104
|
+
folder: entry.folder,
|
|
114
105
|
tokenEnvVar: entry.tokenEnvVar,
|
|
115
|
-
|
|
106
|
+
actorJson,
|
|
107
|
+
dockerContextDir,
|
|
116
108
|
contextPaths,
|
|
117
109
|
};
|
|
118
110
|
};
|
|
119
111
|
|
|
112
|
+
const toLegacyActorConfig = (entry: LoadedActorConfig): ActorConfig => ({
|
|
113
|
+
actorFullName: entry.actorFullName,
|
|
114
|
+
folder: legacyPath(entry.folder),
|
|
115
|
+
tokenEnvVar: entry.tokenEnvVar,
|
|
116
|
+
dockerContextDir: legacyPath(entry.dockerContextDir),
|
|
117
|
+
contextPaths: entry.contextPaths.map(legacyPath),
|
|
118
|
+
});
|
|
119
|
+
|
|
120
120
|
// #endregion
|
|
121
121
|
|
|
122
122
|
export const readConfigFile = async (selection: { actors: string[]; ignore: string[] }): Promise<ActorConfig[]> => {
|
|
@@ -126,7 +126,7 @@ export const readConfigFile = async (selection: { actors: string[]; ignore: stri
|
|
|
126
126
|
const actorConfigs: ActorConfig[] = [];
|
|
127
127
|
for (const parsed of parsedConfigs) {
|
|
128
128
|
// Sequential on purpose: the first actor with a problem should be the one reported.
|
|
129
|
-
actorConfigs.push(loadActorConfig(parsed));
|
|
129
|
+
actorConfigs.push(toLegacyActorConfig(loadActorConfig(parsed)));
|
|
130
130
|
}
|
|
131
131
|
|
|
132
132
|
return selectActors(selection, actorConfigs);
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import z from 'zod';
|
|
2
2
|
|
|
3
|
-
import {
|
|
3
|
+
import { ExistingDirSchema, RelativeDirSchema } from '../path/schema.js';
|
|
4
|
+
import {
|
|
5
|
+
CONFIG_FILE_STRATEGY,
|
|
6
|
+
type ResolvedActorConfig,
|
|
7
|
+
type StrategyParser,
|
|
8
|
+
type ValidatedActorConfig,
|
|
9
|
+
} from './structures/base.js';
|
|
4
10
|
import { GROUPED_PARSER } from './structures/grouped.js';
|
|
5
11
|
import { LEGACY_PARSER } from './structures/legacy.js';
|
|
6
12
|
|
|
@@ -32,32 +38,34 @@ const normalizeFolder = (folder: string): string => {
|
|
|
32
38
|
return stripped === '.' ? '' : stripped;
|
|
33
39
|
};
|
|
34
40
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
const seenActorFullNames = new Set<string>();
|
|
47
|
-
const errors: string[] = [];
|
|
48
|
-
|
|
41
|
+
const ValidatedConfigSchema = z.array(
|
|
42
|
+
z.object({
|
|
43
|
+
actorFullName: z.string(),
|
|
44
|
+
folder: ExistingDirSchema,
|
|
45
|
+
tokenEnvVar: z.string(),
|
|
46
|
+
overrideActorContext: z.array(RelativeDirSchema).optional(),
|
|
47
|
+
}),
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
/** Normalize and validate paths, then check collisions using their final spelling. */
|
|
51
|
+
function verifyConfiguration(body: ResolvedActorConfig[]): ValidatedActorConfig[] {
|
|
49
52
|
const normalized = body.map((entry) => ({
|
|
50
53
|
...entry,
|
|
51
54
|
folder: normalizeFolder(entry.folder),
|
|
52
55
|
overrideActorContext: entry.overrideActorContext?.map(stripTrailingSlash),
|
|
53
56
|
}));
|
|
57
|
+
const parsed = ValidatedConfigSchema.safeParse(normalized);
|
|
58
|
+
if (!parsed.success) throw new Error(z.prettifyError(parsed.error));
|
|
54
59
|
|
|
55
|
-
|
|
60
|
+
const seenFolders = new Set<string>();
|
|
61
|
+
const seenActorFullNames = new Set<string>();
|
|
62
|
+
const errors: string[] = [];
|
|
63
|
+
for (const [index, entry] of parsed.data.entries()) {
|
|
56
64
|
// TODO: drop folder uniqueness (this requires several changes on other places)
|
|
57
|
-
if (seenFolders.has(entry.folder)) {
|
|
65
|
+
if (seenFolders.has(entry.folder.path)) {
|
|
58
66
|
errors.push(`Duplicate folder "${body[index].folder}". Each actor must have a unique folder.`);
|
|
59
67
|
} else {
|
|
60
|
-
seenFolders.add(entry.folder);
|
|
68
|
+
seenFolders.add(entry.folder.path);
|
|
61
69
|
}
|
|
62
70
|
|
|
63
71
|
if (seenActorFullNames.has(entry.actorFullName)) {
|
|
@@ -67,11 +75,8 @@ function verifyConfiguration(body: ResolvedActorConfig[]): ResolvedActorConfig[]
|
|
|
67
75
|
}
|
|
68
76
|
}
|
|
69
77
|
|
|
70
|
-
if (errors.length > 0)
|
|
71
|
-
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
return normalized;
|
|
78
|
+
if (errors.length > 0) throw new Error(errors.join('\n'));
|
|
79
|
+
return parsed.data;
|
|
75
80
|
}
|
|
76
81
|
|
|
77
82
|
// eslint-disable-next-line no-underscore-dangle
|
|
@@ -80,10 +85,10 @@ export const _privates = {
|
|
|
80
85
|
verifyConfiguration,
|
|
81
86
|
};
|
|
82
87
|
|
|
83
|
-
export function parseConfigFile(body: Record<string, unknown>):
|
|
84
|
-
const
|
|
88
|
+
export function parseConfigFile(body: Record<string, unknown>): ValidatedActorConfig[] {
|
|
89
|
+
const { mode, ...rest } = body;
|
|
90
|
+
const strategy = selectStrategy({ mode });
|
|
85
91
|
|
|
86
|
-
const resolved = strategy.parse(
|
|
87
|
-
|
|
88
|
-
return validated;
|
|
92
|
+
const resolved = strategy.parse(rest);
|
|
93
|
+
return verifyConfiguration(resolved);
|
|
89
94
|
}
|
|
@@ -2,6 +2,8 @@ import { prettifyError, type ZodType } from 'zod';
|
|
|
2
2
|
|
|
3
3
|
import { ACTOR_NAME, USERNAME } from '@apify/consts';
|
|
4
4
|
|
|
5
|
+
import type { ExistingDir, RelativeDir } from '../../path/repo-relative.js';
|
|
6
|
+
|
|
5
7
|
function stripRegexAnchor(regex: string): string {
|
|
6
8
|
return regex.replace(/^\^/, '').replace(/\$$/, '');
|
|
7
9
|
}
|
|
@@ -21,6 +23,11 @@ export interface ResolvedActorConfig {
|
|
|
21
23
|
overrideActorContext: string[] | undefined;
|
|
22
24
|
}
|
|
23
25
|
|
|
26
|
+
export interface ValidatedActorConfig extends Omit<ResolvedActorConfig, 'folder' | 'overrideActorContext'> {
|
|
27
|
+
folder: ExistingDir;
|
|
28
|
+
overrideActorContext?: RelativeDir[];
|
|
29
|
+
}
|
|
30
|
+
|
|
24
31
|
export enum CONFIG_FILE_STRATEGY {
|
|
25
32
|
LEGACY = 'legacy',
|
|
26
33
|
GROUPED = 'grouped',
|
package/bin/utils/json-file.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import { readFileSync
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
|
|
3
|
+
import type { ExistingFile } from './path/repo-relative.js';
|
|
2
4
|
|
|
3
5
|
/**
|
|
4
6
|
* Why a JSON object could not be produced from a path. Callers that already have their own
|
|
5
7
|
* user-facing wording (e.g. the config file reader) switch on this instead of re-deriving the
|
|
6
8
|
* cause from an error message.
|
|
7
9
|
*/
|
|
8
|
-
export type JsonObjectReadFailureReason = '
|
|
10
|
+
export type JsonObjectReadFailureReason = 'unreadable' | 'invalid-json' | 'not-an-object';
|
|
9
11
|
|
|
10
12
|
export type JsonObjectReadResult =
|
|
11
13
|
| { success: true; contents: Record<string, unknown> }
|
|
@@ -21,34 +23,29 @@ const failed = (reason: JsonObjectReadFailureReason, message: string): JsonObjec
|
|
|
21
23
|
* Reads a file and parses it as a JSON object, reporting every way that can go wrong as a value
|
|
22
24
|
* instead of a thrown error — the caller decides which failures are fatal and how to word them.
|
|
23
25
|
*/
|
|
24
|
-
export function safeReadJsonObjectFile(
|
|
26
|
+
export function safeReadJsonObjectFile(filePath: ExistingFile): JsonObjectReadResult {
|
|
25
27
|
let file: string;
|
|
26
28
|
try {
|
|
27
|
-
|
|
28
|
-
const stats = statSync(path);
|
|
29
|
-
if (!stats.isFile()) {
|
|
30
|
-
return failed('not-a-file', `Expected ${path} to be a file`);
|
|
31
|
-
}
|
|
32
|
-
file = readFileSync(path, 'utf8');
|
|
29
|
+
file = readFileSync(filePath.path, 'utf8');
|
|
33
30
|
} catch {
|
|
34
|
-
return failed('unreadable', `Expected ${
|
|
31
|
+
return failed('unreadable', `Expected ${filePath} to exist or be readable`);
|
|
35
32
|
}
|
|
36
33
|
|
|
37
34
|
let parsed: unknown;
|
|
38
35
|
try {
|
|
39
36
|
parsed = JSON.parse(file);
|
|
40
37
|
} catch (err) {
|
|
41
|
-
return failed('invalid-json', `Failed to parse ${
|
|
38
|
+
return failed('invalid-json', `Failed to parse ${filePath}: ${err}`);
|
|
42
39
|
}
|
|
43
40
|
|
|
44
41
|
if (parsed === null) {
|
|
45
|
-
return failed('not-an-object', `Expected ${
|
|
42
|
+
return failed('not-an-object', `Expected ${filePath} to be a JSON object, not null`);
|
|
46
43
|
}
|
|
47
44
|
if (Array.isArray(parsed)) {
|
|
48
|
-
return failed('not-an-object', `Expected ${
|
|
45
|
+
return failed('not-an-object', `Expected ${filePath} to be a JSON object, not an array`);
|
|
49
46
|
}
|
|
50
47
|
if (typeof parsed !== 'object') {
|
|
51
|
-
return failed('not-an-object', `Expected ${
|
|
48
|
+
return failed('not-an-object', `Expected ${filePath} to be a JSON object`);
|
|
52
49
|
}
|
|
53
50
|
|
|
54
51
|
return { success: true, contents: parsed as Record<string, unknown> };
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/* eslint-disable max-classes-per-file */
|
|
2
|
+
import { statSync } from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
|
|
5
|
+
// Git and the config file always use "/", so paths are POSIX regardless of the host OS.
|
|
6
|
+
const { posix } = path;
|
|
7
|
+
|
|
8
|
+
// A file path cannot end in "/", ".", "..", or be empty: each of those names a directory.
|
|
9
|
+
const assertNamesFile = (rawPath: string): void => {
|
|
10
|
+
const lastSegment = rawPath.slice(rawPath.lastIndexOf('/') + 1);
|
|
11
|
+
if (lastSegment === '' || lastSegment === '.' || lastSegment === '..') {
|
|
12
|
+
throw new Error(`Expected a file path, got directory path "${rawPath}".`);
|
|
13
|
+
}
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
// Shared by the file and directory paths. Use it as a parameter type where either kind is fine.
|
|
17
|
+
export abstract class AbstractPath {
|
|
18
|
+
readonly #path: string;
|
|
19
|
+
constructor(rawPath: string) {
|
|
20
|
+
if (posix.isAbsolute(rawPath)) {
|
|
21
|
+
throw new Error(`Expected a relative path, got absolute path "${rawPath}".`);
|
|
22
|
+
}
|
|
23
|
+
const normalized = posix.normalize(rawPath);
|
|
24
|
+
// normalize keeps a trailing slash ("actors/foo/"), which would make equal paths compare unequal.
|
|
25
|
+
this.#path = normalized.endsWith('/') ? normalized.slice(0, -1) : normalized;
|
|
26
|
+
this.assertNoEscape(rawPath);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// A file and a directory are never equal, even when spelled the same.
|
|
30
|
+
isEqualTo(other: AbstractPath): boolean {
|
|
31
|
+
// inheritance safe check, at least one of the two is a subclass of the other
|
|
32
|
+
const isRelativeOf = this instanceof other.constructor || other instanceof this.constructor;
|
|
33
|
+
return isRelativeOf && this.#path === other.#path;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
get path(): string {
|
|
37
|
+
return this.#path;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
toString(): string {
|
|
41
|
+
return this.path;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
toJSON(): string {
|
|
45
|
+
return this.path;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
private assertNoEscape(raw: string): void {
|
|
49
|
+
if (this.#path === '..' || this.#path.startsWith('../')) {
|
|
50
|
+
throw new Error(`Path "${raw}" escapes the repo root.`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export class RelativeDir extends AbstractPath {
|
|
56
|
+
static readonly ROOT = new RelativeDir('.');
|
|
57
|
+
|
|
58
|
+
joinDir(relativePath: string): RelativeDir {
|
|
59
|
+
return new RelativeDir(this.join(relativePath));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
joinFile(relativePath: string): RelativeFile {
|
|
63
|
+
// Must be checked before joining: posix.join('actors/foo', '.') is "actors/foo", which looks like a file.
|
|
64
|
+
assertNamesFile(relativePath);
|
|
65
|
+
return new RelativeFile(this.join(relativePath));
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// True when `other` is this directory itself or lies inside it.
|
|
69
|
+
contains(other: AbstractPath): boolean {
|
|
70
|
+
// the root contains every path
|
|
71
|
+
if (this.path === '.') return true;
|
|
72
|
+
return this.isEqualTo(other) || other.path.startsWith(`${this.path}/`);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
isStrictAncestorOf(other: AbstractPath): boolean {
|
|
76
|
+
// you are not an ancestor of yourself
|
|
77
|
+
return !this.isEqualTo(other) && this.contains(other);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Throws unless this path exists and is a directory. Paths resolve against the process working
|
|
81
|
+
// directory, which is the repo root.
|
|
82
|
+
assertIsDir(): this {
|
|
83
|
+
// `stat`, not `lstat`: a symlink must resolve to its target, matching what reading it does.
|
|
84
|
+
const stats = statSync(this.path, { throwIfNoEntry: false });
|
|
85
|
+
if (!stats) throw new Error(`Expected directory "${this}" to exist.`);
|
|
86
|
+
if (!stats.isDirectory()) throw new Error(`Expected "${this}" to be a directory.`);
|
|
87
|
+
return this;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
private join(relativePath: string): string {
|
|
91
|
+
// Must be checked here: posix.join('.', '/etc') is "etc", so the constructor would not catch it.
|
|
92
|
+
if (posix.isAbsolute(relativePath)) {
|
|
93
|
+
throw new Error(`Expected a relative path to join with "${this}", got absolute path "${relativePath}".`);
|
|
94
|
+
}
|
|
95
|
+
return posix.join(this.path, relativePath);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export class ExistingDir extends RelativeDir {
|
|
100
|
+
constructor(rawPath: string) {
|
|
101
|
+
super(rawPath);
|
|
102
|
+
this.assertIsDir();
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
static initialize(dir: RelativeDir): ExistingDir {
|
|
106
|
+
return new ExistingDir(dir.path);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export class RelativeFile extends AbstractPath {
|
|
111
|
+
constructor(rawPath: string) {
|
|
112
|
+
assertNamesFile(rawPath);
|
|
113
|
+
super(rawPath);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// The directory containing this file.
|
|
117
|
+
get parent(): RelativeDir {
|
|
118
|
+
return new RelativeDir(posix.dirname(this.path));
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Resolves against the file's directory, the way a path written inside the file is meant
|
|
122
|
+
// (e.g. `dockerContextDir: ".."` in `.actor/actor.json` is the actor folder).
|
|
123
|
+
joinDir(relativePath: string): RelativeDir {
|
|
124
|
+
return this.parent.joinDir(relativePath);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// Resolves against the file's directory, e.g. a sibling file.
|
|
128
|
+
joinFile(relativePath: string): RelativeFile {
|
|
129
|
+
return this.parent.joinFile(relativePath);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
isWithin(scope: RelativeDir): boolean {
|
|
133
|
+
return scope.contains(this);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Throws unless this path exists and is a file. Paths resolve against the process working
|
|
137
|
+
// directory, which is the repo root.
|
|
138
|
+
assertIsFile(): this {
|
|
139
|
+
// `stat`, not `lstat`: a symlink must resolve to its target, matching what reading it does.
|
|
140
|
+
const stats = statSync(this.path, { throwIfNoEntry: false });
|
|
141
|
+
if (!stats) throw new Error(`Expected file "${this}" to exist.`);
|
|
142
|
+
if (!stats.isFile()) throw new Error(`Expected "${this}" to be a file.`);
|
|
143
|
+
return this;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export class ExistingFile extends RelativeFile {
|
|
148
|
+
constructor(rawPath: string) {
|
|
149
|
+
super(rawPath);
|
|
150
|
+
this.assertIsFile();
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
static initialize(file: RelativeFile): ExistingFile {
|
|
154
|
+
return new ExistingFile(file.path);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import z from 'zod';
|
|
2
|
+
|
|
3
|
+
import { ExistingDir, ExistingFile, RelativeDir, RelativeFile } from './repo-relative.js';
|
|
4
|
+
|
|
5
|
+
const parsePath = <T>(construct: (value: string) => T) =>
|
|
6
|
+
z.string().transform((value, ctx): T | typeof z.NEVER => {
|
|
7
|
+
try {
|
|
8
|
+
return construct(value);
|
|
9
|
+
} catch (error) {
|
|
10
|
+
ctx.addIssue({ code: 'custom', message: error instanceof Error ? error.message : String(error) });
|
|
11
|
+
return z.NEVER;
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
export const RelativeDirSchema = parsePath((value) => new RelativeDir(value));
|
|
16
|
+
export const ExistingDirSchema = parsePath((value) => new ExistingDir(value));
|
|
17
|
+
export const RelativeFileSchema = parsePath((value) => new RelativeFile(value));
|
|
18
|
+
export const ExistingFileSchema = parsePath((value) => new ExistingFile(value));
|