apify-test-tools 0.8.7-beta.1 → 0.8.7-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/README.md +83 -13
- package/bin/actor-filtering.ts +21 -0
- package/bin/build-from-local.ts +35 -34
- package/bin/build.ts +55 -59
- package/bin/diff-changes.ts +166 -125
- package/bin/dockerignore.ts +50 -0
- package/bin/git.ts +5 -1
- package/bin/main.ts +66 -62
- package/bin/path-utils.ts +13 -0
- package/bin/test-report.ts +5 -5
- package/bin/types.ts +19 -5
- package/bin/utils.ts +124 -96
- package/dist/bin/actor-filtering.d.ts +13 -0
- package/dist/bin/actor-filtering.d.ts.map +1 -0
- package/dist/bin/actor-filtering.js +19 -0
- package/dist/bin/actor-filtering.js.map +1 -0
- package/dist/bin/build-from-local.d.ts +2 -2
- package/dist/bin/build-from-local.d.ts.map +1 -1
- package/dist/bin/build-from-local.js +33 -28
- package/dist/bin/build-from-local.js.map +1 -1
- package/dist/bin/build.d.ts +5 -8
- package/dist/bin/build.d.ts.map +1 -1
- package/dist/bin/build.js +49 -56
- package/dist/bin/build.js.map +1 -1
- package/dist/bin/diff-changes.d.ts +0 -6
- package/dist/bin/diff-changes.d.ts.map +1 -1
- package/dist/bin/diff-changes.js +129 -89
- package/dist/bin/diff-changes.js.map +1 -1
- package/dist/bin/dockerignore.d.ts +17 -0
- package/dist/bin/dockerignore.d.ts.map +1 -0
- package/dist/bin/dockerignore.js +41 -0
- package/dist/bin/dockerignore.js.map +1 -0
- package/dist/bin/git.d.ts +1 -1
- package/dist/bin/git.d.ts.map +1 -1
- package/dist/bin/git.js +1 -1
- package/dist/bin/git.js.map +1 -1
- package/dist/bin/main.d.ts +20 -1
- package/dist/bin/main.d.ts.map +1 -1
- package/dist/bin/main.js +57 -38
- package/dist/bin/main.js.map +1 -1
- package/dist/bin/path-utils.d.ts +4 -0
- package/dist/bin/path-utils.d.ts.map +1 -0
- package/dist/bin/path-utils.js +9 -0
- package/dist/bin/path-utils.js.map +1 -0
- package/dist/bin/test-report.js +3 -3
- package/dist/bin/test-report.js.map +1 -1
- package/dist/bin/types.d.ts +17 -4
- package/dist/bin/types.d.ts.map +1 -1
- package/dist/bin/utils.d.ts +5 -19
- package/dist/bin/utils.d.ts.map +1 -1
- package/dist/bin/utils.js +84 -82
- package/dist/bin/utils.js.map +1 -1
- package/dist/lib/lib.d.ts +10 -4
- package/dist/lib/lib.d.ts.map +1 -1
- package/dist/lib/lib.js +28 -22
- package/dist/lib/lib.js.map +1 -1
- package/dist/lib/types.d.ts +2 -2
- package/dist/lib/types.d.ts.map +1 -1
- package/dist/lib/utils.d.ts +1 -1
- package/dist/lib/utils.d.ts.map +1 -1
- package/dist/lib/utils.js +4 -4
- package/dist/lib/utils.js.map +1 -1
- package/dist/test/unit/bin/actor-filtering.test.d.ts +2 -0
- package/dist/test/unit/bin/actor-filtering.test.d.ts.map +1 -0
- package/dist/test/unit/bin/actor-filtering.test.js +41 -0
- package/dist/test/unit/bin/actor-filtering.test.js.map +1 -0
- package/dist/test/unit/bin/build-from-local.test.js +55 -86
- package/dist/test/unit/bin/build-from-local.test.js.map +1 -1
- package/dist/test/unit/bin/diff-changes.test.js +358 -36
- package/dist/test/unit/bin/diff-changes.test.js.map +1 -1
- package/dist/test/unit/bin/dockerignore.test.d.ts +2 -0
- package/dist/test/unit/bin/dockerignore.test.d.ts.map +1 -0
- package/dist/test/unit/bin/dockerignore.test.js +102 -0
- package/dist/test/unit/bin/dockerignore.test.js.map +1 -0
- package/dist/test/unit/bin/path-utils.test.d.ts +2 -0
- package/dist/test/unit/bin/path-utils.test.d.ts.map +1 -0
- package/dist/test/unit/bin/path-utils.test.js +14 -0
- package/dist/test/unit/bin/path-utils.test.js.map +1 -0
- package/dist/test/unit/bin/utils.test.d.ts +2 -0
- package/dist/test/unit/bin/utils.test.d.ts.map +1 -0
- package/dist/test/unit/bin/utils.test.js +321 -0
- package/dist/test/unit/bin/utils.test.js.map +1 -0
- package/dist/test/unit/should-built-and-test.test.js +83 -32
- package/dist/test/unit/should-built-and-test.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/lib/lib.ts +28 -23
- package/lib/types.ts +2 -2
- package/lib/utils.ts +4 -4
- package/package.json +6 -5
- package/test/unit/bin/actor-filtering.test.ts +52 -0
- package/test/unit/bin/build-from-local.test.ts +62 -121
- package/test/unit/bin/diff-changes.test.ts +397 -41
- package/test/unit/bin/dockerignore.test.ts +123 -0
- package/test/unit/bin/path-utils.test.ts +17 -0
- package/test/unit/bin/utils.test.ts +381 -0
- package/test/unit/should-built-and-test.test.ts +89 -32
package/bin/diff-changes.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { isCosmeticOnlyJsonSchemaChange } from './diff-json-schema.js';
|
|
2
|
+
import { type DockerIgnoreMatcher, loadDockerIgnore } from './dockerignore.js';
|
|
3
|
+
import { findContainingScope, hoistPath, isPathWithinScope } from './path-utils.js';
|
|
2
4
|
import type { ActorConfig, Commit } from './types.js';
|
|
3
5
|
|
|
4
6
|
interface ShouldBuildAndTestOptions {
|
|
@@ -8,89 +10,160 @@ interface ShouldBuildAndTestOptions {
|
|
|
8
10
|
commits: Commit[];
|
|
9
11
|
}
|
|
10
12
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
13
|
+
const IGNORED_TOP_LEVEL_FILES = [
|
|
14
|
+
'.vscode/',
|
|
15
|
+
'.gitignore',
|
|
16
|
+
'.husky/',
|
|
17
|
+
'.eslintrc',
|
|
18
|
+
'eslint.config.mjs',
|
|
19
|
+
'.prettierrc',
|
|
20
|
+
'.editorconfig',
|
|
21
|
+
];
|
|
22
|
+
|
|
23
|
+
// Expects an already-hoisted path (relative to the matched context entry, see findContainingScope).
|
|
24
|
+
const isIgnoredTopLevelFile = (hoistedLowercaseFilePath: string): boolean =>
|
|
25
|
+
IGNORED_TOP_LEVEL_FILES.some((pattern) => hoistedLowercaseFilePath.startsWith(pattern));
|
|
26
|
+
|
|
27
|
+
type FileChangeForActor =
|
|
28
|
+
| { impact: 'ignored' }
|
|
29
|
+
| { impact: 'outside-context' }
|
|
30
|
+
| { impact: 'cosmetic'; semanticallyVerified: boolean }
|
|
31
|
+
| { impact: 'functional' };
|
|
21
32
|
|
|
22
33
|
/**
|
|
23
|
-
*
|
|
34
|
+
* Classify a single file change for a single actor.
|
|
35
|
+
*
|
|
36
|
+
* Steps (in order):
|
|
37
|
+
* 1. CHANGELOG.md, by filename, anywhere → cosmetic. There is a single repo-wide shared changelog,
|
|
38
|
+
* not one per actor, so it applies to every actor regardless of context/folder.
|
|
39
|
+
* 2. Context matching (actorConfig.contextPaths) → outside-context if no match
|
|
40
|
+
* 3. Hardcoded ignore list, checked against the path hoisted relative to the matched context entry → ignored
|
|
41
|
+
* 4. .dockerignore filtering (patterns relative to dockerContextDir), skipped for the actor's own `.actor/`
|
|
42
|
+
* dir → ignored if matched
|
|
43
|
+
* 5. README.md by filename → cosmetic if inside the actor's own folder, otherwise ignored
|
|
44
|
+
* 6. .json inside the actor's own `.actor/` dir with only cosmetic schema diffs → cosmetic (semantically verified)
|
|
45
|
+
* 7. Everything else → functional
|
|
24
46
|
*/
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
'.husky/',
|
|
32
|
-
'.eslintrc',
|
|
33
|
-
'eslint.config.mjs',
|
|
34
|
-
'.prettierrc',
|
|
35
|
-
'.editorconfig',
|
|
36
|
-
'.actor/',
|
|
37
|
-
];
|
|
38
|
-
// Strip out deprecated /code and /shared folders, treat them as top-level code
|
|
39
|
-
const sanitizedLowercaseFilePath = lowercaseFilePath.replace(/^code\//, '').replace(/^shared\//, '');
|
|
40
|
-
|
|
41
|
-
return IGNORED_TOP_LEVEL_FILES.some((ignoredFile) => sanitizedLowercaseFilePath.startsWith(ignoredFile));
|
|
42
|
-
};
|
|
43
|
-
|
|
44
|
-
type FileChange =
|
|
45
|
-
| { impact: 'ignored' }
|
|
46
|
-
// Only things that influence how the Actor looks - e.g. README and CHANGELOG files, schema titles, descriptions, reordering, etc. We only need to rebuild on release
|
|
47
|
-
| { impact: 'cosmetic'; semanticallyVerified: boolean; includes: 'all-actors' | ActorConfig }
|
|
48
|
-
// Influences how the Actor works - we need to run tests
|
|
49
|
-
| {
|
|
50
|
-
impact: 'functional';
|
|
51
|
-
includes: 'all-actors' | ActorConfig;
|
|
52
|
-
};
|
|
53
|
-
|
|
54
|
-
const classifyFileChange = (originalFilePath: string, actorConfigs: ActorConfig[], commits: Commit[]): FileChange => {
|
|
55
|
-
// Lowercase for case-insensitive matching; keep original for git show (case-sensitive on Linux)
|
|
47
|
+
const classifyFileChange = (
|
|
48
|
+
originalFilePath: string,
|
|
49
|
+
actorConfig: ActorConfig,
|
|
50
|
+
commits: Commit[],
|
|
51
|
+
dockerIgnoreMatcher: DockerIgnoreMatcher,
|
|
52
|
+
): FileChangeForActor => {
|
|
56
53
|
const lowercaseFilePath = originalFilePath.toLowerCase();
|
|
57
|
-
|
|
54
|
+
|
|
55
|
+
// TODO: hardcodes that there's a single repo-wide changelog belonging to every actor. Should instead
|
|
56
|
+
// be derived from parsing actor.json (readme, changelog, schema paths), see
|
|
57
|
+
// https://github.com/apify/apify-test-tools/issues/106
|
|
58
|
+
if (lowercaseFilePath.endsWith('changelog.md')) {
|
|
59
|
+
return { impact: 'cosmetic', semanticallyVerified: false };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const lowercaseContextPaths = actorConfig.contextPaths.map((contextPath) => contextPath.toLowerCase());
|
|
63
|
+
|
|
64
|
+
const matchedContext = findContainingScope(lowercaseFilePath, lowercaseContextPaths);
|
|
65
|
+
if (matchedContext === undefined) {
|
|
66
|
+
return { impact: 'outside-context' };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const hoistedFilePath = hoistPath(lowercaseFilePath, matchedContext);
|
|
70
|
+
if (isIgnoredTopLevelFile(hoistedFilePath)) {
|
|
58
71
|
return { impact: 'ignored' };
|
|
59
72
|
}
|
|
60
73
|
|
|
61
|
-
|
|
62
|
-
|
|
74
|
+
const lowercaseFolder = actorConfig.folder.toLowerCase();
|
|
75
|
+
const actorDotDir = lowercaseFolder ? `${lowercaseFolder}/.actor` : '.actor';
|
|
76
|
+
const isUnderActorDotDir = isPathWithinScope(lowercaseFilePath, actorDotDir);
|
|
77
|
+
|
|
78
|
+
// .actor/ can legitimately be listed in .dockerignore (the Apify platform evaluates it before
|
|
79
|
+
// the Docker build, so excluding it from the build context is a valid caching optimization) —
|
|
80
|
+
// that shouldn't cause changes to .actor/ itself to be ignored here.
|
|
81
|
+
if (!isUnderActorDotDir && dockerIgnoreMatcher(originalFilePath)) {
|
|
82
|
+
return { impact: 'ignored' };
|
|
63
83
|
}
|
|
64
84
|
|
|
65
|
-
const
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
actorName: actorFolderInfo.actorName,
|
|
76
|
-
lowercaseFilePath,
|
|
77
|
-
},
|
|
78
|
-
);
|
|
79
|
-
return { impact: 'ignored' };
|
|
80
|
-
}
|
|
81
|
-
if (lowercaseFilePath.endsWith('readme.md')) {
|
|
82
|
-
return { impact: 'cosmetic', semanticallyVerified: false, includes: actorConfigChanged };
|
|
85
|
+
const isInActorFolder = isPathWithinScope(lowercaseFilePath, lowercaseFolder);
|
|
86
|
+
|
|
87
|
+
if (lowercaseFilePath.endsWith('readme.md')) {
|
|
88
|
+
return isInActorFolder ? { impact: 'cosmetic', semanticallyVerified: false } : { impact: 'ignored' };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
if (lowercaseFilePath.endsWith('.json') && isUnderActorDotDir) {
|
|
92
|
+
const isCosmetic = isCosmeticOnlyJsonSchemaChange(commits, originalFilePath);
|
|
93
|
+
if (isCosmetic) {
|
|
94
|
+
return { impact: 'cosmetic', semanticallyVerified: true };
|
|
83
95
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
return { impact: 'functional' };
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Check if a file falls inside another actor's folder.
|
|
103
|
+
* Root actors (folder === "") never exclude files from siblings.
|
|
104
|
+
*/
|
|
105
|
+
const isExcludedBySibling = (lowercaseFilePath: string, actor: ActorConfig, allActors: ActorConfig[]): boolean => {
|
|
106
|
+
return allActors.some(
|
|
107
|
+
(other) =>
|
|
108
|
+
other.folder !== actor.folder &&
|
|
109
|
+
other.folder !== '' &&
|
|
110
|
+
isPathWithinScope(lowercaseFilePath, other.folder.toLowerCase()),
|
|
111
|
+
);
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
type ActorChangeEntry = {
|
|
115
|
+
actorConfig: ActorConfig;
|
|
116
|
+
files: string[];
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
type ChangeGroup = { actors: string[]; files: string[] };
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Maps each changed file to the set of actor names it triggered a change for.
|
|
123
|
+
*/
|
|
124
|
+
const buildFileToActorsMap = (actorsChangedMap: Map<string, ActorChangeEntry>): Map<string, Set<string>> => {
|
|
125
|
+
const fileToActors = new Map<string, Set<string>>();
|
|
126
|
+
for (const { actorConfig, files } of actorsChangedMap.values()) {
|
|
127
|
+
for (const file of files) {
|
|
128
|
+
const actors = fileToActors.get(file) ?? new Set<string>();
|
|
129
|
+
actors.add(actorConfig.actorFullName);
|
|
130
|
+
fileToActors.set(file, actors);
|
|
87
131
|
}
|
|
132
|
+
}
|
|
133
|
+
return fileToActors;
|
|
134
|
+
};
|
|
88
135
|
|
|
89
|
-
|
|
136
|
+
/**
|
|
137
|
+
* Groups files by their identical actor-set (files triggering a change for the exact same
|
|
138
|
+
* actors are grouped together), then orders the groups by descending actor-set size
|
|
139
|
+
* (most-shared groups first), breaking ties alphabetically by actor names.
|
|
140
|
+
*/
|
|
141
|
+
const groupFilesByActorSet = (fileToActors: Map<string, Set<string>>): ChangeGroup[] => {
|
|
142
|
+
const groupsByKey = new Map<string, ChangeGroup>();
|
|
143
|
+
for (const [file, actorsSet] of fileToActors) {
|
|
144
|
+
const actors = Array.from(actorsSet).sort();
|
|
145
|
+
const key = actors.join(',');
|
|
146
|
+
const group = groupsByKey.get(key) ?? { actors, files: [] };
|
|
147
|
+
group.files.push(file);
|
|
148
|
+
groupsByKey.set(key, group);
|
|
90
149
|
}
|
|
91
150
|
|
|
92
|
-
|
|
93
|
-
|
|
151
|
+
return Array.from(groupsByKey.values()).sort((groupA, groupB) => {
|
|
152
|
+
if (groupB.actors.length !== groupA.actors.length) {
|
|
153
|
+
return groupB.actors.length - groupA.actors.length;
|
|
154
|
+
}
|
|
155
|
+
return groupA.actors.join(',').localeCompare(groupB.actors.join(','));
|
|
156
|
+
});
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
const logChangeGroups = (groups: ChangeGroup[]): void => {
|
|
160
|
+
for (const { actors, files } of groups) {
|
|
161
|
+
if (actors.length > 1) {
|
|
162
|
+
console.error(`[DIFF]: Shared changes for actors ${actors.join(', ')}: ${files.join(', ')}`);
|
|
163
|
+
} else {
|
|
164
|
+
console.error(`[DIFF]: Changes specific to actor ${actors[0]}: ${files.join(', ')}`);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
94
167
|
};
|
|
95
168
|
|
|
96
169
|
export const getChangedActors = ({
|
|
@@ -99,72 +172,40 @@ export const getChangedActors = ({
|
|
|
99
172
|
isLatest = false,
|
|
100
173
|
commits,
|
|
101
174
|
}: ShouldBuildAndTestOptions): ActorConfig[] => {
|
|
102
|
-
|
|
103
|
-
const actorsChangedMap = new Map<string, ActorConfig>();
|
|
104
|
-
|
|
105
|
-
const actorConfigsWithoutStandalone = actorConfigs.filter(({ isStandalone }) => !isStandalone);
|
|
175
|
+
const actorsChangedMap = new Map<string, ActorChangeEntry>();
|
|
106
176
|
|
|
107
|
-
for (const
|
|
108
|
-
const
|
|
109
|
-
if (fileChange.impact === 'ignored') {
|
|
110
|
-
continue;
|
|
111
|
-
}
|
|
177
|
+
for (const actorConfig of actorConfigs) {
|
|
178
|
+
const dockerIgnoreMatcher = loadDockerIgnore(actorConfig.dockerContextDir);
|
|
112
179
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
}
|
|
180
|
+
for (const originalFilePath of filepathsChanged) {
|
|
181
|
+
const lowercaseFilePath = originalFilePath.toLowerCase();
|
|
116
182
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
} else if (fileChange.includes === 'all-actors') {
|
|
120
|
-
// Standalone Actors are handled always via specific actors change, not all-actors
|
|
121
|
-
for (const actorConfig of actorConfigsWithoutStandalone) {
|
|
122
|
-
actorsChangedMap.set(actorConfig.folder, actorConfig);
|
|
183
|
+
if (isExcludedBySibling(lowercaseFilePath, actorConfig, actorConfigs)) {
|
|
184
|
+
continue;
|
|
123
185
|
}
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
186
|
|
|
127
|
-
|
|
187
|
+
const change = classifyFileChange(originalFilePath, actorConfig, commits, dockerIgnoreMatcher);
|
|
128
188
|
|
|
129
|
-
|
|
130
|
-
|
|
189
|
+
if (change.impact === 'ignored' || change.impact === 'outside-context') continue;
|
|
190
|
+
if (change.impact === 'cosmetic' && !isLatest) continue;
|
|
131
191
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
const cosmeticChanges = filepathsChanged
|
|
138
|
-
.map((file) => ({ file, change: classifyFileChange(file, actorConfigs, commits) }))
|
|
139
|
-
.filter(({ change }) => change.impact === 'cosmetic') as {
|
|
140
|
-
file: string;
|
|
141
|
-
change: Extract<FileChange, { impact: 'cosmetic' }>;
|
|
142
|
-
}[];
|
|
143
|
-
const semanticallyVerifiedFiles = cosmeticChanges
|
|
144
|
-
.filter(({ change }) => change.semanticallyVerified)
|
|
145
|
-
.map(({ file }) => file);
|
|
146
|
-
const inherentlyCosmeticFiles = cosmeticChanges
|
|
147
|
-
.filter(({ change }) => !change.semanticallyVerified)
|
|
148
|
-
.map(({ file }) => file);
|
|
149
|
-
console.error(
|
|
150
|
-
`[DIFF]: Cosmetic-only JSON schema changes (semantically verified, only trigger release build): ${formatFiles(semanticallyVerifiedFiles)}`,
|
|
151
|
-
);
|
|
152
|
-
console.error(
|
|
153
|
-
`[DIFF]: Inherently cosmetic files (README, CHANGELOG — only trigger release build): ${formatFiles(inherentlyCosmeticFiles)}`,
|
|
154
|
-
);
|
|
192
|
+
const entry = actorsChangedMap.get(actorConfig.folder) ?? { actorConfig, files: [] };
|
|
193
|
+
entry.files.push(originalFilePath);
|
|
194
|
+
actorsChangedMap.set(actorConfig.folder, entry);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
155
197
|
|
|
156
|
-
const
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
198
|
+
const actorsChanged = Array.from(actorsChangedMap.values()).map((entry) => entry.actorConfig);
|
|
199
|
+
|
|
200
|
+
// Log changes grouped by actor set, so changes shared across actors are logged once
|
|
201
|
+
// instead of being repeated per actor.
|
|
202
|
+
const fileToActors = buildFileToActorsMap(actorsChangedMap);
|
|
203
|
+
const groups = groupFilesByActorSet(fileToActors);
|
|
204
|
+
logChangeGroups(groups);
|
|
160
205
|
|
|
161
206
|
if (actorsChanged.length > 0) {
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
.filter((config) => config.isStandalone)
|
|
165
|
-
.map((config) => config.actorName);
|
|
166
|
-
console.error(`[DIFF]: MiniActors to be built and tested: ${miniactors.join(', ')}`);
|
|
167
|
-
console.error(`[DIFF]: Standalone Actors to be built and tested: ${standaloneActors.join(', ')}`);
|
|
207
|
+
const actors = actorsChanged.map((config) => config.actorFullName);
|
|
208
|
+
console.error(`[DIFF]: Actors to be built and tested: ${actors.join(', ')}`);
|
|
168
209
|
} else {
|
|
169
210
|
console.error(`[DIFF]: No relevant files changed, skipping builds and tests`);
|
|
170
211
|
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
|
|
4
|
+
import ignore from 'ignore';
|
|
5
|
+
|
|
6
|
+
import { hoistPath, isPathWithinScope } from './path-utils.js';
|
|
7
|
+
|
|
8
|
+
export type DockerIgnoreMatcher = (repoRelativePath: string) => boolean;
|
|
9
|
+
|
|
10
|
+
// Docker normalizes each pattern before matching, so a leading "./" (as in the common "./node_modules"
|
|
11
|
+
// style) is a no-op for Docker. The `ignore` package has no such normalization — it treats "./" as
|
|
12
|
+
// literal pattern text that can never match a real path, so a .dockerignore written in that style
|
|
13
|
+
// would otherwise silently match nothing. Strip it here (after any negation prefix) so the pattern
|
|
14
|
+
// behaves the way Docker itself would apply it.
|
|
15
|
+
const normalizeDockerignorePattern = (line: string): string => line.replace(/^(!?)(?:\.\/)+/, '$1');
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Reads `.dockerignore` from `absoluteRootDir` and returns a matcher for paths relative to
|
|
19
|
+
* `hoistFrom`. `hoistFrom` defaults to '', meaning callers already pass paths relative to
|
|
20
|
+
* `absoluteRootDir` directly — isPathWithinScope/hoistPath both treat '' as "matches everything" /
|
|
21
|
+
* identity, so the scope-check and hoist collapse to a no-op in that case, not via a branch.
|
|
22
|
+
*
|
|
23
|
+
* Returns a no-op matcher (always returns false) when the file is absent.
|
|
24
|
+
*/
|
|
25
|
+
export const buildDockerIgnoreMatcher = (absoluteRootDir: string, hoistFrom = ''): DockerIgnoreMatcher => {
|
|
26
|
+
let content: string;
|
|
27
|
+
try {
|
|
28
|
+
content = fs.readFileSync(path.join(absoluteRootDir, '.dockerignore'), 'utf-8');
|
|
29
|
+
} catch {
|
|
30
|
+
return () => false;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const matcher = ignore().add(content.split('\n').map(normalizeDockerignorePattern).join('\n'));
|
|
34
|
+
|
|
35
|
+
return (filePath: string): boolean => {
|
|
36
|
+
if (!isPathWithinScope(filePath.toLowerCase(), hoistFrom.toLowerCase())) {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return matcher.ignores(hoistPath(filePath, hoistFrom));
|
|
41
|
+
};
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Load .dockerignore from the root of an actor's dockerContextDir and return a matcher
|
|
46
|
+
* that accepts repo-root-relative file paths. Patterns are resolved relative to
|
|
47
|
+
* dockerContextDir, matching Docker's own behavior.
|
|
48
|
+
*/
|
|
49
|
+
export const loadDockerIgnore = (dockerContextDir: string): DockerIgnoreMatcher =>
|
|
50
|
+
buildDockerIgnoreMatcher(path.resolve(dockerContextDir), dockerContextDir);
|
package/bin/git.ts
CHANGED
|
@@ -94,7 +94,11 @@ const fetchAllBranchCommits = (sourceBranch: string, targetBranch: string): Comm
|
|
|
94
94
|
* Gets the commits between sourceBranch and targetBranch (exclusive).
|
|
95
95
|
* - If baseCommit is provided, only returns commits after the baseCommit.
|
|
96
96
|
*/
|
|
97
|
-
export const getCommits = ({
|
|
97
|
+
export const getCommits = ({
|
|
98
|
+
sourceBranch,
|
|
99
|
+
targetBranch,
|
|
100
|
+
baseCommit,
|
|
101
|
+
}: Pick<Config, 'sourceBranch' | 'targetBranch' | 'baseCommit'>): Commit[] => {
|
|
98
102
|
const baseCommitSha = parseBaseCommit(baseCommit);
|
|
99
103
|
const commits = fetchAllBranchCommits(sourceBranch, targetBranch);
|
|
100
104
|
|
package/bin/main.ts
CHANGED
|
@@ -14,14 +14,14 @@ import { getPushData } from './github.js';
|
|
|
14
14
|
import { notifyToSlack } from './slack.js';
|
|
15
15
|
import { reportTestResults } from './test-report.js';
|
|
16
16
|
import type { Config } from './types.js';
|
|
17
|
-
import {
|
|
17
|
+
import { readConfigFile, setCwd, spawnCommandInGhWorkspace } from './utils.js';
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
20
|
* Middlewares to be run before every command execution
|
|
21
21
|
*/
|
|
22
22
|
const middlewares = [setCwd];
|
|
23
23
|
|
|
24
|
-
const buildOptions = (y: Argv) => {
|
|
24
|
+
export const buildOptions = <T>(y: Argv<T>) => {
|
|
25
25
|
return y
|
|
26
26
|
.option('target-branch', {
|
|
27
27
|
type: 'string',
|
|
@@ -37,27 +37,44 @@ const buildOptions = (y: Argv) => {
|
|
|
37
37
|
})
|
|
38
38
|
.option('base-commit', {
|
|
39
39
|
type: 'string',
|
|
40
|
+
demandOption: false,
|
|
40
41
|
});
|
|
41
42
|
};
|
|
42
43
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
/**
|
|
45
|
+
* Actor-selection flags, applied to every command that reads the actor config so a caller can
|
|
46
|
+
* narrow the set it operates on (e.g. two-stage releases: `--ignore X`, then `--actors X`).
|
|
47
|
+
* Kept separate from `buildOptions` so the read-only git commands don't advertise flags they ignore.
|
|
48
|
+
*/
|
|
49
|
+
export const actorSelectionOptions = <T>(y: Argv<T>) => {
|
|
50
|
+
return y
|
|
51
|
+
.option('actors', {
|
|
52
|
+
type: 'string',
|
|
53
|
+
array: true,
|
|
54
|
+
default: [] as string[],
|
|
55
|
+
})
|
|
56
|
+
.option('ignore', {
|
|
57
|
+
type: 'string',
|
|
58
|
+
array: true,
|
|
59
|
+
default: [] as string[],
|
|
60
|
+
});
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
const resolveChangedActors = async (config: Config, { isLatest }: { isLatest: boolean }) => {
|
|
64
|
+
const actorConfigs = await readConfigFile(config);
|
|
48
65
|
|
|
49
|
-
// This is an optimization for the common case where a branch only has cosmetic changes but had to
|
|
66
|
+
// This is an optimization for the common case where a branch only has cosmetic changes but had to smerge in
|
|
50
67
|
// functional changes from master (being up-to-date is a CI requirement). Master is already validated, and
|
|
51
68
|
// since the branch has no functional changes of its own, there is nothing new to validate.
|
|
52
69
|
// Exception: if the branch has any functional changes alongside the merge, we must re-test — even
|
|
53
70
|
// individually validated changes can have novel interactions when combined.
|
|
54
|
-
if (hasMergeFromTarget(sourceBranch, targetBranch)) {
|
|
71
|
+
if (hasMergeFromTarget(config.sourceBranch, config.targetBranch)) {
|
|
55
72
|
console.error(
|
|
56
73
|
'[MERGE-FROM-TARGET-OPTIMIZATION]: There is merge from target branch, checking if there are no functional changes in our own branch. If so, we can skip tests',
|
|
57
74
|
);
|
|
58
|
-
const branchOnlyFiles = getBranchOnlyChangedFiles(sourceBranch, targetBranch);
|
|
75
|
+
const branchOnlyFiles = getBranchOnlyChangedFiles(config.sourceBranch, config.targetBranch);
|
|
59
76
|
// Omit baseCommit to get full branch history. Validated functional commits can still interact with merged ones
|
|
60
|
-
const allBranchCommits = getCommits({
|
|
77
|
+
const allBranchCommits = getCommits({ ...config, baseCommit: undefined });
|
|
61
78
|
const branchOnlyActorsChanged = getChangedActors({
|
|
62
79
|
filepathsChanged: branchOnlyFiles,
|
|
63
80
|
actorConfigs,
|
|
@@ -73,7 +90,7 @@ const resolveChangedActors = async (
|
|
|
73
90
|
}
|
|
74
91
|
|
|
75
92
|
// If the optimization doesn't apply, we check all branch commits including merges for full coverage. We don't reuse the merge optimization results because here we can apply baseCommit and check merge commits (they might be functional or just cosmetic)
|
|
76
|
-
const commits = getCommits(
|
|
93
|
+
const commits = getCommits(config);
|
|
77
94
|
const changedFiles = getChangedFiles(commits);
|
|
78
95
|
return getChangedActors({ filepathsChanged: changedFiles, actorConfigs, isLatest, commits });
|
|
79
96
|
};
|
|
@@ -103,22 +120,19 @@ await yargs()
|
|
|
103
120
|
const changedFiles = getChangedFiles(commits);
|
|
104
121
|
console.log(JSON.stringify(changedFiles));
|
|
105
122
|
})
|
|
123
|
+
.command('get-actor-configs', '', actorSelectionOptions, async ({ actors, ignore }) => {
|
|
124
|
+
const actorConfigs = await readConfigFile({ actors, ignore });
|
|
125
|
+
console.log(JSON.stringify(actorConfigs));
|
|
126
|
+
})
|
|
106
127
|
.command(
|
|
107
|
-
'get-
|
|
128
|
+
'get-affected-actors',
|
|
108
129
|
'',
|
|
109
|
-
(
|
|
110
|
-
async () => {
|
|
111
|
-
const
|
|
112
|
-
console.log(JSON.stringify(
|
|
130
|
+
(args) => actorSelectionOptions(buildOptions(args)),
|
|
131
|
+
async (config) => {
|
|
132
|
+
const actorsChanged = await resolveChangedActors(config, { isLatest: false });
|
|
133
|
+
console.log(JSON.stringify(actorsChanged));
|
|
113
134
|
},
|
|
114
135
|
)
|
|
115
|
-
.command('get-affected-actors', '', buildOptions, async ({ targetBranch, sourceBranch, baseCommit }) => {
|
|
116
|
-
const actorsChanged = await resolveChangedActors(
|
|
117
|
-
{ targetBranch, sourceBranch, baseCommit },
|
|
118
|
-
{ isLatest: false },
|
|
119
|
-
);
|
|
120
|
-
console.log(JSON.stringify(actorsChanged));
|
|
121
|
-
})
|
|
122
136
|
.command(
|
|
123
137
|
'report-tests',
|
|
124
138
|
'',
|
|
@@ -135,12 +149,9 @@ await yargs()
|
|
|
135
149
|
.command(
|
|
136
150
|
'build',
|
|
137
151
|
'',
|
|
138
|
-
(args) => buildOptions(args).option('dry-run', { type: 'boolean', default: false }),
|
|
139
|
-
async (
|
|
140
|
-
const actorsChanged = await resolveChangedActors(
|
|
141
|
-
{ targetBranch, sourceBranch, baseCommit },
|
|
142
|
-
{ isLatest: false },
|
|
143
|
-
);
|
|
152
|
+
(args) => actorSelectionOptions(buildOptions(args)).option('dry-run', { type: 'boolean', default: false }),
|
|
153
|
+
async (config) => {
|
|
154
|
+
const actorsChanged = await resolveChangedActors(config, { isLatest: false });
|
|
144
155
|
// https://github.com/apify-store/google-maps#:actors/lukaskrivka_google-maps-with-contact-details
|
|
145
156
|
// git@github.com:apify-store/google-maps#:actors/lukaskrivka_google-maps-with-contact-details
|
|
146
157
|
const repoUrl = spawnCommandInGhWorkspace(`git remote get-url origin`).replace(
|
|
@@ -151,9 +162,9 @@ await yargs()
|
|
|
151
162
|
const builds = await runBuilds({
|
|
152
163
|
repoUrl,
|
|
153
164
|
actorConfigs: actorsChanged,
|
|
154
|
-
branch: sourceBranch.replace('origin/', ''),
|
|
155
|
-
dryRun,
|
|
156
|
-
useDockerCache,
|
|
165
|
+
branch: config.sourceBranch.replace('origin/', ''),
|
|
166
|
+
dryRun: config.dryRun,
|
|
167
|
+
useDockerCache: config.useDockerCache,
|
|
157
168
|
});
|
|
158
169
|
console.log(JSON.stringify(builds));
|
|
159
170
|
},
|
|
@@ -162,7 +173,7 @@ await yargs()
|
|
|
162
173
|
'release',
|
|
163
174
|
'',
|
|
164
175
|
(args) =>
|
|
165
|
-
args
|
|
176
|
+
actorSelectionOptions(args)
|
|
166
177
|
.option('push-event-path', { type: 'string', demandOption: true })
|
|
167
178
|
.option('dry-run', { type: 'boolean', default: false })
|
|
168
179
|
.option('report-slack-channel', { type: 'string' })
|
|
@@ -173,7 +184,7 @@ await yargs()
|
|
|
173
184
|
args.pushEventPath,
|
|
174
185
|
);
|
|
175
186
|
const isLatest = true;
|
|
176
|
-
const actorConfigs = await
|
|
187
|
+
const actorConfigs = await readConfigFile(args);
|
|
177
188
|
const actorsChanged = getChangedActors({
|
|
178
189
|
filepathsChanged: changedFiles,
|
|
179
190
|
actorConfigs,
|
|
@@ -206,37 +217,30 @@ await yargs()
|
|
|
206
217
|
.command(
|
|
207
218
|
'build-from-local',
|
|
208
219
|
'',
|
|
209
|
-
(args) =>
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
type: 'string',
|
|
213
|
-
description:
|
|
214
|
-
'Comma-separated actor names (owner/name) to build. Defaults to all actors in the repo.',
|
|
215
|
-
})
|
|
216
|
-
.option('dry-run', { type: 'boolean', default: false }),
|
|
217
|
-
async ({ actors, dryRun }) => {
|
|
218
|
-
const allActorConfigs = await getRepoActors();
|
|
219
|
-
const actorConfigs = actors
|
|
220
|
-
? actors.split(',').map((name) => {
|
|
221
|
-
const trimmed = name.trim();
|
|
222
|
-
const config = allActorConfigs.find((c) => c.actorName === trimmed);
|
|
223
|
-
if (!config) throw new Error(`Actor "${trimmed}" not found in repo`);
|
|
224
|
-
return config;
|
|
225
|
-
})
|
|
226
|
-
: allActorConfigs;
|
|
220
|
+
(args) => actorSelectionOptions(args).option('dry-run', { type: 'boolean', default: false }),
|
|
221
|
+
async ({ actors, ignore, dryRun }) => {
|
|
222
|
+
const actorConfigs = await readConfigFile({ actors, ignore });
|
|
227
223
|
const builds = await runBuildsFromLocal({ actorConfigs, dryRun });
|
|
228
224
|
console.log(JSON.stringify(builds));
|
|
229
225
|
},
|
|
230
226
|
)
|
|
231
|
-
.command(
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
async () => {
|
|
236
|
-
const actorConfigs = await getRepoActors();
|
|
237
|
-
await deleteOldBuilds(actorConfigs);
|
|
238
|
-
},
|
|
239
|
-
)
|
|
227
|
+
.command('delete-old-builds', '', actorSelectionOptions, async ({ actors, ignore }) => {
|
|
228
|
+
const actorConfigs = await readConfigFile({ actors, ignore });
|
|
229
|
+
await deleteOldBuilds(actorConfigs);
|
|
230
|
+
})
|
|
240
231
|
.strictCommands()
|
|
241
232
|
.demandCommand(1, 'Command is required')
|
|
233
|
+
.fail((msg, err, yargsInstance) => {
|
|
234
|
+
// Errors thrown from a command handler (e.g. an unknown actor passed to --actors/--ignore,
|
|
235
|
+
// or a missing config file) arrive here as `err`. A malformed selection must fail loudly
|
|
236
|
+
// rather than silently operate on the wrong set of actors — print the message, no stack.
|
|
237
|
+
if (err) {
|
|
238
|
+
console.error(`[ERROR]: ${err.message}`);
|
|
239
|
+
} else {
|
|
240
|
+
// Argument-parsing/validation failure — keep yargs' usage output.
|
|
241
|
+
console.error(yargsInstance.help());
|
|
242
|
+
console.error(`\n${msg}`);
|
|
243
|
+
}
|
|
244
|
+
process.exit(1);
|
|
245
|
+
})
|
|
242
246
|
.parse(hideBin(process.argv));
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// filePath and scopePath are both repo-root-relative POSIX paths, already normalized (no trailing slashes).
|
|
2
|
+
// scopePath === '' means "matches everything" — the caller's own scope (a context path, an actor's folder,
|
|
3
|
+
// a sibling's folder, a docker context dir), not necessarily the repo root.
|
|
4
|
+
export const isPathWithinScope = (filePath: string, scopePath: string): boolean =>
|
|
5
|
+
scopePath === '' || filePath === scopePath || filePath.startsWith(`${scopePath}/`);
|
|
6
|
+
|
|
7
|
+
// Returns the single scopePaths entry filePath falls under, if any.
|
|
8
|
+
export const findContainingScope = (filePath: string, scopePaths: string[]): string | undefined =>
|
|
9
|
+
scopePaths.find((scopePath) => isPathWithinScope(filePath, scopePath));
|
|
10
|
+
|
|
11
|
+
// Returns filePath relative to scopePath (assumes isPathWithinScope(filePath, scopePath) is already true).
|
|
12
|
+
export const hoistPath = (filePath: string, scopePath: string): string =>
|
|
13
|
+
scopePath === '' ? filePath : filePath.slice(scopePath.length + 1);
|