apify-test-tools 0.5.7-beta.3 → 0.5.7-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/_check_code.yaml +17 -18
- package/.husky/pre-commit +13 -0
- package/CHANGELOG.md +3 -0
- package/CONTRIBUTING.md +5 -0
- package/README.md +39 -43
- package/bin/build.ts +138 -87
- package/bin/diff-changes.ts +52 -23
- package/bin/diff-json-schema.ts +10 -3
- package/bin/git.ts +6 -8
- package/bin/github.ts +7 -6
- package/bin/main.ts +14 -8
- package/bin/slack.ts +6 -2
- package/bin/test-report.ts +55 -44
- package/bin/types.ts +56 -59
- package/bin/utils.ts +17 -12
- package/dist/bin/build.d.ts +1 -1
- package/dist/bin/build.d.ts.map +1 -1
- package/dist/bin/build.js +49 -31
- package/dist/bin/build.js.map +1 -1
- package/dist/bin/diff-changes.d.ts +1 -1
- package/dist/bin/diff-changes.d.ts.map +1 -1
- package/dist/bin/diff-changes.js +15 -3
- package/dist/bin/diff-changes.js.map +1 -1
- package/dist/bin/diff-json-schema.d.ts.map +1 -1
- package/dist/bin/diff-json-schema.js +6 -1
- package/dist/bin/diff-json-schema.js.map +1 -1
- package/dist/bin/git.d.ts +0 -1
- package/dist/bin/git.d.ts.map +1 -1
- package/dist/bin/git.js +0 -5
- package/dist/bin/git.js.map +1 -1
- package/dist/bin/github.d.ts +1 -3
- package/dist/bin/github.d.ts.map +1 -1
- package/dist/bin/github.js +5 -5
- package/dist/bin/github.js.map +1 -1
- package/dist/bin/main.js +7 -1
- package/dist/bin/main.js.map +1 -1
- package/dist/bin/slack.d.ts +1 -1
- package/dist/bin/slack.d.ts.map +1 -1
- package/dist/bin/slack.js +1 -1
- package/dist/bin/slack.js.map +1 -1
- package/dist/bin/test-report.d.ts.map +1 -1
- package/dist/bin/test-report.js +9 -3
- package/dist/bin/test-report.js.map +1 -1
- package/dist/bin/types.d.ts +0 -1
- package/dist/bin/types.d.ts.map +1 -1
- package/dist/bin/utils.d.ts.map +1 -1
- package/dist/bin/utils.js +11 -7
- package/dist/bin/utils.js.map +1 -1
- package/dist/lib/extend-expect.d.ts.map +1 -1
- package/dist/lib/extend-expect.js +13 -11
- package/dist/lib/extend-expect.js.map +1 -1
- package/dist/lib/lib.d.ts +3 -2
- package/dist/lib/lib.d.ts.map +1 -1
- package/dist/lib/lib.js +7 -3
- package/dist/lib/lib.js.map +1 -1
- package/dist/lib/run-test-result.d.ts +1 -1
- package/dist/lib/run-test-result.d.ts.map +1 -1
- package/dist/lib/run-test-result.js.map +1 -1
- package/dist/lib/types.d.ts +2 -3
- package/dist/lib/types.d.ts.map +1 -1
- package/dist/lib/utils.d.ts +1 -1
- package/dist/lib/utils.d.ts.map +1 -1
- package/dist/lib/utils.js +4 -1
- package/dist/lib/utils.js.map +1 -1
- package/dist/test/unit/bin/diff-changes.test.js +14 -7
- package/dist/test/unit/bin/diff-changes.test.js.map +1 -1
- package/dist/test/unit/bin/diff.test.js +14 -4
- package/dist/test/unit/bin/diff.test.js.map +1 -1
- package/dist/test/unit/bin/git.test.js.map +1 -1
- package/dist/test/unit/custom-matchers.test.js +3 -8
- package/dist/test/unit/custom-matchers.test.js.map +1 -1
- package/dist/test/unit/should-built-and-test.test.js +75 -18
- package/dist/test/unit/should-built-and-test.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/eslint.config.mjs +4 -2
- package/knip.json +1 -1
- package/lib/extend-expect.ts +23 -21
- package/lib/lib.ts +17 -17
- package/lib/run-test-result.ts +6 -2
- package/lib/types.ts +5 -8
- package/lib/utils.ts +13 -9
- package/package.json +10 -3
- package/test/tsconfig.json +2 -4
- package/test/unit/bin/diff-changes.test.ts +14 -7
- package/test/unit/bin/diff.test.ts +19 -21
- package/test/unit/bin/git.test.ts +3 -1
- package/test/unit/custom-matchers.test.ts +10 -9
- package/test/unit/should-built-and-test.test.ts +77 -20
- package/bin/consts.ts +0 -6
- package/dist/bin/consts.d.ts +0 -4
- package/dist/bin/consts.d.ts.map +0 -1
- package/dist/bin/consts.js +0 -6
- package/dist/bin/consts.js.map +0 -1
- package/example-github-events/pull-request.json +0 -523
- package/example-github-events/push.json +0 -216
package/bin/build.ts
CHANGED
|
@@ -1,25 +1,36 @@
|
|
|
1
|
+
import type { Build } from 'apify-client';
|
|
2
|
+
import { ApifyClient } from 'apify-client';
|
|
3
|
+
|
|
1
4
|
import { ACTOR_SOURCE_TYPES } from '@apify/consts';
|
|
2
|
-
|
|
5
|
+
|
|
3
6
|
import type { ActorConfig, BuildData } from './types.js';
|
|
4
7
|
import { getEnvVar } from './utils.js';
|
|
5
8
|
|
|
6
9
|
type BuildPrActorOptions = {
|
|
7
|
-
buildTag?: string
|
|
8
|
-
versionNumber: string
|
|
9
|
-
gitRepoUrl: string
|
|
10
|
-
actorName: string
|
|
11
|
-
}
|
|
10
|
+
buildTag?: string;
|
|
11
|
+
versionNumber: string;
|
|
12
|
+
gitRepoUrl: string;
|
|
13
|
+
actorName: string;
|
|
14
|
+
};
|
|
12
15
|
class ApifyBuilder {
|
|
13
|
-
|
|
14
|
-
|
|
16
|
+
private constructor(
|
|
17
|
+
private readonly apifyClient: ApifyClient,
|
|
18
|
+
private readonly actorName: string,
|
|
19
|
+
) {}
|
|
15
20
|
|
|
16
21
|
// Usually 'latest' but not necessarily (can be e.g. 'version-0')
|
|
17
|
-
getDefaultVersionAndTag = async (): Promise<{
|
|
22
|
+
getDefaultVersionAndTag = async (): Promise<{
|
|
23
|
+
defaultBuildNumber: string;
|
|
24
|
+
defaultVersionNumber: string;
|
|
25
|
+
defaultBuildTag: string;
|
|
26
|
+
}> => {
|
|
18
27
|
const actorClient = this.apifyClient.actor(this.actorName);
|
|
19
28
|
const actorInfo = await actorClient.get();
|
|
20
29
|
|
|
21
30
|
if (!actorInfo) {
|
|
22
|
-
throw new Error(
|
|
31
|
+
throw new Error(
|
|
32
|
+
`[${this.actorName}] not found. It is not published or we are missing token to access it privately or its name is misspelled`,
|
|
33
|
+
);
|
|
23
34
|
}
|
|
24
35
|
|
|
25
36
|
const defaultBuildTag = actorInfo.defaultRunOptions.build;
|
|
@@ -27,8 +38,10 @@ class ApifyBuilder {
|
|
|
27
38
|
|
|
28
39
|
// We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
|
|
29
40
|
if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
|
|
30
|
-
throw new Error(
|
|
31
|
-
|
|
41
|
+
throw new Error(
|
|
42
|
+
`[${this.actorName}] Default build is a build number, not a tag. While this could work, ` +
|
|
43
|
+
`we want to have a default as tag so this is often an accidental misconfiguration from the dev`,
|
|
44
|
+
);
|
|
32
45
|
}
|
|
33
46
|
// I reported that buildNumber should probably not be optional
|
|
34
47
|
const defaultBuildNumber = actorInfo.taggedBuilds![defaultBuildTag].buildNumber!;
|
|
@@ -38,17 +51,15 @@ class ApifyBuilder {
|
|
|
38
51
|
return { defaultBuildNumber, defaultVersionNumber, defaultBuildTag };
|
|
39
52
|
};
|
|
40
53
|
|
|
41
|
-
startActorBuild = async ({
|
|
42
|
-
buildTag,
|
|
43
|
-
versionNumber,
|
|
44
|
-
gitRepoUrl,
|
|
45
|
-
}: BuildPrActorOptions): Promise<BuildData> => {
|
|
54
|
+
startActorBuild = async ({ buildTag, versionNumber, gitRepoUrl }: BuildPrActorOptions): Promise<BuildData> => {
|
|
46
55
|
const actorClient = this.apifyClient.actor(this.actorName);
|
|
47
56
|
const actorInfo = await actorClient.get();
|
|
48
57
|
if (!actorInfo) {
|
|
49
|
-
throw new Error(
|
|
50
|
-
|
|
51
|
-
|
|
58
|
+
throw new Error(
|
|
59
|
+
`No actor named '${this.actorName}' was found on the platform. If this` +
|
|
60
|
+
' is unexpected, make sure the actor you are targeting is spelled the' +
|
|
61
|
+
' same as the folder in the repository.',
|
|
62
|
+
);
|
|
52
63
|
}
|
|
53
64
|
|
|
54
65
|
// NOTE: I couldn't find this type, so I had to extract it :(
|
|
@@ -83,8 +94,9 @@ class ApifyBuilder {
|
|
|
83
94
|
const build = await this.apifyClient.build(buildId).waitForFinish();
|
|
84
95
|
const versionNumber = build.buildNumber;
|
|
85
96
|
if (build.status === 'FAILED' || build.status === 'TIMED-OUT') {
|
|
86
|
-
const message =
|
|
87
|
-
|
|
97
|
+
const message =
|
|
98
|
+
`[BUILD][${actorName}]: Build ${buildId} (${versionNumber}) failed. ` +
|
|
99
|
+
`Not continuing with other builds and tests.`;
|
|
88
100
|
console.error(`[${this.actorName}]: ${versionNumber}`);
|
|
89
101
|
throw new Error(message);
|
|
90
102
|
}
|
|
@@ -93,8 +105,8 @@ class ApifyBuilder {
|
|
|
93
105
|
};
|
|
94
106
|
|
|
95
107
|
/**
|
|
96
|
-
|
|
97
|
-
|
|
108
|
+
* Create ApifyBuilder with actor owner's token
|
|
109
|
+
*/
|
|
98
110
|
static fromActorName = (actorName: string): ApifyBuilder => {
|
|
99
111
|
const username = actorName.split('/')[0];
|
|
100
112
|
// GitHib secrets only allow word characters (alphanum + underscore)
|
|
@@ -102,8 +114,10 @@ class ApifyBuilder {
|
|
|
102
114
|
const usernameEnvVar = `APIFY_TOKEN_${usernameInGitHubSecretsFormat}`;
|
|
103
115
|
const token = process.env[usernameEnvVar];
|
|
104
116
|
if (!token) {
|
|
105
|
-
throw new Error(
|
|
106
|
-
|
|
117
|
+
throw new Error(
|
|
118
|
+
`Cannot find Apify API token for username: ${username}. ` +
|
|
119
|
+
`Have you set secret env var to this GitHub repo with key: ${usernameEnvVar}?`,
|
|
120
|
+
);
|
|
107
121
|
}
|
|
108
122
|
const apifyClient = new ApifyClient({ token });
|
|
109
123
|
const builder = new ApifyBuilder(apifyClient, actorName);
|
|
@@ -118,7 +132,21 @@ class ApifyBuilder {
|
|
|
118
132
|
// Even though we don't version our current Actors, if we ever such Actors to GitHub CI, we would accidentally delete old supported versions
|
|
119
133
|
// This hardcoded solution is not ideal, but it should prevent most imaginable cases
|
|
120
134
|
// All currently popular versioned Actors use `version-${number}` format
|
|
121
|
-
const PROTECTED_TAGS_PREFIX = [
|
|
135
|
+
const PROTECTED_TAGS_PREFIX = [
|
|
136
|
+
'latest',
|
|
137
|
+
'v-',
|
|
138
|
+
'version',
|
|
139
|
+
'v0',
|
|
140
|
+
'v1',
|
|
141
|
+
'v2',
|
|
142
|
+
'v3',
|
|
143
|
+
'v4',
|
|
144
|
+
'v5',
|
|
145
|
+
'v6',
|
|
146
|
+
'v7',
|
|
147
|
+
'v8',
|
|
148
|
+
'v9',
|
|
149
|
+
];
|
|
122
150
|
|
|
123
151
|
// We don't want to be too short because we might to debug something
|
|
124
152
|
// but also not too long because it increases the risk of users using outdated versions
|
|
@@ -132,43 +160,58 @@ class ApifyBuilder {
|
|
|
132
160
|
|
|
133
161
|
const allTags = Object.keys(actorInfo.taggedBuilds ?? {});
|
|
134
162
|
const protectedTags = allTags.filter((tag) => PROTECTED_TAGS_PREFIX.some((prefix) => tag.startsWith(prefix)));
|
|
135
|
-
const protectedBuildNumbers = protectedTags.map((tag) => ({
|
|
163
|
+
const protectedBuildNumbers = protectedTags.map((tag) => ({
|
|
164
|
+
buildNumber: actorInfo.taggedBuilds![tag]!.buildNumber,
|
|
165
|
+
tag,
|
|
166
|
+
}));
|
|
136
167
|
|
|
137
|
-
const { items } =
|
|
168
|
+
const { items } = await this.apifyClient.actor(this.actorName).builds().list();
|
|
138
169
|
|
|
139
170
|
// Deleting default build throws an error, so we skip it
|
|
140
|
-
const { defaultBuildNumber, defaultBuildTag } = await ApifyBuilder.fromActorName(
|
|
171
|
+
const { defaultBuildNumber, defaultBuildTag } = await ApifyBuilder.fromActorName(
|
|
172
|
+
this.actorName,
|
|
173
|
+
).getDefaultVersionAndTag();
|
|
141
174
|
|
|
142
175
|
const daysAgoUnixProd = Date.now() - DEFAULT_DAYS_BACK_PROD_VERSIONS * 24 * 60 * 60 * 1000;
|
|
143
176
|
const daysAgoUnixDevel = Date.now() - DEFAULT_DAYS_BACK_DEVEL * 24 * 60 * 60 * 1000;
|
|
144
177
|
|
|
145
178
|
// Fixing API client missing buildNumber field
|
|
146
|
-
type CorrectBuildColletionItem = typeof items[0] & { buildNumber: string };
|
|
179
|
+
type CorrectBuildColletionItem = (typeof items)[0] & { buildNumber: string };
|
|
147
180
|
const buildsToDelete = (items as CorrectBuildColletionItem[]).filter((build) => {
|
|
148
181
|
if (build.buildNumber === defaultBuildNumber) {
|
|
149
|
-
console.error(
|
|
150
|
-
|
|
182
|
+
console.error(
|
|
183
|
+
`[DELETE OLD BUILDS][${this.actorName}]: Skipping default build ${defaultBuildNumber} (${defaultBuildTag}). ` +
|
|
184
|
+
`We never delete default builds`,
|
|
185
|
+
);
|
|
151
186
|
return false;
|
|
152
187
|
}
|
|
153
188
|
|
|
154
|
-
const protectedTagFound = protectedBuildNumbers.find(
|
|
189
|
+
const protectedTagFound = protectedBuildNumbers.find(
|
|
190
|
+
(protectedBuildNumber) => protectedBuildNumber.buildNumber === build.buildNumber,
|
|
191
|
+
);
|
|
155
192
|
if (protectedTagFound) {
|
|
156
|
-
console.error(
|
|
193
|
+
console.error(
|
|
194
|
+
`[DELETE OLD BUILDS][${this.actorName}]: Skipping protected build ${protectedTagFound.buildNumber} (${protectedTagFound.tag}).`,
|
|
195
|
+
);
|
|
157
196
|
return false;
|
|
158
197
|
}
|
|
159
198
|
|
|
160
199
|
if (taggedDevelBuildNumber && build.buildNumber === taggedDevelBuildNumber) {
|
|
161
200
|
const shouldDeleteDevelBuild = build.startedAt.getTime() < daysAgoUnixDevel;
|
|
162
201
|
if (shouldDeleteDevelBuild) {
|
|
163
|
-
console.error(
|
|
202
|
+
console.error(
|
|
203
|
+
`[DELETE OLD BUILDS][${this.actorName}]: Removing olf devel build ${taggedDevelBuildNumber}.`,
|
|
204
|
+
);
|
|
164
205
|
}
|
|
165
206
|
return shouldDeleteDevelBuild;
|
|
166
207
|
}
|
|
167
208
|
return build.startedAt.getTime() < daysAgoUnixProd;
|
|
168
209
|
});
|
|
169
210
|
|
|
170
|
-
console.error(
|
|
171
|
-
|
|
211
|
+
console.error(
|
|
212
|
+
`[DELETE OLD BUILDS][${this.actorName}]: Deleting ${buildsToDelete.length} old builds that are non-default and ` +
|
|
213
|
+
`older than 30 days from total ${items.length}`,
|
|
214
|
+
);
|
|
172
215
|
for (const build of buildsToDelete) {
|
|
173
216
|
await this.apifyClient.build(build.id).delete();
|
|
174
217
|
}
|
|
@@ -176,20 +219,14 @@ class ApifyBuilder {
|
|
|
176
219
|
}
|
|
177
220
|
|
|
178
221
|
type RunBuildsOptions = {
|
|
179
|
-
actorConfigs: ActorConfig[]
|
|
180
|
-
isLatest?: boolean
|
|
181
|
-
repoUrl: string
|
|
182
|
-
branch: string
|
|
183
|
-
dryRun: boolean
|
|
184
|
-
}
|
|
222
|
+
actorConfigs: ActorConfig[];
|
|
223
|
+
isLatest?: boolean;
|
|
224
|
+
repoUrl: string;
|
|
225
|
+
branch: string;
|
|
226
|
+
dryRun: boolean;
|
|
227
|
+
};
|
|
185
228
|
|
|
186
|
-
export const runBuilds = async ({
|
|
187
|
-
repoUrl,
|
|
188
|
-
branch,
|
|
189
|
-
actorConfigs,
|
|
190
|
-
isLatest = false,
|
|
191
|
-
dryRun,
|
|
192
|
-
}: RunBuildsOptions) => {
|
|
229
|
+
export const runBuilds = async ({ repoUrl, branch, actorConfigs, isLatest = false, dryRun }: RunBuildsOptions) => {
|
|
193
230
|
const buildConfigs: BuildPrActorOptions[] = [];
|
|
194
231
|
|
|
195
232
|
const circleActors = isLatest ? await findCircleApifyManaged(actorConfigs) : [];
|
|
@@ -199,7 +236,8 @@ export const runBuilds = async ({
|
|
|
199
236
|
let buildTag: string | undefined;
|
|
200
237
|
|
|
201
238
|
if (isLatest) {
|
|
202
|
-
const { defaultVersionNumber, defaultBuildTag } =
|
|
239
|
+
const { defaultVersionNumber, defaultBuildTag } =
|
|
240
|
+
await ApifyBuilder.fromActorName(actorName).getDefaultVersionAndTag();
|
|
203
241
|
versionNumber = defaultVersionNumber;
|
|
204
242
|
buildTag = defaultBuildTag;
|
|
205
243
|
} else {
|
|
@@ -217,25 +255,29 @@ export const runBuilds = async ({
|
|
|
217
255
|
if (dryRun) {
|
|
218
256
|
return buildConfigs;
|
|
219
257
|
}
|
|
220
|
-
console.error(
|
|
221
|
-
console.error(
|
|
222
|
-
const startedBuilds = await Promise.all(
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
258
|
+
console.error('=========================================');
|
|
259
|
+
console.error('STARTED BUILDS:');
|
|
260
|
+
const startedBuilds = await Promise.all(
|
|
261
|
+
buildConfigs.map(async (buildConfig) => {
|
|
262
|
+
const builder = ApifyBuilder.fromActorName(buildConfig.actorName);
|
|
263
|
+
const buildData = await builder.startActorBuild(buildConfig);
|
|
264
|
+
return buildData;
|
|
265
|
+
}),
|
|
266
|
+
);
|
|
267
|
+
console.error('=========================================');
|
|
268
|
+
console.error('FINISHED BUILDS:');
|
|
269
|
+
await Promise.all(
|
|
270
|
+
startedBuilds.map(async (buildData) => {
|
|
271
|
+
const builder = ApifyBuilder.fromActorName(buildData.actorName);
|
|
272
|
+
await builder.waitForBuildToFinish(buildData.buildId, buildData.actorName);
|
|
273
|
+
}),
|
|
274
|
+
);
|
|
275
|
+
console.error('=========================================');
|
|
276
|
+
console.error('SUMMARY:');
|
|
235
277
|
for (const buildData of startedBuilds.sort((a, b) => a.actorName.localeCompare(b.actorName))) {
|
|
236
278
|
console.error(`[${buildData.actorName}]: ${buildData.buildNumber} `);
|
|
237
279
|
}
|
|
238
|
-
console.error(
|
|
280
|
+
console.error('=========================================');
|
|
239
281
|
|
|
240
282
|
return startedBuilds;
|
|
241
283
|
};
|
|
@@ -259,27 +301,36 @@ const findCircleApifyManaged = async (actorConfigs: ActorConfig[]): Promise<Acto
|
|
|
259
301
|
|
|
260
302
|
const { items: circleActors } = await client.actors().list();
|
|
261
303
|
|
|
262
|
-
const actorsToBuild = circleActors
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
304
|
+
const actorsToBuild = circleActors
|
|
305
|
+
.map<ActorConfig | undefined>((circleActor) => {
|
|
306
|
+
// They prefix all with apify-managed---, I communicated with Jacques to keep doing that
|
|
307
|
+
let actorConfigFound = actorConfigs.find(
|
|
308
|
+
(actorConfig) =>
|
|
309
|
+
circleActor.name.replace('apify-managed---', '') === actorConfig.actorName.split('/')[1],
|
|
310
|
+
);
|
|
311
|
+
|
|
312
|
+
// Hack for bad naming of circ_le/apify-managed-google-search, we don't want to rename now to break customers
|
|
313
|
+
if (!actorConfigFound && circleActor.name === 'apify-managed-google-search') {
|
|
314
|
+
actorConfigFound = actorConfigs.find(
|
|
315
|
+
(actorConfig) => actorConfig.actorName.split('/')[1] === 'google-search-scraper',
|
|
316
|
+
);
|
|
317
|
+
}
|
|
270
318
|
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
319
|
+
if (actorConfigFound) {
|
|
320
|
+
return {
|
|
321
|
+
// We point the circle Actor to the repo folder
|
|
322
|
+
actorName: `${circleActor.username}/${circleActor.name}`,
|
|
323
|
+
folder: actorConfigFound.folder,
|
|
324
|
+
isStandalone: actorConfigFound.isStandalone,
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
return undefined;
|
|
328
|
+
})
|
|
329
|
+
.filter((config) => config !== undefined);
|
|
281
330
|
|
|
282
|
-
console.error(
|
|
331
|
+
console.error(
|
|
332
|
+
`Found ${actorsToBuild.length} circ_le actors that match Actors we built out of total ${circleActors.length} circ_le actors`,
|
|
333
|
+
);
|
|
283
334
|
console.error(`All circ_le actors: ${circleActors.map((actor) => actor.name).join(', ')}`);
|
|
284
335
|
console.error(`circ_le Actors to build: ${actorsToBuild.map((actor) => actor.actorName).join(', ')}`);
|
|
285
336
|
|
package/bin/diff-changes.ts
CHANGED
|
@@ -8,35 +8,48 @@ interface ShouldBuildAndTestOptions {
|
|
|
8
8
|
commits: Commit[];
|
|
9
9
|
}
|
|
10
10
|
|
|
11
|
-
export const maybeParseActorFolder = (
|
|
11
|
+
export const maybeParseActorFolder = (
|
|
12
|
+
lowercaseFilePath: string,
|
|
13
|
+
): { isActorFolder: true; actorName: string } | { isActorFolder: false } => {
|
|
12
14
|
const match = lowercaseFilePath.match(/^(?:standalone-)?actors\/([^/]+)\/.+/);
|
|
13
15
|
if (match) {
|
|
14
16
|
// Some usernames weirdly use underscores, e.g. google_maps_email_extractor_standby-contact-details-scraper so we only need replace the last one
|
|
15
17
|
return { isActorFolder: true, actorName: match[1].replace(/_(?=[^_]*$)/, '/') };
|
|
16
18
|
}
|
|
17
19
|
return { isActorFolder: false };
|
|
18
|
-
}
|
|
20
|
+
};
|
|
19
21
|
|
|
20
22
|
/**
|
|
21
23
|
* Also works for folders
|
|
22
24
|
*/
|
|
23
25
|
const isIgnoredTopLevelFile = (lowercaseFilePath: string) => {
|
|
24
26
|
// On top level, we should only have dev-only readme and .actor/ is just for apify push CLI (real Actor configs are in /actors)
|
|
25
|
-
const IGNORED_TOP_LEVEL_FILES = [
|
|
27
|
+
const IGNORED_TOP_LEVEL_FILES = [
|
|
28
|
+
'.vscode/',
|
|
29
|
+
'.gitignore',
|
|
30
|
+
'readme.md',
|
|
31
|
+
'.husky/',
|
|
32
|
+
'.eslintrc',
|
|
33
|
+
'eslint.config.mjs',
|
|
34
|
+
'.prettierrc',
|
|
35
|
+
'.editorconfig',
|
|
36
|
+
'.actor/',
|
|
37
|
+
];
|
|
26
38
|
// Strip out deprecated /code and /shared folders, treat them as top-level code
|
|
27
39
|
const sanitizedLowercaseFilePath = lowercaseFilePath.replace(/^code\//, '').replace(/^shared\//, '');
|
|
28
40
|
|
|
29
41
|
return IGNORED_TOP_LEVEL_FILES.some((ignoredFile) => sanitizedLowercaseFilePath.startsWith(ignoredFile));
|
|
30
42
|
};
|
|
31
43
|
|
|
32
|
-
type FileChange =
|
|
33
|
-
{ impact: 'ignored' }
|
|
44
|
+
type FileChange =
|
|
45
|
+
| { impact: 'ignored' }
|
|
34
46
|
// Only things that influence how the Actor looks - e.g. README and CHANGELOG files, schema titles, descriptions, reordering, etc. We only need to rebuild on release
|
|
35
|
-
{ impact: 'cosmetic'
|
|
47
|
+
| { impact: 'cosmetic'; includes: 'all-actors' | ActorConfig }
|
|
36
48
|
// Influences how the Actor works - we need to run tests
|
|
37
|
-
{
|
|
38
|
-
|
|
39
|
-
|
|
49
|
+
| {
|
|
50
|
+
impact: 'functional';
|
|
51
|
+
includes: 'all-actors' | ActorConfig;
|
|
52
|
+
};
|
|
40
53
|
|
|
41
54
|
const classifyFileChange = (lowercaseFilePath: string, actorConfigs: ActorConfig[], commits: Commit[]): FileChange => {
|
|
42
55
|
if (isIgnoredTopLevelFile(lowercaseFilePath)) {
|
|
@@ -49,13 +62,18 @@ const classifyFileChange = (lowercaseFilePath: string, actorConfigs: ActorConfig
|
|
|
49
62
|
|
|
50
63
|
const actorFolderInfo = maybeParseActorFolder(lowercaseFilePath);
|
|
51
64
|
if (actorFolderInfo.isActorFolder) {
|
|
52
|
-
const actorConfigChanged = actorConfigs.find(
|
|
65
|
+
const actorConfigChanged = actorConfigs.find(
|
|
66
|
+
({ actorName }) => actorName.toLowerCase() === actorFolderInfo.actorName,
|
|
67
|
+
);
|
|
53
68
|
// This is some super weird case that happened once in the past but I don't remember the context anymore
|
|
54
69
|
if (actorConfigChanged === undefined) {
|
|
55
|
-
console.error(
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
70
|
+
console.error(
|
|
71
|
+
'SHOULD NEVER HAPPEN: changes was found in an actor folder which no longer exists in the current commit, skipping this file',
|
|
72
|
+
{
|
|
73
|
+
actorName: actorFolderInfo.actorName,
|
|
74
|
+
lowercaseFilePath,
|
|
75
|
+
},
|
|
76
|
+
);
|
|
59
77
|
return { impact: 'ignored' };
|
|
60
78
|
}
|
|
61
79
|
if (lowercaseFilePath.endsWith('readme.md')) {
|
|
@@ -70,11 +88,14 @@ const classifyFileChange = (lowercaseFilePath: string, actorConfigs: ActorConfig
|
|
|
70
88
|
|
|
71
89
|
// For any other files, we assume they can interact with the code
|
|
72
90
|
return { impact: 'functional', includes: 'all-actors' };
|
|
73
|
-
}
|
|
91
|
+
};
|
|
74
92
|
|
|
75
|
-
export const getChangedActors = (
|
|
76
|
-
|
|
77
|
-
|
|
93
|
+
export const getChangedActors = ({
|
|
94
|
+
filepathsChanged,
|
|
95
|
+
actorConfigs,
|
|
96
|
+
isLatest = false,
|
|
97
|
+
commits,
|
|
98
|
+
}: ShouldBuildAndTestOptions): ActorConfig[] => {
|
|
78
99
|
// folder -> ActorConfig
|
|
79
100
|
const actorsChangedMap = new Map<string, ActorConfig>();
|
|
80
101
|
|
|
@@ -105,23 +126,31 @@ export const getChangedActors = (
|
|
|
105
126
|
const actorsChanged = Array.from(actorsChangedMap.values());
|
|
106
127
|
|
|
107
128
|
// All below here is just for logging
|
|
108
|
-
const ignoredFilesChanged = lowercaseFiles.filter(
|
|
129
|
+
const ignoredFilesChanged = lowercaseFiles.filter(
|
|
130
|
+
(file) => classifyFileChange(file, actorConfigs, commits).impact === 'ignored',
|
|
131
|
+
);
|
|
109
132
|
console.error(`[DIFF]: Ignored files (don't trigger test or build): ${ignoredFilesChanged.join(', ')}`);
|
|
110
133
|
|
|
111
|
-
const cosmeticFilesChanged = lowercaseFiles.filter(
|
|
134
|
+
const cosmeticFilesChanged = lowercaseFiles.filter(
|
|
135
|
+
(file) => classifyFileChange(file, actorConfigs, commits).impact === 'cosmetic',
|
|
136
|
+
);
|
|
112
137
|
console.error(`[DIFF]: Cosmetic files (should only trigger release build): ${cosmeticFilesChanged.join(', ')}`);
|
|
113
138
|
|
|
114
|
-
const functionalFilesChanged = lowercaseFiles.filter(
|
|
139
|
+
const functionalFilesChanged = lowercaseFiles.filter(
|
|
140
|
+
(file) => classifyFileChange(file, actorConfigs, commits).impact === 'functional',
|
|
141
|
+
);
|
|
115
142
|
console.error(`[DIFF]: Functional files (trigger test & release build): ${functionalFilesChanged.join(', ')}`);
|
|
116
143
|
|
|
117
144
|
if (actorsChanged.length > 0) {
|
|
118
145
|
const miniactors = actorsChanged.filter((config) => !config.isStandalone).map((config) => config.actorName);
|
|
119
|
-
const standaloneActors = actorsChanged
|
|
146
|
+
const standaloneActors = actorsChanged
|
|
147
|
+
.filter((config) => config.isStandalone)
|
|
148
|
+
.map((config) => config.actorName);
|
|
120
149
|
console.error(`[DIFF]: MiniActors to be built and tested: ${miniactors.join(', ')}`);
|
|
121
150
|
console.error(`[DIFF]: Standalone Actors to be built and tested: ${standaloneActors.join(', ')}`);
|
|
122
151
|
} else {
|
|
123
152
|
console.error(`[DIFF]: No relevant files changed, skipping builds and tests`);
|
|
124
153
|
}
|
|
125
|
-
|
|
154
|
+
|
|
126
155
|
return actorsChanged;
|
|
127
156
|
};
|
package/bin/diff-json-schema.ts
CHANGED
|
@@ -2,7 +2,12 @@ import type { Commit } from './types.js';
|
|
|
2
2
|
import { spawnCommandInGhWorkspace } from './utils.js';
|
|
3
3
|
|
|
4
4
|
const COSMETIC_JSON_FIELD_NAMES = new Set([
|
|
5
|
-
'title',
|
|
5
|
+
'title',
|
|
6
|
+
'description',
|
|
7
|
+
'example',
|
|
8
|
+
'enumTitles',
|
|
9
|
+
'sectionCaption',
|
|
10
|
+
'sectionDescription',
|
|
6
11
|
]);
|
|
7
12
|
|
|
8
13
|
const isPlainObject = (val: unknown): val is Record<string, unknown> =>
|
|
@@ -32,11 +37,13 @@ export const isCosmeticOnlyJsonSchemaChange = (commits: Commit[], changedFilepat
|
|
|
32
37
|
try {
|
|
33
38
|
const oldContent = spawnCommandInGhWorkspace(`git show ${oldRef}:${changedFilepath}`);
|
|
34
39
|
const newContent = spawnCommandInGhWorkspace(`git show ${newRef}:${changedFilepath}`);
|
|
35
|
-
|
|
40
|
+
|
|
36
41
|
oldJson = JSON.parse(oldContent);
|
|
37
42
|
newJson = JSON.parse(newContent);
|
|
38
43
|
} catch {
|
|
39
|
-
console.error(
|
|
44
|
+
console.error(
|
|
45
|
+
`Failed to get or parse JSON content for ${changedFilepath} at refs ${oldRef} and ${newRef}, maybe it is new file or deleted? Treating it as a non-cosmetic change.`,
|
|
46
|
+
);
|
|
40
47
|
return false;
|
|
41
48
|
}
|
|
42
49
|
return isCosmeticObjectChange(oldJson, newJson);
|
package/bin/git.ts
CHANGED
|
@@ -33,22 +33,20 @@ export const getCommits = ({ sourceBranch, targetBranch, baseCommit: baseCommitS
|
|
|
33
33
|
const hasBaseCommit = baseCommitIndex !== -1;
|
|
34
34
|
if (hasBaseCommit) {
|
|
35
35
|
const commitsUpToBaseCommit = commits.slice(baseCommitIndex + 1);
|
|
36
|
-
console.error(
|
|
36
|
+
console.error(
|
|
37
|
+
`Found base commit ${baseCommitSha} at index ${baseCommitIndex}, returning ${commitsUpToBaseCommit.length} commits after it`,
|
|
38
|
+
);
|
|
37
39
|
console.error(`Commits being returned: ${commitsUpToBaseCommit.map((c) => c.sha).join(', ')}`);
|
|
38
40
|
return commitsUpToBaseCommit;
|
|
39
41
|
}
|
|
40
42
|
|
|
41
|
-
console.error(
|
|
43
|
+
console.error(
|
|
44
|
+
`Base commit ${baseCommitSha} not found in the commit range, returning all ${commits.length} commits`,
|
|
45
|
+
);
|
|
42
46
|
console.error(`Commits being returned: ${commits.map((c) => c.sha).join(', ')}`);
|
|
43
47
|
return commits;
|
|
44
48
|
};
|
|
45
49
|
|
|
46
|
-
export const getCommitInfo = (commitSha: string): Commit => {
|
|
47
|
-
const commitString = spawnCommandInGhWorkspace(`git log -1 --pretty=format:'${GIT_LOG_FORMAT}' ${commitSha}`);
|
|
48
|
-
const commit = parseCommit(commitString);
|
|
49
|
-
return commit;
|
|
50
|
-
};
|
|
51
|
-
|
|
52
50
|
export const parseCommit = (commitString: string): Commit => {
|
|
53
51
|
const splits = commitString.split(GIT_FORMAT_SEPARATOR);
|
|
54
52
|
if (splits.length !== 4) {
|
package/bin/github.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import fs from 'fs/promises';
|
|
2
|
-
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
|
|
3
|
+
import type { Commit, GitHubEventPush } from './types.js';
|
|
3
4
|
import { spawnCommandInGhWorkspace } from './utils.js';
|
|
4
5
|
|
|
5
6
|
export const getPushData = async (path: string) => {
|
|
@@ -10,8 +11,8 @@ export const getPushData = async (path: string) => {
|
|
|
10
11
|
repository: { ssh_url: repoUrl, name },
|
|
11
12
|
head_commit: { author },
|
|
12
13
|
} = event;
|
|
13
|
-
const commits: Commit[] = ghCommits.map(({ author, message, id, timestamp }) => ({
|
|
14
|
-
author:
|
|
14
|
+
const commits: Commit[] = ghCommits.map(({ author: commitAuthor, message, id, timestamp }) => ({
|
|
15
|
+
author: commitAuthor.username,
|
|
15
16
|
sha: id,
|
|
16
17
|
message,
|
|
17
18
|
date: timestamp,
|
|
@@ -29,7 +30,7 @@ export const getPushData = async (path: string) => {
|
|
|
29
30
|
};
|
|
30
31
|
};
|
|
31
32
|
|
|
32
|
-
|
|
33
|
+
const loadGitHubEvent = async (path: string): Promise<GitHubEventPush> => {
|
|
33
34
|
const pushEvent = JSON.parse((await fs.readFile(path)).toString()) as GitHubEventPush;
|
|
34
35
|
return pushEvent;
|
|
35
36
|
};
|
|
@@ -39,7 +40,7 @@ const getChangedFiles = ({ after, before }: GitHubEventPush): string[] => {
|
|
|
39
40
|
return changedFiles.split('\n');
|
|
40
41
|
};
|
|
41
42
|
|
|
42
|
-
|
|
43
|
+
const getChangelogChanges = (changedFiles: string[], event: GitHubEventPush): string | null => {
|
|
43
44
|
const changelogPath = 'CHANGELOG.md';
|
|
44
45
|
if (!changedFiles.includes(changelogPath)) {
|
|
45
46
|
return null;
|