apify-test-tools 0.10.2-beta.6 → 0.10.2-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/cli/apify-test-tools.mjs +62 -62
- package/CHANGELOG.md +1 -0
- package/bin/git.ts +5 -1
- package/dist/bin/git.d.ts.map +1 -1
- package/dist/bin/git.js +4 -1
- package/dist/bin/git.js.map +1 -1
- package/dist/lib/extend-expect.d.ts.map +1 -1
- package/dist/lib/extend-expect.js +37 -218
- package/dist/lib/extend-expect.js.map +1 -1
- package/dist/lib/lib.d.ts +1 -2
- package/dist/lib/lib.d.ts.map +1 -1
- package/dist/lib/lib.js +15 -22
- package/dist/lib/lib.js.map +1 -1
- package/dist/lib/run-checks.d.ts +14 -0
- package/dist/lib/run-checks.d.ts.map +1 -0
- package/dist/lib/run-checks.js +82 -0
- package/dist/lib/run-checks.js.map +1 -0
- package/dist/lib/run-test-result.d.ts +3 -6
- package/dist/lib/run-test-result.d.ts.map +1 -1
- package/dist/lib/run-test-result.js +41 -54
- package/dist/lib/run-test-result.js.map +1 -1
- package/dist/test/unit/bin/affected-actors-cli.test.js +6 -0
- package/dist/test/unit/bin/affected-actors-cli.test.js.map +1 -1
- package/dist/test/unit/bin/git.test.js +3 -3
- package/dist/test/unit/bin/git.test.js.map +1 -1
- package/dist/test/unit/run-checks.test.d.ts +2 -0
- package/dist/test/unit/run-checks.test.d.ts.map +1 -0
- package/dist/test/unit/run-checks.test.js +62 -0
- package/dist/test/unit/run-checks.test.js.map +1 -0
- package/dist/test/unit/run-test-result.test.d.ts +2 -0
- package/dist/test/unit/run-test-result.test.d.ts.map +1 -0
- package/dist/test/unit/run-test-result.test.js +34 -0
- package/dist/test/unit/run-test-result.test.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/lib/extend-expect.ts +48 -258
- package/lib/lib.ts +24 -28
- package/lib/run-checks.ts +101 -0
- package/lib/run-test-result.ts +45 -55
- package/package.json +1 -1
- package/test/unit/bin/affected-actors-cli.test.ts +8 -0
- package/test/unit/bin/git.test.ts +8 -5
- package/test/unit/run-checks.test.ts +76 -0
- package/test/unit/run-test-result.test.ts +44 -0
package/lib/extend-expect.ts
CHANGED
|
@@ -1,106 +1,45 @@
|
|
|
1
|
+
import { inspect } from 'node:util';
|
|
2
|
+
|
|
1
3
|
import type { Assertion, ExpectStatic } from 'vitest';
|
|
2
4
|
|
|
3
|
-
import {
|
|
5
|
+
import { checkRun } from './run-checks.js';
|
|
4
6
|
import type { RunTestResult } from './run-test-result.js';
|
|
5
|
-
import type {
|
|
7
|
+
import type { ToFinishWithOptions } from './types.js';
|
|
8
|
+
|
|
9
|
+
/** Matchers that only check the type or shape of a value. */
|
|
10
|
+
const SHAPE_MATCHERS = {
|
|
11
|
+
toBeArray: ['an array', (actual: unknown) => Array.isArray(actual)],
|
|
12
|
+
toBeBoolean: ['a boolean', (actual: unknown) => typeof actual === 'boolean'],
|
|
13
|
+
toBeEmptyArray: ['an empty array', (actual: unknown) => Array.isArray(actual) && actual.length === 0],
|
|
14
|
+
toBeNonEmptyArray: ['a non-empty array', (actual: unknown) => Array.isArray(actual) && actual.length > 0],
|
|
15
|
+
toBeNonEmptyString: ['a non-empty string', (actual: unknown) => typeof actual === 'string' && actual.length > 0],
|
|
16
|
+
toBeNumber: ['a number', (actual: unknown) => typeof actual === 'number'],
|
|
17
|
+
toBeFalse: ['false', (actual: unknown) => actual === false],
|
|
18
|
+
toBeTrue: ['true', (actual: unknown) => actual === true],
|
|
19
|
+
toBeNonEmptyObject: ['a non-empty object', (actual: unknown) => isObject(actual) && Object.keys(actual).length > 0],
|
|
20
|
+
toBeObject: ['an object', isObject],
|
|
21
|
+
toBeString: ['a string', (actual: unknown) => typeof actual === 'string'],
|
|
22
|
+
toBeWholeNumber: ['a whole number', (actual: unknown) => Number.isInteger(actual)],
|
|
23
|
+
} as const satisfies Record<string, readonly [string, (actual: unknown) => boolean]>;
|
|
24
|
+
|
|
25
|
+
function isObject(actual: unknown): actual is object {
|
|
26
|
+
return typeof actual === 'object' && !Array.isArray(actual) && actual !== null;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const shapeMatchers = Object.fromEntries(
|
|
30
|
+
Object.entries(SHAPE_MATCHERS).map(([name, [description, predicate]]) => [
|
|
31
|
+
name,
|
|
32
|
+
(actual: unknown) => ({
|
|
33
|
+
pass: predicate(actual),
|
|
34
|
+
message: () => `Expected ${inspect(actual)} to be ${description}`,
|
|
35
|
+
actual,
|
|
36
|
+
}),
|
|
37
|
+
]),
|
|
38
|
+
);
|
|
6
39
|
|
|
7
40
|
export const extendExpect = (expect: ExpectStatic): ExpectStatic => {
|
|
8
41
|
expect.extend({
|
|
9
|
-
|
|
10
|
-
const pass = Array.isArray(actual);
|
|
11
|
-
return {
|
|
12
|
-
pass,
|
|
13
|
-
message: () => `Expected "${actual}" to be a array`,
|
|
14
|
-
};
|
|
15
|
-
},
|
|
16
|
-
toBeBoolean: (received) => {
|
|
17
|
-
const actual = typeof received;
|
|
18
|
-
const expected = 'boolean';
|
|
19
|
-
return {
|
|
20
|
-
pass: actual === expected,
|
|
21
|
-
message: () => `Expected "${actual}" to be a boolean`,
|
|
22
|
-
expected: 'boolean',
|
|
23
|
-
};
|
|
24
|
-
},
|
|
25
|
-
toBeEmptyArray: (actual) => {
|
|
26
|
-
const pass = Array.isArray(actual) && actual.length === 0;
|
|
27
|
-
return {
|
|
28
|
-
pass,
|
|
29
|
-
message: () => `Expected ${actual} to be an empty array`,
|
|
30
|
-
actual,
|
|
31
|
-
expected: [],
|
|
32
|
-
};
|
|
33
|
-
},
|
|
34
|
-
toBeNonEmptyArray: (actual) => {
|
|
35
|
-
const pass = Array.isArray(actual) && actual.length > 0;
|
|
36
|
-
return {
|
|
37
|
-
pass,
|
|
38
|
-
message: () => `Expected ${actual} to be a non-empty array`,
|
|
39
|
-
};
|
|
40
|
-
},
|
|
41
|
-
toBeNonEmptyString: (actual) => {
|
|
42
|
-
const pass = typeof actual === 'string' && actual.length > 0;
|
|
43
|
-
return {
|
|
44
|
-
pass,
|
|
45
|
-
message: () => `Expected "${actual}" to be a non-empty string`,
|
|
46
|
-
};
|
|
47
|
-
},
|
|
48
|
-
toBeNumber: (actual) => {
|
|
49
|
-
const pass = typeof actual === 'number';
|
|
50
|
-
return {
|
|
51
|
-
pass,
|
|
52
|
-
message: () => `Expected "${actual}" to be a number`,
|
|
53
|
-
};
|
|
54
|
-
},
|
|
55
|
-
toBeNonEmptyObject: (actual) => {
|
|
56
|
-
const isObject = typeof actual === 'object' && !Array.isArray(actual) && actual !== null;
|
|
57
|
-
const isNonEmpty = Object.entries(actual).length > 0;
|
|
58
|
-
const pass = isObject && isNonEmpty;
|
|
59
|
-
return {
|
|
60
|
-
pass,
|
|
61
|
-
message: () => `Expected "${actual}" to be a non-emtpy object`,
|
|
62
|
-
};
|
|
63
|
-
},
|
|
64
|
-
toBeObject: (actual) => {
|
|
65
|
-
const pass = typeof actual === 'object' && !Array.isArray(actual) && actual !== null;
|
|
66
|
-
return {
|
|
67
|
-
pass,
|
|
68
|
-
message: () => `Expected "${actual}" to be an object`,
|
|
69
|
-
};
|
|
70
|
-
},
|
|
71
|
-
toBeString: (received) => {
|
|
72
|
-
const actual = typeof received;
|
|
73
|
-
const expected = 'string';
|
|
74
|
-
return {
|
|
75
|
-
pass: actual === expected,
|
|
76
|
-
message: () => `Expected "${actual}" to be a string`,
|
|
77
|
-
actual,
|
|
78
|
-
expected,
|
|
79
|
-
};
|
|
80
|
-
},
|
|
81
|
-
toBeTrue: (actual) => {
|
|
82
|
-
const pass = actual === true;
|
|
83
|
-
return {
|
|
84
|
-
pass,
|
|
85
|
-
message: () => `Expected "${actual}" to be true`,
|
|
86
|
-
expected: true,
|
|
87
|
-
actual,
|
|
88
|
-
};
|
|
89
|
-
},
|
|
90
|
-
toBeFalse: (actual) => {
|
|
91
|
-
const pass = actual === false;
|
|
92
|
-
return {
|
|
93
|
-
pass,
|
|
94
|
-
message: () => `Expected "${actual}" to be false`,
|
|
95
|
-
};
|
|
96
|
-
},
|
|
97
|
-
toBeWholeNumber: (received) => {
|
|
98
|
-
const pass = Number.isInteger(received);
|
|
99
|
-
return {
|
|
100
|
-
pass,
|
|
101
|
-
message: () => `Expected "${received}" to be a whole number`,
|
|
102
|
-
};
|
|
103
|
-
},
|
|
42
|
+
...shapeMatchers,
|
|
104
43
|
toBeWithinRange: (received, lower, upper) => {
|
|
105
44
|
const pass = received >= lower && received <= upper;
|
|
106
45
|
return {
|
|
@@ -112,130 +51,19 @@ export const extendExpect = (expect: ExpectStatic): ExpectStatic => {
|
|
|
112
51
|
received: RunTestResult,
|
|
113
52
|
userOptions: ToFinishWithOptions<PpeEvent>,
|
|
114
53
|
) => {
|
|
115
|
-
const
|
|
116
|
-
|
|
117
|
-
...userOptions,
|
|
118
|
-
};
|
|
119
|
-
|
|
120
|
-
const failedAssertions: string[] = [];
|
|
121
|
-
|
|
122
|
-
const diffs: Diffs = {
|
|
123
|
-
pass: true,
|
|
124
|
-
actual: ['Run:'],
|
|
125
|
-
expected: ['Run:'],
|
|
126
|
-
};
|
|
127
|
-
{
|
|
128
|
-
const expected = options?.status;
|
|
129
|
-
const actual = received.status;
|
|
130
|
-
const statusTest = expected === actual;
|
|
131
|
-
if (statusTest === false) {
|
|
132
|
-
diffs.pass = false;
|
|
133
|
-
diffs.actual.push(`status=${actual}`);
|
|
134
|
-
diffs.expected.push(`status=${expected}`);
|
|
135
|
-
failedAssertions.push(`Failed status check, expected "${expected}", got "${actual}".`);
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
const {
|
|
140
|
-
chargedEventCounts,
|
|
141
|
-
stats: { durationMillis },
|
|
142
|
-
} = await received.getRunInfo();
|
|
143
|
-
const datasetItemCount = (await received.getDataset()).items.length;
|
|
144
|
-
const stats = await received.getStatistics();
|
|
145
|
-
|
|
146
|
-
const checkInterval = (
|
|
147
|
-
value: number | undefined,
|
|
148
|
-
key: ['datasetItemCount', 'duration', 'failedRequests', 'requestsRetries'][number],
|
|
149
|
-
label: string,
|
|
150
|
-
) => {
|
|
151
|
-
const result = isWithinInterval(diffs, value, options, key);
|
|
152
|
-
if (result === false) {
|
|
153
|
-
failedAssertions.push(
|
|
154
|
-
`Failed ${label} check, expected ${JSON.stringify(options[key])}, got ${value}.`,
|
|
155
|
-
);
|
|
156
|
-
}
|
|
157
|
-
};
|
|
158
|
-
|
|
159
|
-
checkInterval(datasetItemCount, 'datasetItemCount', 'dataset item count');
|
|
160
|
-
checkInterval(durationMillis, 'duration', 'duration');
|
|
161
|
-
checkInterval(stats?.requestsFailed, 'failedRequests', 'failed requests');
|
|
162
|
-
checkInterval(stats?.requestsRetries, 'requestsRetries', 'requests retries');
|
|
163
|
-
|
|
164
|
-
if (options.maxRetriesPerRequest !== null) {
|
|
165
|
-
const maxRetriesPerRequestObserved = (stats?.requestRetryHistogram ?? [0]).length - 1;
|
|
166
|
-
if (maxRetriesPerRequestObserved > options.maxRetriesPerRequest) {
|
|
167
|
-
diffs.pass = false;
|
|
168
|
-
diffs.actual.push(`maxRetriesPerRequest=${maxRetriesPerRequestObserved}`);
|
|
169
|
-
diffs.expected.push(`maxRetriesPerRequest<=${options.maxRetriesPerRequest}`);
|
|
170
|
-
failedAssertions.push(
|
|
171
|
-
`Failed max retries observed check, expected <=${options.maxRetriesPerRequest}, got ${maxRetriesPerRequestObserved}.`,
|
|
172
|
-
);
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
const ppeDiffs: Diffs = {
|
|
176
|
-
pass: true,
|
|
177
|
-
actual: ['PPE Events:'],
|
|
178
|
-
expected: ['PPE Events:'],
|
|
179
|
-
};
|
|
180
|
-
|
|
181
|
-
if (options.chargedEventCounts) {
|
|
182
|
-
const expected = options.chargedEventCounts;
|
|
183
|
-
const actual = chargedEventCounts ?? {};
|
|
184
|
-
const uniquePpeEvents = new Set([
|
|
185
|
-
...Array.from(Object.keys(expected)),
|
|
186
|
-
...Array.from(Object.keys(actual)),
|
|
187
|
-
]);
|
|
188
|
-
|
|
189
|
-
for (const ppeEvent of uniquePpeEvents) {
|
|
190
|
-
if (!(ppeEvent in expected)) {
|
|
191
|
-
isWithinInterval(ppeDiffs, actual[ppeEvent], { [ppeEvent]: 0 }, ppeEvent);
|
|
192
|
-
continue;
|
|
193
|
-
}
|
|
194
|
-
if (!(ppeEvent in actual)) {
|
|
195
|
-
ppeDiffs.pass = false;
|
|
196
|
-
ppeDiffs.actual.push(`${ppeEvent}=X`);
|
|
197
|
-
ppeDiffs.expected.push(`${ppeEvent}=${expected[ppeEvent as PpeEvent]}`);
|
|
198
|
-
continue;
|
|
199
|
-
}
|
|
200
|
-
isWithinInterval(ppeDiffs, actual[ppeEvent], expected, ppeEvent as PpeEvent);
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
if (ppeDiffs.pass === false) {
|
|
204
|
-
diffs.pass = ppeDiffs.pass;
|
|
205
|
-
diffs.actual.push(ppeDiffs.actual.join('\n '));
|
|
206
|
-
diffs.expected.push(ppeDiffs.expected.join('\n '));
|
|
207
|
-
failedAssertions.push(
|
|
208
|
-
`Failed PPE event counts check, expected ${JSON.stringify(
|
|
209
|
-
options.chargedEventCounts,
|
|
210
|
-
)}, got ${JSON.stringify(chargedEventCounts)}.`,
|
|
211
|
-
);
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
{
|
|
216
|
-
const { forbiddenLogs } = options;
|
|
217
|
-
const log = await received.getLog();
|
|
218
|
-
const occuredLogs = [];
|
|
219
|
-
for (const forbiddenLog of forbiddenLogs) {
|
|
220
|
-
if (log.includes(forbiddenLog)) {
|
|
221
|
-
occuredLogs.push(forbiddenLog);
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
if (occuredLogs.length > 0) {
|
|
225
|
-
diffs.pass = false;
|
|
226
|
-
diffs.actual.push(` logs=[${occuredLogs.join(', ')}]`);
|
|
227
|
-
diffs.expected.push(` logs=[]`);
|
|
228
|
-
failedAssertions.push(
|
|
229
|
-
`Failed forbidden logs check, expected [] but got [${occuredLogs.join(', ')}].`,
|
|
230
|
-
);
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
|
|
54
|
+
const checks = await checkRun(received, userOptions);
|
|
55
|
+
const failed = checks.filter(({ pass }) => !pass);
|
|
234
56
|
return {
|
|
235
|
-
pass:
|
|
236
|
-
message: () =>
|
|
237
|
-
|
|
238
|
-
|
|
57
|
+
pass: failed.length === 0,
|
|
58
|
+
message: () =>
|
|
59
|
+
`Run did not finish as expected. Failed assertions: ${failed
|
|
60
|
+
.map(
|
|
61
|
+
({ name, expected, actual }) =>
|
|
62
|
+
`Failed ${name} check, expected ${expected}, got ${actual}.`,
|
|
63
|
+
)
|
|
64
|
+
.join(' ')}`,
|
|
65
|
+
actual: ['Run:', ...failed.map(({ name, actual }) => `${name}=${actual}`)].join('\n '),
|
|
66
|
+
expected: ['Run:', ...failed.map(({ name, expected }) => `${name}=${expected}`)].join('\n '),
|
|
239
67
|
};
|
|
240
68
|
},
|
|
241
69
|
toStartWith: (received, prefix) => {
|
|
@@ -267,41 +95,3 @@ export const extendExpect = (expect: ExpectStatic): ExpectStatic => {
|
|
|
267
95
|
|
|
268
96
|
return softExpect as unknown as ExpectStatic;
|
|
269
97
|
};
|
|
270
|
-
|
|
271
|
-
type Diffs = {
|
|
272
|
-
pass: boolean;
|
|
273
|
-
actual: string[];
|
|
274
|
-
expected: string[];
|
|
275
|
-
};
|
|
276
|
-
|
|
277
|
-
const isWithinInterval = <T extends string>(
|
|
278
|
-
diffs: Diffs,
|
|
279
|
-
actual: number | undefined,
|
|
280
|
-
options: Record<T, Interval | null>,
|
|
281
|
-
intervalOption: T,
|
|
282
|
-
) => {
|
|
283
|
-
const expected = options[intervalOption];
|
|
284
|
-
if (expected === null) {
|
|
285
|
-
// check is disabled if expected value is null
|
|
286
|
-
return undefined;
|
|
287
|
-
}
|
|
288
|
-
if (typeof expected === 'number') {
|
|
289
|
-
if (actual !== expected) {
|
|
290
|
-
// eslint-disable-next-line no-param-reassign --- Don't want to touch this code :)
|
|
291
|
-
diffs.pass = false;
|
|
292
|
-
diffs.actual.push(`${intervalOption}=${actual}`);
|
|
293
|
-
diffs.expected.push(`${intervalOption}=${expected}`);
|
|
294
|
-
return false;
|
|
295
|
-
}
|
|
296
|
-
} else if (typeof expected === 'object') {
|
|
297
|
-
const { min, max } = expected;
|
|
298
|
-
if (actual === undefined || (min !== undefined && actual < min) || (max !== undefined && actual > max)) {
|
|
299
|
-
// eslint-disable-next-line no-param-reassign
|
|
300
|
-
diffs.pass = false;
|
|
301
|
-
diffs.actual.push(`${intervalOption}=${actual}`);
|
|
302
|
-
diffs.expected.push(`${intervalOption}=<${min ?? ''},${max ?? ''}>`);
|
|
303
|
-
return false;
|
|
304
|
-
}
|
|
305
|
-
}
|
|
306
|
-
return undefined;
|
|
307
|
-
};
|
package/lib/lib.ts
CHANGED
|
@@ -53,27 +53,36 @@ export const describe = (name: string, fn?: SuiteFactory<object>, options: Actor
|
|
|
53
53
|
};
|
|
54
54
|
|
|
55
55
|
/**
|
|
56
|
-
*
|
|
56
|
+
* Registers a vitest test for an Actor, which only runs when that Actor was built (or for the scheduled runs).
|
|
57
|
+
* `body` gets the test context with the soft `expect` and the full test name.
|
|
57
58
|
*/
|
|
58
|
-
|
|
59
|
+
const defineActorTest = (
|
|
59
60
|
actorId: string,
|
|
60
61
|
testName: string,
|
|
61
|
-
|
|
62
|
-
|
|
62
|
+
testOptions: ActorTestOptions | undefined,
|
|
63
|
+
body: (context: TestContext, name: string) => Promise<void>,
|
|
63
64
|
) => {
|
|
64
65
|
const options = { ...DEFAULT_TEST_OPTIONS, ...testOptions };
|
|
65
|
-
|
|
66
66
|
const name = `${actorId}: ${testName}`;
|
|
67
67
|
// `RUN_ALL_PLATFORM_TESTS` is needed for the scheduled tests, which have no `ACTOR_BUILDS` to match the
|
|
68
68
|
// tests against - without it, every test would be filtered out as an actor we didn't build.
|
|
69
69
|
const shouldRun = !!RUN_ALL_PLATFORM_TESTS || config.has(actorId);
|
|
70
|
-
vitestTest.runIf(shouldRun)(name, options, async
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
70
|
+
vitestTest.runIf(shouldRun)(name, options, async (context) => {
|
|
71
|
+
await body({ ...context, expect: extendExpect(context.expect) }, name);
|
|
72
|
+
});
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* @param actorId - The actor's raw platform ID or its full name (`owner/name`, e.g. `"apify/web-scraper"`).
|
|
77
|
+
*/
|
|
78
|
+
export const testActor = <T>(
|
|
79
|
+
actorId: string,
|
|
80
|
+
testName: string,
|
|
81
|
+
fn: TestFunction<{ run: ReturnType<typeof createStartRunFn<T>> }>,
|
|
82
|
+
testOptions?: ActorTestOptions,
|
|
83
|
+
) => {
|
|
84
|
+
defineActorTest(actorId, testName, testOptions, async (context) => {
|
|
85
|
+
await fn({ ...context, run: createStartRunFn(actorId, context) });
|
|
77
86
|
});
|
|
78
87
|
};
|
|
79
88
|
|
|
@@ -82,8 +91,7 @@ export const testActor = <T>(
|
|
|
82
91
|
* `callStandby` function, which calls the task's `standbyUrl`.
|
|
83
92
|
*
|
|
84
93
|
* Using task is just current shortcoming of standby feature but ideally we would use Actor directly
|
|
85
|
-
|
|
86
|
-
/**
|
|
94
|
+
*
|
|
87
95
|
* @param actorId - The actor's raw platform ID or its full name (`owner/name`, e.g. `"apify/web-scraper"`).
|
|
88
96
|
*/
|
|
89
97
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -93,24 +101,12 @@ export const testStandbyActor = <I = any, O = any>(
|
|
|
93
101
|
fn: TestFunction<{ callStandby: ReturnType<typeof createStartStandbyFn<I, O>> }>,
|
|
94
102
|
testOptions?: ActorTestOptions,
|
|
95
103
|
) => {
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
const name = `${actorId}: ${testName}`;
|
|
99
|
-
// `RUN_ALL_PLATFORM_TESTS` is needed for the scheduled tests, which have no `ACTOR_BUILDS` to match the
|
|
100
|
-
// tests against - without it, every test would be filtered out as an actor we didn't build.
|
|
101
|
-
const shouldRun = !!RUN_ALL_PLATFORM_TESTS || config.has(actorId);
|
|
102
|
-
|
|
103
|
-
vitestTest.runIf(shouldRun)(name, options, async <T extends TestContext>(context: T) => {
|
|
104
|
+
defineActorTest(actorId, testName, testOptions, async (context, name) => {
|
|
104
105
|
const standbyTask = await createStandbyTask(actorId, config.get(actorId)?.buildNumber);
|
|
105
|
-
const { expect, ...rest } = context;
|
|
106
106
|
|
|
107
107
|
// NOTE: we wrap `fn` in try/finally so cleanup (deleting the task) always runs afterwards
|
|
108
108
|
try {
|
|
109
|
-
await fn({
|
|
110
|
-
expect: extendExpect(expect),
|
|
111
|
-
callStandby: createStartStandbyFn(standbyTask, context, name),
|
|
112
|
-
...rest,
|
|
113
|
-
});
|
|
109
|
+
await fn({ ...context, callStandby: createStartStandbyFn(standbyTask, context, name) });
|
|
114
110
|
} finally {
|
|
115
111
|
const { taskId } = standbyTask;
|
|
116
112
|
// we want to delete the task at the end of the test
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { TO_FINISH_WITH_OPTIONS } from './consts.js';
|
|
2
|
+
import type { RunTestResult } from './run-test-result.js';
|
|
3
|
+
import type { Interval, ToFinishWithOptions } from './types.js';
|
|
4
|
+
|
|
5
|
+
type Check = {
|
|
6
|
+
name: string;
|
|
7
|
+
pass: boolean;
|
|
8
|
+
actual: string;
|
|
9
|
+
expected: string;
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
const isWithinInterval = (actual: number | undefined, interval: Interval): boolean => {
|
|
13
|
+
if (typeof interval === 'number') {
|
|
14
|
+
return actual === interval;
|
|
15
|
+
}
|
|
16
|
+
const { min, max } = interval;
|
|
17
|
+
return actual !== undefined && (min === undefined || actual >= min) && (max === undefined || actual <= max);
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
const describeInterval = (interval: Interval): string => {
|
|
21
|
+
return typeof interval === 'number' ? `${interval}` : `<${interval.min ?? ''},${interval.max ?? ''}>`;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
const intervalCheck = (name: string, actual: number | undefined, interval: Interval | null | undefined): Check[] => {
|
|
25
|
+
// `null` disables the check, and so does an explicit `undefined`, e.g. from a caller forwarding an optional setting
|
|
26
|
+
if (interval == null) {
|
|
27
|
+
return [];
|
|
28
|
+
}
|
|
29
|
+
return [
|
|
30
|
+
{
|
|
31
|
+
name,
|
|
32
|
+
pass: isWithinInterval(actual, interval),
|
|
33
|
+
actual: `${actual}`,
|
|
34
|
+
expected: describeInterval(interval),
|
|
35
|
+
},
|
|
36
|
+
];
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
const ppeChecks = (expected: Record<string, Interval>, actual: Record<string, number> = {}): Check[] => {
|
|
40
|
+
const events = new Set([...Object.keys(expected), ...Object.keys(actual)]);
|
|
41
|
+
return [...events].flatMap((event): Check[] => {
|
|
42
|
+
const name = `chargedEventCounts.${event}`;
|
|
43
|
+
// charged events the test didn't list are expected to not happen at all
|
|
44
|
+
if (!(event in expected)) {
|
|
45
|
+
return intervalCheck(name, actual[event], 0);
|
|
46
|
+
}
|
|
47
|
+
// an expected event that was never charged is a failure, even for an interval that includes 0
|
|
48
|
+
if (!(event in actual)) {
|
|
49
|
+
return [{ name, pass: false, actual: 'X', expected: describeInterval(expected[event]) }];
|
|
50
|
+
}
|
|
51
|
+
return intervalCheck(name, actual[event], expected[event]);
|
|
52
|
+
});
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Evaluates every `toFinishWith` expectation against the run, so a failing assertion reports all of them at once.
|
|
57
|
+
*/
|
|
58
|
+
export const checkRun = async <PpeEvent extends string>(
|
|
59
|
+
run: RunTestResult,
|
|
60
|
+
userOptions: ToFinishWithOptions<PpeEvent>,
|
|
61
|
+
): Promise<Check[]> => {
|
|
62
|
+
const options = { ...TO_FINISH_WITH_OPTIONS, ...userOptions };
|
|
63
|
+
|
|
64
|
+
// The dataset's item count lags the run by 10-20 seconds, so counting the downloaded items is the only way to
|
|
65
|
+
// be in sync. Don't swap it for the count from the dataset info.
|
|
66
|
+
const [runInfo, { items }, stats, log] = await Promise.all([
|
|
67
|
+
run.getRunInfo(),
|
|
68
|
+
run.getDataset(),
|
|
69
|
+
run.getStatistics(),
|
|
70
|
+
run.getLog(),
|
|
71
|
+
]);
|
|
72
|
+
|
|
73
|
+
// the histogram has one bucket per retry count, so its length - 1 is the most retries any request needed
|
|
74
|
+
const maxRetriesPerRequest = (stats?.requestRetryHistogram ?? [0]).length - 1;
|
|
75
|
+
const occurredLogs = options.forbiddenLogs.filter((forbidden) => log.includes(forbidden));
|
|
76
|
+
|
|
77
|
+
return [
|
|
78
|
+
{ name: 'status', pass: run.status === options.status, actual: run.status, expected: options.status },
|
|
79
|
+
...intervalCheck('datasetItemCount', items.length, options.datasetItemCount),
|
|
80
|
+
...intervalCheck('duration', runInfo.stats.durationMillis, options.duration),
|
|
81
|
+
...intervalCheck('failedRequests', stats?.requestsFailed, options.failedRequests),
|
|
82
|
+
...intervalCheck('requestsRetries', stats?.requestsRetries, options.requestsRetries),
|
|
83
|
+
...(options.maxRetriesPerRequest == null
|
|
84
|
+
? []
|
|
85
|
+
: [
|
|
86
|
+
{
|
|
87
|
+
name: 'maxRetriesPerRequest',
|
|
88
|
+
pass: maxRetriesPerRequest <= options.maxRetriesPerRequest,
|
|
89
|
+
actual: `${maxRetriesPerRequest}`,
|
|
90
|
+
expected: `<=${options.maxRetriesPerRequest}`,
|
|
91
|
+
},
|
|
92
|
+
]),
|
|
93
|
+
...(options.chargedEventCounts ? ppeChecks(options.chargedEventCounts, runInfo.chargedEventCounts) : []),
|
|
94
|
+
{
|
|
95
|
+
name: 'forbiddenLogs',
|
|
96
|
+
pass: occurredLogs.length === 0,
|
|
97
|
+
actual: `[${occurredLogs.join(', ')}]`,
|
|
98
|
+
expected: '[]',
|
|
99
|
+
},
|
|
100
|
+
];
|
|
101
|
+
};
|
package/lib/run-test-result.ts
CHANGED
|
@@ -2,14 +2,25 @@ import type { ActorRun, ApifyClient, KeyValueStoreClient } from 'apify-client';
|
|
|
2
2
|
|
|
3
3
|
import type { Dataset, SdkCrawlerStatistics } from './types.js';
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
5
|
+
/** Items fetched per request, so no single response can exceed Node's string limit on large datasets. */
|
|
6
|
+
const DATASET_PAGE_SIZE = 1000;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Runs `fn` at most once and shares its promise, so repeated and concurrent calls are one request.
|
|
10
|
+
* A failure is not cached, the next call retries.
|
|
11
|
+
*/
|
|
12
|
+
const once = <T>(fn: () => Promise<T>): (() => Promise<T>) => {
|
|
13
|
+
let promise: Promise<T> | undefined;
|
|
14
|
+
return async () => {
|
|
15
|
+
promise ??= fn().catch((error) => {
|
|
16
|
+
promise = undefined;
|
|
17
|
+
throw error;
|
|
18
|
+
});
|
|
19
|
+
return promise;
|
|
20
|
+
};
|
|
21
|
+
};
|
|
12
22
|
|
|
23
|
+
export class RunTestResult {
|
|
13
24
|
constructor(
|
|
14
25
|
private readonly apifyClient: ApifyClient,
|
|
15
26
|
private readonly run: ActorRun,
|
|
@@ -17,62 +28,41 @@ export class RunTestResult {
|
|
|
17
28
|
/**/
|
|
18
29
|
}
|
|
19
30
|
|
|
20
|
-
getStatistics = async (): Promise<SdkCrawlerStatistics | undefined> => {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
const kvs = this.apifyClient.keyValueStore(this.run.defaultKeyValueStoreId);
|
|
25
|
-
const stats = await kvs.getRecord('SDK_CRAWLER_STATISTICS_0');
|
|
26
|
-
this.statistics = stats?.value as unknown as SdkCrawlerStatistics;
|
|
27
|
-
return this.statistics;
|
|
28
|
-
};
|
|
31
|
+
getStatistics = once(async (): Promise<SdkCrawlerStatistics | undefined> => {
|
|
32
|
+
const record = await this.getKeyValueStoreClient().getRecord('SDK_CRAWLER_STATISTICS_0');
|
|
33
|
+
return record?.value as SdkCrawlerStatistics | undefined;
|
|
34
|
+
});
|
|
29
35
|
|
|
30
|
-
getLog = async (): Promise<string> => {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
}
|
|
34
|
-
const runLog = await this.apifyClient.run(this.id).log().get();
|
|
35
|
-
this.log = runLog;
|
|
36
|
-
return runLog as string;
|
|
37
|
-
};
|
|
36
|
+
getLog = once(async (): Promise<string> => {
|
|
37
|
+
return (await this.apifyClient.run(this.id).log().get()) as string;
|
|
38
|
+
});
|
|
38
39
|
|
|
39
|
-
|
|
40
|
-
if (this.dataset) {
|
|
41
|
-
return this.dataset as Dataset<T>;
|
|
42
|
-
}
|
|
40
|
+
private readonly datasetItems = once(async (): Promise<unknown[]> => {
|
|
43
41
|
const dataset = this.apifyClient.dataset(this.run.defaultDatasetId);
|
|
44
|
-
const items =
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
42
|
+
const items: unknown[] = [];
|
|
43
|
+
for (;;) {
|
|
44
|
+
const page = await dataset.listItems({ offset: items.length, limit: DATASET_PAGE_SIZE });
|
|
45
|
+
items.push(...page.items);
|
|
46
|
+
if (page.items.length === 0 || items.length >= page.total) {
|
|
47
|
+
return items;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
getDataset = async <T>(): Promise<Dataset<T>> => ({ items: (await this.datasetItems()) as T[] });
|
|
48
53
|
|
|
49
54
|
getKeyValueStoreClient = (): KeyValueStoreClient => {
|
|
50
|
-
|
|
51
|
-
return this.keyValueStoreClient;
|
|
52
|
-
}
|
|
53
|
-
const keyValueStoreClient = this.apifyClient.keyValueStore(this.run.defaultKeyValueStoreId);
|
|
54
|
-
this.keyValueStoreClient = keyValueStoreClient;
|
|
55
|
-
return keyValueStoreClient;
|
|
55
|
+
return this.apifyClient.keyValueStore(this.run.defaultKeyValueStoreId);
|
|
56
56
|
};
|
|
57
57
|
|
|
58
|
-
|
|
59
|
-
if (this.input) {
|
|
60
|
-
return this.input as T;
|
|
61
|
-
}
|
|
62
|
-
const kvs = this.apifyClient.keyValueStore(this.run.defaultKeyValueStoreId);
|
|
63
|
-
const input = await kvs.getRecord('INPUT');
|
|
64
|
-
this.input = input?.value;
|
|
65
|
-
return this.input as T;
|
|
66
|
-
};
|
|
58
|
+
private readonly inputRecord = once(async () => (await this.getKeyValueStoreClient().getRecord('INPUT'))?.value);
|
|
67
59
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
return this.runInfo!;
|
|
75
|
-
};
|
|
60
|
+
getInput = async <T>(): Promise<T> => (await this.inputRecord()) as T;
|
|
61
|
+
|
|
62
|
+
/** Fetches the run again, so unlike `id` and `status` it also reflects what changed after the run finished. */
|
|
63
|
+
getRunInfo = once(async (): Promise<ActorRun> => {
|
|
64
|
+
return (await this.apifyClient.run(this.run.id).get())!;
|
|
65
|
+
});
|
|
76
66
|
|
|
77
67
|
get id(): string {
|
|
78
68
|
return this.run.id;
|
package/package.json
CHANGED
|
@@ -174,6 +174,14 @@ describe('get-affected-actors CLI on real git histories', () => {
|
|
|
174
174
|
});
|
|
175
175
|
});
|
|
176
176
|
|
|
177
|
+
it('counts a moved file for both the Actor it left and the one it joined', async () => {
|
|
178
|
+
git('mv', 'actors/a/main.ts', 'actors/c/moved.ts');
|
|
179
|
+
git('commit', '-q', '-m', 'refactor: move a file from a to c');
|
|
180
|
+
|
|
181
|
+
// With rename detection, git would list only the new path, and A would look unchanged
|
|
182
|
+
expect(getAffectedActors('--base', 'main')).toStrictEqual({ functional: ['a', 'c'], cosmetic: [] });
|
|
183
|
+
});
|
|
184
|
+
|
|
177
185
|
it('checks the whole branch on a rerun or when --since is not in the history', async () => {
|
|
178
186
|
await commit('feat(a): validated change', 'actors/a/main.ts');
|
|
179
187
|
const head = await commit('docs(a): readme', 'actors/a/README.md');
|