apify-test-tools 0.3.0 → 0.5.0

package/lib/consts.ts

@@ -12,4 +12,5 @@ export const TO_FINISH_WITH_OPTIONS: ToFinishWithOptionsWithDefaults = {
         'ReferenceError',
         'TypeError',
     ],
+    maxRetriesPerRequest: null,
 };

package/lib/extend-expect.ts

@@ -117,6 +117,8 @@ export const extendExpect = (expect: ExpectStatic): ExpectStatic => {
                 ...userOptions,
             };
 
+            const failedAssertions: string[] = [];
+
             const diffs: Diffs = {
                 pass: true,
                 actual: ['Run:'],
@@ -125,9 +127,13 @@ export const extendExpect = (expect: ExpectStatic): ExpectStatic => {
             {
                 const expected = options?.status;
                 const actual = received.status;
-                diffs.pass = expected === actual;
-                diffs.actual.push(`status=${actual}`);
-                diffs.expected.push(`status=${expected}`);
+                const statusTest = expected === actual;
+                if (statusTest === false) {
+                    diffs.pass = false;
+                    diffs.actual.push(`status=${actual}`);
+                    diffs.expected.push(`status=${expected}`);
+                    failedAssertions.push(`Failed status check, expected "${expected}", got "${actual}".`);
+                }
             }
 
             const {
@@ -135,12 +141,25 @@ export const extendExpect = (expect: ExpectStatic): ExpectStatic => {
                 stats: { durationMillis },
             } = await received.getRunInfo();
             const datasetItemCount = (await received.getDataset()).items.length;
-            const stats = (await received.getStatistics());
+            const stats = await received.getStatistics();
 
-            isWithinInterval(diffs, datasetItemCount, options, 'datasetItemCount');
-            isWithinInterval(diffs, durationMillis, options, 'duration');
-            isWithinInterval(diffs, stats?.requestsFailed, options, 'failedRequests');
-            isWithinInterval(diffs, stats?.requestsRetries, options, 'requestsRetries');
+            const checkInterval = (
+                value: number | undefined,
+                key: ['datasetItemCount', 'duration', 'failedRequests', 'requestsRetries', 'maxRetriesPerRequest'][number],
+                label: string,
+            ) => {
+                const result = isWithinInterval(diffs, value, options, key);
+                if (result === false) {
+                    failedAssertions.push(`Failed ${label} check, expected ${JSON.stringify(options[key])}, got ${value}.`);
+                }
+            };
+
+            checkInterval(datasetItemCount, 'datasetItemCount', 'dataset item count');
+            checkInterval(durationMillis, 'duration', 'duration');
+            checkInterval(stats?.requestsFailed, 'failedRequests', 'failed requests');
+            checkInterval(stats?.requestsRetries, 'requestsRetries', 'requests retries');
+            const maxRetriesObserved = (stats?.requestRetryHistogram ?? [0]).length - 1;
+            checkInterval(maxRetriesObserved, 'maxRetriesPerRequest', 'max retries per request');
 
             const ppeDiffs: Diffs = {
                 pass: true,
@@ -175,9 +194,12 @@ export const extendExpect = (expect: ExpectStatic): ExpectStatic => {
                     isWithinInterval(ppeDiffs, actual[ppeEvent], expected, ppeEvent as PpeEvent);
                 }
 
-                diffs.pass = diffs.pass && ppeDiffs.pass;
-                diffs.actual.push(ppeDiffs.actual.join('\n    '));
-                diffs.expected.push(ppeDiffs.expected.join('\n    '));
+                if (ppeDiffs.pass === false) {
+                    diffs.pass = ppeDiffs.pass;
+                    diffs.actual.push(ppeDiffs.actual.join('\n    '));
+                    diffs.expected.push(ppeDiffs.expected.join('\n    '));
+                    failedAssertions.push(`Failed PPE event counts check, expected ${JSON.stringify(options.chargedEventCounts)}, got ${JSON.stringify(chargedEventCounts)}.`);
+                }
             }
 
             {
@@ -193,12 +215,13 @@ export const extendExpect = (expect: ExpectStatic): ExpectStatic => {
                     diffs.pass = false;
                     diffs.actual.push(` logs=[${occuredLogs.join(', ')}]`);
                     diffs.expected.push(` logs=[]`);
+                    failedAssertions.push(`Failed forbidden logs check, expected [] but got [${occuredLogs.join(', ')}].`);
                 }
             }
 
             return {
                 pass: diffs.pass,
-                message: () => `Run ${received.id} didn't finish as expected`,
+                message: () => `Run did not finish as expected. Failed assertions: ${failedAssertions.join(' ',)}`,
                 actual: diffs.actual.join('\n  '),
                 expected: diffs.expected.join('\n  '),
             };
@@ -256,6 +279,7 @@ const isWithinInterval = <T extends string>(
             diffs.pass = false;
             diffs.actual.push(`${intervalOption}=${actual}`);
             diffs.expected.push(`${intervalOption}=${expected}`);
+            return false
         }
     } else if (typeof expected === 'object') {
         const { min, max } = expected;
@@ -263,6 +287,8 @@ const isWithinInterval = <T extends string>(
             diffs.pass = false;
             diffs.actual.push(`${intervalOption}=${actual}`);
             diffs.expected.push(`${intervalOption}=<${min ?? ''},${max ?? ''}>`);
+            return false
         }
     }
+    return
 };

package/lib/lib.ts

@@ -5,10 +5,9 @@ import {
     TestFunction,
     test as vitestTest,
     SuiteFactory,
-    TestOptions,
     TestContext,
 } from 'vitest';
-import type { ActorBuild, RunOptions } from './types';
+import type { ActorBuild, ActorTestOptions, RunOptions } from './types';
 import { RunTestResult } from './run-test-result.js';
 import { extendExpect } from './extend-expect.js';
 import { getActorPrefilledInput, sleep } from './utils.js';
@@ -38,7 +37,7 @@ export { ExpectStatic };
 const { TESTER_APIFY_TOKEN, RUN_PLATFORM_TESTS, RUN_ALL_PLATFORM_TESTS } = process.env;
 const apifyClient = new ApifyClient({ token: TESTER_APIFY_TOKEN });
 
-const DEFAULT_TEST_OPTIONS: TestOptions = {
+const DEFAULT_TEST_OPTIONS: ActorTestOptions = {
     // we want to run tests concurrently
     concurrent: true,
     // test should finish within 1 hour
@@ -48,12 +47,12 @@ const DEFAULT_TEST_OPTIONS: TestOptions = {
 export const describe = (
     name: string,
     fn?: SuiteFactory<object>,
-    options: TestOptions = DEFAULT_TEST_OPTIONS,
+    options: ActorTestOptions = DEFAULT_TEST_OPTIONS,
 ) => {
     vitestDescribe.runIf(!!RUN_PLATFORM_TESTS || !!RUN_ALL_PLATFORM_TESTS)(name, options, fn);
 };
 
-const DEFAULT_TEST_ACTOR_OPTIONS: TestOptions = {
+const DEFAULT_TEST_ACTOR_OPTIONS: ActorTestOptions = {
     retry: 1,
 };
 
@@ -61,7 +60,7 @@ export const testActor = <T>(
     actorName: string,
     testName: string,
     fn: TestFunction<{ run: ReturnType<typeof createStartRunFn<T>> }>,
-    testOptions?: TestOptions,
+    testOptions?: ActorTestOptions,
 ) => {
     const options = {
         ...DEFAULT_TEST_ACTOR_OPTIONS,
@@ -90,7 +89,7 @@ export const testStandbyActor = <I = any, O = any>(
     actorName: string,
     testName: string,
     fn: TestFunction<{ callStandby: ReturnType<typeof createStartStandbyFn<I, O>> }>,
-    testOptions?: TestOptions,
+    testOptions?: ActorTestOptions,
 ) => {
     const options = {
         ...DEFAULT_TEST_ACTOR_OPTIONS,

package/lib/types.ts

@@ -1,5 +1,5 @@
 import { ActorCallOptions } from 'apify-client';
-import { Assertion } from 'vitest';
+import { Assertion, TestOptions } from 'vitest';
 
 export type ActorBuild = {
     buildId: string;
@@ -46,6 +46,7 @@ export type ToFinishWithOptionsWithDefaults = {
     failedRequests: Interval | null
     requestsRetries: Interval | null
     forbiddenLogs: string[]
+    maxRetriesPerRequest: number | null
 }
 export type IntervalOption<PpeEvent extends string> = keyof Pick<
     ToFinishWithOptions<PpeEvent>,
@@ -123,6 +124,7 @@ export interface ActorMatchers<R = unknown> {
      * - `failedRequests` (default: `0`)
      * - `requestsRetries` (default: `{ max: 3 }`)
      * - `forbiddenLogs` (default: `['ReferenceError', 'TypeError']`)
+     * - `maxRetriesPerRequest` (not checked by default)
      * - `datasetItemCount` (required)
      * - `chargedEventCounts`
      */
@@ -131,6 +133,17 @@ export interface ActorMatchers<R = unknown> {
     hard: <T>(actual: T, message?: string) => Assertion
 }
 
+export type ActorTestOptions = Omit<TestOptions, 'retry'> & {
+    /**
+     * Times to retry the test if fails. Useful for making flaky tests more stable.
+     * When retries is up, the last test error will be thrown.
+     *
+     * @default 1
+     */
+    // we are just extending the docs here to replace the default value, otherwise it's the exact same
+    retry?: TestOptions['retry'];
+};
+
 declare module 'vitest' {
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     interface Assertion<T = any> extends ActorMatchers<T> { }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "apify-test-tools",
-    "version": "0.3.0",
+    "version": "0.5.0",
     "type": "module",
     "description": "TBD",
     "engines": {