apify-test-tools 0.10.2-beta.2 → 0.10.2-beta.21

package/test/unit/bin/test-files-glob.test.ts

@@ -0,0 +1,118 @@
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+import { createTestFilesMatcher, resolveTestFilesGlob } from '../../../bin/test-files-glob.js';
+
+describe('createTestFilesMatcher', () => {
+    it('matches repo-root-relative paths, ignoring case and including dot files', () => {
+        const isMatch = createTestFilesMatcher('test/platform/{daily,hourly}/**');
+        expect(isMatch('test/platform/daily/a.test.ts')).toBe(true);
+        expect(isMatch('Test/Platform/Hourly/.b.test.ts')).toBe(true);
+        expect(isMatch('test/platform/daily-without-pr/a.test.ts')).toBe(false);
+        expect(isMatch('test/platform/utils.ts')).toBe(false);
+    });
+
+    it('matches nothing without a glob', () => {
+        expect(createTestFilesMatcher(undefined)('test/platform/a.test.ts')).toBe(false);
+    });
+
+    it('rejects an absolute glob', () => {
+        expect(() => createTestFilesMatcher('/repo/test/platform/**')).toThrow('must be relative to the repo root');
+    });
+});
+
+describe('resolveTestFilesGlob', () => {
+    let repoDir: string;
+
+    beforeAll(async () => {
+        repoDir = await mkdtemp(path.join(os.tmpdir(), 'apify-test-files-glob-'));
+        for (const file of [
+            'test/platform/utils.ts',
+            'test/platform/daily/a.test.ts',
+            'test/platform/daily-without-pr/b.test.ts',
+            'test/platform/hourly/nested/c.test.ts',
+            'test/platform/node_modules/d.test.ts',
+            'other/test/platform/daily/e.test.ts',
+        ]) {
+            await mkdir(path.join(repoDir, path.dirname(file)), { recursive: true });
+            await writeFile(path.join(repoDir, file), '');
+        }
+    });
+
+    afterAll(async () => {
+        if (repoDir) await rm(repoDir, { recursive: true, force: true });
+    });
+
+    const files = async (glob: string) => (await resolveTestFilesGlob(glob, repoDir)).files;
+    const paths = async (glob: string) => (await resolveTestFilesGlob(glob, repoDir)).paths;
+
+    it('lists the matched files from the repo root, sorted, skipping node_modules', async () => {
+        expect(await files('test/platform/**')).toEqual([
+            'test/platform/daily-without-pr/b.test.ts',
+            'test/platform/daily/a.test.ts',
+            'test/platform/hourly/nested/c.test.ts',
+            'test/platform/utils.ts',
+        ]);
+    });
+
+    it('selects folders with a brace pattern', async () => {
+        expect(await files('test/platform/{daily,hourly}/**')).toEqual([
+            'test/platform/daily/a.test.ts',
+            'test/platform/hourly/nested/c.test.ts',
+        ]);
+    });
+
+    it('accepts a leading ./', async () => {
+        expect(await files('./test/platform/daily/**')).toEqual(['test/platform/daily/a.test.ts']);
+    });
+
+    it('walks the whole repo when the glob has no static base', async () => {
+        expect(await files('**/daily/*.test.ts')).toEqual([
+            'other/test/platform/daily/e.test.ts',
+            'test/platform/daily/a.test.ts',
+        ]);
+    });
+
+    it('resolves a glob without wildcards to the file itself', async () => {
+        expect(await resolveTestFilesGlob('test/platform/utils.ts', repoDir)).toStrictEqual({
+            files: ['test/platform/utils.ts'],
+            paths: ['test/platform/utils.ts'],
+        });
+    });
+
+    it('matches no file with a glob relative to test/platform or a folder without **', async () => {
+        expect(await files('daily/**')).toEqual([]);
+        expect(await files('test/platform/daily')).toEqual([]);
+    });
+
+    it('gives a folder whose files all match as one path, ending with / so it is not a prefix of a sibling', async () => {
+        expect(await paths('test/platform/{daily,hourly}/**')).toEqual([
+            'test/platform/daily/',
+            'test/platform/hourly/',
+        ]);
+    });
+
+    it('lists the matched files one by one in a folder that also holds files the glob leaves out', async () => {
+        // test/platform/utils.ts is left out, so test/platform/ can't be one path
+        expect(await paths('test/platform/**/*.test.ts')).toEqual([
+            'test/platform/daily-without-pr/',
+            'test/platform/daily/',
+            'test/platform/hourly/',
+        ]);
+        // other/ holds only the matched file, so it is the highest folder that can be one path
+        expect(await paths('**/daily/*.test.ts')).toEqual(['other/', 'test/platform/daily/']);
+    });
+
+    it('never gives a folder that holds a skipped node_modules as one path', async () => {
+        // test/platform/node_modules/ is not walked, so test/platform/ might hold files the glob leaves out
+        expect(await paths('test/platform/**')).toEqual([
+            'test/platform/daily-without-pr/',
+            'test/platform/daily/',
+            'test/platform/hourly/',
+            'test/platform/utils.ts',
+        ]);
+    });
+});

package/test/unit/bin/test-report.test.ts

@@ -0,0 +1,118 @@
+import { describe, expect, it } from 'vitest';
+
+import { toTestReport, type VitestJsonReport } from '../../../bin/test-report.js';
+
+const cwd = '/repo';
+const run = { actorId: 'owner/a', runId: 'run1', runLink: 'https://console.apify.com/view/runs/run1' };
+
+const vitestReport = (testResults: VitestJsonReport['testResults'], success = false): VitestJsonReport => ({
+    success,
+    testResults,
+});
+
+describe('toTestReport', () => {
+    it('reads each test with its status, first failure lines and run', () => {
+        const report = toTestReport(
+            vitestReport([
+                {
+                    name: '/repo/test/platform/a.test.ts',
+                    status: 'failed',
+                    message: '',
+                    assertionResults: [
+                        {
+                            fullName: 'owner/a: works',
+                            status: 'passed',
+                            duration: 12.6,
+                            failureMessages: [],
+                            meta: run,
+                        },
+                        {
+                            fullName: 'owner/a: fails',
+                            status: 'failed',
+                            duration: 3,
+                            // Every failure counts, also the same one twice (e.g. a retry)
+                            failureMessages: ['AssertionError: boom\n    at a.test.ts:1:1', 'AssertionError: boom\n'],
+                            meta: {},
+                        },
+                        { fullName: 'owner/a: other', status: 'pending', failureMessages: null },
+                    ],
+                },
+            ]),
+            { cwd, exitCode: 1 },
+        );
+        expect(report).toStrictEqual({
+            kind: 'test-report',
+            version: 1,
+            success: false,
+            summary: { total: 3, passed: 1, failed: 1, skipped: 1 },
+            tests: [
+                {
+                    name: 'owner/a: works',
+                    file: 'test/platform/a.test.ts',
+                    status: 'passed',
+                    durationMs: 13,
+                    failureMessages: [],
+                    run,
+                },
+                {
+                    name: 'owner/a: fails',
+                    file: 'test/platform/a.test.ts',
+                    status: 'failed',
+                    durationMs: 3,
+                    failureMessages: ['AssertionError: boom', 'AssertionError: boom'],
+                    run: null,
+                },
+                {
+                    name: 'owner/a: other',
+                    file: 'test/platform/a.test.ts',
+                    status: 'skipped',
+                    durationMs: null,
+                    failureMessages: [],
+                    run: null,
+                },
+            ],
+            errors: [],
+        });
+    });
+
+    it('reports a test file that failed to load, which has no test to blame', () => {
+        const report = toTestReport(
+            vitestReport([
+                {
+                    name: '/repo/test/platform/b.test.ts',
+                    status: 'failed',
+                    message: "Cannot find package 'foo'\nmore",
+                    assertionResults: [],
+                },
+            ]),
+            { cwd, exitCode: 1 },
+        );
+        expect(report.success).toBe(false);
+        expect(report.errors).toStrictEqual(["test/platform/b.test.ts: Cannot find package 'foo'"]);
+    });
+
+    it('fails when vitest does, even if every test passed', () => {
+        const report = toTestReport(vitestReport([], true), { cwd, exitCode: 1 });
+        expect(report.success).toBe(false);
+        expect(report.errors).toStrictEqual(['Vitest exited with code 1, see its output above']);
+    });
+
+    it('passes when every test passed or was skipped', () => {
+        const report = toTestReport(
+            vitestReport(
+                [
+                    {
+                        name: '/repo/test/platform/a.test.ts',
+                        status: 'passed',
+                        message: '',
+                        assertionResults: [{ fullName: 'skipped', status: 'skipped', failureMessages: [] }],
+                    },
+                ],
+                true,
+            ),
+            { cwd, exitCode: 0 },
+        );
+        expect(report.success).toBe(true);
+        expect(report.summary).toStrictEqual({ total: 1, passed: 0, failed: 0, skipped: 1 });
+    });
+});

package/test/unit/bin/utils/config/load-config.test.ts

@@ -4,7 +4,12 @@ import path from 'node:path';
 
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 
-import { CONFIG_FILE_NAME, loadActorConfig, readConfigFile } from '../../../../../bin/utils/config/load-config.js';
+import {
+    CONFIG_FILE_NAME,
+    loadActorConfig,
+    readBaseConfigFile,
+    readConfigFile,
+} from '../../../../../bin/utils/config/load-config.js';
 import type { ResolvedActorConfig } from '../../../../../bin/utils/config/structures/base.js';
 import { ExistingDir, ExistingFile, RelativeDir, RelativeFile } from '../../../../../bin/utils/path/repo-relative.js';
 
@@ -265,3 +270,23 @@ describe('readConfigFile', () => {
         await expect(readConfigFile(emptyActorSelection)).rejects.toThrow(/at actors/);
     });
 });
+
+describe('readBaseConfigFile', () => {
+    it('returns non-file settings without checking Actor paths or reading actor.json', async () => {
+        await writeFiles({
+            [CONFIG_FILE_NAME]: validConfig([
+                {
+                    folder: 'missing-actor-dir',
+                    actorFullName: 'myteam/shopify',
+                    tokenEnvVar: 'APIFY_TOKEN',
+                    overrideActorContext: ['../outside'],
+                },
+            ]),
+        });
+
+        await expect(readBaseConfigFile()).resolves.toEqual([
+            { actorFullName: 'myteam/shopify', tokenEnvVar: 'APIFY_TOKEN' },
+        ]);
+        await expect(readConfigFile(emptyActorSelection)).rejects.toThrow();
+    });
+});

package/test/unit/bin/utils/config/parser.test.ts

@@ -4,13 +4,23 @@ import path from 'node:path';
 
 import { afterAll, beforeAll, describe, expect, it } from 'vitest';
 
-import { _privates, parseConfigFile } from '../../../../../bin/utils/config/parser.js';
-import { CONFIG_FILE_STRATEGY } from '../../../../../bin/utils/config/structures/base.js';
+import {
+    _privates,
+    parseConfigFile,
+    parseRepoSettings,
+    parseResolvedConfigFile,
+} from '../../../../../bin/utils/config/parser.js';
+import { CONFIG_FILE_STRATEGY, type ResolvedActorConfig } from '../../../../../bin/utils/config/structures/base.js';
 import { GROUPED_PARSER } from '../../../../../bin/utils/config/structures/grouped.js';
 import { LEGACY_PARSER } from '../../../../../bin/utils/config/structures/legacy.js';
 import { ExistingDir, RelativeDir } from '../../../../../bin/utils/path/repo-relative.js';
 
-const { selectStrategy, verifyConfiguration } = _privates;
+const { selectStrategy, initializePaths, verifyConfiguration: verifyInitializedConfiguration } = _privates;
+const verifyConfiguration = (body: ResolvedActorConfig[]) => {
+    const initialized = initializePaths(body);
+    verifyInitializedConfiguration(initialized);
+    return initialized;
+};
 
 let repoDir: string;
 const originalCwd = process.cwd();
@@ -95,13 +105,13 @@ describe('verifyConfiguration', () => {
         ).toThrow(/Duplicate folder/);
     });
 
-    it('quotes a duplicate folder the way it was written, not the way it normalized', () => {
+    it('reports the normalized path for a duplicate folder', () => {
         expect(() =>
             verifyConfiguration([
                 actor({ folder: '.', actorFullName: 'myteam/actor-a' }),
                 actor({ folder: './', actorFullName: 'myteam/actor-b' }),
             ]),
-        ).toThrow('Duplicate folder "./"');
+        ).toThrow('Duplicate folder "."');
     });
 
     it('rejects two entries pointing at the same actor', () => {
@@ -150,6 +160,19 @@ describe('verifyConfiguration', () => {
 });
 
 describe('parseConfigFile', () => {
+    it('shares strategy resolution with parsing that does not initialize paths', () => {
+        const body = { actors: [actor({ folder: 'missing-actor-dir', overrideActorContext: ['../outside'] })] };
+        expect(parseResolvedConfigFile(body)).toEqual([
+            {
+                actorFullName: 'myteam/shopify',
+                folder: 'missing-actor-dir',
+                tokenEnvVar: 'APIFY_TOKEN',
+                overrideActorContext: ['../outside'],
+            },
+        ]);
+        expect(() => parseConfigFile(body)).toThrow();
+    });
+
     it('validates and normalizes paths after resolving the config', () => {
         const [parsed] = parseConfigFile({ actors: [actor({ folder: './bin/', overrideActorContext: ['test/'] })] });
         expect(parsed.folder).toBeInstanceOf(ExistingDir);
@@ -210,3 +233,21 @@ describe('parseConfigFile', () => {
         ).toThrow('Duplicate folder "test"');
     });
 });
+
+describe('parseRepoSettings', () => {
+    it('defaults platformTestsGlob to test/platform/**', () => {
+        expect(parseRepoSettings({ actors: [actor()] })).toStrictEqual({ platformTestsGlob: 'test/platform/**' });
+    });
+
+    it('reads platformTestsGlob next to the Actor entries, in any mode', () => {
+        expect(parseRepoSettings({ platformTestsGlob: 'e2e/**', actors: [actor()] })).toStrictEqual({
+            platformTestsGlob: 'e2e/**',
+        });
+        expect(parseConfigFile({ platformTestsGlob: 'e2e/**', actors: [actor({ folder: 'bin' })] })).toHaveLength(1);
+    });
+
+    it('rejects an empty or non-string platformTestsGlob', () => {
+        expect(() => parseRepoSettings({ platformTestsGlob: '' })).toThrow(/platformTestsGlob/);
+        expect(() => parseRepoSettings({ platformTestsGlob: ['test/**'] })).toThrow(/platformTestsGlob/);
+    });
+});

package/test/unit/bin/utils/config/structures/grouped.test.ts

@@ -68,6 +68,18 @@ describe('GROUPED_PARSER', () => {
         expect(() => GROUPED_PARSER.parse({})).toThrow();
     });
 
+    it('rejects an empty group token name', () => {
+        expect(() => GROUPED_PARSER.parse({ groups: { myteam: group({ tokenEnvVar: '' }) } })).toThrow(
+            /at groups\.myteam\.tokenEnvVar/,
+        );
+    });
+
+    it('rejects an empty actor token override instead of falling back to the group token', () => {
+        expect(() =>
+            GROUPED_PARSER.parse({ groups: { myteam: group({ actors: [actor({ tokenEnvVar: '' })] }) } }),
+        ).toThrow(/at groups\.myteam\.actors\[0\]\.tokenEnvVar/);
+    });
+
     it('reports problems from every group at once instead of only the first', () => {
         const message = (() => {
             try {

package/test/unit/bin/utils/config/structures/legacy.test.ts

@@ -32,6 +32,7 @@ describe('LEGACY_PARSER', () => {
         ['folder is missing', { folder: undefined }, /at actors\[0\]\.folder/],
         ['folder is not a string', { folder: 123 }, /at actors\[0\]\.folder/],
         ['tokenEnvVar is missing', { tokenEnvVar: undefined }, /at actors\[0\]\.tokenEnvVar/],
+        ['tokenEnvVar is empty', { tokenEnvVar: '' }, /at actors\[0\]\.tokenEnvVar/],
         ['overrideActorContext is not an array', { overrideActorContext: 'packages' }, /expected array/],
         ['overrideActorContext holds non-strings', { overrideActorContext: [123] }, /overrideActorContext\[0\]/],
     ])('rejects an entry where %s', (_name, fields, expected) => {

package/test/unit/github-scripts/mask-config-secrets.test.ts

@@ -0,0 +1,61 @@
+import { spawnSync } from 'node:child_process';
+import process from 'node:process';
+import { fileURLToPath } from 'node:url';
+
+import { describe, expect, it } from 'vitest';
+
+const SCRIPT = fileURLToPath(new URL('../../../.github/scripts/mask-config-secrets.sh', import.meta.url));
+
+const run = (configSecrets?: string) =>
+    spawnSync('bash', [SCRIPT], {
+        encoding: 'utf8',
+        env: { PATH: process.env.PATH, ...(configSecrets === undefined ? {} : { CONFIG_SECRETS: configSecrets }) },
+    });
+
+const masks = (stdout: string) => stdout.split('\n').filter((line) => line !== '');
+
+describe('mask-config-secrets.sh', () => {
+    it('registers every value as a mask', () => {
+        const { status, stdout, stderr } = run(JSON.stringify({ TOKEN_ONE: 'one', TOKEN_TWO: 'two' }));
+        expect(status, stderr).toBe(0);
+        expect(masks(stdout)).toStrictEqual(['::add-mask::one', '::add-mask::two']);
+    });
+
+    it('registers a multi-line value line by line, skipping empty lines', () => {
+        const { status, stdout, stderr } = run(
+            JSON.stringify({ DEPLOY_KEY: '-----BEGIN KEY-----\nabc\n\n-----END KEY-----\n' }),
+        );
+        expect(status, stderr).toBe(0);
+        expect(masks(stdout)).toStrictEqual([
+            '::add-mask::-----BEGIN KEY-----',
+            '::add-mask::abc',
+            '::add-mask::-----END KEY-----',
+        ]);
+    });
+
+    it('skips values that are not strings', () => {
+        const { status, stdout, stderr } = run(JSON.stringify({ TOKEN_ONE: 'one', COUNT: 1 }));
+        expect(status, stderr).toBe(0);
+        expect(masks(stdout)).toStrictEqual(['::add-mask::one']);
+    });
+
+    it.each([
+        ['is unset', undefined],
+        ['is empty', ''],
+    ])('does nothing when CONFIG_SECRETS %s', (_, configSecrets) => {
+        const { status, stdout, stderr } = run(configSecrets);
+        expect(status, stderr).toBe(0);
+        expect(stdout).toBe('');
+    });
+
+    it.each([
+        ['is not JSON', 'apify_api_xyzzy'],
+        ['is a JSON string', '"apify_api_xyzzy"'],
+        ['is a JSON array', '["apify_api_xyzzy"]'],
+    ])('fails without printing the value when the secret %s', (_, configSecrets) => {
+        const { status, stdout, stderr } = run(configSecrets);
+        expect(status).toBe(1);
+        expect(stdout).toContain('::error::The CONFIG_SECRETS secret must be a JSON object.');
+        expect(stdout + stderr).not.toContain('xyzzy');
+    });
+});

package/test/unit/github-scripts/run-with-apify-tokens.test.ts

@@ -0,0 +1,115 @@
+import { spawnSync } from 'node:child_process';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath } from 'node:url';
+
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const SCRIPT = fileURLToPath(new URL('../../../.github/scripts/run-with-apify-tokens.mjs', import.meta.url));
+
+// Three Actors, each with its own token, so a test can leave any of them out.
+const CONFIG = {
+    actors: [
+        { folder: 'actors/one', actorFullName: 'team/one', tokenEnvVar: 'TOKEN_ONE' },
+        { folder: 'actors/two', actorFullName: 'team/two', tokenEnvVar: 'TOKEN_TWO' },
+        { folder: 'actors/three', actorFullName: 'team/three', tokenEnvVar: 'TOKEN_THREE' },
+    ],
+};
+
+// The command the wrapper runs prints what it was handed, so a test sees exactly what got through.
+const PRINT_ENV = `console.log(JSON.stringify({
+    TOKEN_ONE: process.env.TOKEN_ONE,
+    TOKEN_TWO: process.env.TOKEN_TWO,
+    TOKEN_THREE: process.env.TOKEN_THREE,
+    NPM_TOKEN: process.env.NPM_TOKEN,
+    WORKFLOW_SECRETS: process.env.WORKFLOW_SECRETS,
+}))`;
+
+let cwd: string;
+
+// `workflowSecrets` is what the step's `toJSON(secrets)` holds inside the reusable workflow.
+const run = (workflowSecrets: Record<string, string>) => {
+    const result = spawnSync(process.execPath, [SCRIPT, process.execPath, '-e', PRINT_ENV], {
+        cwd,
+        encoding: 'utf8',
+        env: { PATH: process.env.PATH, WORKFLOW_SECRETS: JSON.stringify(workflowSecrets) },
+    });
+    return { ...result, env: result.status === 0 ? JSON.parse(result.stdout) : undefined };
+};
+
+describe('run-with-apify-tokens.mjs', () => {
+    beforeAll(async () => {
+        cwd = await mkdtemp(path.join(os.tmpdir(), 'run-with-apify-tokens-'));
+        await writeFile(path.join(cwd, 'apify-test-tools.config.json'), JSON.stringify(CONFIG));
+    });
+
+    afterAll(async () => {
+        await rm(cwd, { recursive: true, force: true });
+    });
+
+    it('passes on inherited secrets, as `secrets: inherit` sends them', () => {
+        const { status, stderr, env } = run({
+            TOKEN_ONE: 'one',
+            TOKEN_TWO: 'two',
+            TOKEN_THREE: 'three',
+            NPM_TOKEN: 'npm',
+        });
+        expect(status, stderr).toBe(0);
+        expect(env).toStrictEqual({ TOKEN_ONE: 'one', TOKEN_TWO: 'two', TOKEN_THREE: 'three' });
+    });
+
+    it('unpacks the tokens from the CONFIG_SECRETS secret', () => {
+        const { status, stderr, env } = run({
+            CONFIG_SECRETS: JSON.stringify({ TOKEN_ONE: 'one', TOKEN_TWO: 'two', TOKEN_THREE: 'three' }),
+            NPM_TOKEN: 'npm',
+        });
+        expect(status, stderr).toBe(0);
+        expect(env).toStrictEqual({ TOKEN_ONE: 'one', TOKEN_TWO: 'two', TOKEN_THREE: 'three' });
+    });
+
+    it('passes on only the config names when CONFIG_SECRETS holds every secret of the caller', () => {
+        const { status, stderr, env } = run({
+            CONFIG_SECRETS: JSON.stringify({
+                TOKEN_ONE: 'one',
+                TOKEN_TWO: 'two',
+                TOKEN_THREE: 'three',
+                NPM_TOKEN: 'npm',
+                github_token: 'gh',
+            }),
+        });
+        expect(status, stderr).toBe(0);
+        expect(env).toStrictEqual({ TOKEN_ONE: 'one', TOKEN_TWO: 'two', TOKEN_THREE: 'three' });
+    });
+
+    it('prefers CONFIG_SECRETS over an inherited secret of the same name', () => {
+        const { status, stderr, env } = run({
+            CONFIG_SECRETS: JSON.stringify({ TOKEN_ONE: 'from-config-secrets' }),
+            TOKEN_ONE: 'inherited',
+            TOKEN_TWO: 'inherited',
+        });
+        expect(status, stderr).toBe(0);
+        expect(env).toStrictEqual({ TOKEN_ONE: 'from-config-secrets', TOKEN_TWO: 'inherited' });
+    });
+
+    it('warns about a token the config declares but nothing passes, and runs anyway', () => {
+        const { status, stderr, env } = run({
+            CONFIG_SECRETS: JSON.stringify({ TOKEN_ONE: 'one', TOKEN_TWO: 'two' }),
+        });
+        expect(status, stderr).toBe(0);
+        expect(stderr).toContain('declares TOKEN_THREE, but no such secret was passed');
+        expect(env).toStrictEqual({ TOKEN_ONE: 'one', TOKEN_TWO: 'two' });
+    });
+
+    it.each([
+        ['is not JSON', 'apify_api_xyzzy', 'The CONFIG_SECRETS secret is not valid JSON.'],
+        ['is a JSON string', '"apify_api_xyzzy"', 'The CONFIG_SECRETS secret must be a JSON object.'],
+        ['is a JSON array', '["apify_api_xyzzy"]', 'The CONFIG_SECRETS secret must be a JSON object.'],
+    ])('fails without printing the value when CONFIG_SECRETS %s', (_, packed, message) => {
+        const { status, stdout, stderr } = run({ CONFIG_SECRETS: packed });
+        expect(status).toBe(1);
+        expect(stderr).toContain(message);
+        expect(stdout + stderr).not.toContain('xyzzy');
+    });
+});

package/test/unit/run-checks.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, it } from 'vitest';
+
+import { checkRun } from '../../lib/run-checks.js';
+import type { RunTestResult } from '../../lib/run-test-result.js';
+
+const fakeRun = (overrides: { status?: string; log?: string; stats?: object; chargedEventCounts?: object } = {}) =>
+    ({
+        status: overrides.status ?? 'SUCCEEDED',
+        getRunInfo: async () => ({
+            stats: { durationMillis: 5_000 },
+            chargedEventCounts: overrides.chargedEventCounts,
+        }),
+        getDataset: async () => ({ items: new Array(10) }),
+        getStatistics: async () => ({
+            requestsFailed: 0,
+            requestsRetries: 1,
+            requestRetryHistogram: [5, 1],
+            ...overrides.stats,
+        }),
+        getLog: async () => overrides.log ?? 'all good',
+    }) as unknown as RunTestResult;
+
+const failures = async (run: RunTestResult, options: Parameters<typeof checkRun>[1]) =>
+    (await checkRun(run, options)).filter(({ pass }) => !pass).map(({ name }) => name);
+
+describe('checkRun()', () => {
+    it('passes a run matching the defaults', async () => {
+        expect(await failures(fakeRun(), { datasetItemCount: 10 })).toEqual([]);
+    });
+
+    it('reports every failing check at once', async () => {
+        const run = fakeRun({ status: 'FAILED', log: 'TypeError: x', stats: { requestsFailed: 2 } });
+        expect(await failures(run, { datasetItemCount: { min: 11 } })).toEqual([
+            'status',
+            'datasetItemCount',
+            'failedRequests',
+            'forbiddenLogs',
+        ]);
+    });
+
+    it('disables a check with null', async () => {
+        const run = fakeRun({ stats: { requestsFailed: 2 } });
+        expect(await failures(run, { datasetItemCount: 10, failedRequests: null })).toEqual([]);
+    });
+
+    it('treats explicitly undefined options as disabled checks', async () => {
+        const options = { datasetItemCount: 10, duration: undefined, maxRetriesPerRequest: undefined };
+        expect(await failures(fakeRun(), options)).toEqual([]);
+    });
+
+    it('checks the longest retry chain only when asked to', async () => {
+        expect(await failures(fakeRun(), { datasetItemCount: 10, maxRetriesPerRequest: 0 })).toEqual([
+            'maxRetriesPerRequest',
+        ]);
+        expect(await failures(fakeRun(), { datasetItemCount: 10, maxRetriesPerRequest: 1 })).toEqual([]);
+    });
+
+    describe('chargedEventCounts', () => {
+        const run = fakeRun({ chargedEventCounts: { start: 1, item: 7 } });
+
+        it('passes matching counts and intervals', async () => {
+            const options = { datasetItemCount: 10, chargedEventCounts: { start: 1, item: { min: 5 } } };
+            expect(await failures(run, options)).toEqual([]);
+        });
+
+        it('expects unlisted charged events to not happen', async () => {
+            const options = { datasetItemCount: 10, chargedEventCounts: { start: 1 } as Record<string, number> };
+            expect(await failures(run, options)).toEqual(['chargedEventCounts.item']);
+        });
+
+        it('fails an expected event that was never charged', async () => {
+            const options = { datasetItemCount: 10, chargedEventCounts: { start: 1, item: 7, extra: { min: 0 } } };
+            expect(await failures(run, options)).toEqual(['chargedEventCounts.extra']);
+        });
+    });
+});