apify-test-tools 0.10.1-beta.4 → 0.10.1-beta.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "apify-test-tools",
-  "version": "0.10.1-beta.4",
+  "version": "0.10.1-beta.6",
   "type": "module",
   "description": "TBD",
   "repository": {

package/platform-tests-investigate-and-fix/.claude-plugin/plugin.json

@@ -0,0 +1,4 @@
+{
+    "name": "platform-tests-investigate-and-fix",
+    "description": "Skills for the platform-tests Claude investigate-and-fix workflow"
+}

package/platform-tests-investigate-and-fix/skills/measure-flakiness/SKILL.md

@@ -0,0 +1,28 @@
+---
+name: measure-flakiness
+description: Re-runs the failing Actor run many times in parallel to measure how often the failure happens. Use only when you are about to create a new issue for a failing platform test and its failure rate is not already obvious.
+---
+
+# Measure flakiness
+
+Use this only when both are true:
+
+- You are about to create a new issue — no existing issue already covers this failure.
+- How often the failure happens isn't obvious. Blocking, too few scraped items or a missing field can happen in 1% of runs or in all of them, and one failed run doesn't tell which.
+
+## Steps
+
+The Apify API token is in the `TESTER_APIFY_TOKEN` environment variable. It is always set, so there's no need to check for it.
+
+1. From the failing run (the run the failed test started; its link is in the test log), get the Actor ID, build number, memory, timeout and input. For example, you can get the first four from `GET https://api.apify.com/v2/actor-runs/{runId}`.
+2. Decide how to tell whether a finished run hit the same failure. Check what the failed test assertion checked — e.g. the run's status, its dataset item count against the expected minimum, a field missing from items, a log line.
+3. Start runs of the same Actor at once, with the same build, memory, timeout and input. For example, you can start each with `POST https://api.apify.com/v2/acts/{actorId}/runs`. If a run can't start because the account's memory limit is reached, start it once earlier runs finish.
+
+    Decide on your own how many runs you need, up to !`echo "$MAX_FLAKINESS_RUNS"` in total. The point isn't a statistically significant rate, but a rough idea of whether the failure is frequent, infrequent or a once-in-a-lifetime event. For example:
+    - Blocking that looks persistent: if around 20 runs all get blocked the same way, that's enough.
+    - A rare flake that's hard to reproduce, e.g. one item out of hundreds missing a field: start a lot of runs — it may take a hundred runs, and after that another hundred, to see it even a few times.
+
+4. Wait until every run has finished, then apply the check from step 2 to each. If there were no or almost no reproductions and you can't tell yet how often the failure happens, start more runs and check them the same way, but up to !`echo "$MAX_FLAKINESS_RUNS"` in total.
+5. Add a short `## Flakiness` section to the issue, right after the Summary:
+    - The rate, e.g. "Fails in 28 of 47 runs (60%)". Leave out runs that failed for another reason, e.g. never started — only mention how many there were.
+    - Links to up to 2 runs that hit the failure.

package/test/unit/bin/build-from-local.test.ts

@@ -1,5 +1,3 @@
-import type * as ChildProcessModule from 'node:child_process';
-import { spawnSync } from 'node:child_process';
 import fs from 'node:fs/promises';
 import os from 'node:os';
 import path from 'node:path';
@@ -17,20 +15,8 @@ import {
 } from '../../../bin/build-from-local.js';
 import * as Utils from '../../../bin/utils.js';
 
-// Defaults to the real spawnSync so `git init`/`git ls-files` calls made by the code under test
-// (and by test setup below) actually run — individual tests override this via mockReturnValue
-// where they need to fake git's output, and vi.restoreAllMocks() reverts back to this passthrough.
-vi.mock('node:child_process', async (importOriginal) => {
-    const actual = await importOriginal<typeof ChildProcessModule>();
-    return { ...actual, spawnSync: vi.fn(actual.spawnSync) };
-});
-
 const mkTempDir = async (prefix: string) => fs.mkdtemp(path.join(os.tmpdir(), prefix));
 
-const initGitRepo = (dir: string) => {
-    spawnSync('git', ['init', '-q'], { cwd: dir });
-};
-
 describe('build-from-local helpers', () => {
     const tempDirs: string[] = [];
 
@@ -43,15 +29,13 @@ describe('build-from-local helpers', () => {
         it('drops secret-pattern files and gitignored files, keeps everything else', async () => {
             const rootDir = await mkTempDir('apify-test-tools-collect-');
             tempDirs.push(rootDir);
-            initGitRepo(rootDir);
 
             await fs.writeFile(path.join(rootDir, 'main.js'), 'console.log(1)');
             await fs.writeFile(path.join(rootDir, '.env'), 'SECRET=1');
             await fs.mkdir(path.join(rootDir, 'sub'));
             await fs.writeFile(path.join(rootDir, 'sub', 'ignored.log'), 'log');
 
-            // Only the gitignore side is mocked — isSecretFile runs for real, so this also
-            // proves the secret-pattern backstop applies independently of .gitignore.
+            vi.spyOn(Utils, 'listRepoFilePaths').mockResolvedValue(['main.js', '.env', 'sub/ignored.log']);
             vi.spyOn(Utils, 'getGitignoredPaths').mockImplementation(
                 async (relativePaths) => new Set(relativePaths.filter((p) => p.endsWith('.log'))),
             );
@@ -146,7 +130,6 @@ describe('build-from-local helpers', () => {
         it("always collects the actor's own .actor/actor.json for a monorepo actor, since flattenMonorepoContext depends on it", async () => {
             const repoRoot = await mkTempDir('apify-test-tools-collect-source-');
             tempDirs.push(repoRoot);
-            initGitRepo(repoRoot);
 
             const originalCwd = process.cwd();
             // We simulate the working directory being the repo root, since collectSourceFiles uses relative paths to the repo root.
@@ -161,6 +144,14 @@ describe('build-from-local helpers', () => {
                 );
                 await fs.writeFile(path.join(cwd, 'package.json'), '{}');
 
+                vi.spyOn(Utils, 'listRepoFilePaths').mockResolvedValue([
+                    'actors/owner_actor/.actor/actor.json',
+                    'package.json',
+                ]);
+                vi.spyOn(Utils, 'getGitignoredPaths').mockResolvedValue(
+                    new Set(['actors/owner_actor/.actor/actor.json']),
+                );
+
                 const sourceFiles = await collectSourceFiles('owner/actor', actorDir);
                 expect(sourceFiles.map((file) => file.name)).toContain('.actor/actor.json');
             } finally {

package/test/unit/bin/git-command.test.ts

@@ -1,71 +1,30 @@
-import os from 'node:os';
-import path from 'node:path';
-
 import { describe, expect, it } from 'vitest';
 
-import { getGitignoredPaths, GitCommandError, listRepoFilePaths, runGitCommand } from '../../../bin/utils.js';
+import { GitCommandError, runGitCommand } from '../../../bin/utils.js';
 
 describe('runGitCommand', () => {
-    it('returns trimmed Git output', async () => {
-        await expect(
-            runGitCommand(['rev-parse', '--is-inside-work-tree'], {
-                cwd: process.cwd(),
-                operation: 'check whether the current directory is in a Git repository',
-            }),
-        ).resolves.toBe('true');
+    it('returns trimmed Git output without requiring a repository', async () => {
+        await expect(runGitCommand(['--version'])).resolves.toMatch(/^git version /);
     });
 
-    it('rejects on git exit code, not stderr', async () => {
-        const error: unknown = await runGitCommand(['rev-parse', '--verify', '--quiet'], {
-            cwd: process.cwd(),
-            operation: 'verify a commit',
+    it('wraps a Git failure with the command and operation', async () => {
+        const error: unknown = await runGitCommand(['--definitely-invalid-option'], {
+            operation: 'check error handling',
         }).catch((failure: unknown) => failure);
+
         expect(error).toBeInstanceOf(GitCommandError);
         if (error instanceof GitCommandError) {
-            expect(error.exitCode).toBe(1);
-            expect(error.gitError).toContain('Command failed: git rev-parse --verify --quiet');
-            expect(error.message).toContain('Failed to verify a commit.\nCommand:');
-            expect(error.stack).toContain('Failed to verify a commit.');
+            expect(error.exitCode).not.toBe(0);
+            expect(error.message).toContain('Failed to check error handling.\n');
+            expect(error.message).toContain('Command: git "--definitely-invalid-option"\n');
+            expect(error.gitError).toContain('--definitely-invalid-option');
             expect(error.cause).toBeInstanceOf(Error);
         }
     });
 
-    it('passes arguments literally and rejects Git failures', async () => {
-        await expect(
-            runGitCommand(['rev-parse', '--verify', 'HEAD; echo injected'], {
-                operation: 'verify a commit reference',
-            }),
-        ).rejects.toThrow('Command: git "rev-parse" "--verify" "HEAD; echo injected"\nGit error:');
-    });
-});
-
-describe('listRepoFilePaths', () => {
-    it('lists files under a subdirectory relative to the repository root', async () => {
-        const repoRoot = process.cwd();
-        await expect(listRepoFilePaths(repoRoot, path.join(repoRoot, 'bin'))).resolves.toContain('bin/utils.ts');
-    });
-});
-
-describe('getGitignoredPaths', () => {
-    it('returns an empty set for no paths', async () => {
-        await expect(getGitignoredPaths([])).resolves.toStrictEqual(new Set());
-    });
-
-    it('returns only paths ignored by the repository rules', async () => {
-        await expect(
-            getGitignoredPaths(['node_modules/example.js', 'bin/utils.ts', '.DS_Store'], process.cwd()),
-        ).resolves.toStrictEqual(new Set(['node_modules/example.js', '.DS_Store']));
-    });
-
-    it('returns an empty set when Git exits with code 1', async () => {
-        await expect(getGitignoredPaths(['bin/utils.ts'], process.cwd())).resolves.toStrictEqual(new Set());
-    });
-
-    it('rejects unexpected Git errors', async () => {
-        await expect(getGitignoredPaths(['bin/utils.ts'], os.tmpdir())).rejects.toThrow(
-            'Failed to check which repository paths are ignored.\n' +
-                'Command: git "check-ignore" "--" "bin/utils.ts"\n' +
-                'Git error: fatal: not a git repository',
+    it('passes arguments literally instead of running them through a shell', async () => {
+        await expect(runGitCommand(['--version; echo injected'])).rejects.toThrow(
+            'Command: git "--version; echo injected"\nGit error:',
         );
     });
 });