apify-test-tools 0.10.1-beta.3 → 0.10.1-beta.5

package/.claude-plugin/marketplace.json

@@ -0,0 +1,12 @@
+{
+    "name": "apify-test-tools",
+    "owner": {
+        "name": "Apify"
+    },
+    "plugins": [
+        {
+            "name": "platform-tests-investigate-and-fix",
+            "source": "./platform-tests-investigate-and-fix"
+        }
+    ]
+}

package/.github/workflows/public_platform-tests-claude-investigate-and-fix.yaml

@@ -39,16 +39,17 @@ on:
                 type: string
                 default: sonnet
                 description: Claude model to use for investigation and fix
+            max_flakiness_runs:
+                required: false
+                type: number
+                default: 200
+                description: The most times Claude may re-run the failing Actor run to measure how often the failure happens, when that isn't obvious. Claude decides how many of them it needs.
             investigate_prompt:
                 required: false
                 type: string
                 default: |
                     Investigate the failing E2E platform test and open a GitHub issue documenting your findings. Start by reviewing the test logs, error messages, and recent workflow run history to determine the root cause.
 
-                    The failure will typically fall into one of these categories:
-                    - Flaky test: The test is non-deterministic or depends on timing/external state and fails intermittently. Check past workflow runs to see if this test has failed before.
-                    - Real bug: Genuine broken functionality — either a pre-existing issue or an environment/dependency change outside the codebase (e.g. a website changed its structure, an external API changed behavior).
-
                     Use available tools to check previous workflow runs, git history, and existing GitHub issues to gather context and avoid creating duplicate issues. For deeper context,
                     use tools to investigate the run on Apify platform, review its logs, datasets, key-value stores.
 
@@ -56,9 +57,12 @@ on:
                     - Debug snapshots: often stored as a field on dataset items (e.g. `_debugSnapshots`) or as records in a key-value store, keyed so they can be matched to the log line for the failing request.
                     - Any other debug artifacts the Actor produced (extra logs, error/state fields on output items) that narrow down the failing request.
 
+                    Before you create a new issue, check whether the `platform-tests-investigate-and-fix:measure-flakiness` skill applies, and follow it if it does.
+
                     When you create the issue, write it like an incident postmortem: a short summary and impact up top for anyone skimming, full technical detail below for whoever fixes it. Don't water down the engineering detail to make the top readable.
                     - Title: short and clean, describing the actual problem in plain terms — no dates, run numbers, or other noise. Prefix it with a conventional-commit-style type, e.g. "fix: blocking issue on detail endpoint".
                     - Body, section 1 ("## Summary"): one line stating what's broken plus its impact, in plain language a PM can understand at a glance — no jargon, internal field names, or stack traces (e.g. "Scraping product detail pages fails when a listing has no reviews, affecting N% of runs").
+                    - If you measured flakiness, a "## Flakiness" section right after the Summary, as the `measure-flakiness` skill describes.
                     - A `---` divider, then section 2 ("## Details"): the engineering bug report — root cause, relevant log lines, links to the failing run, and any other internals an engineer needs. If a single item/request failed among otherwise-good ones, show or link that specific one (e.g. the dataset item index/id, or its matching debug snapshot) rather than just describing the run as failed.
 
                     If you create a new issue, you MUST write its issue number (just the number, nothing else) to $GITHUB_WORKSPACE/issue-number.txt as the very last thing you do. If you skip creating one because a duplicate already exists, do not create the file.
@@ -67,9 +71,12 @@ on:
             ANTHROPIC_API_KEY:
                 required: true
                 description: Claude API key
-            TESTER_APIFY_TOKEN_READ_ONLY:
+            TESTER_APIFY_TOKEN:
                 required: true
-                description: Apify token used only for read-only investigation — reading the failing run, its log and its storages.
+                description: Apify token for investigating the failing run — reading it, its log and its storages — and re-running it to measure how often the failure happens.
+            NPM_TOKEN:
+                required: false
+                description: Read-only token for installing private npm packages before Claude runs.
 
 jobs:
     investigate:
@@ -89,7 +96,8 @@ jobs:
               id: claude
               uses: anthropics/claude-code-action@8251c103ac8c1d761882c86aba1412c7f583c844 # v1.0.213
               env:
-                  TESTER_APIFY_TOKEN_READ_ONLY: ${{ secrets.TESTER_APIFY_TOKEN_READ_ONLY }}
+                  TESTER_APIFY_TOKEN: ${{ secrets.TESTER_APIFY_TOKEN }}
+                  MAX_FLAKINESS_RUNS: ${{ inputs.max_flakiness_runs }}
               with:
                   prompt: |
                       The failing E2E platform test ran in workflow run ${{ github.run_id }} of this repo.
@@ -97,13 +105,15 @@ jobs:
                       ${{ inputs.investigate_prompt }}
                   github_token: ${{ secrets.GITHUB_TOKEN }}
                   anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+                  plugin_marketplaces: https://github.com/apify/apify-test-tools.git
+                  plugins: platform-tests-investigate-and-fix@apify-test-tools
                   # The `tools` query parameter below selects which tools the Apify MCP server
                   # exposes: reading an Actor run — its status, exit code, storage ids and log — and
                   # reading that run's dataset items and its key-value store keys and records, e.g.
                   # INPUT, OUTPUT and error snapshots. WARN: the server adds `abort-actor-run` on top
                   # even when not listed here, see https://github.com/apify/apify-mcp-server
                   claude_args: |
-                      --mcp-config '{"mcpServers":{"apify":{"type":"http","url":"https://mcp.apify.com/?tools=get-actor-run,get-actor-log,get-dataset-list,get-dataset,get-dataset-items,get-key-value-store-list,get-key-value-store,get-key-value-store-keys,get-key-value-store-record","headers":{"Authorization":"Bearer ${TESTER_APIFY_TOKEN_READ_ONLY}"}}}}'
+                      --mcp-config '{"mcpServers":{"apify":{"type":"http","url":"https://mcp.apify.com/?tools=get-actor-run,get-actor-run-log,get-dataset-list,get-dataset,get-dataset-items,get-key-value-store-list,get-key-value-store,get-key-value-store-keys,get-key-value-store-record","headers":{"Authorization":"Bearer ${TESTER_APIFY_TOKEN}"}}}}'
                       --model ${{ inputs.model }}
                       --dangerously-skip-permissions
 
@@ -166,8 +176,10 @@ jobs:
             id-token: write
             actions: read
         steps:
-            - name: Checkout repository
-              uses: actions/checkout@v5
+            - name: Setup repository and dependencies
+              uses: apify/apify-test-tools/.github/actions/checkout-restore-dependencies@workflows-v0
+              with:
+                  npm-token: ${{ secrets.NPM_TOKEN }}
 
             - name: Run Claude Code
               id: claude

package/CHANGELOG.md

@@ -5,10 +5,17 @@ All notable changes to this project will be documented in this file.
 <!-- git-cliff-unreleased-start -->
 ## 0.10.1 - **not yet released**
 
+### 🚀 Features
+
+- **relative-path:** Path safety and manipulation lib ([#154](https://github.com/apify/apify-test-tools/pull/154)) ([c151a1c](https://github.com/apify/apify-test-tools/commit/c151a1c2755fef98b335d48a3461f5042b90def9)) by [@JuanGalilea](https://github.com/JuanGalilea)
+- **logger:** Add logger and ban console statements ([#170](https://github.com/apify/apify-test-tools/pull/170)) ([b295d36](https://github.com/apify/apify-test-tools/commit/b295d363276c92fc6c30dd34d7a90acef5d380bf)) by [@JuanGalilea](https://github.com/JuanGalilea), closes [#156](https://github.com/apify/apify-test-tools/issues/156)
+- Give claude investigator a skill to test flakiness ([#184](https://github.com/apify/apify-test-tools/pull/184)) ([b490298](https://github.com/apify/apify-test-tools/commit/b4902987dc2ae55882463ff8223e1e9f25b8b497)) by [@foxt451](https://github.com/foxt451), closes [#162](https://github.com/apify/apify-test-tools/issues/162)
+
 ### 🐛 Bug Fixes
 
 - **internal-workflows:** Wait out npm&#x27;s async publish before holding the major tag ([#160](https://github.com/apify/apify-test-tools/pull/160)) ([78434f0](https://github.com/apify/apify-test-tools/commit/78434f03d288ecbd383d55d56a1610f06ded298f)) by [@metalwarrior665](https://github.com/metalwarrior665)
 - Skip builds and tests when the net diff is empty ([#171](https://github.com/apify/apify-test-tools/pull/171)) ([874d70c](https://github.com/apify/apify-test-tools/commit/874d70ca5506f7ec742a1808c9bf6c670f08552c)) by [@artogahr](https://github.com/artogahr)
+- **workflows:** Add npm install step to Claude fix workflow ([#161](https://github.com/apify/apify-test-tools/pull/161)) ([d8884e0](https://github.com/apify/apify-test-tools/commit/d8884e07f778de14c8d9d7f0ec9e0a1672c224ad)) by [@foxt451](https://github.com/foxt451), closes [#39](https://github.com/apify/apify-test-tools/issues/39)
 
 
 <!-- git-cliff-unreleased-end -->

package/CONTRIBUTING.md

@@ -14,6 +14,15 @@ The package consists of three parts:
 - `bin/slack.ts` sending notifications to slack
 - `bin/test-report.ts` processing vitest's test reports
 
+CLI command results use `writeJson` from `bin/output.ts` to write newline-terminated JSON to stdout.
+Use the CLI `logger` from `bin/logger.ts` for diagnostics; every level writes to stderr.
+ESLint rejects `console.log` throughout the codebase and all direct `console` calls in `bin/`.
+The importable library in `lib/` keeps its own diagnostics; CLI log levels apply only to `bin/`.
+
+The CLI's global `--log-level` option accepts `debug`, `info` (default), `warn`, `error`, or `silent`.
+It filters diagnostics without affecting command results. Use `logger.debug` for detailed troubleshooting,
+`logger.info` for progress, `logger.warn` for recoverable problems, and `logger.error` for failures.
+
 ## Test library
 
 - `lib/extend-expect.ts` - custom matchers

package/README.md

@@ -258,13 +258,12 @@ it at a branch only to test a prompt change.
 Callers pass `secrets: inherit`. The workflows do not turn every inherited secret into job-wide
 environment variables, so a secret is only visible to the step that needs it:
 
-| Secret                                                                    | Reaches                                                                                                                                                                                                                |
-| ------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `NPM_TOKEN`                                                               | dependency install steps only, as both `NPM_TOKEN` and `NODE_AUTH_TOKEN`. Use a read-only token: npm granular tokens can be read-only, classic automation tokens can publish.                                          |
-| the Actor tokens named by `tokenEnvVar` in `apify-test-tools.config.json` | the `build`, `release`, and `delete-old-builds` steps only                                                                                                                                                             |
-| `TESTER_APIFY_TOKEN`                                                      | the vitest step only                                                                                                                                                                                                   |
-| `SLACK_TOKEN_TESTS_BOT` / `SLACK_TOKEN_RELEASES_BOT`                      | the reporting and release steps only                                                                                                                                                                                   |
-| `TESTER_APIFY_TOKEN_READ_ONLY`                                            | the Claude investigation step only, as the Apify MCP server's bearer token. Required by `public_platform-tests-claude-investigate-and-fix`. Use a read-only token: it reads the failing run, its log and its storages. |
+| Secret                                                                    | Reaches                                                                                                                                                                                                                      |
+| ------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `NPM_TOKEN`                                                               | dependency install steps only, as both `NPM_TOKEN` and `NODE_AUTH_TOKEN`. Use a read-only token: npm granular tokens can be read-only, classic automation tokens can publish.                                                |
+| the Actor tokens named by `tokenEnvVar` in `apify-test-tools.config.json` | the `build`, `release`, and `delete-old-builds` steps only                                                                                                                                                                   |
+| `TESTER_APIFY_TOKEN`                                                      | the vitest step, and the Claude investigation step: the Apify MCP server's bearer token for reading the failing run, its log and its storages, and Claude's token for re-running it to measure how often the failure happens |
+| `SLACK_TOKEN_TESTS_BOT` / `SLACK_TOKEN_RELEASES_BOT`                      | the reporting and release steps only                                                                                                                                                                                         |
 
 The Actor tokens are the one set that cannot be listed in the workflow, because each Actor names its
 own token via `tokenEnvVar` in `apify-test-tools.config.json`. Those steps pass

package/bin/build-from-local.ts

@@ -8,6 +8,7 @@ import { ActorSourceType } from 'apify-client';
 
 import { dryRunBuildData, LOCAL_SOURCE_VERSION_NUMBER, runAndSummarizeBuilds } from './build.js';
 import { buildDockerIgnoreMatcher } from './dockerignore.js';
+import { logger } from './logger.js';
 import { isPathWithinScope } from './path-utils.js';
 import type { ActorConfig, BuildData } from './types.js';
 import type { SourceFile } from './utils.js';
@@ -34,7 +35,7 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P
     const isMonorepoActor = !!contextAbsDir && isOutsideDir(contextAbsDir, absActorDir);
 
     const dockerContextDirAbs = isMonorepoActor ? contextAbsDir! : absActorDir;
-    const keptFilePaths = collectNonIgnoredFiles(dockerContextDirAbs, repoRoot);
+    const keptFilePaths = await collectNonIgnoredFiles(dockerContextDirAbs, repoRoot);
 
     if (!isMonorepoActor) {
         return Promise.all(keptFilePaths.map(async (filePath) => readSourceFile(filePath, dockerContextDirAbs)));
@@ -64,9 +65,9 @@ export const collectSourceFiles = async (actorName: string, actorDir: string): P
 // regardless of .gitignore/.dockerignore, matching Apify CLI's own behavior. Files matching the
 // hardcoded secret-pattern backstop (keys, certs, .env variants) are dropped unconditionally,
 // .actor/ included, since those should never ship regardless of what the ignore files say.
-export const collectNonIgnoredFiles = (dockerContextDir: string, repoRoot: string): string[] => {
-    const relativePaths = listRepoFilePaths(repoRoot, dockerContextDir);
-    const ignoredPaths = getGitignoredPaths(relativePaths);
+export const collectNonIgnoredFiles = async (dockerContextDir: string, repoRoot: string): Promise<string[]> => {
+    const relativePaths = await listRepoFilePaths(repoRoot, dockerContextDir);
+    const ignoredPaths = await getGitignoredPaths(relativePaths, repoRoot);
     const rootRelativePaths = new Map(
         relativePaths.map((relPath) => [
             relPath,
@@ -104,7 +105,7 @@ export const flattenMonorepoContext = async (
     actorJson: Record<string, unknown>,
     keptContextFiles: string[],
 ): Promise<{ tempDir: string; filePaths: string[] }> => {
-    console.error(`[${actorName}]: monorepo actor detected — flattening from Docker context`);
+    logger.info(`[${actorName}]: monorepo actor detected — flattening from Docker context`);
 
     const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), `apify-build-${actorName.replace('/', '_')}-`));
     const filePaths: string[] = [];
@@ -194,9 +195,9 @@ export const runBuildsFromLocal = async ({
     dryRun: boolean;
 }): Promise<BuildData[]> => {
     if (dryRun) {
-        console.error('[DRY RUN] Would build from local source:');
+        logger.info('[DRY RUN] Would build from local source:');
         for (const { actorFullName, folder } of actorConfigs) {
-            console.error(`  ${actorFullName} (${folder})`);
+            logger.info(`  ${actorFullName} (${folder})`);
         }
         return actorConfigs.map(({ actorFullName }) => dryRunBuildData(actorFullName, LOCAL_SOURCE_VERSION_NUMBER));
     }

package/bin/build.ts

@@ -2,6 +2,7 @@ import type * as ApifyClientTypes from 'apify-client';
 import { ActorSourceType, ApifyClient } from 'apify-client';
 
 import { normalizeRepoUrl } from './git.js';
+import { logger } from './logger.js';
 import type { ActorConfig, BuildData } from './types.js';
 
 type BuildPrActorOptions = {
@@ -28,7 +29,7 @@ const ACTOR_SETUP_REQUIREMENT =
  */
 export const resolveDefaultVersion = (actorFullName: string, actorInfo: ApifyClientTypes.Actor) => {
     const defaultBuildTag = actorInfo.defaultRunOptions.build;
-    console.error(`Default build tag for ${actorFullName} is ${defaultBuildTag}`);
+    logger.info(`Default build tag for ${actorFullName} is ${defaultBuildTag}`);
 
     // We could technically allow this but in most cases this is accidentally set wrongly and there is a workaround
     if (defaultBuildTag.match(/\d+\.\d+\.\d+/)) {
@@ -43,7 +44,7 @@ export const resolveDefaultVersion = (actorFullName: string, actorInfo: ApifyCli
         throw new Error(`[${actorFullName}] No build found for tag "${defaultBuildTag}". ${ACTOR_SETUP_REQUIREMENT}`);
     }
     const defaultVersionNumber = defaultBuildNumber.match(/(\d+\.\d+)\.\d+/)![1];
-    console.error(`Default version for ${actorFullName} is ${defaultVersionNumber}`);
+    logger.info(`Default version for ${actorFullName} is ${defaultVersionNumber}`);
 
     const defaultVersion = actorInfo.versions.find((version) => version.versionNumber === defaultVersionNumber);
 
@@ -139,7 +140,7 @@ export class ApifyBuilder {
         // We also get back actId so the testing actor can both match by actor ID and name
         const { id, actId, buildNumber } = await actorClient.build(versionNumber, { useCache });
 
-        console.error(`[${this.actorFullName}]: ${id} (${buildNumber})`);
+        logger.info(`[${this.actorFullName}]: ${id} (${buildNumber})`);
         return { buildId: id, actorRawId: actId, buildNumber, actorFullName: this.actorFullName };
     };
 
@@ -147,20 +148,20 @@ export class ApifyBuilder {
         const build = await this.apifyClient.build(buildId).waitForFinish();
         const versionNumber = build.buildNumber;
         if (build.status === 'FAILED' || build.status === 'TIMED-OUT') {
-            console.error(`[${this.actorFullName}]: ${versionNumber}`);
+            logger.error(`[${this.actorFullName}]: ${versionNumber}`);
             try {
                 const log = await this.apifyClient.build(buildId).log().get();
                 const logTail = log?.split('\n').slice(-40).join('\n');
-                console.error(`\n--- BUILD LOG (last 40 lines) ---\n${logTail}\n---`);
+                logger.error(`\n--- BUILD LOG (last 40 lines) ---\n${logTail}\n---`);
             } catch (err) {
-                console.error(`[${this.actorFullName}]: Failed to fetch build log: ${err}`);
+                logger.error(`[${this.actorFullName}]: Failed to fetch build log: ${err}`);
             }
             throw new Error(
                 `[BUILD][${this.actorFullName}]: Build ${buildId} (${versionNumber}) failed. ` +
                     `Not continuing with other builds and tests.`,
             );
         }
-        console.error(`[${this.actorFullName}]: ${versionNumber}`);
+        logger.info(`[${this.actorFullName}]: ${versionNumber}`);
         return build;
     };
 
@@ -227,7 +228,7 @@ export class ApifyBuilder {
         type CorrectBuildColletionItem = (typeof items)[0] & { buildNumber: string };
         const buildsToDelete = (items as CorrectBuildColletionItem[]).filter((build) => {
             if (build.buildNumber === defaultBuildNumber) {
-                console.error(
+                logger.info(
                     `[DELETE OLD BUILDS][${this.actorFullName}]: Skipping default build ${defaultBuildNumber} (${defaultBuildTag}). ` +
                         `We never delete default builds`,
                 );
@@ -238,7 +239,7 @@ export class ApifyBuilder {
                 (protectedBuildNumber) => protectedBuildNumber.buildNumber === build.buildNumber,
             );
             if (protectedTagFound) {
-                console.error(
+                logger.info(
                     `[DELETE OLD BUILDS][${this.actorFullName}]: Skipping protected build ${protectedTagFound.buildNumber} (${protectedTagFound.tag}).`,
                 );
                 return false;
@@ -247,7 +248,7 @@ export class ApifyBuilder {
             if (taggedDevelBuildNumber && build.buildNumber === taggedDevelBuildNumber) {
                 const shouldDeleteDevelBuild = build.startedAt.getTime() < daysAgoUnixDevel;
                 if (shouldDeleteDevelBuild) {
-                    console.error(
+                    logger.info(
                         `[DELETE OLD BUILDS][${this.actorFullName}]: Removing olf devel build ${taggedDevelBuildNumber}.`,
                     );
                 }
@@ -256,7 +257,7 @@ export class ApifyBuilder {
             return build.startedAt.getTime() < daysAgoUnixProd;
         });
 
-        console.error(
+        logger.info(
             `[DELETE OLD BUILDS][${this.actorFullName}]: Deleting ${buildsToDelete.length} old builds that are non-default and ` +
                 `older than 30 days from total ${items.length}`,
         );
@@ -271,8 +272,8 @@ export const waitAndSummarizeBuilds = async (
     buildersMap: Map<string, ApifyBuilder>,
     label: string,
 ): Promise<BuildData[]> => {
-    console.error('=========================================');
-    console.error(`FINISHED ${label}:`);
+    logger.info('=========================================');
+    logger.info(`FINISHED ${label}:`);
     await Promise.all(
         startedBuilds.map(async (buildData) => {
             const builder = buildersMap.get(buildData.actorFullName)!;
@@ -280,12 +281,12 @@ export const waitAndSummarizeBuilds = async (
         }),
     );
 
-    console.error('=========================================');
-    console.error('SUMMARY:');
+    logger.info('=========================================');
+    logger.info('SUMMARY:');
     for (const buildData of startedBuilds.sort((a, b) => a.actorFullName.localeCompare(b.actorFullName))) {
-        console.error(`[${buildData.actorFullName}]: ${buildData.buildNumber}`);
+        logger.info(`[${buildData.actorFullName}]: ${buildData.buildNumber}`);
     }
-    console.error('=========================================');
+    logger.info('=========================================');
 
     return startedBuilds;
 };
@@ -298,8 +299,8 @@ export const runAndSummarizeBuilds = async (
     const buildersByActorFullName = new Map<string, ApifyBuilder>(
         actorConfigs.map((actorConfig) => [actorConfig.actorFullName, ApifyBuilder.fromActorConfig(actorConfig)]),
     );
-    console.error('=========================================');
-    console.error(`STARTED ${label}:`);
+    logger.info('=========================================');
+    logger.info(`STARTED ${label}:`);
     const startedBuilds = await Promise.all(
         actorConfigs.map(async (actorConfig) =>
             buildOneActor(actorConfig, buildersByActorFullName.get(actorConfig.actorFullName)!),
@@ -366,9 +367,9 @@ export const runBuilds = async ({
     );
 
     if (dryRun) {
-        console.error('[DRY RUN] Would build:');
+        logger.info('[DRY RUN] Would build:');
         for (const { actorConfig, versionNumber } of buildConfigs) {
-            console.error(`  ${actorConfig.actorFullName} (${versionNumber})`);
+            logger.info(`  ${actorConfig.actorFullName} (${versionNumber})`);
         }
         return buildConfigs.map(({ actorConfig, versionNumber }) =>
             dryRunBuildData(actorConfig.actorFullName, versionNumber),

package/bin/diff-changes.ts

@@ -1,5 +1,6 @@
 import { isCosmeticOnlyJsonSchemaChange } from './diff-json-schema.js';
 import { type DockerIgnoreMatcher, loadDockerIgnore } from './dockerignore.js';
+import { logger } from './logger.js';
 import { findContainingScope, hoistPath, isPathWithinScope } from './path-utils.js';
 import type { ActorConfig, Commit } from './types.js';
 
@@ -44,12 +45,12 @@ type FileChangeForActor =
  * 6. .json inside the actor's own `.actor/` dir with only cosmetic schema diffs → cosmetic (semantically verified)
  * 7. Everything else → functional
  */
-const classifyFileChange = (
+const classifyFileChange = async (
     originalFilePath: string,
     actorConfig: ActorConfig,
     commits: Commit[],
     dockerIgnoreMatcher: DockerIgnoreMatcher,
-): FileChangeForActor => {
+): Promise<FileChangeForActor> => {
     const lowercaseFilePath = originalFilePath.toLowerCase();
 
     // TODO: hardcodes that there's a single repo-wide changelog belonging to every actor. Should instead
@@ -89,7 +90,7 @@ const classifyFileChange = (
     }
 
     if (lowercaseFilePath.endsWith('.json') && isUnderActorDotDir) {
-        const isCosmetic = isCosmeticOnlyJsonSchemaChange(commits, originalFilePath);
+        const isCosmetic = await isCosmeticOnlyJsonSchemaChange(commits, originalFilePath);
         if (isCosmetic) {
             return { impact: 'cosmetic', semanticallyVerified: true };
         }
@@ -159,19 +160,19 @@ const groupFilesByActorSet = (fileToActors: Map<string, Set<string>>): ChangeGro
 const logChangeGroups = (groups: ChangeGroup[]): void => {
     for (const { actors, files } of groups) {
         if (actors.length > 1) {
-            console.error(`[DIFF]: Shared changes for actors ${actors.join(', ')}: ${files.join(', ')}`);
+            logger.info(`[DIFF]: Shared changes for actors ${actors.join(', ')}: ${files.join(', ')}`);
         } else {
-            console.error(`[DIFF]: Changes specific to actor ${actors[0]}: ${files.join(', ')}`);
+            logger.info(`[DIFF]: Changes specific to actor ${actors[0]}: ${files.join(', ')}`);
         }
     }
 };
 
-export const getChangedActors = ({
+export const getChangedActors = async ({
     filepathsChanged,
     actorConfigs,
     isLatest = false,
     commits,
-}: ShouldBuildAndTestOptions): ActorConfig[] => {
+}: ShouldBuildAndTestOptions): Promise<ActorConfig[]> => {
     const actorsChangedMap = new Map<string, ActorChangeEntry>();
 
     for (const actorConfig of actorConfigs) {
@@ -184,7 +185,7 @@ export const getChangedActors = ({
                 continue;
             }
 
-            const change = classifyFileChange(originalFilePath, actorConfig, commits, dockerIgnoreMatcher);
+            const change = await classifyFileChange(originalFilePath, actorConfig, commits, dockerIgnoreMatcher);
 
             if (change.impact === 'ignored' || change.impact === 'outside-context') continue;
             if (change.impact === 'cosmetic' && !isLatest) continue;
@@ -205,9 +206,9 @@ export const getChangedActors = ({
 
     if (actorsChanged.length > 0) {
         const actors = actorsChanged.map((config) => config.actorFullName);
-        console.error(`[DIFF]: Actors to be built and tested: ${actors.join(', ')}`);
+        logger.info(`[DIFF]: Actors to be built and tested: ${actors.join(', ')}`);
     } else {
-        console.error(`[DIFF]: No relevant files changed, skipping builds and tests`);
+        logger.info(`[DIFF]: No relevant files changed, skipping builds and tests`);
     }
 
     return actorsChanged;

package/bin/diff-json-schema.ts

@@ -1,5 +1,6 @@
+import { logger } from './logger.js';
 import type { Commit } from './types.js';
-import { spawnCommand } from './utils.js';
+import { runGitCommand } from './utils.js';
 
 const COSMETIC_JSON_FIELD_NAMES = new Set([
     'title',
@@ -28,21 +29,22 @@ const isCosmeticObjectChange = (oldVal: unknown, newVal: unknown, currentKey?: s
  * Returns true if the two JSON strings differ only in cosmetic fields
  * (title, description, example, enumTitles, sectionCaption, sectionDescription).
  */
-export const isCosmeticOnlyJsonSchemaChange = (commits: Commit[], changedFilepath: string): boolean => {
+export const isCosmeticOnlyJsonSchemaChange = async (commits: Commit[], changedFilepath: string): Promise<boolean> => {
     // TODO: validate this is the right commit range
     const oldRef = `${commits[0].sha}~`;
     const newRef = commits[commits.length - 1].sha;
-    let oldJson: unknown;
-    let newJson: unknown;
-    try {
-        const oldContent = spawnCommand(`git show ${oldRef}:${changedFilepath}`);
-        const newContent = spawnCommand(`git show ${newRef}:${changedFilepath}`);
 
-        oldJson = JSON.parse(oldContent);
-        newJson = JSON.parse(newContent);
-    } catch {
-        console.error(
-            `Failed to get or parse JSON content for ${changedFilepath} at refs ${oldRef} and ${newRef}, maybe it is new file or deleted? Treating it as a non-cosmetic change.`,
+    const [oldJson, newJson] = await Promise.all([
+        runGitCommand(['show', `${oldRef}:${changedFilepath}`])
+            .then(JSON.parse)
+            .catch(() => undefined),
+        runGitCommand(['show', `${newRef}:${changedFilepath}`])
+            .then(JSON.parse)
+            .catch(() => undefined),
+    ]);
+    if (!oldJson || !newJson) {
+        logger.info(
+            `Failed to get JSON content for ${changedFilepath} at refs ${oldRef} and ${newRef}, maybe it is new file or deleted? Treating it as a non-cosmetic change.`,
         );
         return false;
     }