@diffci.com/diffci 0.1.5 → 0.1.7

package/README.md

@@ -6,7 +6,8 @@
 
 **Find test-selection opportunities in your CI before changing what it runs.** DiffCI analyzes a
 commit's changes and dependency graph, then reports which test files it would select, why it falls
-back to a full run, and whether it can propose a test command. The CLI and Action are observation-only.
+back to a full run, and whether it can propose a test command. The `observe` command and Action are observation-only;
+the opt-in `pilot` and `verify-savings` commands execute tests.
 
 From an existing repository checkout, with Node.js 22.5+ and Git installed:
 
@@ -14,6 +15,23 @@ From an existing repository checkout, with Node.js 22.5+ and Git installed:
 npx @diffci.com/diffci@latest observe --no-send
 ```
 
+For the fastest self-serve runtime pilot, run one paired check from the repository root:
+
+```bash
+npx @diffci.com/diffci@latest pilot --full "npm test"
+```
+
+On Windows PowerShell, quote the package name:
+
+```powershell
+npx '@diffci.com/diffci@latest' pilot --full "npm test"
+```
+
+This executes the full and selected commands sequentially, and writes `diffci-observe.json`,
+`diffci-savings.json`, and `diffci-savings.md` to a sibling `diffci-output` folder outside the checkout.
+The commands you supply may create files or otherwise change the checkout. One paired run is preliminary
+timing evidence; repeat comparisons and account for cache effects before claiming savings.
+
 **Upgrade from 0.1.3:** tests excluded by a source-only `tsconfig.json` could be discovered without
 their dependency edges, producing an incomplete selection. This is fixed in **0.1.4**. Revalidate
 affected observations before using them as opportunity evidence; see the
@@ -34,8 +52,8 @@ not Cal.com's production savings or a prediction for your repository.
 Selection counts alone do not establish runtime savings. Observation mode measures neither the
 selected test execution nor realized savings.
 
-For a self-serve paired runtime check, run `observe` first and then run `verify-savings` against the
-observation report. It compares your normal full command with
+For an advanced paired runtime check, you can still run `observe` first and then run `verify-savings`
+against the observation report. It compares your normal full command with
 DiffCI's proposed selected command and writes JSON plus Markdown evidence; see
 [`docs/npm-adoption.md`](docs/npm-adoption.md#self-serve-runtime-pilot).
 

package/dist-client/src/client/cli.js

@@ -28,7 +28,7 @@
 import { execFileSync } from "node:child_process";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
-import { dirname, join, resolve } from "node:path";
+import { basename, dirname, join, resolve } from "node:path";
 import { observe, isInsideRepository } from "./observe.js";
 import { submitObservation } from "./submit.js";
 import { formatVerifySavingsSummary, runVerifySavings, writeVerifySavingsReport } from "./verify-savings.js";
@@ -286,9 +286,65 @@ function runVerifySavingsCommand(flags, env) {
         console.log(`  markdown: ${options.markdown}`);
     return report.comparison.fullCommandSucceeded && report.comparison.selectedCommandSucceeded ? 0 : 1;
 }
+function defaultPilotOutputDir(repoPath) {
+    return join(dirname(repoPath), "diffci-output");
+}
+async function runPilot(flags, env) {
+    const full = stringFlag(flags, "full");
+    if (!full) {
+        console.error('--full <command> is required, for example: diffci pilot --full "npm test"');
+        return 1;
+    }
+    const repoPath = resolve(stringFlag(flags, "repo") ?? env.GITHUB_WORKSPACE ?? process.cwd());
+    const outDir = resolve(stringFlag(flags, "out-dir") ?? defaultPilotOutputDir(repoPath));
+    if (isInsideRepository(repoPath, outDir) || outDir === repoPath) {
+        console.error(`Refusing to write pilot reports inside the repository: ${outDir}. Pass --out-dir with a path outside the checkout.`);
+        return 2;
+    }
+    const label = stringFlag(flags, "label") ?? basename(repoPath);
+    const observationPath = join(outDir, "diffci-observe.json");
+    const savingsPath = join(outDir, "diffci-savings.json");
+    const markdownPath = join(outDir, "diffci-savings.md");
+    const identity = observerIdentity();
+    const observation = await observe({
+        repoPath,
+        env: env,
+        version: identity.version,
+        engineSha: identity.sha,
+        baseOverride: stringFlag(flags, "base"),
+        headOverride: stringFlag(flags, "head"),
+        redactPaths: flags["redact-paths"] === true,
+        reportPath: observationPath,
+    });
+    mkdirSync(outDir, { recursive: true });
+    writeFileSync(observationPath, `${JSON.stringify(observation, null, 2)}\n`, "utf8");
+    console.log(summarise(observation));
+    console.log(`  observation report: ${observationPath}`);
+    if (observation.status !== "OBSERVED") {
+        console.log("DiffCI pilot stopped before timing because observation did not produce a selectable report.");
+        return 1;
+    }
+    const savings = runVerifySavings({
+        full,
+        selectedFromReport: observationPath,
+        out: savingsPath,
+        markdown: markdownPath,
+        label,
+        cwd: repoPath,
+        timeoutMs: numberFlag(flags, "timeout-ms") ?? 30 * 60 * 1000,
+        analysisOverheadMs: numberFlag(flags, "analysis-overhead-ms"),
+        tailBytes: numberFlag(flags, "tail-bytes") ?? 12_000,
+    });
+    writeVerifySavingsReport(savings, { out: savingsPath, markdown: markdownPath });
+    console.log(formatVerifySavingsSummary(savings));
+    console.log(`  savings report: ${savingsPath}`);
+    console.log(`  markdown: ${markdownPath}`);
+    return savings.comparison.fullCommandSucceeded && savings.comparison.selectedCommandSucceeded ? 0 : 1;
+}
 const USAGE = `diffci - observation-only change-aware CI analysis
 
 Usage:
+  diffci pilot --full <command> [--repo <path>] [--out-dir <dir>] [--label <name>]
   diffci observe [--repo <path>] [--out <file>] [--base <sha> --head <sha>]
                  [--redact-paths] [--json] [--quiet] [--fail-on-error]
                  [--api-url <url> --api-token <token>] [--no-send]
@@ -298,6 +354,7 @@ Usage:
   diffci verify-workflow [--repo <path>]
   diffci version
 
+pilot runs observe and verify-savings together, writing reports to ../diffci-output by default.
 observe analyses the checkout and writes one JSON report. It runs nothing and changes nothing.
 verify-savings runs both commands and reports measured paired runtime; it is an opt-in pilot command.
 verify-workflow checks that the job running DiffCI cannot affect any other job, and exits 1 if it can.
@@ -315,6 +372,9 @@ async function main() {
         return;
     }
     switch (command) {
+        case "pilot":
+            process.exitCode = await runPilot(flags, env);
+            return;
         case "observe":
             process.exitCode = await runObserve(flags, env);
             return;
@@ -333,8 +393,15 @@ async function main() {
     }
 }
 main().catch((error) => {
-    // Reaching here means a defect outside observe()'s own guard. It still must not take a build down:
-    // the failure is printed, and the exit code stays 0 unless the caller asked otherwise.
-    console.error(`DiffCI observer failed: ${error instanceof Error ? error.message : String(error)}`);
-    process.exitCode = process.argv.includes("--fail-on-error") ? 1 : 0;
+    const command = process.argv[2];
+    const message = error instanceof Error ? error.message : String(error);
+    if (command === "observe") {
+        // Reaching here means a defect outside observe()'s own guard. It still must not take a build down:
+        // the failure is printed, and the exit code stays 0 unless the caller asked otherwise.
+        console.error(`DiffCI observer failed: ${message}`);
+        process.exitCode = process.argv.includes("--fail-on-error") ? 1 : 0;
+        return;
+    }
+    console.error(`DiffCI ${command ?? "command"} failed: ${message}`);
+    process.exitCode = 1;
 });

package/dist-client/src/client/verify-savings.js

@@ -11,11 +11,11 @@ function readSelectionFromObservation(path) {
     const parsed = JSON.parse(readFileSync(absolutePath, "utf8"));
     if (parsed.status !== "OBSERVED")
         throw new Error(`--selected-from-report requires an OBSERVED report; got ${String(parsed.status)}`);
-    const command = Array.isArray(parsed.result?.proposedCommands) && typeof parsed.result.proposedCommands[0] === "string"
-        ? parsed.result.proposedCommands[0]
-        : undefined;
-    if (!command)
-        throw new Error("--selected-from-report did not contain result.proposedCommands[0]");
+    const commands = parsed.result?.proposedCommands;
+    if (!Array.isArray(commands) || commands.length !== 1 || typeof commands[0] !== "string" || !commands[0].trim()) {
+        throw new Error("--selected-from-report requires exactly one non-empty proposed command; use --selected with an explicit command covering the complete selection for multi-command plans");
+    }
+    const command = commands[0];
     const selectedTests = Array.isArray(parsed.result?.selectedTests) ? parsed.result.selectedTests : undefined;
     return {
         command,
@@ -122,7 +122,9 @@ export function renderVerifySavingsMarkdown(report) {
     const title = report.label ? `# DiffCI Verify Savings: ${report.label}` : "# DiffCI Verify Savings";
     const warning = report.comparison.missedFailureSignal
         ? "\n> WARNING: Full failed while selected passed. Do not treat this selected command as safe until the full-run failure is understood.\n"
-        : "";
+        : !report.comparison.fullCommandSucceeded || !report.comparison.selectedCommandSucceeded
+            ? "\n> WARNING: One or both commands failed. This comparison is invalid as savings evidence; timings below are diagnostic only.\n"
+            : "";
     const selectionCounts = report.selectedTestCount !== undefined && report.totalTestCount !== undefined
         ? `\nSelected tests: ${report.selectedTestCount} of ${report.totalTestCount}\n`
         : "";
@@ -137,6 +139,7 @@ This report compares a full command with a selected command on the same checkout
 
 ## Result
 
+${!report.comparison.fullCommandSucceeded || !report.comparison.selectedCommandSucceeded ? "Comparison invalid: command failure. Do not interpret the timing difference as savings.\n" : ""}
 | Measure | Value |
 | --- | ---: |
 | Full runtime | ${formatMs(report.comparison.fullWallMs)} |
@@ -183,6 +186,10 @@ export function writeVerifySavingsReport(report, paths) {
         writeText(paths.markdown, renderVerifySavingsMarkdown(report));
 }
 export function formatVerifySavingsSummary(report) {
+    if (!report.comparison.fullCommandSucceeded || !report.comparison.selectedCommandSucceeded) {
+        return "DiffCI verify-savings: comparison invalid because one or both commands failed" +
+            (report.comparison.missedFailureSignal ? "\n  warning: full failed while selected passed; inspect outputs before claiming safety" : "");
+    }
     const lines = [
         `DiffCI verify-savings: ${report.comparison.deltaMs >= 0 ? "faster" : "slower"} by ${formatMs(Math.abs(report.comparison.deltaMs))}`,
     ];

package/docs/npm-adoption.md

@@ -62,7 +62,7 @@ This is a paired local measurement, not a production-savings claim.
 Step 1: create an observation report without sending it anywhere.
 
 ```bash
-npx @diffci.com/diffci@latest observe --no-send --out ./diffci-observation.json
+npx @diffci.com/diffci@latest observe --no-send --out ../diffci-output/diffci-observation.json
 ```
 
 Step 2: run the paired pilot.
@@ -72,13 +72,17 @@ npx @diffci.com/diffci@latest verify-savings \
   --label owner/repo \
   --repo /path/to/their/repo \
   --full "npm test" \
-  --selected-from-report /path/to/their/repo/diffci-observation.json \
-  --out ./diffci-verify-savings.json \
-  --markdown ./diffci-verify-savings.md
+  --selected-from-report ../diffci-output/diffci-observation.json \
+  --out ../diffci-output/diffci-verify-savings.json \
+  --markdown ../diffci-output/diffci-verify-savings.md
 ```
 
 What the report means:
 
+Run both steps from the same repository root with the same checked-out revision. These commands
+execute repository code. The full run can warm caches for the selected run, so repeat comparisons
+with controlled cache state before drawing conclusions. A passing pair does not establish selection safety.
+
 - Full runtime is measured from `--full`.
 - Selected runtime is measured from DiffCI's proposed command in the observation report.
 - DiffCI analysis overhead is imported from `timings.totalMs` in the observation report unless

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@diffci.com/diffci",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "DiffCI - deterministic change-aware CI planning",
   "license": "AGPL-3.0-only",
   "repository": {