@diffci.com/diffci 0.1.6 → 0.1.7

package/README.md

@@ -6,7 +6,8 @@
 
 **Find test-selection opportunities in your CI before changing what it runs.** DiffCI analyzes a
 commit's changes and dependency graph, then reports which test files it would select, why it falls
-back to a full run, and whether it can propose a test command. The CLI and Action are observation-only.
+back to a full run, and whether it can propose a test command. The `observe` command and Action are observation-only;
+the opt-in `pilot` and `verify-savings` commands execute tests.
 
 From an existing repository checkout, with Node.js 22.5+ and Git installed:
 
@@ -26,8 +27,10 @@ On Windows PowerShell, quote the package name:
 npx '@diffci.com/diffci@latest' pilot --full "npm test"
 ```
 
-This writes `diffci-observe.json`, `diffci-savings.json`, and `diffci-savings.md` to a sibling
-`diffci-output` folder outside the checkout, so it does not dirty the repository.
+This executes the full and selected commands sequentially, and writes `diffci-observe.json`,
+`diffci-savings.json`, and `diffci-savings.md` to a sibling `diffci-output` folder outside the checkout.
+The commands you supply may create files or otherwise change the checkout. One paired run is preliminary
+timing evidence; repeat comparisons and account for cache effects before claiming savings.
 
 **Upgrade from 0.1.3:** tests excluded by a source-only `tsconfig.json` could be discovered without
 their dependency edges, producing an incomplete selection. This is fixed in **0.1.4**. Revalidate

package/dist-client/src/client/cli.js

@@ -393,8 +393,15 @@ async function main() {
     }
 }
 main().catch((error) => {
-    // Reaching here means a defect outside observe()'s own guard. It still must not take a build down:
-    // the failure is printed, and the exit code stays 0 unless the caller asked otherwise.
-    console.error(`DiffCI observer failed: ${error instanceof Error ? error.message : String(error)}`);
-    process.exitCode = process.argv.includes("--fail-on-error") ? 1 : 0;
+    const command = process.argv[2];
+    const message = error instanceof Error ? error.message : String(error);
+    if (command === "observe") {
+        // Reaching here means a defect outside observe()'s own guard. It still must not take a build down:
+        // the failure is printed, and the exit code stays 0 unless the caller asked otherwise.
+        console.error(`DiffCI observer failed: ${message}`);
+        process.exitCode = process.argv.includes("--fail-on-error") ? 1 : 0;
+        return;
+    }
+    console.error(`DiffCI ${command ?? "command"} failed: ${message}`);
+    process.exitCode = 1;
 });

package/dist-client/src/client/verify-savings.js

@@ -11,11 +11,11 @@ function readSelectionFromObservation(path) {
     const parsed = JSON.parse(readFileSync(absolutePath, "utf8"));
     if (parsed.status !== "OBSERVED")
         throw new Error(`--selected-from-report requires an OBSERVED report; got ${String(parsed.status)}`);
-    const command = Array.isArray(parsed.result?.proposedCommands) && typeof parsed.result.proposedCommands[0] === "string"
-        ? parsed.result.proposedCommands[0]
-        : undefined;
-    if (!command)
-        throw new Error("--selected-from-report did not contain result.proposedCommands[0]");
+    const commands = parsed.result?.proposedCommands;
+    if (!Array.isArray(commands) || commands.length !== 1 || typeof commands[0] !== "string" || !commands[0].trim()) {
+        throw new Error("--selected-from-report requires exactly one non-empty proposed command; use --selected with an explicit command covering the complete selection for multi-command plans");
+    }
+    const command = commands[0];
     const selectedTests = Array.isArray(parsed.result?.selectedTests) ? parsed.result.selectedTests : undefined;
     return {
         command,
@@ -122,7 +122,9 @@ export function renderVerifySavingsMarkdown(report) {
     const title = report.label ? `# DiffCI Verify Savings: ${report.label}` : "# DiffCI Verify Savings";
     const warning = report.comparison.missedFailureSignal
         ? "\n> WARNING: Full failed while selected passed. Do not treat this selected command as safe until the full-run failure is understood.\n"
-        : "";
+        : !report.comparison.fullCommandSucceeded || !report.comparison.selectedCommandSucceeded
+            ? "\n> WARNING: One or both commands failed. This comparison is invalid as savings evidence; timings below are diagnostic only.\n"
+            : "";
     const selectionCounts = report.selectedTestCount !== undefined && report.totalTestCount !== undefined
         ? `\nSelected tests: ${report.selectedTestCount} of ${report.totalTestCount}\n`
         : "";
@@ -137,6 +139,7 @@ This report compares a full command with a selected command on the same checkout
 
 ## Result
 
+${!report.comparison.fullCommandSucceeded || !report.comparison.selectedCommandSucceeded ? "Comparison invalid: command failure. Do not interpret the timing difference as savings.\n" : ""}
 | Measure | Value |
 | --- | ---: |
 | Full runtime | ${formatMs(report.comparison.fullWallMs)} |
@@ -183,6 +186,10 @@ export function writeVerifySavingsReport(report, paths) {
         writeText(paths.markdown, renderVerifySavingsMarkdown(report));
 }
 export function formatVerifySavingsSummary(report) {
+    if (!report.comparison.fullCommandSucceeded || !report.comparison.selectedCommandSucceeded) {
+        return "DiffCI verify-savings: comparison invalid because one or both commands failed" +
+            (report.comparison.missedFailureSignal ? "\n  warning: full failed while selected passed; inspect outputs before claiming safety" : "");
+    }
     const lines = [
         `DiffCI verify-savings: ${report.comparison.deltaMs >= 0 ? "faster" : "slower"} by ${formatMs(Math.abs(report.comparison.deltaMs))}`,
     ];

package/docs/npm-adoption.md

@@ -62,7 +62,7 @@ This is a paired local measurement, not a production-savings claim.
 Step 1: create an observation report without sending it anywhere.
 
 ```bash
-npx @diffci.com/diffci@latest observe --no-send --out ./diffci-observation.json
+npx @diffci.com/diffci@latest observe --no-send --out ../diffci-output/diffci-observation.json
 ```
 
 Step 2: run the paired pilot.
@@ -72,13 +72,17 @@ npx @diffci.com/diffci@latest verify-savings \
   --label owner/repo \
   --repo /path/to/their/repo \
   --full "npm test" \
-  --selected-from-report /path/to/their/repo/diffci-observation.json \
-  --out ./diffci-verify-savings.json \
-  --markdown ./diffci-verify-savings.md
+  --selected-from-report ../diffci-output/diffci-observation.json \
+  --out ../diffci-output/diffci-verify-savings.json \
+  --markdown ../diffci-output/diffci-verify-savings.md
 ```
 
 What the report means:
 
+Run both steps from the same repository root with the same checked-out revision. These commands
+execute repository code. The full run can warm caches for the selected run, so repeat comparisons
+with controlled cache state before drawing conclusions. A passing pair does not establish selection safety.
+
 - Full runtime is measured from `--full`.
 - Selected runtime is measured from DiffCI's proposed command in the observation report.
 - DiffCI analysis overhead is imported from `timings.totalMs` in the observation report unless

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@diffci.com/diffci",
-  "version": "0.1.6",
+  "version": "0.1.7",
   "description": "DiffCI - deterministic change-aware CI planning",
   "license": "AGPL-3.0-only",
   "repository": {