@haystackeditor/cli 0.15.26 → 0.15.27

package/dist/utils/design-verifier-result.js

@@ -0,0 +1,299 @@
+import { isDeepStrictEqual } from 'node:util';
+function findingExplanation(kind) {
+    if (kind === 'declared-change-not-observed') {
+        return 'The intended behavior change was not observed on the proposed revision.';
+    }
+    if (kind === 'unexpected-change') {
+        return 'The proposed revision changed behavior that was registered to remain stable.';
+    }
+    return 'Executed behavior contradicted the registered expectation.';
+}
+function sameRecordedValue(left, right) {
+    return isDeepStrictEqual(left, right);
+}
+function executionSummaries(value) {
+    if (!Array.isArray(value))
+        return [];
+    return value.filter((execution) => !!execution && typeof execution === 'object' &&
+        ['base1', 'base2', 'head'].includes(execution.side) &&
+        ['completed', 'failed', 'missing'].includes(execution.status) &&
+        Array.isArray(execution.diagnostics) && Array.isArray(execution.outputExcerpt));
+}
+export function buildDesignVerifierAgentReport(run) {
+    const repository = `${run.owner}/${run.repo}`;
+    if (run.status !== 'succeeded' || !run.result) {
+        return {
+            ok: false,
+            status: 'failed',
+            verdict: 'inconclusive',
+            headline: 'VERIFICATION INCONCLUSIVE',
+            explanation: run.error || 'The verifier did not produce a terminal behavioral verdict.',
+            runId: run.runId,
+            repository,
+            pullRequest: run.pr,
+            revisions: { base: run.baseSha, head: run.headSha },
+            durationMs: run.durationMs,
+            timings: null,
+            changedFiles: [],
+            cases: [],
+            inspectCommand: `haystack design-verify ${repository}#${run.pr} --run ${run.runId}`,
+            replay: null,
+            error: run.error,
+            exitCode: 2,
+        };
+    }
+    const verdict = run.result.verdict || {};
+    const summary = verdict.summary || {};
+    const findings = Array.isArray(verdict.findings) ? verdict.findings : [];
+    const confirmations = Array.isArray(verdict.confirmations) ? verdict.confirmations : [];
+    const findingByCase = new Map(findings.map((finding) => [finding.caseId, finding]));
+    const confirmedIds = new Set(confirmations.map((confirmation) => confirmation.caseId));
+    const scenarioByCase = new Map(run.result.scenarios.map((scenario) => [scenario.caseId, scenario]));
+    const probeByCase = new Map(run.result.probes.map((probe) => [probe.caseId, probe]));
+    const cases = run.result.cases.map((testCase) => {
+        const id = String(testCase.id || '');
+        const finding = findingByCase.get(id);
+        const scenario = scenarioByCase.get(id);
+        const probe = probeByCase.get(id);
+        const observations = (Array.isArray(scenario?.divergences) ? scenario.divergences : []).map((item) => ({
+            path: String(item.display || (Array.isArray(item.path) ? item.path.join('.') : 'observation')),
+            base: item.base,
+            baseRepeat: item.base2,
+            head: item.head,
+        }));
+        return {
+            id,
+            title: String(testCase.title || scenario?.scenario || id),
+            story: String(testCase.story || ''),
+            expectedHead: String(testCase.expected_head_observation || ''),
+            verdict: finding
+                ? 'finding'
+                : confirmedIds.has(id)
+                    ? 'confirmed'
+                    : scenario?.status === 'clean'
+                        ? 'clean'
+                        : 'inconclusive',
+            findingKind: finding?.kind || null,
+            baseStable: observations.length
+                ? observations.every((item) => sameRecordedValue(item.base, item.baseRepeat))
+                : scenario?.status === 'clean' ? true : null,
+            observations,
+            generatedFiles: probe?.files || [],
+            executedCommand: probe?.run || null,
+            executions: executionSummaries(scenario?.executions),
+            infrastructure: scenario?.infrastructureReceipt &&
+                typeof scenario.infrastructureReceipt === 'object'
+                ? {
+                    verified: scenario.infrastructureReceipt.status === 'passed',
+                    recipeId: typeof scenario.infrastructureReceipt.recipeId === 'string'
+                        ? scenario.infrastructureReceipt.recipeId
+                        : null,
+                    freshInstances: Array.isArray(scenario.infrastructureReceipt.environmentInstanceIds)
+                        ? scenario.infrastructureReceipt.environmentInstanceIds.length
+                        : 0,
+                    baseRepeat: typeof scenario.infrastructureReceipt.baseRepeat === 'string'
+                        ? scenario.infrastructureReceipt.baseRepeat
+                        : null,
+                    errors: Array.isArray(scenario.infrastructureReceipt.errors)
+                        ? scenario.infrastructureReceipt.errors.filter((item) => typeof item === 'string')
+                        : [],
+                }
+                : null,
+        };
+    });
+    let agentVerdict;
+    let headline;
+    let explanation;
+    let exitCode;
+    if (findings.length > 0) {
+        agentVerdict = 'fix_needed';
+        headline = 'FIX NEEDED';
+        explanation = findingExplanation(String(findings[0]?.kind || ''));
+        exitCode = 1;
+    }
+    else if (summary.caseDesignOutcome === 'no-supported-cases' || Number(summary.scenarios || 0) === 0) {
+        agentVerdict = 'inconclusive';
+        headline = 'VERIFICATION INCONCLUSIVE';
+        explanation = 'The verifier could not design and execute a supported behavioral case.';
+        exitCode = 2;
+    }
+    else if (confirmations.length > 0) {
+        agentVerdict = 'fix_confirmed';
+        headline = 'FIX CONFIRMED';
+        explanation = 'The intended behavior change was observed on head and reproduced across two stable base runs.';
+        exitCode = 0;
+    }
+    else {
+        agentVerdict = 'no_unexpected_change';
+        headline = 'NO UNEXPECTED CHANGE';
+        explanation = 'Executed cases found no behavior change outside the registered expectations.';
+        exitCode = 0;
+    }
+    return {
+        ok: exitCode === 0,
+        status: 'complete',
+        verdict: agentVerdict,
+        headline,
+        explanation,
+        runId: run.runId,
+        repository,
+        pullRequest: run.pr,
+        revisions: { base: run.baseSha, head: run.headSha },
+        durationMs: run.durationMs,
+        timings: run.result.timings || null,
+        changedFiles: run.result.changedFiles,
+        cases,
+        inspectCommand: `haystack design-verify ${repository}#${run.pr} --run ${run.runId}`,
+        replay: run.result.replay,
+        error: null,
+        exitCode,
+    };
+}
+function displayValue(value) {
+    const encoded = JSON.stringify(value);
+    return encoded === undefined ? String(value) : encoded;
+}
+function durationLabel(durationMs) {
+    if (durationMs === null)
+        return '';
+    const totalSeconds = Math.round(durationMs / 1000);
+    const minutes = Math.floor(totalSeconds / 60);
+    const seconds = totalSeconds % 60;
+    return minutes ? `${minutes}m ${seconds}s` : `${seconds}s`;
+}
+function preciseDurationLabel(durationMs) {
+    if (durationMs === null)
+        return 'not recorded';
+    if (durationMs < 1000)
+        return `${Number(durationMs.toFixed(3))}ms`;
+    const minutes = Math.floor(durationMs / 60_000);
+    const seconds = Number(((durationMs % 60_000) / 1000).toFixed(3));
+    return minutes ? `${minutes}m ${seconds}s` : `${seconds}s`;
+}
+function timingStageLabel(stage) {
+    if (stage === 'blast-radius')
+        return 'blast radius';
+    if (stage === 'case-design')
+        return 'case design';
+    return stage;
+}
+function commandLabel(command) {
+    if (!command)
+        return null;
+    return command.argv.map((arg) => (/^[A-Za-z0-9_./:=+-]+$/.test(arg) ? arg : JSON.stringify(arg))).join(' ');
+}
+function executionSideLabel(side) {
+    if (side === 'base1')
+        return 'base';
+    if (side === 'base2')
+        return 'base repeat';
+    return 'head';
+}
+function executionNeedsAttention(execution) {
+    return execution.status !== 'completed' || execution.diagnostics.length > 0;
+}
+export function formatDesignVerifierAgentReport(report) {
+    const timing = durationLabel(report.durationMs);
+    const lines = [
+        `${report.exitCode === 0 ? '✓' : report.exitCode === 1 ? '✗' : '?'} ${report.headline}${timing ? `  ${timing}` : ''}`,
+        '',
+        report.explanation,
+    ];
+    if (report.revisions.base || report.revisions.head) {
+        lines.push('', `Revisions: base ${report.revisions.base || 'unknown'} (executed twice)`, `           head ${report.revisions.head || 'unknown'} (executed once)`);
+    }
+    if (report.timings) {
+        lines.push('', 'Timing breakdown:');
+        if (report.timings.admissionMs !== null) {
+            lines.push(`  admission: ${preciseDurationLabel(report.timings.admissionMs)}`);
+        }
+        for (const stage of report.timings.stages) {
+            lines.push(`  ${timingStageLabel(stage.stage)}: ${preciseDurationLabel(stage.durationMs)} (${stage.status})`);
+        }
+        lines.push(`  pipeline total: ${preciseDurationLabel(report.timings.pipelineDurationMs)}`);
+        if (report.timings.unattributedMs !== null) {
+            lines.push(`  orchestration/finalization not recorded in a phase: ${preciseDurationLabel(report.timings.unattributedMs)}`);
+        }
+    }
+    for (const testCase of report.cases) {
+        lines.push('', `Test: ${testCase.title}`);
+        if (testCase.story)
+            lines.push(`  Behavior: ${testCase.story}`);
+        if (testCase.expectedHead)
+            lines.push(`  Expected on head: ${testCase.expectedHead}`);
+        if (testCase.observations.length) {
+            lines.push('  Observed:');
+            for (const observation of testCase.observations) {
+                lines.push(`    ${observation.path}`);
+                lines.push(`      base: ${displayValue(observation.base)} (repeat: ${displayValue(observation.baseRepeat)})`);
+                lines.push(`      head: ${displayValue(observation.head)}`);
+            }
+        }
+        if (testCase.baseStable !== null) {
+            lines.push(`  Base stability: ${testCase.baseStable ? 'passed; both runs agreed' : 'failed; base runs disagreed'}`);
+        }
+        if (testCase.infrastructure) {
+            lines.push(testCase.infrastructure.verified
+                ? `  Infrastructure: verified; same recipe, ${testCase.infrastructure.freshInstances} fresh instances`
+                : `  Infrastructure: unverified; ${testCase.infrastructure.errors.join('; ') || 'receipt missing'}`);
+        }
+        const executionsWithDiagnostics = testCase.executions.filter(executionNeedsAttention);
+        if (executionsWithDiagnostics.length) {
+            lines.push('  Execution diagnostics:');
+            for (const execution of executionsWithDiagnostics) {
+                const exit = execution.commandExitCode === null ? 'exit unknown' : `exit ${execution.commandExitCode}`;
+                const elapsed = execution.durationMs === null ? '' : `, ${durationLabel(execution.durationMs)}`;
+                lines.push(`    ${executionSideLabel(execution.side)} (${exit}${elapsed}):`);
+                if (execution.scenarioError)
+                    lines.push(`      Scenario error: ${execution.scenarioError}`);
+                for (const diagnostic of execution.diagnostics) {
+                    const identity = [diagnostic.type, diagnostic.code].filter(Boolean).join(', ');
+                    lines.push(`      ${diagnostic.kind}${identity ? ` (${identity})` : ''}: ${diagnostic.message}`);
+                }
+                if (execution.diagnosticsOmitted > 0) {
+                    lines.push(`      ... ${execution.diagnosticsOmitted} additional diagnostics omitted`);
+                }
+                if (execution.outputExcerpt.length) {
+                    lines.push('      Relevant command output:');
+                    for (const output of execution.outputExcerpt) {
+                        lines.push(`        ${output.stream}: ${output.text}`);
+                    }
+                    if (execution.outputLinesOmitted > 0) {
+                        lines.push(`        ... ${execution.outputLinesOmitted} output lines omitted`);
+                    }
+                }
+                lines.push(`      Full output artifact: ${execution.outputArtifact}`);
+                lines.push(`      Structured trace artifact: ${execution.traceArtifact}`);
+            }
+        }
+        if (testCase.generatedFiles.length)
+            lines.push(`  Generated probe artifact: ${testCase.generatedFiles.join(', ')}`);
+        const command = commandLabel(testCase.executedCommand);
+        if (command)
+            lines.push(`  Executed in the sandbox: ${command}`);
+    }
+    if (report.changedFiles.length) {
+        lines.push('', 'Changed files to inspect:', ...report.changedFiles.map((file) => `  ${file}`));
+    }
+    lines.push('', `Inspect this exact result again: ${report.inspectCommand}`, `Run: ${report.runId}`);
+    return lines.join('\n');
+}
+export function designVerifierStageLabel(stage) {
+    if (!stage)
+        return 'Waiting for the run to start';
+    if (stage === 'prepare' || stage === 'bake')
+        return 'Preparing the repository';
+    if (stage === 'blast-radius')
+        return 'Mapping affected behavior';
+    if (stage === 'prefetch')
+        return 'Retrieving relevant failure patterns';
+    if (stage === 'stage1')
+        return 'Designing behavioral test cases';
+    if (stage.startsWith('build:'))
+        return 'Building an executable probe';
+    if (stage.startsWith('execute:'))
+        return 'Running base/base/head comparisons';
+    if (stage === 'verdict')
+        return 'Comparing observations';
+    return stage;
+}

package/dist/utils/haystack-api.d.ts

@@ -1,3 +1,5 @@
+export declare function haystackApiBase(): string;
+export declare function haystackApiUrl(path: string): string;
 export interface UserInstallation {
     id: number;
     account?: {

package/dist/utils/haystack-api.js

@@ -1,4 +1,11 @@
-const HAYSTACK_API_BASE = process.env.HAYSTACK_API_BASE ?? 'https://haystackeditor.com';
+export function haystackApiBase() {
+    return process.env.HAYSTACK_API_BASE ?? 'https://haystackeditor.com';
+}
+export function haystackApiUrl(path) {
+    if (!path.startsWith('/'))
+        throw new Error('Haystack API path must be absolute');
+    return `${haystackApiBase()}${path}`;
+}
 /**
  * Typed error for non-OK Haystack API responses. Callers classify by
  * `status` (never by message substring) and the message always carries a
@@ -70,7 +77,7 @@ export async function classifyHttpError(response, context) {
     return new HaystackApiError(response.status, `${context}: unexpected response (HTTP ${response.status})${suffix}.`);
 }
 export async function haystackJson(path, token, init = {}, acceptedStatuses = []) {
-    const response = await fetch(`${HAYSTACK_API_BASE}${path}`, {
+    const response = await fetch(haystackApiUrl(path), {
         ...init,
         headers: headers(token, init.headers),
     });
@@ -91,7 +98,7 @@ export async function fetchGitHubJson(path, token, init = {}) {
 }
 export async function fetchAnalysisArtifact(owner, repo, prNumber, file, token) {
     const ref = encodeURIComponent(`${owner}/${repo}#${prNumber}`);
-    const response = await fetch(`${HAYSTACK_API_BASE}/api/analysis/${ref}/${file}`, {
+    const response = await fetch(haystackApiUrl(`/api/analysis/${ref}/${file}`), {
         headers: headers(token, { 'Cache-Control': 'no-cache' }),
     });
     if (response.status === 404)

package/dist/utils/hooks.js

@@ -169,10 +169,11 @@ export async function copyHookFiles(hooksDir) {
     await copyDirRecursive(path.join(assetsDir, 'truncation-checker'), path.join(hooksDir, 'truncation-checker'));
     // Copy hooks package.json
     await fs.copyFile(path.join(assetsDir, 'package.json'), path.join(hooksDir, 'package.json'));
+    await fs.copyFile(path.join(assetsDir, 'package-lock.json'), path.join(hooksDir, 'package-lock.json'));
 }
 /** Install npm dependencies in the hooks directory */
 export async function installHookDeps(hooksDir) {
-    execSync('npm install --no-fund --no-audit', {
+    execSync('npm ci --ignore-scripts --no-fund --no-audit', {
         cwd: hooksDir,
         stdio: 'pipe',
         timeout: 60_000,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@haystackeditor/cli",
-  "version": "0.15.26",
+  "version": "0.15.27",
   "description": "Set up Haystack for your project — automated PR review, triage, and merge queue",
   "type": "module",
   "bin": {
@@ -10,10 +10,13 @@
     "build": "rm -rf dist && tsc && cp -r src/assets dist/assets",
     "dev": "tsc --watch",
     "start": "node dist/index.js",
+    "benchmark:telemetry": "pnpm build && node scripts/benchmark-telemetry-overhead.mjs",
+    "benchmark:telemetry:corpus": "pnpm build && node scripts/benchmark-open-source-telemetry.mjs",
     "check:schemas": "node scripts/check-schemas.mjs",
     "release": "node scripts/release.mjs",
     "prepublishOnly": "node scripts/no-direct-publish.mjs",
-    "test": "pnpm build && vitest run && node --test tests/*.test.mjs"
+    "test": "pnpm build && vitest run && node --test tests/*.test.mjs",
+    "test:package": "node scripts/package-smoke.mjs"
   },
   "keywords": [
     "haystack",
@@ -35,6 +38,7 @@
     "url": "https://github.com/haystackeditor/haystack-review/issues"
   },
   "dependencies": {
+    "@babel/core": "7.29.0",
     "@modelcontextprotocol/sdk": "1.26.0",
     "chalk": "5.6.2",
     "commander": "12.1.0",
@@ -56,6 +60,7 @@
     "dist",
     "schemas"
   ],
+  "bundledDependencies": [],
   "engines": {
     "node": ">=18"
   }