@haystackeditor/cli 0.25.1 → 0.26.0

package/dist/utils/design-verifier-result.js

@@ -1,424 +0,0 @@
-import { isDeepStrictEqual } from 'node:util';
-function findingExplanation(kind) {
-    if (kind === 'declared-change-not-observed') {
-        return 'The registered intended behavior change was not observed; investigate the captured outputs before concluding why.';
-    }
-    if (kind === 'unexpected-change') {
-        return 'The proposed revision produced a difference outside this case’s registered intent; this is an anomaly candidate for investigation.';
-    }
-    if (kind === 'intended-change-control-confounded') {
-        return 'The intended difference was observed, but control-before also varied on an intersecting output path; inspect it before claiming causality.';
-    }
-    return 'Executed behavior produced a suspicious difference against the registered expectation.';
-}
-function sameRecordedValue(left, right) {
-    return isDeepStrictEqual(left, right);
-}
-function executionSummaries(value) {
-    if (!Array.isArray(value))
-        return [];
-    return value.filter((execution) => !!execution && typeof execution === 'object' &&
-        ['base1', 'base2', 'head'].includes(execution.side) &&
-        ['completed', 'failed', 'missing'].includes(execution.status) &&
-        Array.isArray(execution.diagnostics) && Array.isArray(execution.outputExcerpt));
-}
-function universeSummaries(value, caseId) {
-    if (!Array.isArray(value))
-        return [];
-    return value.filter((universe) => universe?.version === 'dv-case-universe-summary-v1' && universe.caseId === caseId &&
-        /^universe-[0-9a-f]{24}$/.test(universe.universeId || '') &&
-        ['control-before', 'before', 'after'].includes(universe.role) &&
-        universe.lease && typeof universe.lease.available === 'boolean');
-}
-function runHealthOf(value) {
-    if (!value || typeof value !== 'object' || Array.isArray(value))
-        return null;
-    const health = value;
-    if (health.version !== 'dv-run-health-v1' ||
-        !['healthy', 'degraded', 'invalid'].includes(health.status) ||
-        !Number.isInteger(health.admitted) || health.admitted < 0 ||
-        !Array.isArray(health.losses))
-        return null;
-    const losses = health.losses.filter((loss) => loss && typeof loss === 'object' &&
-        typeof loss.code === 'string' && loss.code.length > 0 &&
-        ['harness', 'case'].includes(loss.owner) &&
-        ['degraded', 'invalid'].includes(loss.impact) &&
-        Number.isInteger(loss.count) && loss.count > 0 &&
-        [loss.caseIds, loss.obligationIds, loss.slotIds, loss.sides]
-            .every(items => Array.isArray(items) && items.every(item => typeof item === 'string')));
-    if (losses.length !== health.losses.length)
-        return null;
-    const invalidHarnessLoss = losses.some(loss => loss.owner === 'harness' && loss.impact === 'invalid');
-    const expectedStatus = health.admitted === 0 || invalidHarnessLoss
-        ? 'invalid'
-        : losses.length > 0 ? 'degraded' : 'healthy';
-    if (health.status !== expectedStatus)
-        return null;
-    return {
-        version: health.version,
-        status: health.status,
-        admitted: health.admitted,
-        losses,
-    };
-}
-function missingRunHealth() {
-    return {
-        version: 'dv-run-health-v1',
-        status: 'invalid',
-        admitted: 0,
-        losses: [{
-                code: 'run-health-missing',
-                owner: 'harness',
-                impact: 'invalid',
-                count: 1,
-                caseIds: [],
-                obligationIds: [],
-                slotIds: [],
-                sides: [],
-            }],
-    };
-}
-export function buildDesignVerifierAgentReport(run) {
-    const repository = `${run.owner}/${run.repo}`;
-    if (run.status !== 'succeeded' || !run.result) {
-        return {
-            ok: false,
-            status: 'failed',
-            verdict: 'inconclusive',
-            headline: 'VERIFICATION INCONCLUSIVE',
-            explanation: run.error || 'The verifier did not produce a terminal behavioral verdict.',
-            runId: run.runId,
-            repository,
-            pullRequest: run.pr,
-            revisions: { base: run.baseSha, head: run.headSha },
-            durationMs: run.durationMs,
-            timings: null,
-            changedFiles: [],
-            cases: [],
-            runHealth: null,
-            inspectCommand: `haystack design-verify ${repository}#${run.pr} --run ${run.runId}`,
-            replay: null,
-            error: run.error,
-            exitCode: 2,
-        };
-    }
-    const verdict = run.result.verdict || {};
-    const summary = verdict.summary || {};
-    const runHealth = runHealthOf(verdict.runHealth) || missingRunHealth();
-    const findings = Array.isArray(verdict.findings) ? verdict.findings : [];
-    const confirmations = Array.isArray(verdict.confirmations) ? verdict.confirmations : [];
-    const findingsByCase = new Map();
-    for (const finding of findings) {
-        const caseFindings = findingsByCase.get(String(finding?.caseId || '')) || [];
-        caseFindings.push(finding);
-        findingsByCase.set(String(finding?.caseId || ''), caseFindings);
-    }
-    const confirmedIds = new Set(confirmations.map((confirmation) => confirmation.caseId));
-    const scenarioByCase = new Map(run.result.scenarios.map((scenario) => [scenario.caseId, scenario]));
-    const probeByCase = new Map(run.result.probes.map((probe) => [probe.caseId, probe]));
-    const cases = run.result.cases.map((testCase) => {
-        const id = String(testCase.id || '');
-        const caseFindings = findingsByCase.get(id) || [];
-        const finding = caseFindings[0];
-        const scenario = scenarioByCase.get(id);
-        const probe = probeByCase.get(id);
-        const observations = (Array.isArray(scenario?.divergences) ? scenario.divergences : []).map((item) => ({
-            path: String(item.display || (Array.isArray(item.path) ? item.path.join('.') : 'observation')),
-            base: item.base,
-            baseRepeat: item.base2,
-            head: item.head,
-            controlObservedAtSamePath: item.controlObservedAtSamePath === true,
-            controlObservedAtAncestorPath: item.controlObservedAtAncestorPath === true,
-            controlObservedAtDescendantPath: item.controlObservedAtDescendantPath === true,
-        }));
-        const hasControlEvidence = typeof scenario?.noisePathCount === 'number' ||
-            scenario?.controlScenarioErrorVariation != null;
-        return {
-            id,
-            title: String(testCase.title || scenario?.scenario || id),
-            story: String(testCase.story || ''),
-            expectedHead: String(testCase.expected_head_observation || ''),
-            verdict: caseFindings.length
-                ? 'finding'
-                : confirmedIds.has(id)
-                    ? 'confirmed'
-                    : scenario?.status === 'clean'
-                        ? 'clean'
-                        : 'inconclusive',
-            findingKind: finding?.kind || null,
-            findingKinds: caseFindings
-                .map((item) => String(item?.kind || ''))
-                .filter(Boolean),
-            baseStable: hasControlEvidence
-                ? scenario.noisePathCount === 0 && scenario.controlScenarioErrorVariation == null
-                : observations.length
-                    ? observations.every((item) => sameRecordedValue(item.base, item.baseRepeat))
-                    : scenario?.status === 'clean' ? true : null,
-            observations,
-            generatedFiles: probe?.files || [],
-            executedCommand: probe?.run || null,
-            executions: executionSummaries(scenario?.executions),
-            universes: universeSummaries(scenario?.universes, id),
-            infrastructure: scenario?.infrastructureReceipt &&
-                typeof scenario.infrastructureReceipt === 'object'
-                ? {
-                    verified: scenario.infrastructureReceipt.status === 'passed',
-                    recipeId: typeof scenario.infrastructureReceipt.recipeId === 'string'
-                        ? scenario.infrastructureReceipt.recipeId
-                        : null,
-                    freshInstances: Array.isArray(scenario.infrastructureReceipt.environmentInstanceIds)
-                        ? scenario.infrastructureReceipt.environmentInstanceIds.length
-                        : 0,
-                    baseRepeat: typeof scenario.infrastructureReceipt.baseRepeat === 'string'
-                        ? scenario.infrastructureReceipt.baseRepeat
-                        : null,
-                    errors: Array.isArray(scenario.infrastructureReceipt.errors)
-                        ? scenario.infrastructureReceipt.errors.filter((item) => typeof item === 'string')
-                        : [],
-                }
-                : null,
-        };
-    });
-    let agentVerdict;
-    let headline;
-    let explanation;
-    let exitCode;
-    if (runHealth.status === 'invalid') {
-        agentVerdict = 'inconclusive';
-        headline = 'VERIFICATION INCONCLUSIVE';
-        explanation = 'No admissible evidence — this is not a pass.';
-        exitCode = 2;
-    }
-    else if (runHealth.status === 'degraded') {
-        agentVerdict = 'inconclusive';
-        headline = 'VERIFICATION INCONCLUSIVE';
-        explanation = 'Evidence losses degraded this run; inspect the loss table before drawing a conclusion.';
-        exitCode = 2;
-    }
-    else if (findings.length > 0) {
-        agentVerdict = 'anomaly_candidate';
-        headline = 'ANOMALY CANDIDATE';
-        explanation = findingExplanation(String(findings[0]?.kind || ''));
-        exitCode = 1;
-    }
-    else if (summary.caseDesignOutcome === 'no-supported-cases' || Number(summary.scenarios || 0) === 0) {
-        agentVerdict = 'inconclusive';
-        headline = 'VERIFICATION INCONCLUSIVE';
-        explanation = 'The verifier could not design and execute a supported behavioral case.';
-        exitCode = 2;
-    }
-    else if (confirmations.length > 0) {
-        agentVerdict = 'fix_confirmed';
-        headline = 'FIX CONFIRMED';
-        explanation = 'The intended behavior change was observed on output paths that did not vary between control-before and before.';
-        exitCode = 0;
-    }
-    else {
-        agentVerdict = 'no_unexpected_change';
-        headline = 'NO UNEXPECTED CHANGE';
-        explanation = 'The executed cases found no observed difference outside their registered expectations.';
-        exitCode = 0;
-    }
-    return {
-        ok: exitCode === 0,
-        status: 'complete',
-        verdict: agentVerdict,
-        headline,
-        explanation,
-        runId: run.runId,
-        repository,
-        pullRequest: run.pr,
-        revisions: { base: run.baseSha, head: run.headSha },
-        durationMs: run.durationMs,
-        timings: run.result.timings || null,
-        changedFiles: run.result.changedFiles,
-        cases,
-        runHealth,
-        inspectCommand: `haystack design-verify ${repository}#${run.pr} --run ${run.runId}`,
-        replay: run.result.replay,
-        error: null,
-        exitCode,
-    };
-}
-function displayValue(value) {
-    const encoded = JSON.stringify(value);
-    return encoded === undefined ? String(value) : encoded;
-}
-function durationLabel(durationMs) {
-    if (durationMs === null)
-        return '';
-    const totalSeconds = Math.round(durationMs / 1000);
-    const minutes = Math.floor(totalSeconds / 60);
-    const seconds = totalSeconds % 60;
-    return minutes ? `${minutes}m ${seconds}s` : `${seconds}s`;
-}
-function preciseDurationLabel(durationMs) {
-    if (durationMs === null)
-        return 'not recorded';
-    if (durationMs < 1000)
-        return `${Number(durationMs.toFixed(3))}ms`;
-    const minutes = Math.floor(durationMs / 60_000);
-    const seconds = Number(((durationMs % 60_000) / 1000).toFixed(3));
-    return minutes ? `${minutes}m ${seconds}s` : `${seconds}s`;
-}
-function timingStageLabel(stage) {
-    if (stage === 'blast-radius')
-        return 'blast radius';
-    if (stage === 'case-design')
-        return 'case design';
-    return stage;
-}
-function commandLabel(command) {
-    if (!command)
-        return null;
-    return command.argv.map((arg) => (/^[A-Za-z0-9_./:=+-]+$/.test(arg) ? arg : JSON.stringify(arg))).join(' ');
-}
-function executionSideLabel(side) {
-    if (side === 'base1')
-        return 'before';
-    if (side === 'base2')
-        return 'control-before';
-    return 'after';
-}
-function executionNeedsAttention(execution) {
-    return execution.status !== 'completed' || execution.diagnostics.length > 0;
-}
-export function formatDesignVerifierAgentReport(report) {
-    const timing = durationLabel(report.durationMs);
-    const lines = [
-        `${report.exitCode === 0 ? '✓' : report.exitCode === 1 ? '✗' : '?'} ${report.headline}${timing ? `  ${timing}` : ''}`,
-        '',
-        report.explanation,
-    ];
-    if (report.revisions.base || report.revisions.head) {
-        lines.push('', `Revisions: before ${report.revisions.base || 'unknown'} (control + comparison)`, `           after  ${report.revisions.head || 'unknown'} (comparison)`);
-    }
-    if (report.timings) {
-        lines.push('', 'Timing breakdown:');
-        if (report.timings.admissionMs !== null) {
-            lines.push(`  admission: ${preciseDurationLabel(report.timings.admissionMs)}`);
-        }
-        for (const stage of report.timings.stages) {
-            lines.push(`  ${timingStageLabel(stage.stage)}: ${preciseDurationLabel(stage.durationMs)} (${stage.status})`);
-        }
-        lines.push(`  pipeline total: ${preciseDurationLabel(report.timings.pipelineDurationMs)}`);
-        if (report.timings.unattributedMs !== null) {
-            lines.push(`  orchestration/finalization not recorded in a phase: ${preciseDurationLabel(report.timings.unattributedMs)}`);
-        }
-    }
-    if (report.runHealth && report.runHealth.status !== 'healthy') {
-        lines.push('', `Run health: ${report.runHealth.status}; admitted evidence: ${report.runHealth.admitted}`);
-        if (report.runHealth.losses.length > 0) {
-            lines.push('Evidence loss table:', '  owner   impact    count  loss / stable identifiers');
-            for (const loss of report.runHealth.losses) {
-                const identifiers = [
-                    ...loss.caseIds,
-                    ...loss.obligationIds,
-                    ...loss.slotIds,
-                    ...loss.sides,
-                ];
-                lines.push(`  ${loss.owner.padEnd(7)} ${loss.impact.padEnd(9)} ${String(loss.count).padStart(5)}  ` +
-                    `${loss.code}${identifiers.length ? `: ${identifiers.join(', ')}` : ''}`);
-            }
-        }
-    }
-    for (const testCase of report.cases) {
-        lines.push('', `Test: ${testCase.title}`);
-        if (testCase.story)
-            lines.push(`  Behavior: ${testCase.story}`);
-        if (testCase.expectedHead)
-            lines.push(`  Expected on head: ${testCase.expectedHead}`);
-        if (testCase.findingKinds.length > 0) {
-            lines.push(`  Candidate reasons: ${testCase.findingKinds.join(', ')}`);
-        }
-        if (testCase.observations.length) {
-            lines.push('  Observed:');
-            for (const observation of testCase.observations) {
-                lines.push(`    ${observation.path}`);
-                lines.push(`      control-before: ${displayValue(observation.baseRepeat)}`);
-                lines.push(`      before: ${displayValue(observation.base)}`);
-                lines.push(`      after: ${displayValue(observation.head)}`);
-                if (observation.controlObservedAtSamePath || observation.controlObservedAtAncestorPath ||
-                    observation.controlObservedAtDescendantPath) {
-                    lines.push(`      control evidence: ${observation.controlObservedAtSamePath
-                        ? 'variation observed at this same path'
-                        : observation.controlObservedAtAncestorPath
-                            ? 'variation observed at an ancestor path'
-                            : 'variation observed at a descendant path'}`);
-                }
-            }
-        }
-        if (testCase.baseStable !== null) {
-            lines.push(`  Control evidence: ${testCase.baseStable
-                ? 'control-before and before agreed'
-                : 'control-before and before varied; inspect alongside the difference'}`);
-        }
-        if (testCase.infrastructure) {
-            lines.push(testCase.infrastructure.verified
-                ? `  Infrastructure: verified; same recipe, ${testCase.infrastructure.freshInstances} fresh instances`
-                : `  Infrastructure: unverified; ${testCase.infrastructure.errors.join('; ') || 'receipt missing'}`);
-        }
-        const executionsWithDiagnostics = testCase.executions.filter(executionNeedsAttention);
-        if (executionsWithDiagnostics.length) {
-            lines.push('  Execution diagnostics:');
-            for (const execution of executionsWithDiagnostics) {
-                const exit = execution.commandExitCode === null ? 'exit unknown' : `exit ${execution.commandExitCode}`;
-                const elapsed = execution.durationMs === null ? '' : `, ${durationLabel(execution.durationMs)}`;
-                lines.push(`    ${executionSideLabel(execution.side)} (${exit}${elapsed}):`);
-                if (execution.scenarioError)
-                    lines.push(`      Scenario error: ${execution.scenarioError}`);
-                for (const diagnostic of execution.diagnostics) {
-                    const identity = [diagnostic.type, diagnostic.code].filter(Boolean).join(', ');
-                    lines.push(`      ${diagnostic.kind}${identity ? ` (${identity})` : ''}: ${diagnostic.message}`);
-                }
-                if (execution.diagnosticsOmitted > 0) {
-                    lines.push(`      ... ${execution.diagnosticsOmitted} additional diagnostics omitted`);
-                }
-                if (execution.outputExcerpt.length) {
-                    lines.push('      Relevant command output:');
-                    for (const output of execution.outputExcerpt) {
-                        lines.push(`        ${output.stream}: ${output.text}`);
-                    }
-                    if (execution.outputLinesOmitted > 0) {
-                        lines.push(`        ... ${execution.outputLinesOmitted} output lines omitted`);
-                    }
-                }
-                lines.push(`      Full output artifact: ${execution.outputArtifact}`);
-                lines.push(`      Structured trace artifact: ${execution.traceArtifact}`);
-            }
-        }
-        if (testCase.generatedFiles.length)
-            lines.push(`  Generated probe artifact: ${testCase.generatedFiles.join(', ')}`);
-        const command = commandLabel(testCase.executedCommand);
-        if (command)
-            lines.push(`  Executed in the sandbox: ${command}`);
-        for (const universe of testCase.universes.filter(item => item.lease.available)) {
-            lines.push(`  Replay ${universe.role}: haystack design-verify ${report.repository}#${report.pullRequest}` +
-                ` --run ${report.runId} --case ${testCase.id} --universe ${universe.role} --replay`);
-        }
-    }
-    if (report.changedFiles.length) {
-        lines.push('', 'Changed files to inspect:', ...report.changedFiles.map((file) => `  ${file}`));
-    }
-    lines.push('', `Inspect this exact result again: ${report.inspectCommand}`, `Run: ${report.runId}`);
-    return lines.join('\n');
-}
-export function designVerifierStageLabel(stage) {
-    if (!stage)
-        return 'Waiting for the run to start';
-    if (stage === 'prepare' || stage === 'bake')
-        return 'Preparing the repository';
-    if (stage === 'blast-radius')
-        return 'Mapping affected behavior';
-    if (stage === 'prefetch')
-        return 'Retrieving relevant failure patterns';
-    if (stage === 'stage1')
-        return 'Designing behavioral test cases';
-    if (stage.startsWith('build:'))
-        return 'Building an executable probe';
-    if (stage.startsWith('execute:'))
-        return 'Running control-before/before/after comparisons';
-    if (stage === 'verdict')
-        return 'Comparing observations';
-    return stage;
-}