@haystackeditor/cli 0.15.25 → 0.15.26

package/dist/commands/verify-hosted-reproducibility.d.ts

@@ -4,11 +4,13 @@ export interface HostedReproducibilityOptions {
     head: string;
     runs?: string;
     seriesKey?: string;
+    freshPlans?: boolean;
     account?: string;
     interval?: string;
     timeout?: string;
     json?: boolean;
 }
+type HostedReproducibilityMode = 'exact_plan_replay' | 'fresh_plans';
 interface HostedReproducibilityRun {
     state: HostedRunState;
     timedOut: boolean;
@@ -19,6 +21,7 @@ interface HostedReproducibilityIdentity {
     baseCommit: string;
     headCommit: string;
     requestedRuns?: number;
+    mode?: HostedReproducibilityMode;
 }
 interface ParsedAssessment {
     schemaVersion: 8 | 9;
@@ -60,6 +63,7 @@ export interface HostedReproducibilityReport {
     repository: string;
     base_commit: string;
     head_commit: string;
+    reproducibility_mode: HostedReproducibilityMode;
     requested_runs: number;
     launched_runs: number;
     unstarted_runs: number;
@@ -67,7 +71,7 @@ export interface HostedReproducibilityReport {
     run_ids: string[];
     replay_source_run_id: string | null;
     plan_replay_verified: boolean;
-    planner_reproducibility_tested: false;
+    planner_reproducibility_tested: boolean;
     planner_reproducible: boolean;
     assessment_reproducible: boolean;
     reproducible: boolean;
@@ -78,7 +82,7 @@ export interface HostedReproducibilityReport {
     inconclusive_cell_ids: string[];
     unrunnable_runs: RunFailure[];
 }
-export declare function hostedReproducibilityRunKey(seriesKey: string, repository: string, baseCommit: string, headCommit: string, ordinal: number): string;
+export declare function hostedReproducibilityRunKey(seriesKey: string, repository: string, baseCommit: string, headCommit: string, ordinal: number, mode?: HostedReproducibilityMode): string;
 export declare function analyzeHostedReproducibility(runs: HostedReproducibilityRun[], identity: HostedReproducibilityIdentity): HostedReproducibilityReport;
 export declare function isHostedReproducibilityReplaySource(run: HostedReproducibilityRun, identity: HostedReproducibilityIdentity): boolean;
 export declare function verifyHostedReproducibilityCommand(repositoryArg: string, options: HostedReproducibilityOptions): Promise<void>;

package/dist/commands/verify-hosted-reproducibility.js

@@ -69,15 +69,24 @@ function requireSeriesKey(value) {
     }
     return key;
 }
-export function hostedReproducibilityRunKey(seriesKey, repository, baseCommit, headCommit, ordinal) {
+export function hostedReproducibilityRunKey(seriesKey, repository, baseCommit, headCommit, ordinal, mode = 'exact_plan_replay') {
     const digest = createHash('sha256')
-        .update(JSON.stringify([
-        seriesKey,
-        repository,
-        baseCommit,
-        headCommit,
-        ordinal,
-    ]))
+        .update(JSON.stringify(mode === 'exact_plan_replay'
+        ? [
+            seriesKey,
+            repository,
+            baseCommit,
+            headCommit,
+            ordinal,
+        ]
+        : [
+            mode,
+            seriesKey,
+            repository,
+            baseCommit,
+            headCommit,
+            ordinal,
+        ]))
         .digest('hex');
     return `repro:${digest}`;
 }
@@ -371,10 +380,17 @@ export function analyzeHostedReproducibility(runs, identity) {
             });
         }
     }
+    const requestedRuns = identity.requestedRuns ?? runs.length;
+    const allRequestedRunsLaunched = runs.length === requestedRuns;
     const planVariants = groupedPlanVariants(parsed);
     const assessmentVariants = groupedOutcomeVariants(parsed);
-    const replaySourceRunId = parsed[0]?.runId ?? null;
-    const planReplayVerified = parsed.length === runs.length
+    const mode = identity.mode ?? 'exact_plan_replay';
+    const replaySourceRunId = mode === 'exact_plan_replay'
+        ? parsed[0]?.runId ?? null
+        : null;
+    const planReplayVerified = mode === 'exact_plan_replay'
+        && allRequestedRunsLaunched
+        && parsed.length === runs.length
         && parsed.length > 1
         && parsed[0]?.assessment.schemaVersion === 8
         && parsed[0]?.assessment.replaySourceRunId === null
@@ -384,7 +400,17 @@ export function analyzeHostedReproducibility(runs, identity) {
                 === parsed[0].assessment.schemaVersion
             && item.assessment.riskPlanSha256
                 === parsed[0].assessment.riskPlanSha256);
-    const assessmentReproducible = planReplayVerified && assessmentVariants.length === 1;
+    const plannerReproducibilityTested = mode === 'fresh_plans' && parsed.length > 1;
+    const plannerReproducible = plannerReproducibilityTested
+        && allRequestedRunsLaunched
+        && parsed.length === runs.length
+        && parsed.every(item => item.assessment.schemaVersion === 8
+            && item.assessment.replaySourceRunId === null)
+        && planVariants.length === 1;
+    const assessmentReproducible = (mode === 'exact_plan_replay'
+        ? planReplayVerified
+        : plannerReproducible)
+        && assessmentVariants.length === 1;
     const planDriftCellIds = driftIds(parsed, assessment => new Map(assessment.planIdentities.map(plan => [
         plan.cellId,
         { capabilityId: plan.capabilityId },
@@ -401,12 +427,12 @@ export function analyzeHostedReproducibility(runs, identity) {
     const inconclusiveCellIds = [...new Set(parsed.flatMap(item => item.assessment.outcomes
             .filter(outcome => outcome.status === 'inconclusive')
             .map(outcome => outcome.cellId)))].sort();
-    const requestedRuns = identity.requestedRuns ?? runs.length;
     return {
         series_key: identity.seriesKey,
         repository: identity.repository,
         base_commit: identity.baseCommit,
         head_commit: identity.headCommit,
+        reproducibility_mode: mode,
         requested_runs: requestedRuns,
         launched_runs: runs.length,
         unstarted_runs: Math.max(0, requestedRuns - runs.length),
@@ -414,10 +440,12 @@ export function analyzeHostedReproducibility(runs, identity) {
         run_ids: runs.map(run => run.state.runId),
         replay_source_run_id: replaySourceRunId,
         plan_replay_verified: planReplayVerified,
-        planner_reproducibility_tested: false,
-        planner_reproducible: false,
+        planner_reproducibility_tested: plannerReproducibilityTested,
+        planner_reproducible: plannerReproducible,
         assessment_reproducible: assessmentReproducible,
-        reproducible: planReplayVerified
+        reproducible: (mode === 'exact_plan_replay'
+            ? planReplayVerified
+            : plannerReproducible)
             && assessmentReproducible
             && inconclusiveCellIds.length === 0,
         plan_variants: planVariants,
@@ -444,9 +472,20 @@ function renderHostedReproducibility(report) {
     const icon = report.reproducible ? chalk.green('✓') : chalk.yellow('!');
     console.log(`${icon} Hosted reproducibility: ${report.reproducible ? 'reproducible' : 'not established'}`);
     console.log(`  ${chalk.dim('Runs:')} ${report.analyzable_runs}/${report.requested_runs} analyzable; ${report.launched_runs} launched`);
+    console.log(`  ${chalk.dim('Mode:')} ${report.reproducibility_mode === 'fresh_plans'
+        ? 'independent fresh plans'
+        : 'exact plan replay'}`);
     console.log(`  ${chalk.dim('Plan variants:')} ${report.plan_variants.length}`);
-    console.log(`  ${chalk.dim('Exact plan replay:')} ${report.plan_replay_verified ? 'verified' : 'not verified'}`);
-    console.log(`  ${chalk.dim('Independent planner reproducibility:')} not tested`);
+    console.log(`  ${chalk.dim('Exact plan replay:')} ${report.reproducibility_mode === 'fresh_plans'
+        ? 'not requested'
+        : report.plan_replay_verified
+            ? 'verified'
+            : 'not verified'}`);
+    console.log(`  ${chalk.dim('Independent planner reproducibility:')} ${report.planner_reproducibility_tested
+        ? report.planner_reproducible
+            ? 'verified'
+            : 'not reproducible'
+        : 'not tested'}`);
     console.log(`  ${chalk.dim('Assessment variants:')} ${report.assessment_variants.length}`);
     if (report.plan_drift_cell_ids.length > 0) {
         console.log(`  ${chalk.dim('Plan drift:')} ${report.plan_drift_cell_ids.length} stable cell id(s)`);
@@ -465,6 +504,9 @@ export async function verifyHostedReproducibilityCommand(repositoryArg, options)
     const repository = parseHostedRepository(repositoryArg);
     const requestedRuns = boundedInteger(options.runs, DEFAULT_RUNS, '--runs', 2, MAX_RUNS);
     const seriesKey = requireSeriesKey(options.seriesKey);
+    const mode = options.freshPlans
+        ? 'fresh_plans'
+        : 'exact_plan_replay';
     const auth = await resolveAuthContext({
         preferredLogin: options.account,
         owner: repository.owner,
@@ -479,8 +521,8 @@ export async function verifyHostedReproducibilityCommand(repositoryArg, options)
         const started = await startHostedVerification(repositoryArg, {
             base: options.base,
             head: options.head,
-            idempotencyKey: hostedReproducibilityRunKey(seriesKey, repository.fullName, options.base, options.head, index + 1),
-            replayPlanFrom: index === 0
+            idempotencyKey: hostedReproducibilityRunKey(seriesKey, repository.fullName, options.base, options.head, index + 1, mode),
+            replayPlanFrom: options.freshPlans || index === 0
                 ? undefined
                 : runs[0]?.state.runId,
             wait: false,
@@ -497,7 +539,9 @@ export async function verifyHostedReproducibilityCommand(repositoryArg, options)
             timedOut = waited.timedOut;
         }
         runs.push({ state, timedOut });
-        if (index === 0
+        if (!options.freshPlans
+            &&
+                index === 0
             && !isHostedReproducibilityReplaySource(runs[0], {
                 seriesKey,
                 repository: repository.fullName,
@@ -516,6 +560,7 @@ export async function verifyHostedReproducibilityCommand(repositoryArg, options)
         baseCommit: options.base,
         headCommit: options.head,
         requestedRuns,
+        mode,
     });
     if (options.json) {
         process.stdout.write(`${JSON.stringify(withSchema('cloud-verifier', {

package/dist/index.js

@@ -262,12 +262,13 @@ Example:
 });
 hostedVerify
     .command('reproducibility')
-    .description('Replay one exact hosted risk plan and report executor or assessment drift')
+    .description('Replay one exact hosted risk plan or run fresh plans to report drift')
     .argument('<repository>', 'Exact GitHub owner/repository name')
     .option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
     .option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
     .option('--runs <n>', 'Number of sequential production runs (default 3; min 2, max 10)')
     .option('--series-key <key>', 'Stable series key for resumable idempotent retries')
+    .option('--fresh-plans', 'Run the planner independently for every run instead of replaying the first plan')
     .option('--account <login>', 'Use a specific saved Haystack account')
     .option('--interval <seconds>', 'Polling interval for each run (default 5)')
     .option('--timeout <minutes>', 'Maximum wait per run (default 35)')
@@ -278,8 +279,14 @@ validated plan against the same repository and commit pair with a distinct,
 series-derived idempotency key. A reproducible verdict requires schema-9 replay
 provenance, stable assessment outcomes, and no inconclusive cells.
 
+With --fresh-plans, every run plans independently. A reproducible verdict then
+requires one stable-id plan variant, one assessment variant, no replay
+provenance, and no inconclusive cells. Use this mode to detect planner drift;
+use the default mode to isolate executor or assessment drift.
+
 Example:
   haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --runs 3
+  haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --runs 3 --fresh-plans
 `)
     .action(async (repository, _options, cmd) => {
     const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@haystackeditor/cli",
-  "version": "0.15.25",
+  "version": "0.15.26",
   "description": "Set up Haystack for your project — automated PR review, triage, and merge queue",
   "type": "module",
   "bin": {