@haystackeditor/cli 0.15.24 → 0.15.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -5
- package/dist/commands/verify-hosted-mcp.js +6 -0
- package/dist/commands/verify-hosted-reproducibility.d.ts +23 -7
- package/dist/commands/verify-hosted-reproducibility.js +166 -17
- package/dist/commands/verify-hosted.d.ts +2 -1
- package/dist/commands/verify-hosted.js +13 -4
- package/dist/index.js +16 -5
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -105,6 +105,21 @@ haystack verify hosted status cv_<48-lowercase-hex-characters> \
|
|
|
105
105
|
--json
|
|
106
106
|
```
|
|
107
107
|
|
|
108
|
+
To execute a completed run's exact validated risk plan again, keep the same
|
|
109
|
+
repository and commit pair and pass:
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
haystack verify hosted start owner/repo \
|
|
113
|
+
--base <40-character-commit-sha> \
|
|
114
|
+
--head <40-character-commit-sha> \
|
|
115
|
+
--replay-plan-from cv_<48-lowercase-hex-characters> \
|
|
116
|
+
--json
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
A successful replay emits a schema-9 terminal result whose
|
|
120
|
+
`riskPlanReplayReceipt` binds the source run and canonical risk-plan hash. The
|
|
121
|
+
replay does not mint or deliver a planning-model credential.
|
|
122
|
+
|
|
108
123
|
Agents can use the same production lifecycle over stdio MCP:
|
|
109
124
|
|
|
110
125
|
```bash
|
|
@@ -116,7 +131,8 @@ The hosted server exposes only `verify_start`, `verify_status`, and
|
|
|
116
131
|
credentials, repository source, provider metadata, and sandbox mutation are
|
|
117
132
|
not exposed.
|
|
118
133
|
|
|
119
|
-
|
|
134
|
+
Freeze one production plan and repeat its exact execution to detect executor or
|
|
135
|
+
assessment drift:
|
|
120
136
|
|
|
121
137
|
```bash
|
|
122
138
|
haystack verify hosted reproducibility owner/repo \
|
|
@@ -126,10 +142,12 @@ haystack verify hosted reproducibility owner/repo \
|
|
|
126
142
|
--json
|
|
127
143
|
```
|
|
128
144
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
145
|
+
The first run creates the source plan; every later run names it through
|
|
146
|
+
`replayPlanFromRunId`. Each run gets a distinct idempotency key derived from one
|
|
147
|
+
printed series key. Pass `--series-key <key>` to resume the same series without
|
|
148
|
+
creating duplicate runs. The report verifies schema-9 replay provenance and
|
|
149
|
+
compares stable risk-cell, capability, and comparison IDs. Missing replay
|
|
150
|
+
provenance, assessment drift, timeouts, unsupported receipts, and inconclusive
|
|
133
151
|
cells all prevent a reproducible verdict and exit 2.
|
|
134
152
|
|
|
135
153
|
**`haystack setup --json`** speaks NDJSON: events (`question`, `permission`,
|
|
@@ -53,6 +53,10 @@ export const HOSTED_VERIFY_TOOLS = [
|
|
|
53
53
|
pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$',
|
|
54
54
|
description: 'Optional stable retry key. Omit to derive it from the exact repository and commits.',
|
|
55
55
|
},
|
|
56
|
+
replay_plan_from_run_id: {
|
|
57
|
+
...runIdInput,
|
|
58
|
+
description: 'Optional completed owned run whose exact validated risk plan must be replayed.',
|
|
59
|
+
},
|
|
56
60
|
account: accountInput,
|
|
57
61
|
},
|
|
58
62
|
required: ['repository', 'base_commit', 'head_commit'],
|
|
@@ -120,6 +124,7 @@ function assertToolInputFields(name, input) {
|
|
|
120
124
|
'base_commit',
|
|
121
125
|
'head_commit',
|
|
122
126
|
'idempotency_key',
|
|
127
|
+
'replay_plan_from_run_id',
|
|
123
128
|
'repository',
|
|
124
129
|
]);
|
|
125
130
|
return;
|
|
@@ -147,6 +152,7 @@ export async function executeHostedVerifyTool(name, input, context) {
|
|
|
147
152
|
base: requireInputString(input, 'base_commit'),
|
|
148
153
|
head: requireInputString(input, 'head_commit'),
|
|
149
154
|
idempotencyKey: optionalInputString(input, 'idempotency_key'),
|
|
155
|
+
replayPlanFrom: optionalInputString(input, 'replay_plan_from_run_id'),
|
|
150
156
|
wait: false,
|
|
151
157
|
}, context.token);
|
|
152
158
|
return buildHostedJson('start', started.state, {
|
|
@@ -4,16 +4,30 @@ export interface HostedReproducibilityOptions {
|
|
|
4
4
|
head: string;
|
|
5
5
|
runs?: string;
|
|
6
6
|
seriesKey?: string;
|
|
7
|
+
freshPlans?: boolean;
|
|
7
8
|
account?: string;
|
|
8
9
|
interval?: string;
|
|
9
10
|
timeout?: string;
|
|
10
11
|
json?: boolean;
|
|
11
12
|
}
|
|
13
|
+
type HostedReproducibilityMode = 'exact_plan_replay' | 'fresh_plans';
|
|
12
14
|
interface HostedReproducibilityRun {
|
|
13
15
|
state: HostedRunState;
|
|
14
16
|
timedOut: boolean;
|
|
15
17
|
}
|
|
18
|
+
interface HostedReproducibilityIdentity {
|
|
19
|
+
seriesKey: string;
|
|
20
|
+
repository: string;
|
|
21
|
+
baseCommit: string;
|
|
22
|
+
headCommit: string;
|
|
23
|
+
requestedRuns?: number;
|
|
24
|
+
mode?: HostedReproducibilityMode;
|
|
25
|
+
}
|
|
16
26
|
interface ParsedAssessment {
|
|
27
|
+
schemaVersion: 8 | 9;
|
|
28
|
+
riskPlanSha256: string;
|
|
29
|
+
replaySourceRunId: string | null;
|
|
30
|
+
replaySourceResultSchemaVersion: 8 | 9 | null;
|
|
17
31
|
planIdentities: Array<{
|
|
18
32
|
cellId: string;
|
|
19
33
|
capabilityId: string | null;
|
|
@@ -49,9 +63,15 @@ export interface HostedReproducibilityReport {
|
|
|
49
63
|
repository: string;
|
|
50
64
|
base_commit: string;
|
|
51
65
|
head_commit: string;
|
|
66
|
+
reproducibility_mode: HostedReproducibilityMode;
|
|
52
67
|
requested_runs: number;
|
|
68
|
+
launched_runs: number;
|
|
69
|
+
unstarted_runs: number;
|
|
53
70
|
analyzable_runs: number;
|
|
54
71
|
run_ids: string[];
|
|
72
|
+
replay_source_run_id: string | null;
|
|
73
|
+
plan_replay_verified: boolean;
|
|
74
|
+
planner_reproducibility_tested: boolean;
|
|
55
75
|
planner_reproducible: boolean;
|
|
56
76
|
assessment_reproducible: boolean;
|
|
57
77
|
reproducible: boolean;
|
|
@@ -62,12 +82,8 @@ export interface HostedReproducibilityReport {
|
|
|
62
82
|
inconclusive_cell_ids: string[];
|
|
63
83
|
unrunnable_runs: RunFailure[];
|
|
64
84
|
}
|
|
65
|
-
export declare function hostedReproducibilityRunKey(seriesKey: string, repository: string, baseCommit: string, headCommit: string, ordinal: number): string;
|
|
66
|
-
export declare function analyzeHostedReproducibility(runs: HostedReproducibilityRun[], identity:
|
|
67
|
-
|
|
68
|
-
repository: string;
|
|
69
|
-
baseCommit: string;
|
|
70
|
-
headCommit: string;
|
|
71
|
-
}): HostedReproducibilityReport;
|
|
85
|
+
export declare function hostedReproducibilityRunKey(seriesKey: string, repository: string, baseCommit: string, headCommit: string, ordinal: number, mode?: HostedReproducibilityMode): string;
|
|
86
|
+
export declare function analyzeHostedReproducibility(runs: HostedReproducibilityRun[], identity: HostedReproducibilityIdentity): HostedReproducibilityReport;
|
|
87
|
+
export declare function isHostedReproducibilityReplaySource(run: HostedReproducibilityRun, identity: HostedReproducibilityIdentity): boolean;
|
|
72
88
|
export declare function verifyHostedReproducibilityCommand(repositoryArg: string, options: HostedReproducibilityOptions): Promise<void>;
|
|
73
89
|
export {};
|
|
@@ -6,6 +6,7 @@ import { hostedPollingOptions, parseHostedRepository, startHostedVerification, w
|
|
|
6
6
|
const CELL_ID = /^cell_[0-9a-f]{32}$/;
|
|
7
7
|
const CAPABILITY_ID = /^repository_driver_[0-9a-f]{64}$/;
|
|
8
8
|
const COMPARISON_ID = /^driver_comparison_[0-9a-f]{64}$/;
|
|
9
|
+
const RUN_ID = /^cv_[0-9a-f]{48}$/;
|
|
9
10
|
const SHA256 = /^[0-9a-f]{64}$/;
|
|
10
11
|
const SERIES_KEY = /^[A-Za-z0-9][A-Za-z0-9._:-]{7,63}$/;
|
|
11
12
|
const TERMINAL_STATUSES = new Set([
|
|
@@ -68,21 +69,41 @@ function requireSeriesKey(value) {
|
|
|
68
69
|
}
|
|
69
70
|
return key;
|
|
70
71
|
}
|
|
71
|
-
export function hostedReproducibilityRunKey(seriesKey, repository, baseCommit, headCommit, ordinal) {
|
|
72
|
+
export function hostedReproducibilityRunKey(seriesKey, repository, baseCommit, headCommit, ordinal, mode = 'exact_plan_replay') {
|
|
72
73
|
const digest = createHash('sha256')
|
|
73
|
-
.update(JSON.stringify(
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
74
|
+
.update(JSON.stringify(mode === 'exact_plan_replay'
|
|
75
|
+
? [
|
|
76
|
+
seriesKey,
|
|
77
|
+
repository,
|
|
78
|
+
baseCommit,
|
|
79
|
+
headCommit,
|
|
80
|
+
ordinal,
|
|
81
|
+
]
|
|
82
|
+
: [
|
|
83
|
+
mode,
|
|
84
|
+
seriesKey,
|
|
85
|
+
repository,
|
|
86
|
+
baseCommit,
|
|
87
|
+
headCommit,
|
|
88
|
+
ordinal,
|
|
89
|
+
]))
|
|
80
90
|
.digest('hex');
|
|
81
91
|
return `repro:${digest}`;
|
|
82
92
|
}
|
|
93
|
+
function stable(value) {
|
|
94
|
+
if (Array.isArray(value))
|
|
95
|
+
return value.map(stable);
|
|
96
|
+
if (value && typeof value === 'object') {
|
|
97
|
+
const object = value;
|
|
98
|
+
return Object.fromEntries(Object.keys(object)
|
|
99
|
+
.sort()
|
|
100
|
+
.map(key => [key, stable(object[key])]));
|
|
101
|
+
}
|
|
102
|
+
return value;
|
|
103
|
+
}
|
|
83
104
|
function fingerprint(value) {
|
|
84
105
|
return createHash('sha256')
|
|
85
|
-
.update(JSON.stringify(value))
|
|
106
|
+
.update(JSON.stringify(stable(value)))
|
|
86
107
|
.digest('hex');
|
|
87
108
|
}
|
|
88
109
|
function expectedCellStatus(reasonCode) {
|
|
@@ -124,11 +145,51 @@ function parseAssessment(state, identity) {
|
|
|
124
145
|
throw new Error('run identity does not match the requested exact launch');
|
|
125
146
|
}
|
|
126
147
|
const output = record(state.output, 'terminal output');
|
|
127
|
-
if (output.schemaVersion !== 8
|
|
148
|
+
if ((output.schemaVersion !== 8 && output.schemaVersion !== 9)
|
|
128
149
|
|| output.status !== 'driver_assessment_recorded') {
|
|
129
|
-
throw new Error('run did not produce a schema-8 driver assessment');
|
|
150
|
+
throw new Error('run did not produce a schema-8 or schema-9 driver assessment');
|
|
130
151
|
}
|
|
131
152
|
const riskPlan = record(output.riskPlanReceipt, 'risk plan receipt');
|
|
153
|
+
const riskPlanSha256 = fingerprint(riskPlan);
|
|
154
|
+
let replaySourceRunId = null;
|
|
155
|
+
let replaySourceResultSchemaVersion = null;
|
|
156
|
+
if (output.schemaVersion === 9) {
|
|
157
|
+
if (output.planningCredentialDelivery !== 'not-required-plan-replay') {
|
|
158
|
+
throw new Error('schema-9 replay used an invalid planning credential mode');
|
|
159
|
+
}
|
|
160
|
+
const replay = record(output.riskPlanReplayReceipt, 'risk plan replay receipt');
|
|
161
|
+
const fields = Object.keys(replay);
|
|
162
|
+
if (fields.length !== 6
|
|
163
|
+
|| fields.some(field => ![
|
|
164
|
+
'schemaVersion',
|
|
165
|
+
'sourceContentPersisted',
|
|
166
|
+
'sourceResultSchemaVersion',
|
|
167
|
+
'sourceRiskPlanReceiptSha256',
|
|
168
|
+
'sourceRunId',
|
|
169
|
+
'status',
|
|
170
|
+
].includes(field))
|
|
171
|
+
|| replay.schemaVersion !== 1
|
|
172
|
+
|| replay.status !== 'risk_plan_replay_verified'
|
|
173
|
+
|| typeof replay.sourceRunId !== 'string'
|
|
174
|
+
|| !RUN_ID.test(replay.sourceRunId)
|
|
175
|
+
|| (replay.sourceResultSchemaVersion !== 8
|
|
176
|
+
&& replay.sourceResultSchemaVersion !== 9)
|
|
177
|
+
|| replay.sourceRiskPlanReceiptSha256 !== riskPlanSha256
|
|
178
|
+
|| typeof replay.sourceRiskPlanReceiptSha256 !== 'string'
|
|
179
|
+
|| !SHA256.test(replay.sourceRiskPlanReceiptSha256)
|
|
180
|
+
|| replay.sourceContentPersisted !== false) {
|
|
181
|
+
throw new Error('risk plan replay receipt is malformed');
|
|
182
|
+
}
|
|
183
|
+
replaySourceRunId = replay.sourceRunId;
|
|
184
|
+
replaySourceResultSchemaVersion = replay.sourceResultSchemaVersion;
|
|
185
|
+
}
|
|
186
|
+
else if (output.riskPlanReplayReceipt !== undefined) {
|
|
187
|
+
throw new Error('schema-8 result contains unversioned replay provenance');
|
|
188
|
+
}
|
|
189
|
+
else if (output.planningCredentialDelivery
|
|
190
|
+
!== 'per-command-env-gateway-placeholder') {
|
|
191
|
+
throw new Error('schema-8 result used an invalid planning credential mode');
|
|
192
|
+
}
|
|
132
193
|
const assessment = record(output.driverAssessmentReceipt, 'driver assessment receipt');
|
|
133
194
|
const planCells = array(riskPlan.cells, 'risk plan cells');
|
|
134
195
|
const assessmentCells = array(assessment.cells, 'assessment cells');
|
|
@@ -219,6 +280,10 @@ function parseAssessment(state, identity) {
|
|
|
219
280
|
throw new Error('driver assessment overall status is inconsistent');
|
|
220
281
|
}
|
|
221
282
|
return {
|
|
283
|
+
schemaVersion: output.schemaVersion,
|
|
284
|
+
riskPlanSha256,
|
|
285
|
+
replaySourceRunId,
|
|
286
|
+
replaySourceResultSchemaVersion,
|
|
222
287
|
planIdentities,
|
|
223
288
|
cellIds,
|
|
224
289
|
status: assessment.status,
|
|
@@ -315,10 +380,37 @@ export function analyzeHostedReproducibility(runs, identity) {
|
|
|
315
380
|
});
|
|
316
381
|
}
|
|
317
382
|
}
|
|
383
|
+
const requestedRuns = identity.requestedRuns ?? runs.length;
|
|
384
|
+
const allRequestedRunsLaunched = runs.length === requestedRuns;
|
|
318
385
|
const planVariants = groupedPlanVariants(parsed);
|
|
319
386
|
const assessmentVariants = groupedOutcomeVariants(parsed);
|
|
320
|
-
const
|
|
321
|
-
const
|
|
387
|
+
const mode = identity.mode ?? 'exact_plan_replay';
|
|
388
|
+
const replaySourceRunId = mode === 'exact_plan_replay'
|
|
389
|
+
? parsed[0]?.runId ?? null
|
|
390
|
+
: null;
|
|
391
|
+
const planReplayVerified = mode === 'exact_plan_replay'
|
|
392
|
+
&& allRequestedRunsLaunched
|
|
393
|
+
&& parsed.length === runs.length
|
|
394
|
+
&& parsed.length > 1
|
|
395
|
+
&& parsed[0]?.assessment.schemaVersion === 8
|
|
396
|
+
&& parsed[0]?.assessment.replaySourceRunId === null
|
|
397
|
+
&& parsed.slice(1).every(item => item.assessment.schemaVersion === 9
|
|
398
|
+
&& item.assessment.replaySourceRunId === replaySourceRunId
|
|
399
|
+
&& item.assessment.replaySourceResultSchemaVersion
|
|
400
|
+
=== parsed[0].assessment.schemaVersion
|
|
401
|
+
&& item.assessment.riskPlanSha256
|
|
402
|
+
=== parsed[0].assessment.riskPlanSha256);
|
|
403
|
+
const plannerReproducibilityTested = mode === 'fresh_plans' && parsed.length > 1;
|
|
404
|
+
const plannerReproducible = plannerReproducibilityTested
|
|
405
|
+
&& allRequestedRunsLaunched
|
|
406
|
+
&& parsed.length === runs.length
|
|
407
|
+
&& parsed.every(item => item.assessment.schemaVersion === 8
|
|
408
|
+
&& item.assessment.replaySourceRunId === null)
|
|
409
|
+
&& planVariants.length === 1;
|
|
410
|
+
const assessmentReproducible = (mode === 'exact_plan_replay'
|
|
411
|
+
? planReplayVerified
|
|
412
|
+
: plannerReproducible)
|
|
413
|
+
&& assessmentVariants.length === 1;
|
|
322
414
|
const planDriftCellIds = driftIds(parsed, assessment => new Map(assessment.planIdentities.map(plan => [
|
|
323
415
|
plan.cellId,
|
|
324
416
|
{ capabilityId: plan.capabilityId },
|
|
@@ -340,12 +432,22 @@ export function analyzeHostedReproducibility(runs, identity) {
|
|
|
340
432
|
repository: identity.repository,
|
|
341
433
|
base_commit: identity.baseCommit,
|
|
342
434
|
head_commit: identity.headCommit,
|
|
343
|
-
|
|
435
|
+
reproducibility_mode: mode,
|
|
436
|
+
requested_runs: requestedRuns,
|
|
437
|
+
launched_runs: runs.length,
|
|
438
|
+
unstarted_runs: Math.max(0, requestedRuns - runs.length),
|
|
344
439
|
analyzable_runs: parsed.length,
|
|
345
440
|
run_ids: runs.map(run => run.state.runId),
|
|
441
|
+
replay_source_run_id: replaySourceRunId,
|
|
442
|
+
plan_replay_verified: planReplayVerified,
|
|
443
|
+
planner_reproducibility_tested: plannerReproducibilityTested,
|
|
346
444
|
planner_reproducible: plannerReproducible,
|
|
347
445
|
assessment_reproducible: assessmentReproducible,
|
|
348
|
-
reproducible:
|
|
446
|
+
reproducible: (mode === 'exact_plan_replay'
|
|
447
|
+
? planReplayVerified
|
|
448
|
+
: plannerReproducible)
|
|
449
|
+
&& assessmentReproducible
|
|
450
|
+
&& inconclusiveCellIds.length === 0,
|
|
349
451
|
plan_variants: planVariants,
|
|
350
452
|
assessment_variants: assessmentVariants,
|
|
351
453
|
plan_drift_cell_ids: planDriftCellIds,
|
|
@@ -354,11 +456,36 @@ export function analyzeHostedReproducibility(runs, identity) {
|
|
|
354
456
|
unrunnable_runs: unrunnableRuns,
|
|
355
457
|
};
|
|
356
458
|
}
|
|
459
|
+
export function isHostedReproducibilityReplaySource(run, identity) {
|
|
460
|
+
if (run.timedOut || run.state.status !== 'complete')
|
|
461
|
+
return false;
|
|
462
|
+
try {
|
|
463
|
+
const assessment = parseAssessment(run.state, identity);
|
|
464
|
+
return assessment.schemaVersion === 8
|
|
465
|
+
&& assessment.replaySourceRunId === null;
|
|
466
|
+
}
|
|
467
|
+
catch {
|
|
468
|
+
return false;
|
|
469
|
+
}
|
|
470
|
+
}
|
|
357
471
|
function renderHostedReproducibility(report) {
|
|
358
472
|
const icon = report.reproducible ? chalk.green('✓') : chalk.yellow('!');
|
|
359
473
|
console.log(`${icon} Hosted reproducibility: ${report.reproducible ? 'reproducible' : 'not established'}`);
|
|
360
|
-
console.log(` ${chalk.dim('Runs:')} ${report.analyzable_runs}/${report.requested_runs} analyzable`);
|
|
474
|
+
console.log(` ${chalk.dim('Runs:')} ${report.analyzable_runs}/${report.requested_runs} analyzable; ${report.launched_runs} launched`);
|
|
475
|
+
console.log(` ${chalk.dim('Mode:')} ${report.reproducibility_mode === 'fresh_plans'
|
|
476
|
+
? 'independent fresh plans'
|
|
477
|
+
: 'exact plan replay'}`);
|
|
361
478
|
console.log(` ${chalk.dim('Plan variants:')} ${report.plan_variants.length}`);
|
|
479
|
+
console.log(` ${chalk.dim('Exact plan replay:')} ${report.reproducibility_mode === 'fresh_plans'
|
|
480
|
+
? 'not requested'
|
|
481
|
+
: report.plan_replay_verified
|
|
482
|
+
? 'verified'
|
|
483
|
+
: 'not verified'}`);
|
|
484
|
+
console.log(` ${chalk.dim('Independent planner reproducibility:')} ${report.planner_reproducibility_tested
|
|
485
|
+
? report.planner_reproducible
|
|
486
|
+
? 'verified'
|
|
487
|
+
: 'not reproducible'
|
|
488
|
+
: 'not tested'}`);
|
|
362
489
|
console.log(` ${chalk.dim('Assessment variants:')} ${report.assessment_variants.length}`);
|
|
363
490
|
if (report.plan_drift_cell_ids.length > 0) {
|
|
364
491
|
console.log(` ${chalk.dim('Plan drift:')} ${report.plan_drift_cell_ids.length} stable cell id(s)`);
|
|
@@ -377,6 +504,9 @@ export async function verifyHostedReproducibilityCommand(repositoryArg, options)
|
|
|
377
504
|
const repository = parseHostedRepository(repositoryArg);
|
|
378
505
|
const requestedRuns = boundedInteger(options.runs, DEFAULT_RUNS, '--runs', 2, MAX_RUNS);
|
|
379
506
|
const seriesKey = requireSeriesKey(options.seriesKey);
|
|
507
|
+
const mode = options.freshPlans
|
|
508
|
+
? 'fresh_plans'
|
|
509
|
+
: 'exact_plan_replay';
|
|
380
510
|
const auth = await resolveAuthContext({
|
|
381
511
|
preferredLogin: options.account,
|
|
382
512
|
owner: repository.owner,
|
|
@@ -391,7 +521,10 @@ export async function verifyHostedReproducibilityCommand(repositoryArg, options)
|
|
|
391
521
|
const started = await startHostedVerification(repositoryArg, {
|
|
392
522
|
base: options.base,
|
|
393
523
|
head: options.head,
|
|
394
|
-
idempotencyKey: hostedReproducibilityRunKey(seriesKey, repository.fullName, options.base, options.head, index + 1),
|
|
524
|
+
idempotencyKey: hostedReproducibilityRunKey(seriesKey, repository.fullName, options.base, options.head, index + 1, mode),
|
|
525
|
+
replayPlanFrom: options.freshPlans || index === 0
|
|
526
|
+
? undefined
|
|
527
|
+
: runs[0]?.state.runId,
|
|
395
528
|
wait: false,
|
|
396
529
|
}, auth.token);
|
|
397
530
|
let state = started.state;
|
|
@@ -406,12 +539,28 @@ export async function verifyHostedReproducibilityCommand(repositoryArg, options)
|
|
|
406
539
|
timedOut = waited.timedOut;
|
|
407
540
|
}
|
|
408
541
|
runs.push({ state, timedOut });
|
|
542
|
+
if (!options.freshPlans
|
|
543
|
+
&&
|
|
544
|
+
index === 0
|
|
545
|
+
&& !isHostedReproducibilityReplaySource(runs[0], {
|
|
546
|
+
seriesKey,
|
|
547
|
+
repository: repository.fullName,
|
|
548
|
+
baseCommit: options.base,
|
|
549
|
+
headCommit: options.head,
|
|
550
|
+
})) {
|
|
551
|
+
if (!options.json) {
|
|
552
|
+
console.error(chalk.yellow('Source run is not replayable; later runs were not launched.'));
|
|
553
|
+
}
|
|
554
|
+
break;
|
|
555
|
+
}
|
|
409
556
|
}
|
|
410
557
|
const report = analyzeHostedReproducibility(runs, {
|
|
411
558
|
seriesKey,
|
|
412
559
|
repository: repository.fullName,
|
|
413
560
|
baseCommit: options.base,
|
|
414
561
|
headCommit: options.head,
|
|
562
|
+
requestedRuns,
|
|
563
|
+
mode,
|
|
415
564
|
});
|
|
416
565
|
if (options.json) {
|
|
417
566
|
process.stdout.write(`${JSON.stringify(withSchema('cloud-verifier', {
|
|
@@ -2,6 +2,7 @@ export interface HostedStartOptions {
|
|
|
2
2
|
base: string;
|
|
3
3
|
head: string;
|
|
4
4
|
idempotencyKey?: string;
|
|
5
|
+
replayPlanFrom?: string;
|
|
5
6
|
account?: string;
|
|
6
7
|
wait?: boolean;
|
|
7
8
|
interval?: string;
|
|
@@ -41,7 +42,7 @@ export declare function parseHostedRepository(value: string): {
|
|
|
41
42
|
repository: string;
|
|
42
43
|
fullName: string;
|
|
43
44
|
};
|
|
44
|
-
export declare function defaultHostedIdempotencyKey(repository: string, baseCommit: string, headCommit: string): string;
|
|
45
|
+
export declare function defaultHostedIdempotencyKey(repository: string, baseCommit: string, headCommit: string, replayPlanFrom?: string): string;
|
|
45
46
|
export declare function hostedPollingOptions(interval: string | undefined, timeout: string | undefined, onStatus?: (state: HostedRunState) => void): HostedPollOptions;
|
|
46
47
|
export declare function startHostedVerification(repositoryArg: string, options: HostedStartOptions, token: string): Promise<{
|
|
47
48
|
state: HostedRunState;
|
|
@@ -121,9 +121,11 @@ function requireCommit(value, flag) {
|
|
|
121
121
|
}
|
|
122
122
|
return value;
|
|
123
123
|
}
|
|
124
|
-
export function defaultHostedIdempotencyKey(repository, baseCommit, headCommit) {
|
|
124
|
+
export function defaultHostedIdempotencyKey(repository, baseCommit, headCommit, replayPlanFrom) {
|
|
125
125
|
const digest = createHash('sha256')
|
|
126
|
-
.update(JSON.stringify(
|
|
126
|
+
.update(JSON.stringify(replayPlanFrom
|
|
127
|
+
? [repository, baseCommit, headCommit, replayPlanFrom]
|
|
128
|
+
: [repository, baseCommit, headCommit]))
|
|
127
129
|
.digest('hex');
|
|
128
130
|
return `cli:${digest}`;
|
|
129
131
|
}
|
|
@@ -153,8 +155,12 @@ export async function startHostedVerification(repositoryArg, options, token) {
|
|
|
153
155
|
const repository = parseHostedRepository(repositoryArg);
|
|
154
156
|
const baseCommit = requireCommit(options.base, '--base');
|
|
155
157
|
const headCommit = requireCommit(options.head, '--head');
|
|
158
|
+
const replayPlanFrom = options.replayPlanFrom;
|
|
159
|
+
if (replayPlanFrom !== undefined && !RUN_ID.test(replayPlanFrom)) {
|
|
160
|
+
throw new Error('--replay-plan-from must match cv_ followed by 48 lowercase hex characters.');
|
|
161
|
+
}
|
|
156
162
|
const idempotencyKey = requireIdempotencyKey(options.idempotencyKey
|
|
157
|
-
?? defaultHostedIdempotencyKey(repository.fullName, baseCommit, headCommit));
|
|
163
|
+
?? defaultHostedIdempotencyKey(repository.fullName, baseCommit, headCommit, replayPlanFrom));
|
|
158
164
|
const response = await haystackJson('/api/agent/cloud-verifier/runs', token, {
|
|
159
165
|
method: 'POST',
|
|
160
166
|
headers: { 'Content-Type': 'application/json' },
|
|
@@ -164,6 +170,9 @@ export async function startHostedVerification(repositoryArg, options, token) {
|
|
|
164
170
|
baseCommit,
|
|
165
171
|
headCommit,
|
|
166
172
|
idempotencyKey,
|
|
173
|
+
...(replayPlanFrom
|
|
174
|
+
? { replayPlanFromRunId: replayPlanFrom }
|
|
175
|
+
: {}),
|
|
167
176
|
}),
|
|
168
177
|
});
|
|
169
178
|
const state = parseHostedRunState(response);
|
|
@@ -200,7 +209,7 @@ function delay(milliseconds) {
|
|
|
200
209
|
export async function waitForHostedVerification(runId, token, options) {
|
|
201
210
|
const startedAt = Date.now();
|
|
202
211
|
let lastStatus;
|
|
203
|
-
|
|
212
|
+
for (;;) {
|
|
204
213
|
const state = await getHostedVerification(runId, token);
|
|
205
214
|
if (state.status !== lastStatus) {
|
|
206
215
|
options.onStatus?.(state);
|
package/dist/index.js
CHANGED
|
@@ -219,6 +219,7 @@ hostedVerify
|
|
|
219
219
|
.option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
|
|
220
220
|
.option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
|
|
221
221
|
.option('--idempotency-key <key>', 'Stable retry key (default: deterministic from repository and commits)')
|
|
222
|
+
.option('--replay-plan-from <run-id>', 'Reuse the exact validated risk plan from a completed owned run')
|
|
222
223
|
.option('--account <login>', 'Use a specific saved Haystack account')
|
|
223
224
|
.option('--no-wait', 'Return after the production workflow is queued')
|
|
224
225
|
.option('--interval <seconds>', 'Polling interval while waiting (default 5)')
|
|
@@ -228,9 +229,12 @@ hostedVerify
|
|
|
228
229
|
The default idempotency key makes an exact retry return the original run. Pass
|
|
229
230
|
--idempotency-key with a new stable key only when you intentionally want a
|
|
230
231
|
fresh production execution of the same commit pair.
|
|
232
|
+
--replay-plan-from requires the same repository and exact commit pair and emits
|
|
233
|
+
a schema-9 terminal receipt with replay provenance.
|
|
231
234
|
|
|
232
235
|
Examples:
|
|
233
236
|
haystack verify hosted start owner/repo --base <sha> --head <sha>
|
|
237
|
+
haystack verify hosted start owner/repo --base <sha> --head <sha> --replay-plan-from cv_<48 lowercase hex characters>
|
|
234
238
|
haystack verify hosted start owner/repo --base <sha> --head <sha> --no-wait --json
|
|
235
239
|
`)
|
|
236
240
|
.action(async (repository, _options, cmd) => {
|
|
@@ -258,24 +262,31 @@ Example:
|
|
|
258
262
|
});
|
|
259
263
|
hostedVerify
|
|
260
264
|
.command('reproducibility')
|
|
261
|
-
.description('
|
|
265
|
+
.description('Replay one exact hosted risk plan or run fresh plans to report drift')
|
|
262
266
|
.argument('<repository>', 'Exact GitHub owner/repository name')
|
|
263
267
|
.option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
|
|
264
268
|
.option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
|
|
265
269
|
.option('--runs <n>', 'Number of sequential production runs (default 3; min 2, max 10)')
|
|
266
270
|
.option('--series-key <key>', 'Stable series key for resumable idempotent retries')
|
|
271
|
+
.option('--fresh-plans', 'Run the planner independently for every run instead of replaying the first plan')
|
|
267
272
|
.option('--account <login>', 'Use a specific saved Haystack account')
|
|
268
273
|
.option('--interval <seconds>', 'Polling interval for each run (default 5)')
|
|
269
274
|
.option('--timeout <minutes>', 'Maximum wait per run (default 35)')
|
|
270
275
|
.option('--json', 'Versioned machine-readable reproducibility report')
|
|
271
276
|
.addHelpText('after', `
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
277
|
+
The first run creates a fresh risk plan. Every later run replays that exact
|
|
278
|
+
validated plan against the same repository and commit pair with a distinct,
|
|
279
|
+
series-derived idempotency key. A reproducible verdict requires schema-9 replay
|
|
280
|
+
provenance, stable assessment outcomes, and no inconclusive cells.
|
|
281
|
+
|
|
282
|
+
With --fresh-plans, every run plans independently. A reproducible verdict then
|
|
283
|
+
requires one stable-id plan variant, one assessment variant, no replay
|
|
284
|
+
provenance, and no inconclusive cells. Use this mode to detect planner drift;
|
|
285
|
+
use the default mode to isolate executor or assessment drift.
|
|
276
286
|
|
|
277
287
|
Example:
|
|
278
288
|
haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --runs 3
|
|
289
|
+
haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --runs 3 --fresh-plans
|
|
279
290
|
`)
|
|
280
291
|
.action(async (repository, _options, cmd) => {
|
|
281
292
|
const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };
|