@haystackeditor/cli 0.16.0 → 0.17.0

package/README.md

@@ -218,7 +218,7 @@ haystack logout
 
 ### `haystack submit`
 
-Create a PR from current changes. Runs pre-PR triage (code review, rules validation, instruction drift), pushes your branch, and opens the PR.
+Create a PR from current changes. Runs pre-PR triage (code review, rules validation), pushes your branch, and opens the PR.
 
 ```bash
 haystack submit                      # Triage -> create PR -> wait for analysis
@@ -229,6 +229,28 @@ haystack submit --force              # Skip triage checks
 haystack submit --no-wait            # Don't wait for analysis results
 ```
 
+**Pre-PR triage**: two checkers run in parallel, each one structured call to
+`gpt-6-astra` over the OpenAI Responses API at reasoning effort `max` (the
+highest the API accepts for that model). The key comes from `OPENAI_API_KEY`
+if set, else from the SSM SecureString `/haystack/secrets/shared/prod/OPENAI_API_KEY`
+(us-west-2) through the default AWS credential chain (e.g. `AWS_PROFILE=prod`).
+With neither, each checker reports `no OpenAI key (...)` and submit continues.
+
+- **code-review** reads `git diff -U10 origin/<base>...HEAD` and reports objective bugs.
+- **rules-validator** runs when `.haystack/pr-rules.yml` or an agent
+  instruction file (`CLAUDE.md`, `AGENTS.md`, `COPILOT.md`, `.cursorrules`,
+  `.github/copilot-instructions.md`) exists, and checks the diff against them.
+
+Each finding carries `file`, `line`, `severity` (`error`/`warning`/`info`),
+`category`, a one-sentence summary and a concrete failure scenario; results
+are saved to `.haystack/triage/<checker>.json`. A finding with severity
+`error` stops the submit (`--force` skips triage). A checker that cannot run
+(missing key, API error, stalled stream) prints `code review failed: <error>`
+and the submit continues. The reviewer sees only the diff, not the rest of the
+repository. `--triage-timeout <sec>` (or `.haystack.json` `triage.timeoutMs`)
+aborts a checker whose response stream has been silent that long (default
+300s); there is no total-time cap.
+
 The `--json` payload (`haystack schema submit`) reports the PR ref, the
 resolved title/body and where each came from, auto-merge/auto-fix state with
 its source (flag vs `.haystack.json`), and the analysis outcome in the shared
@@ -483,13 +505,20 @@ Knobs and what turning them does:
 | `--max-concurrent <n>` | the case count, capped at 500 | Ceiling only. Admitted concurrency is the smaller of it and the fleet's free capacity; the run's receipt reports both. |
 | `--per-case-wall-ms <ms>` | 120000 | A case that overruns is killed and recorded `UNKNOWN`/`wall-budget`, never dropped. Above the MVP's measured 103.8 s maximum, so it does not kill cases running at today's cost. |
 | `--total-budget-ms <ms>` | 1800000 (the contract ceiling) | Bounds launches from the first launch. Cases already launched run to their own wall budget. |
+| `--warm-settle-ms <ms>` | absent (0) | Each arm adopts only a warm clone that has been ready at least this long, 0 to the per-case wall. A clone adopted right after its restore attaches cold (quiet fleet: connectMs 5.8 s at 0 vs 1.1 s after 3.5-6 s of idle); each second of settle can delay a case's first create by up to a second. |
 | `--idempotency-key <key>` | derived from the whole submission | An exact retry returns the original run; a new key is a new execution. |
 | `--interval <seconds>` | 10 | Polling interval while `--watch` is set. |
 | `--max-wall <minutes>` | 45 | Total polling wall. Hitting it prints the snapshot, says `timed_out`, and exits 2. |
 
 `status` exits 0 for a `completed` batch with cleanup proven, and 2 for
 `incomplete`, `cancelled`, unproven cleanup, or a wait that hits `--max-wall`.
-Either way every per-case row is printed.
+Either way every per-case row is printed. A `--json` row from a v4 batch also
+carries `node_ids: { base1, head }` (the node each arm's VM was placed on) and
+`failure: { source, arm, error_class, error_code, node_id }` (the typed fact
+behind an `UNKNOWN` that is not an arrival or wall outcome, and behind an
+`arrival-failed` whose two arms stalled on the same step, `same-step-stall`); each
+is `null` when the outcome has none. A case in `--cases` may carry `repeatIndex` (1-16), which
+must be part of its `contentDigest`, to run as a repeat next to its original.
 
 ### `haystack policy`
 

package/dist/commands/case-batch-contract.js

@@ -15,7 +15,7 @@
  */
 import { createHash } from 'node:crypto';
 /** Bounds, copied verbatim from the frozen contract. */
-export const CASE_BATCH_VERSION = 'case-batch-v1';
+export const CASE_BATCH_VERSION = 'case-batch-v4';
 export const CASE_BATCH_MAX_CONCURRENT_CASES = 500;
 export const CASE_BATCH_MAX_CASES = 5_000;
 export const CASE_BATCH_MAX_INPUT_BYTES = 8 * 1024 * 1024;
@@ -25,6 +25,8 @@ export const CASE_BATCH_MIN_TOTAL_BUDGET_MS = 10_000;
 export const CASE_BATCH_MAX_TOTAL_BUDGET_MS = 30 * 60_000;
 export const CASE_BATCH_VM_SLOTS_PER_CASE = 2;
 export const CASE_BATCH_ID_PATTERN = /^[A-Za-z0-9_-]{1,160}$/u;
+/** v4: `ProductCase.repeatIndex` is an integer from 1 to this; absent is the original experiment. */
+export const CASE_BATCH_MAX_REPEAT_INDEX = 16;
 /** `CombinationSearchStatus`, reused by the batch without change. */
 export const CASE_BATCH_STATUSES = [
     'queued', 'preparing', 'searching', 'draining', 'completed', 'incomplete', 'cancelled',
@@ -41,6 +43,18 @@ export const KNOWN_CASE_UNKNOWN_REASONS = [
     'missing-state', 'missing-action', 'missing-harness', 'arrival-failed',
     'comparison-unavailable', 'infrastructure', 'wall-budget', 'wall-watchdog', 'worker-exited',
 ];
+/** MIRRORED RULES (v2) — `case_batch_contracts.ts`: a start value is a reference id or two joined by
+ * one '='; the action and observation kinds the runtime executes; the execution paths an outcome names. */
+export const CASE_START_VALUE_PATTERN = /^(?=.{1,256}$)[A-Za-z0-9._:/-]+(?:=[A-Za-z0-9._:/-]+)?$/u;
+export const CASE_BATCH_ACTION_KINDS = ['navigate', 'click', 'type', 'wait-for', 'exec', 'raw-pointer', 'key', 'focus',
+    'pointer-move', 'pointer-down', 'pointer-up', 'type-focused'];
+export const CASE_BATCH_OBSERVATION_KINDS = ['dom-inventory', 'visible-text-digest', 'url', 'main-document',
+    'element-count', 'element-state', 'focused-element', 'storage-snapshot'];
+export const CASE_EXECUTION_PATHS = ['resident-sealed-plan', 'sealed-plan', 'record-replay'];
+/** v4: the sources a failure fact names today (`CASE_FAILURE_SOURCES` in the contract). Read-side, a
+ *  source added later is still rendered, as an unrecognised unknown reason is: the list documents, it
+ *  does not refuse. */
+export const KNOWN_CASE_FAILURE_SOURCES = ['control-plane', 'transport', 'harness', 'runner'];
 export class CaseBatchRequestValidationError extends TypeError {
     name = 'CaseBatchRequestValidationError';
 }
@@ -186,7 +200,9 @@ function jsonObject(value, what, maximumBytes = CASE_BATCH_MAX_INPUT_BYTES) {
 export function parseProductCase(value, index) {
     const row = object(value, `case ${index}`);
     keys(row, ['id', 'contentDigest', 'conditionId', 'start', 'actions', 'arrivalCheck',
-        'observations', 'variantOf', 'provenance', 'surfaceId', 'finalControlId', 'staticReach', 'label'], ['arrivalActionIndex']);
+        'observations', 'variantOf', 'provenance', 'surfaceId', 'finalControlId', 'staticReach', 'label'], ['arrivalActionIndex', 'settingsDigest', 'startStateSnapshotIds', 'armCommits', 'publishScreenshots', 'repeatIndex']);
+    if (row.settingsDigest !== undefined && row.startStateSnapshotIds !== undefined)
+        fail(`case ${index} names its start state twice (settingsDigest and startStateSnapshotIds)`);
     const id = text(row.id, `case ${index} id`, 160);
     if (!CASE_BATCH_ID_PATTERN.test(id))
         fail(`case ${index} id (queue identity charset is [A-Za-z0-9_-], 1-160)`);
@@ -199,8 +215,11 @@ export function parseProductCase(value, index) {
     for (const [position, axis] of axes.entries()) {
         if (['__proto__', 'constructor', 'prototype'].includes(axis))
             fail(`case ${index} start (unsafe key)`);
-        start[referenceIdentity(axis, `case ${index} start axis ${position}`)] =
-            referenceIdentity(startRow[axis], `case ${index} start value for axis ${position} (${excerpt(axis)})`);
+        const startValue = startRow[axis];
+        if (typeof startValue !== 'string' || !CASE_START_VALUE_PATTERN.test(startValue))
+            fail(`case ${index} start value for axis ${position} (${excerpt(axis)}) is not a reference id `
+                + `or two joined by one '=': ${excerpt(String(startValue))}`);
+        start[referenceIdentity(axis, `case ${index} start axis ${position}`)] = startValue;
     }
     if (!Array.isArray(row.actions))
         fail(`case ${index} actions`);
@@ -234,13 +253,45 @@ export function parseProductCase(value, index) {
         contentDigest: hex(row.contentDigest, 64, `case ${index} contentDigest`),
         conditionId: referenceIdentity(row.conditionId, `case ${index} conditionId`),
         start,
-        actions: row.actions.map((action, position) => boundedCaseBatchJson(action, `case ${index} action ${position}`, CASE_VALUE_MAX_BYTES)),
+        actions: row.actions.map((action, position) => {
+            const checked = jsonObject(action, `case ${index} action ${position}`, CASE_VALUE_MAX_BYTES);
+            if (typeof checked.kind !== 'string' || !CASE_BATCH_ACTION_KINDS.includes(checked.kind))
+                fail(`case ${index} action ${position} kind ${excerpt(String(checked.kind))} is not one the runtime executes`);
+            if (checked.coverage !== undefined && typeof checked.coverage !== 'boolean')
+                fail(`case ${index} action ${position} coverage must be a boolean`);
+            return checked;
+        }),
         // Optional and additive: refused here when it does not name an action this
         // case carries, rather than at execution where it costs two VMs to find.
         ...(row.arrivalActionIndex === undefined ? {}
             : { arrivalActionIndex: integer(row.arrivalActionIndex, `case ${index} arrivalActionIndex`, 0, row.actions.length - 1) }),
         arrivalCheck: jsonObject(row.arrivalCheck, `case ${index} arrivalCheck`, CASE_VALUE_MAX_BYTES),
-        observations: row.observations.map((observation, position) => jsonObject(observation, `case ${index} observation ${position}`, CASE_VALUE_MAX_BYTES)),
+        observations: row.observations.map((observation, position) => {
+            const checked = jsonObject(observation, `case ${index} observation ${position}`, CASE_VALUE_MAX_BYTES);
+            if (typeof checked.kind !== 'string' || !CASE_BATCH_OBSERVATION_KINDS.includes(checked.kind))
+                fail(`case ${index} observation ${position} kind ${excerpt(String(checked.kind))} is not one the runtime reads`);
+            return checked;
+        }),
+        ...(row.settingsDigest === undefined ? {} : { settingsDigest: hex(row.settingsDigest, 64, `case ${index} settingsDigest`) }),
+        ...(row.startStateSnapshotIds === undefined ? {} : { startStateSnapshotIds: (() => {
+                const ids = object(row.startStateSnapshotIds, `case ${index} startStateSnapshotIds`);
+                keys(ids, ['base1', 'head']);
+                return { base1: hex(ids.base1, 16, `case ${index} startStateSnapshotIds.base1`, 'frz_'),
+                    head: hex(ids.head, 16, `case ${index} startStateSnapshotIds.head`, 'frz_') };
+            })() }),
+        ...(row.armCommits === undefined ? {} : { armCommits: (() => {
+                const commits = object(row.armCommits, `case ${index} armCommits`);
+                keys(commits, ['base1', 'head']);
+                return { base1: hex(commits.base1, 40, `case ${index} armCommits.base1`),
+                    head: hex(commits.head, 40, `case ${index} armCommits.head`) };
+            })() }),
+        ...(row.publishScreenshots === undefined ? {} : { publishScreenshots: (() => {
+                if (typeof row.publishScreenshots !== 'boolean')
+                    fail(`case ${index} publishScreenshots must be a boolean`);
+                return row.publishScreenshots;
+            })() }),
+        ...(row.repeatIndex === undefined ? {}
+            : { repeatIndex: integer(row.repeatIndex, `case ${index} repeatIndex`, 1, CASE_BATCH_MAX_REPEAT_INDEX) }),
         variantOf: nullableIdentity(row.variantOf, `case ${index} variantOf`),
         provenance: jsonObject(row.provenance, `case ${index} provenance`, CASE_VALUE_MAX_BYTES),
         surfaceId: nullableIdentity(row.surfaceId, `case ${index} surfaceId`),
@@ -283,10 +334,12 @@ export function parseProductCases(value) {
     return cases;
 }
 export function parseCaseBatchLimits(value) {
+    const perCaseWallMs = integer(value.perCaseWallMs, 'perCaseWallMs', CASE_BATCH_MIN_CASE_WALL_MS, CASE_BATCH_MAX_CASE_WALL_MS);
     return {
         maxConcurrentCases: integer(value.maxConcurrentCases, 'maxConcurrentCases', 1, CASE_BATCH_MAX_CONCURRENT_CASES),
-        perCaseWallMs: integer(value.perCaseWallMs, 'perCaseWallMs', CASE_BATCH_MIN_CASE_WALL_MS, CASE_BATCH_MAX_CASE_WALL_MS),
+        perCaseWallMs,
         totalBudgetMs: integer(value.totalBudgetMs, 'totalBudgetMs', CASE_BATCH_MIN_TOTAL_BUDGET_MS, CASE_BATCH_MAX_TOTAL_BUDGET_MS),
+        ...(value.warmSettleMs === undefined ? {} : { warmSettleMs: integer(value.warmSettleMs, 'warmSettleMs', 0, perCaseWallMs) }),
     };
 }
 export function parseCaseBatchRepository(value) {
@@ -353,6 +406,14 @@ export function parseCaseBatchSource(value, baseCommit, headCommit) {
 }
 /** Assemble and bound the request exactly as it goes on the wire. */
 export function buildCaseBatchRequest(input) {
+    // v3, as the gateway checks it: every declared arm commit is the batch's base or head commit.
+    for (const [index, row] of input.cases.entries()) {
+        for (const arm of ['base1', 'head']) {
+            const commit = row.armCommits?.[arm];
+            if (commit !== undefined && commit !== input.baseCommit && commit !== input.headCommit)
+                fail(`case ${index} armCommits.${arm} ${commit} is neither --base nor --head`);
+        }
+    }
     const request = {
         repository: input.repository,
         baseCommit: input.baseCommit,
@@ -360,6 +421,7 @@ export function buildCaseBatchRequest(input) {
         idempotencyKey: input.idempotencyKey,
         ...(input.baseWorld ? { baseWorld: input.baseWorld } : {}),
         ...(input.headWorld ? { headWorld: input.headWorld } : {}),
+        ...(input.driver ? { driver: input.driver } : {}),
         combinationsRef: { inline: input.combinations },
         casesRef: { inline: input.cases },
         limits: input.limits,
@@ -405,6 +467,9 @@ const RECORD_ARRAY_MAX = 10_000;
  * string at 128 rather than refusing a reason added later. Looser, never
  * tighter, than the gateway. */
 const RECORD_UNKNOWN_REASON_MAX = 128;
+/** v4: `CASE_FAILURE_FIELD_MAX` in `outcome()`: every node id, failure class and subcode is bounded
+ * by length only. */
+const RECORD_FAILURE_FIELD_MAX = 256;
 /** `validSearchRunId`: `cv_` plus 48 hex characters. */
 const SNAPSHOT_RUN_ID_MAX = 51;
 /** `hex(worlds.base, [16], 'base world', 'frz_')`: 4 + 16. */
@@ -446,6 +511,23 @@ function armPhases(value, what) {
         bindingWaitMs: counter('bindingWaitMs'),
     };
 }
+function caseArmNodeIds(value) {
+    const row = isJsonObject(value) ? value : bad('case outcome nodeIds');
+    return { base1: responseNullableString(row.base1, 'case outcome nodeIds.base1', RECORD_FAILURE_FIELD_MAX),
+        head: responseNullableString(row.head, 'case outcome nodeIds.head', RECORD_FAILURE_FIELD_MAX) };
+}
+function caseFailureFact(value) {
+    const row = isJsonObject(value) ? value : bad('case outcome failure');
+    if (row.arm !== null && row.arm !== 'base1' && row.arm !== 'head')
+        bad('case outcome failure.arm');
+    return {
+        source: responseText(row.source, 'case outcome failure.source', RECORD_FAILURE_FIELD_MAX),
+        arm: row.arm,
+        errorClass: responseText(row.errorClass, 'case outcome failure.errorClass', RECORD_FAILURE_FIELD_MAX),
+        errorCode: responseNullableString(row.errorCode, 'case outcome failure.errorCode', RECORD_FAILURE_FIELD_MAX),
+        nodeId: responseNullableString(row.nodeId, 'case outcome failure.nodeId', RECORD_FAILURE_FIELD_MAX),
+    };
+}
 function caseOutcome(value, caseId) {
     const row = isJsonObject(value) ? value : bad('case outcome');
     if (row.caseId !== caseId)
@@ -477,6 +559,10 @@ function caseOutcome(value, caseId) {
         cleanupProven: row.cleanupProven,
         differingOutcomes: responseStringArray(row.differingOutcomes, 'case outcome differingOutcomes', RECORD_DIFFERING_OUTCOME_MAX),
         detail: responseNullableString(row.detail, 'case outcome detail', RECORD_DETAIL_MAX),
+        ...(typeof row.executionPath === 'string' && CASE_EXECUTION_PATHS.includes(row.executionPath)
+            ? { executionPath: row.executionPath } : {}),
+        ...(row.nodeIds === undefined ? {} : { nodeIds: caseArmNodeIds(row.nodeIds) }),
+        ...(row.failure === undefined ? {} : { failure: row.failure === null ? null : caseFailureFact(row.failure) }),
     };
 }
 export function parseCaseBatchCaseRecord(value) {