create-mercato-app 0.6.8-develop.7042.1.413df59570 → 0.6.8-develop.7046.1.153faed87a

package/agentic/shared/ai/harness/RELEASE.md

@@ -50,9 +50,9 @@ After preflight it runs, in order:
 6. real generated-code execution for OMH-163 and OMH-192 through fixed Jest, OMH-164 through API-only Playwright, and OMH-165 through real-browser Playwright; and
 7. explicit isolated `om-judge-agent-session` for every writable result, composing `om-code-review` and applicable design-system guidance, bound to its passing command attestation, any required generated-test result and artifact hash, and the final target fingerprint.
 
-Each writable target is single-use because fixture preparation marks it disposable. Externally supplied target realpaths must be pairwise disjoint and neither equal to, contain, nor be contained by the controller. A failed deterministic or foundation-validation step prevents model execution. Once fixture preparation succeeds, all four target commands run even when the writable gate itself fails, so every generated target has exact diagnostics. A writable case may declare `timeoutMs` only to raise the release `--case-timeout` floor (never lower it); OMH-185 and its business-language parity case OMH-193 use 600000 ms because the complete module slice exceeded the generic five-minute evaluator default while actively producing source. Generated tests run only after the trusted writable oracle and all four target commands pass; review then requires all applicable gates. A target command or generated-test failure is recorded with its sanitized diagnostic and review is skipped. Other matrix entries continue so the report remains useful.
+Each writable target is single-use because fixture preparation marks it disposable. Externally supplied target realpaths must be pairwise disjoint and neither equal to, contain, nor be contained by the controller. A failed deterministic or foundation-validation step prevents model execution. Once fixture preparation succeeds, all four target commands run even when the writable gate itself fails, so every generated target has exact diagnostics. A writable case may declare `timeoutMs` only to raise the release `--case-timeout` floor (never lower it); OMH-185 and its business-language parity case OMH-193 use 600000 ms because the complete module slice exceeded the generic five-minute evaluator default while actively producing source. Since the schema caps `timeoutMs` at exactly the shipped `--case-timeout` default, a declared value only raises anything for an operator who lowered that default — it is the floor under a lowered budget, not an addition to the shipped one. Generated tests run only after the trusted writable oracle and all four target commands pass; review then requires all applicable gates. A target command or generated-test failure is recorded with its sanitized diagnostic and review is skipped. Other matrix entries continue so the report remains useful.
 
-Routing cases carry no case-local duration budget, and that is a decision rather than an omission. `maxContextFiles` and the byte budgets measure the agent's context discipline, which is intrinsic to the case and portable between machines; duration measures the runner and the attempt count, which are not — the audited cohort spanned 71 s to 231 s for passing routing runs, one case measured 147 s and 132 s on two runs of the same model, and OMH-139 exhausted the evaluator's 300000 ms default outright. The operator budget carries that variance instead of the catalog. On this release path the lever is `--case-timeout` (default 120000 ms), which the release command passes on to the evaluator explicitly for every routing step; because that pass-through marks the timeout explicit, the evaluator's own runner-aware floors — 600000 ms for Claude, 900000 ms for a Codex `gpt-5.4-mini` high-effort run, 300000 ms for every other runner — apply only to direct `evaluate-agent-harness.mjs` invocations and never fire under `yarn harness:release`. The release default sits below the slowest audited passing routing run, so raise `--case-timeout` for a slow model rather than expecting the default to absorb the tail; on the direct evaluator path that run leaves about 62% headroom against the Claude floor and about 23% against the generic 300000 ms default, and OMH-139 shows the latter is not always enough. A declared writable `timeoutMs` is combined with whichever operator value applies as a maximum, so it raises the floor and never lowers it; routing cases declare none, so there the operator value stands alone. Writable cases keep their own `timeoutMs` because a writable one-shot's cost is dominated by the slice it must produce, which the case does define.
+Routing cases carry no case-local duration budget, and that is a decision rather than an omission. `maxContextFiles` and the byte budgets measure the agent's context discipline, which is intrinsic to the case and portable between machines; duration measures the runner and the attempt count, which are not — the audited cohort spanned 71 s to 231 s for passing routing runs, one case measured 147 s and 132 s on two runs of the same model, and OMH-139 exhausted the evaluator's 300000 ms default outright. The operator budget carries that variance instead of the catalog. On this release path the lever is `--case-timeout` (default 600000 ms), which the release command passes on to the evaluator explicitly for every routing step; because that pass-through marks the timeout explicit, the evaluator's own runner-aware floors — 600000 ms for Claude, 900000 ms for a Codex `gpt-5.4-mini` high-effort run, 300000 ms for every other runner — apply only to direct `evaluate-agent-harness.mjs` invocations and never fire under `yarn harness:release`. Keeping that pass-through is deliberate rather than inherited: the routing step derives its own process budget from the same value, as slack plus the sum of the per-case ceilings it hands out, so letting the floors raise the inner ceiling while the outer budget still followed `--case-timeout` would kill an entire routing step instead of failing one slow case. Leaving it explicit also keeps the budget runner-independent, which is what lets the primary and portability lanes in one report be compared as models rather than as budgets. The default clears the slowest audited passing routing run — 231 s against 600000 ms, about 62% headroom — and matches both the Claude floor and the `timeoutMs` ceiling `cases.schema.json` enforces, so the gate carries one upper number instead of three; lower `--case-timeout` when a hung case should fail sooner, and note that the step's process budget drops with it — though at a lowered budget the declared writable ceilings keep their own slots, so the step's budget drops less than proportionally. That value was chosen from evaluator-path measurements — the audited cohort above and the live evidence recorded on #5068 — rather than from a driven `yarn harness:release` run, because the complete gate fails closed off the Linux-with-Bubblewrap host stated above; #5078 records the decision and that deviation, and #5433 carries the unmet measurement forward so the default is confirmed or corrected against the release path's own numbers. `--case-timeout` is one budget for three lanes, not a routing-only lever: the writable and review lanes resolve their own ceilings from the same value, so raising it raises theirs too. A declared writable `timeoutMs` is combined with whichever operator value applies as a maximum, so it raises the floor and never lowers it — but because the schema caps it at the shipped default, it can only raise a budget an operator has lowered; routing cases declare none, so there the operator value always stands alone. Writable cases keep their own `timeoutMs` because a writable one-shot's cost is dominated by the slice it must produce, which the case does define.
 
 UI-routed implementation reviews receive only the bounded backend UI guide and `om-backend-ui-design` design-system references. Non-UI reviews do not receive that extra context.
 

package/agentic/shared/scripts/run-agent-harness-release.mjs

@@ -25,6 +25,11 @@ const GENERATED_TEST_RUNNERS = new Set(['jest', 'playwright-api', 'playwright-br
 const RESULT_LIMIT = 262_144
 const ERROR_LIMIT = 2_000
 const VIOLATION_LIMIT = 300
+// The routing step hands this to the evaluator as --timeout and derives its own process budget from
+// the same value, so the two cannot be stated separately; the help text reads it rather than
+// repeating it (#5078).
+export const DEFAULT_CASE_TIMEOUT_MS = 600_000
+export const ROUTING_STEP_SLACK_MS = 60_000
 const COPY_EXCLUDED_PREFIXES = [
   '.git', '.next', '.turbo', '.cache', 'build', 'coverage', 'dist', 'node_modules', 'out',
   '.ai/framework-context', '.ai/harness/results', '.ai/reports',
@@ -50,7 +55,7 @@ Options:
   --portability-runner <runner> Optional different runner for the 49-case read-only portability lane
   --prepare-targets <absolute>   Clone this fresh scaffold once per writable case under an empty/new directory
   --writable-targets <absolute> JSON map of every writable case to a fresh disposable app
-  --case-timeout <ms>           Per-model invocation timeout floor (default: 120000; a writable case may raise it)
+  --case-timeout <ms>           Per-model invocation timeout floor for the routing, writable, and review lanes (default: ${DEFAULT_CASE_TIMEOUT_MS})
   --validation-timeout <ms>     Timeout for each yarn validation (default: 1800000)
   --acknowledge-writes          Required: fixture preparation and validation commands write files
   --help                        Show this help
@@ -76,7 +81,7 @@ function parseArgs(argv) {
     portabilityRunner: undefined,
     prepareTargets: undefined,
     writableTargets: undefined,
-    caseTimeout: 120_000,
+    caseTimeout: DEFAULT_CASE_TIMEOUT_MS,
     validationTimeout: 1_800_000,
     acknowledgeWrites: false,
     help: false,
@@ -118,6 +123,17 @@ export function effectiveCaseTimeout(cases, caseId, fallback) {
   return Math.max(fallback, Number.isInteger(declared) ? declared : 0)
 }
 
+export function routingInvocation({ evaluator, root, step, cases, caseTimeout }) {
+  const args = [evaluator, '--root', root, '--runner', step.runner]
+  if (step.lane === 'primary') args.push('--all')
+  args.push('--model', step.modelSelector, '--timeout', String(caseTimeout))
+  const timeout = step.expectedCaseIds.reduce(
+    (total, caseId) => total + effectiveCaseTimeout(cases, caseId, caseTimeout),
+    ROUTING_STEP_SLACK_MS,
+  )
+  return { args, timeout }
+}
+
 function readJson(file) {
   return JSON.parse(fs.readFileSync(file, 'utf8'))
 }
@@ -1614,13 +1630,9 @@ export function main(argv = process.argv.slice(2)) {
   } else {
     for (const step of plan.steps.filter((entry) => entry.kind === 'routing')) {
       const before = resultFiles(root)
-      const routingArgs = [evaluator, '--root', root, '--runner', step.runner]
-      if (step.lane === 'primary') routingArgs.push('--all')
-      routingArgs.push('--model', step.modelSelector, '--timeout', String(options.caseTimeout))
-      const routingTimeout = step.expectedCaseIds.reduce(
-        (total, caseId) => total + effectiveCaseTimeout(cases, caseId, options.caseTimeout),
-        60_000,
-      )
+      const { args: routingArgs, timeout: routingTimeout } = routingInvocation({
+        evaluator, root, step, cases, caseTimeout: options.caseTimeout,
+      })
       const execution = execute(process.execPath, routingArgs, root, routingTimeout)
       const artifacts = readNewResults(root, before)
       resultArtifacts.push(...artifacts)