@kontourai/flow-agents 3.3.0 → 3.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/add-to-project.yml +15 -0
- package/.github/workflows/ci.yml +161 -0
- package/CHANGELOG.md +48 -0
- package/CONTEXT.md +5 -1
- package/README.md +19 -8
- package/build/src/builder-flow-run-adapter.d.ts +80 -0
- package/build/src/builder-flow-run-adapter.js +241 -0
- package/build/src/builder-flow-runtime.d.ts +16 -0
- package/build/src/builder-flow-runtime.js +290 -0
- package/build/src/cli/builder-run.d.ts +1 -0
- package/build/src/cli/builder-run.js +27 -0
- package/build/src/cli/effective-backlog-settings.js +70 -2
- package/build/src/cli/init.d.ts +34 -0
- package/build/src/cli/init.js +341 -61
- package/build/src/cli/kit.js +55 -12
- package/build/src/cli/pull-work-provider.js +346 -5
- package/build/src/cli/skill-drift-check.d.ts +1 -0
- package/build/src/cli/skill-drift-check.js +165 -0
- package/build/src/cli/telemetry-doctor.d.ts +37 -0
- package/build/src/cli/telemetry-doctor.js +53 -6
- package/build/src/cli/validate-hook-influence.js +37 -7
- package/build/src/cli/workflow-sidecar.d.ts +93 -8
- package/build/src/cli/workflow-sidecar.js +1175 -158
- package/build/src/cli.js +5 -0
- package/build/src/flow-kit/validate.d.ts +54 -34
- package/build/src/flow-kit/validate.js +237 -26
- package/build/src/index.d.ts +2 -0
- package/build/src/index.js +1 -0
- package/build/src/lib/console-connect-options.d.ts +97 -0
- package/build/src/lib/console-connect-options.js +199 -0
- package/build/src/lib/console-telemetry-validate.d.ts +49 -0
- package/build/src/lib/console-telemetry-validate.js +91 -0
- package/build/src/lib/flow-resolver.d.ts +56 -3
- package/build/src/lib/flow-resolver.js +151 -11
- package/build/src/lib/fs.d.ts +17 -0
- package/build/src/lib/fs.js +172 -0
- package/build/src/lib/local-artifact-root.d.ts +44 -1
- package/build/src/lib/local-artifact-root.js +131 -3
- package/build/src/runtime-adapters.d.ts +39 -3
- package/build/src/runtime-adapters.js +77 -31
- package/build/src/tools/build-universal-bundles.js +40 -2
- package/build/src/tools/codex-agent-routing.d.ts +2 -0
- package/build/src/tools/codex-agent-routing.js +49 -0
- package/build/src/tools/generate-context-map.js +1 -0
- package/build/src/tools/validate-source-tree.js +27 -1
- package/context/scripts/hooks/lib/kit-catalog.js +235 -0
- package/context/scripts/hooks/lib/runnable-command.js +177 -0
- package/context/scripts/hooks/stop-goal-fit.js +278 -48
- package/context/scripts/hooks/workflow-steering.js +121 -21
- package/context/scripts/package.json +3 -0
- package/context/scripts/telemetry/install-console-config.sh +25 -4
- package/context/scripts/telemetry/lib/config.sh +102 -12
- package/context/scripts/telemetry/lib/pricing.sh +50 -0
- package/context/scripts/telemetry/lib/session.sh +3 -0
- package/context/scripts/telemetry/lib/transport.sh +87 -0
- package/context/scripts/telemetry/lib/usage.sh +205 -4
- package/context/scripts/telemetry/telemetry.conf +6 -0
- package/context/scripts/telemetry/telemetry.sh +48 -0
- package/context/settings/workspace-backlog-provider-settings.example.json +48 -0
- package/docs/agent-usage-feedback-loop.md +35 -0
- package/docs/architecture-engine-and-kits.md +110 -0
- package/docs/context-map.md +2 -0
- package/docs/decisions/embeddable-engine.md +152 -0
- package/docs/decisions/index.md +3 -1
- package/docs/decisions/trust-ledger-retention.md +88 -0
- package/docs/decisions/workflow-enforcement.md +31 -9
- package/docs/fixture-ownership.md +3 -0
- package/docs/implementing-trust-reconciliation.md +129 -0
- package/docs/index.md +19 -9
- package/docs/integrations/flow-agents-console.md +167 -0
- package/docs/kit-authoring-guide.md +52 -21
- package/docs/spec/builder-flow-runtime.md +80 -0
- package/docs/spec/runtime-hook-surface.md +45 -1
- package/docs/specs/economics-record-contract.md +270 -0
- package/docs/specs/harness-capability-matrix.md +74 -0
- package/docs/specs/learning-review-proposals-contract.md +340 -0
- package/docs/specs/routing-efficiency-review.md +59 -0
- package/docs/verifiable-trust.md +74 -25
- package/docs/workflow-usage-guide.md +10 -0
- package/evals/acceptance/prove-capture-teeth.sh +132 -0
- package/evals/ci/antigaming-suite.sh +1 -0
- package/evals/ci/run-baseline.sh +72 -4
- package/evals/fixtures/economics/acceptance.json +12 -0
- package/evals/fixtures/economics/agents/tool-worker-1/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-2/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-3/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-4/events.jsonl +1 -0
- package/evals/fixtures/economics/agents/tool-worker-5/events.jsonl +2 -0
- package/evals/fixtures/economics/critique.json +22 -0
- package/evals/fixtures/economics/expected-record.json +71 -0
- package/evals/fixtures/economics/session-usage-event.json +1 -0
- package/evals/fixtures/economics/state.json +11 -0
- package/evals/fixtures/economics/transcript.jsonl +3 -0
- package/evals/fixtures/hook-influence/cases.json +7 -7
- package/evals/fixtures/learning-review-proposals/balanced/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/economics.jsonl +5 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-1/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-2/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-3/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-4/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-5/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/expected-aggregates.json +30 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/expected-aggregates.md +66 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-1/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-1/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-2/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-2/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-3/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-3/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-4/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-4/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-5/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-6/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/repeat-window/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/under-threshold/economics.jsonl +3 -0
- package/evals/fixtures/telemetry/usage-transcript-sample.jsonl +4 -0
- package/evals/fixtures/trust-reconcile-exploits/mcp-degrade.json +42 -0
- package/evals/integration/test_builder_entry_enforcement.sh +241 -0
- package/evals/integration/test_builder_step_producers.sh +18 -10
- package/evals/integration/test_bundle_install.sh +172 -0
- package/evals/integration/test_console_tenant_isolation.sh +167 -0
- package/evals/integration/test_critique_supersession_roundtrip.sh +4 -1
- package/evals/integration/test_dual_emit_flow_step.sh +10 -4
- package/evals/integration/test_economics_record.sh +674 -0
- package/evals/integration/test_effective_backlog_settings.sh +1 -1
- package/evals/integration/test_evidence_capture_hook.sh +17 -2
- package/evals/integration/test_exemption_usage_review.sh +198 -0
- package/evals/integration/test_fixture_retirement_audit.sh +2 -2
- package/evals/integration/test_flow_kit_install_git.sh +83 -0
- package/evals/integration/test_flowdef_session_activation.sh +0 -1
- package/evals/integration/test_flowdef_session_history_preservation.sh +13 -3
- package/evals/integration/test_gate_lockdown.sh +7 -0
- package/evals/integration/test_gate_review_inquiry_records.sh +9 -1
- package/evals/integration/test_goal_fit_hook.sh +2031 -0
- package/evals/integration/test_hook_category_behaviors.sh +8 -1
- package/evals/integration/test_hook_influence_cases.sh +25 -1
- package/evals/integration/test_install_merge.sh +227 -2
- package/evals/integration/test_kit_conformance_levels.sh +6 -6
- package/evals/integration/test_learning_review_proposals.sh +329 -0
- package/evals/integration/test_liveness_conflict_injection.sh +26 -22
- package/evals/integration/test_liveness_console_relay.sh +166 -0
- package/evals/integration/test_liveness_heartbeat.sh +17 -17
- package/evals/integration/test_liveness_worktree_root.sh +575 -0
- package/evals/integration/test_phase_map_and_gate_claim.sh +6 -1
- package/evals/integration/test_publish_delivery.sh +331 -1
- package/evals/integration/test_pull_work_board.sh +200 -0
- package/evals/integration/test_pull_work_provider.sh +1 -1
- package/evals/integration/test_record_check.sh +378 -0
- package/evals/integration/test_routing_efficiency.sh +71 -0
- package/evals/integration/test_runtime_adapter_activation.sh +28 -0
- package/evals/integration/test_session_resume_roundtrip.sh +16 -19
- package/evals/integration/test_skill_drift_check.sh +870 -0
- package/evals/integration/test_telemetry.sh +445 -0
- package/evals/integration/test_telemetry_doctor.sh +66 -0
- package/evals/integration/test_telemetry_usage_pipeline.sh +228 -0
- package/evals/integration/test_trust_reconcile_negatives.sh +30 -13
- package/evals/integration/test_trust_reconcile_trailer_diagnostic.sh +247 -0
- package/evals/integration/test_usage_cost.sh +61 -0
- package/evals/integration/test_workflow_sidecar_writer.sh +1395 -0
- package/evals/integration/test_workflow_steering_hook.sh +157 -16
- package/evals/integration/test_workspace_settings.sh +176 -0
- package/evals/lib/env.sh +26 -0
- package/evals/lib/node.sh +8 -0
- package/evals/run.sh +29 -0
- package/evals/static/test_ci_integration_coverage.sh +115 -0
- package/evals/static/test_declared_scope_forms_documented.sh +114 -0
- package/evals/static/test_universal_bundles.sh +34 -0
- package/evals/static/test_validate_source_kit_asset_scope.sh +259 -0
- package/evals/static/test_workflow_skills.sh +1 -1
- package/kits/builder/flows/build.flow.json +9 -18
- package/kits/builder/flows/publish-learn.flow.json +5 -1
- package/kits/builder/kit.json +120 -0
- package/kits/builder/skills/deliver/SKILL.md +42 -0
- package/kits/builder/skills/evidence-gate/SKILL.md +12 -0
- package/kits/builder/skills/execute-plan/SKILL.md +9 -0
- package/kits/builder/skills/learning-review/SKILL.md +51 -0
- package/kits/builder/skills/plan-work/SKILL.md +17 -20
- package/kits/builder/skills/pull-work/SKILL.md +21 -0
- package/kits/builder/skills/release-readiness/SKILL.md +12 -0
- package/kits/knowledge/kit.json +9 -0
- package/kits/veritas-governance/docs/README.md +35 -7
- package/kits/veritas-governance/fixtures/exemption-review/mixed-fresh-stale.DECLARED.json +14 -0
- package/kits/veritas-governance/kit.json +14 -0
- package/kits/veritas-governance/skills/exemption-usage-review/SKILL.md +128 -0
- package/kits/veritas-governance/skills/exemption-usage-review/review-exemptions.mjs +231 -0
- package/package.json +2 -2
- package/packaging/manifest.json +29 -0
- package/schemas/backlog-provider-settings.schema.json +13 -0
- package/schemas/workflow-state.schema.json +44 -0
- package/scripts/README.md +4 -0
- package/scripts/check-content-boundary.cjs +8 -1
- package/scripts/ci/trust-reconcile.js +136 -0
- package/scripts/hooks/codex-hook-adapter.js +77 -2
- package/scripts/hooks/evidence-capture.js +38 -5
- package/scripts/hooks/lib/codex-exit-code.js +316 -0
- package/scripts/hooks/lib/kit-catalog.js +235 -0
- package/scripts/hooks/lib/liveness-write.js +28 -1
- package/scripts/hooks/lib/local-artifact-paths.js +97 -1
- package/scripts/hooks/lib/runnable-command.js +177 -0
- package/scripts/hooks/lib/skill-drift.js +350 -0
- package/scripts/hooks/stop-goal-fit.js +278 -48
- package/scripts/hooks/workflow-steering.js +121 -21
- package/scripts/install-codex-home.sh +97 -47
- package/scripts/install-merge.js +72 -14
- package/scripts/install-owned-files.js +178 -0
- package/scripts/liveness/relay.sh +84 -0
- package/scripts/telemetry/economics-record.schema.json +145 -0
- package/scripts/telemetry/economics-record.sh +331 -0
- package/scripts/telemetry/install-console-config.sh +25 -4
- package/scripts/telemetry/learning-review-decide.sh +124 -0
- package/scripts/telemetry/learning-review-proposals.schema.json +161 -0
- package/scripts/telemetry/learning-review-proposals.sh +484 -0
- package/scripts/telemetry/lib/config.sh +102 -12
- package/scripts/telemetry/lib/pricing.sh +14 -6
- package/scripts/telemetry/lib/session.sh +3 -0
- package/scripts/telemetry/lib/transport.sh +133 -15
- package/scripts/telemetry/lib/usage.sh +121 -28
- package/scripts/telemetry/routing-efficiency.sh +0 -0
- package/scripts/telemetry/telemetry.conf +6 -0
- package/scripts/telemetry/telemetry.sh +48 -0
- package/src/builder-flow-run-adapter.ts +357 -0
- package/src/builder-flow-runtime.ts +348 -0
- package/src/cli/builder-flow-run-adapter.test.mjs +495 -0
- package/src/cli/builder-flow-runtime.test.mjs +213 -0
- package/src/cli/builder-run.ts +28 -0
- package/src/cli/codex-agent-routing.test.mjs +44 -0
- package/src/cli/codex-exit-code.test.mjs +207 -0
- package/src/cli/console-connect-options.test.mjs +329 -0
- package/src/cli/console-telemetry-validate.test.mjs +157 -0
- package/src/cli/effective-backlog-settings.ts +68 -2
- package/src/cli/flow-resolver-composition.test.mjs +101 -0
- package/src/cli/init.test.mjs +161 -0
- package/src/cli/init.ts +407 -62
- package/src/cli/kit-metadata-security.test.mjs +443 -0
- package/src/cli/kit.ts +50 -12
- package/src/cli/pull-work-provider.ts +377 -3
- package/src/cli/sidecar-pure-helpers.test.mjs +64 -0
- package/src/cli/skill-drift-check.ts +196 -0
- package/src/cli/telemetry-doctor.test.mjs +53 -0
- package/src/cli/telemetry-doctor.ts +50 -7
- package/src/cli/validate-hook-influence.ts +37 -6
- package/src/cli/workflow-sidecar.ts +1150 -151
- package/src/cli.ts +5 -0
- package/src/flow-kit/validate.ts +277 -38
- package/src/index.ts +19 -0
- package/src/lib/console-connect-options.ts +261 -0
- package/src/lib/console-telemetry-validate.ts +88 -0
- package/src/lib/flow-resolver.ts +153 -10
- package/src/lib/fs.ts +160 -0
- package/src/lib/local-artifact-root.ts +129 -3
- package/src/runtime-adapters.ts +113 -33
- package/src/tools/build-universal-bundles.ts +36 -2
- package/src/tools/codex-agent-routing.ts +48 -0
- package/src/tools/generate-context-map.ts +1 -0
- package/src/tools/validate-source-tree.ts +26 -1
|
@@ -2800,6 +2800,52 @@ else
|
|
|
2800
2800
|
_fail "trust.bundle capture-authoritative setup failed: $(cat "$TMPDIR_EVAL/tb-capture-evidence.out" "$TMPDIR_EVAL/tb-capture-evidence.err")"
|
|
2801
2801
|
fi
|
|
2802
2802
|
|
|
2803
|
+
# ─── #470 iteration 2 (finding #2, HIGH): captureByCommand treats ambiguous as
|
|
2804
|
+
# non-confirming, never coerced to "pass" (would reconcile as a verified event + passing:true,
|
|
2805
|
+
# reintroducing the #470 false-pass). A command-log entry whose only capture is
|
|
2806
|
+
# observedResult:"ambiguous" (e.g. a codex no-signal capture, or the #362 grep/diff absence
|
|
2807
|
+
# carve-out) paired with a claimed-pass check must build a bundle claim whose value is the
|
|
2808
|
+
# EXISTING canonical non-confirming status "not_verified", an evidence item stamped
|
|
2809
|
+
# passing:false, and NO verification event for that claim. ------------------------------------
|
|
2810
|
+
TB_AMBIGUOUS_DIR="$TMPDIR_EVAL/repo/.kontourai/flow-agents/trust-bundle-ambiguous"
|
|
2811
|
+
mkdir -p "$TB_AMBIGUOUS_DIR"
|
|
2812
|
+
cp "$ARTIFACT_DIR/auto-sidecars--deliver.md" "$TB_AMBIGUOUS_DIR/trust-bundle-ambiguous--deliver.md"
|
|
2813
|
+
flow_agents_node "$WRITER" init-plan "$TB_AMBIGUOUS_DIR/trust-bundle-ambiguous--deliver.md" \
|
|
2814
|
+
--source-request "Ambiguous-capture non-confirming trust bundle fixture (#470 iteration 2)." \
|
|
2815
|
+
--summary "Ambiguous-capture non-confirming trust bundle fixture." \
|
|
2816
|
+
--next-action "Seed a claimed-pass check whose only capture is observedResult:ambiguous." \
|
|
2817
|
+
--timestamp "2026-05-09T00:00:00Z" >"$TMPDIR_EVAL/tb-ambiguous-init.out" 2>"$TMPDIR_EVAL/tb-ambiguous-init.err"
|
|
2818
|
+
# Deterministic capture log: the command produced NO usable pass/fail signal (codex no-signal
|
|
2819
|
+
# default / #362 absence carve-out), recorded before record-evidence.
|
|
2820
|
+
printf '%s\n' '{"command":"npm test","observedResult":"ambiguous","exitCode":null}' > "$TB_AMBIGUOUS_DIR/command-log.jsonl"
|
|
2821
|
+
if flow_agents_node "$WRITER" record-evidence "$TB_AMBIGUOUS_DIR" \
|
|
2822
|
+
--verdict pass \
|
|
2823
|
+
--check-json '{"id":"tb-ambiguous-check","kind":"test","status":"pass","summary":"Claimed pass.","command":"npm test"}' \
|
|
2824
|
+
--timestamp "2026-05-09T00:01:00Z" >"$TMPDIR_EVAL/tb-ambiguous-evidence.out" 2>"$TMPDIR_EVAL/tb-ambiguous-evidence.err" \
|
|
2825
|
+
&& [[ -f "$TB_AMBIGUOUS_DIR/trust.bundle" ]]; then
|
|
2826
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/tb-ambiguous-assert.err"
|
|
2827
|
+
import { readFileSync } from 'node:fs';
|
|
2828
|
+
const bundle = JSON.parse(readFileSync('${TB_AMBIGUOUS_DIR}/trust.bundle', 'utf8'));
|
|
2829
|
+
const claim = bundle.claims.find((c) => c.subjectId && c.subjectId.endsWith('/tb-ambiguous-check'));
|
|
2830
|
+
if (!claim) { process.stderr.write('missing claim for /tb-ambiguous-check\n'); process.exit(1); }
|
|
2831
|
+
if (claim.value !== 'not_verified') { process.stderr.write('claimed-pass check with captured ambiguous had claim.value ' + claim.value + ', expected not_verified\n'); process.exit(1); }
|
|
2832
|
+
if (claim.status === 'verified') { process.stderr.write('claimed-pass check with captured ambiguous derived claim.status verified -- ambiguous must never confirm\n'); process.exit(1); }
|
|
2833
|
+
const ev = bundle.evidence.find((e) => e.claimId === claim.id);
|
|
2834
|
+
if (!ev) { process.stderr.write('missing evidence item for ambiguous-captured claim\n'); process.exit(1); }
|
|
2835
|
+
if (ev.passing !== false) { process.stderr.write('ambiguous-captured evidence item had passing=' + ev.passing + ', expected false\n'); process.exit(1); }
|
|
2836
|
+
if (ev.execution && ev.execution.isError !== false) { process.stderr.write('ambiguous-captured evidence item had execution.isError=' + (ev.execution && ev.execution.isError) + ', expected false (ambiguous is not an error)\n'); process.exit(1); }
|
|
2837
|
+
const verifiedEvent = bundle.events.find((e) => e.claimId === claim.id && e.status === 'verified');
|
|
2838
|
+
if (verifiedEvent) { process.stderr.write('a verified event was emitted for an ambiguous-captured claim -- #470 false-pass reintroduced\n'); process.exit(1); }
|
|
2839
|
+
NODEOF
|
|
2840
|
+
then
|
|
2841
|
+
_pass "trust.bundle ambiguous non-confirming (#470 iter2): claimed-pass + captured-ambiguous -> claim.value=not_verified, passing:false, no verified event"
|
|
2842
|
+
else
|
|
2843
|
+
_fail "trust.bundle ambiguous non-confirming assertion failed: $(cat "$TMPDIR_EVAL/tb-ambiguous-assert.err")"
|
|
2844
|
+
fi
|
|
2845
|
+
else
|
|
2846
|
+
_fail "trust.bundle ambiguous non-confirming setup failed: $(cat "$TMPDIR_EVAL/tb-ambiguous-evidence.out" "$TMPDIR_EVAL/tb-ambiguous-evidence.err")"
|
|
2847
|
+
fi
|
|
2848
|
+
|
|
2803
2849
|
# ---- #347: hard cutover -- an unstamped (pre-#344) claim in a session trust bundle is a loud,
|
|
2804
2850
|
# typed error; checksFromBundle/critiquesFromBundle carry NO claimType-derivation fallback (that
|
|
2805
2851
|
# fallback WAS the #268 defect kept reachable). ----------------------------------------------
|
|
@@ -3388,6 +3434,1355 @@ else
|
|
|
3388
3434
|
_fail "missing slug error message lacked 'task-slug is required': $(cat "$TMPDIR_EVAL/wi-no-slug.out")"
|
|
3389
3435
|
fi
|
|
3390
3436
|
|
|
3437
|
+
# ─── #270/#298 compose layer: gate-claim accumulation, gate-claim typing survives rebuild, ──
|
|
3438
|
+
# compose-two/three/four-writer round-trip, waiver + artifact_refs/standard_refs round-trip,
|
|
3439
|
+
# runnability rejection at record time (AC1-AC6, AC8, AC10) ──────────────────────────────────
|
|
3440
|
+
COMPOSE_ROOT="$TMPDIR_EVAL/compose-root"
|
|
3441
|
+
COMPOSE_SLUG="compose-270"
|
|
3442
|
+
COMPOSE_DIR="$COMPOSE_ROOT/$COMPOSE_SLUG"
|
|
3443
|
+
mkdir -p "$COMPOSE_ROOT"
|
|
3444
|
+
|
|
3445
|
+
flow_agents_node "$WRITER" ensure-session \
|
|
3446
|
+
--artifact-root "$COMPOSE_ROOT" \
|
|
3447
|
+
--task-slug "$COMPOSE_SLUG" \
|
|
3448
|
+
--actor compose-actor \
|
|
3449
|
+
--flow-id builder.build \
|
|
3450
|
+
--title "Compose layer session" \
|
|
3451
|
+
--source-request "Compose-safe writer layer smoke session." \
|
|
3452
|
+
--summary "Seed session for compose-layer round-trip assertions." \
|
|
3453
|
+
--criterion "Compose layer round-trips losslessly" \
|
|
3454
|
+
--timestamp "2026-07-05T09:00:00Z" >"$TMPDIR_EVAL/compose-seed.out" 2>"$TMPDIR_EVAL/compose-seed.err"
|
|
3455
|
+
|
|
3456
|
+
_compose_claims_json() {
|
|
3457
|
+
cat "$COMPOSE_DIR/trust.bundle"
|
|
3458
|
+
}
|
|
3459
|
+
|
|
3460
|
+
# ─── AC3/AC6: record-evidence #1 — command-backed check carrying artifact_refs/standard_refs ──
|
|
3461
|
+
if flow_agents_node "$WRITER" record-evidence "$COMPOSE_DIR" \
|
|
3462
|
+
--verdict pass \
|
|
3463
|
+
--check-json '{"id":"compose-build-check","kind":"build","status":"pass","summary":"Build passed","command":"npm run build","artifact_refs":[{"kind":"artifact","file":"build/out.js","summary":"build output"}],"standard_refs":[{"standard":"junit","file":"build/out.xml"}]}' \
|
|
3464
|
+
--timestamp "2026-07-05T09:05:00Z" >"$TMPDIR_EVAL/compose-ev1.out" 2>"$TMPDIR_EVAL/compose-ev1.err" \
|
|
3465
|
+
&& [[ -f "$COMPOSE_DIR/trust.bundle" ]]; then
|
|
3466
|
+
_pass "compose: record-evidence #1 (command-backed check w/ artifact_refs/standard_refs) writes trust.bundle"
|
|
3467
|
+
else
|
|
3468
|
+
_fail "compose: record-evidence #1 failed: $(cat "$TMPDIR_EVAL/compose-ev1.out" "$TMPDIR_EVAL/compose-ev1.err")"
|
|
3469
|
+
fi
|
|
3470
|
+
|
|
3471
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/compose-ev1-assert.err"
|
|
3472
|
+
import { readFileSync } from 'node:fs';
|
|
3473
|
+
const bundle = JSON.parse(readFileSync('${COMPOSE_DIR}/trust.bundle', 'utf8'));
|
|
3474
|
+
const claim = bundle.claims.find((c) => c.subjectId && c.subjectId.endsWith('/compose-build-check'));
|
|
3475
|
+
if (!claim) { process.stderr.write('missing compose-build-check claim\n'); process.exit(1); }
|
|
3476
|
+
const md = claim.metadata || {};
|
|
3477
|
+
if (!Array.isArray(md.artifact_refs) || md.artifact_refs.length !== 1) { process.stderr.write('artifact_refs did not stamp onto claim.metadata: ' + JSON.stringify(md.artifact_refs) + '\n'); process.exit(1); }
|
|
3478
|
+
if (!Array.isArray(md.standard_refs) || md.standard_refs.length !== 1) { process.stderr.write('standard_refs did not stamp onto claim.metadata: ' + JSON.stringify(md.standard_refs) + '\n'); process.exit(1); }
|
|
3479
|
+
NODEOF
|
|
3480
|
+
then
|
|
3481
|
+
_pass "AC6: artifact_refs/standard_refs stamp onto claim.metadata on FIRST write (#298, previously silently dropped)"
|
|
3482
|
+
else
|
|
3483
|
+
_fail "AC6: artifact_refs/standard_refs did not stamp onto claim.metadata: $(cat "$TMPDIR_EVAL/compose-ev1-assert.err")"
|
|
3484
|
+
fi
|
|
3485
|
+
|
|
3486
|
+
# ─── AC5: record-evidence #2 — a SEPARATE call records a waived session-local check ──────────
|
|
3487
|
+
if flow_agents_node "$WRITER" record-evidence "$COMPOSE_DIR" \
|
|
3488
|
+
--verdict pass \
|
|
3489
|
+
--check-json '{"id":"compose-manual-check","kind":"external","status":"pass","summary":"Manual review confirmed no regressions"}' \
|
|
3490
|
+
--accepted-gap-reason "manual check waived for compose eval" \
|
|
3491
|
+
--waived-by "compose-tester" \
|
|
3492
|
+
--timestamp "2026-07-05T09:06:00Z" >"$TMPDIR_EVAL/compose-ev2.out" 2>"$TMPDIR_EVAL/compose-ev2.err"; then
|
|
3493
|
+
_pass "compose: record-evidence #2 (waived session-local check) succeeds"
|
|
3494
|
+
else
|
|
3495
|
+
_fail "compose: record-evidence #2 failed: $(cat "$TMPDIR_EVAL/compose-ev2.out" "$TMPDIR_EVAL/compose-ev2.err")"
|
|
3496
|
+
fi
|
|
3497
|
+
|
|
3498
|
+
# ─── AC3: compose-two-writer — check #1 (with artifact_refs/standard_refs) survives the SECOND
|
|
3499
|
+
# record-evidence call (#298's core complaint: record-evidence previously REPLACED all checks) ──
|
|
3500
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/compose-two-writer.err"
|
|
3501
|
+
import { readFileSync } from 'node:fs';
|
|
3502
|
+
const bundle = JSON.parse(readFileSync('${COMPOSE_DIR}/trust.bundle', 'utf8'));
|
|
3503
|
+
const claims = bundle.claims;
|
|
3504
|
+
const buildClaim = claims.find((c) => c.subjectId && c.subjectId.endsWith('/compose-build-check'));
|
|
3505
|
+
const manualClaim = claims.find((c) => c.subjectId && c.subjectId.endsWith('/compose-manual-check'));
|
|
3506
|
+
const acClaim = claims.find((c) => c.metadata && c.metadata.origin === 'acceptance');
|
|
3507
|
+
if (!buildClaim) { process.stderr.write('compose-build-check claim LOST after second record-evidence call (#298 regression)\n'); process.exit(1); }
|
|
3508
|
+
if (!manualClaim) { process.stderr.write('compose-manual-check claim missing after its own record-evidence call\n'); process.exit(1); }
|
|
3509
|
+
if (!acClaim) { process.stderr.write('acceptance criterion claim LOST after second record-evidence call\n'); process.exit(1); }
|
|
3510
|
+
const md = buildClaim.metadata || {};
|
|
3511
|
+
if (!Array.isArray(md.artifact_refs) || md.artifact_refs.length !== 1) { process.stderr.write('artifact_refs did not survive the SECOND writer pass (AC6 second-writer round-trip)\n'); process.exit(1); }
|
|
3512
|
+
if (!Array.isArray(md.standard_refs) || md.standard_refs.length !== 1) { process.stderr.write('standard_refs did not survive the SECOND writer pass\n'); process.exit(1); }
|
|
3513
|
+
const waiverMd = manualClaim.metadata || {};
|
|
3514
|
+
if (!waiverMd.waiver || waiverMd.waiver.approved_by !== 'compose-tester') { process.stderr.write('waiver did not stamp onto compose-manual-check claim\n'); process.exit(1); }
|
|
3515
|
+
if (claims.length !== 3) { process.stderr.write('expected exactly 3 claims after 2 record-evidence calls (2 checks + 1 acceptance criterion), got ' + claims.length + '\n'); process.exit(1); }
|
|
3516
|
+
NODEOF
|
|
3517
|
+
then
|
|
3518
|
+
_pass "AC3: compose-two-writer — record-evidence #1's check (+ artifact_refs/standard_refs) AND the acceptance criterion survive record-evidence #2's call, not replaced"
|
|
3519
|
+
else
|
|
3520
|
+
_fail "AC3: compose-two-writer assertion failed: $(cat "$TMPDIR_EVAL/compose-two-writer.err")"
|
|
3521
|
+
fi
|
|
3522
|
+
|
|
3523
|
+
# ─── AC1/AC10: record-gate-claim (pull-work-gate/selected-work) with --command ───────────────
|
|
3524
|
+
if flow_agents_node "$WRITER" record-gate-claim "$COMPOSE_DIR" \
|
|
3525
|
+
--actor compose-actor \
|
|
3526
|
+
--status pass \
|
|
3527
|
+
--summary "Work item selected for compose session" \
|
|
3528
|
+
--command "npm run validate:source --" \
|
|
3529
|
+
--timestamp "2026-07-05T09:07:00Z" >"$TMPDIR_EVAL/compose-gate1.out" 2>"$TMPDIR_EVAL/compose-gate1.err"; then
|
|
3530
|
+
_pass "compose: record-gate-claim (selected-work) with --command succeeds"
|
|
3531
|
+
else
|
|
3532
|
+
_fail "compose: record-gate-claim (selected-work) failed: $(cat "$TMPDIR_EVAL/compose-gate1.out" "$TMPDIR_EVAL/compose-gate1.err")"
|
|
3533
|
+
fi
|
|
3534
|
+
|
|
3535
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/compose-gate1-assert.err"
|
|
3536
|
+
import { readFileSync } from 'node:fs';
|
|
3537
|
+
const bundle = JSON.parse(readFileSync('${COMPOSE_DIR}/trust.bundle', 'utf8'));
|
|
3538
|
+
const claims = bundle.claims;
|
|
3539
|
+
const buildClaim = claims.find((c) => c.subjectId && c.subjectId.endsWith('/compose-build-check'));
|
|
3540
|
+
const manualClaim = claims.find((c) => c.subjectId && c.subjectId.endsWith('/compose-manual-check'));
|
|
3541
|
+
const gateClaim = claims.find((c) => c.metadata && c.metadata.gate_claim && c.metadata.gate_claim.expectation_id === 'selected-work');
|
|
3542
|
+
if (!buildClaim || !manualClaim) { process.stderr.write('record-gate-claim clobbered prior checks (#270 21-claims-to-1 wipe class)\n'); process.exit(1); }
|
|
3543
|
+
if (!gateClaim) { process.stderr.write('gate claim missing metadata.gate_claim stamp\n'); process.exit(1); }
|
|
3544
|
+
if (gateClaim.claimType !== 'builder.pull-work.selected' || gateClaim.subjectType !== 'work-item') { process.stderr.write('gate claim typed incorrectly: ' + gateClaim.claimType + '/' + gateClaim.subjectType + '\n'); process.exit(1); }
|
|
3545
|
+
const ev = bundle.evidence.find((e) => e.claimId === gateClaim.id);
|
|
3546
|
+
if (!ev || !ev.execution || ev.execution.label !== 'npm run validate:source --') { process.stderr.write('AC10: --command did not stamp execution.label on the gate claim evidence\n'); process.exit(1); }
|
|
3547
|
+
if (claims.length !== 4) { process.stderr.write('expected exactly 4 claims after record-gate-claim (2 checks + 1 acceptance + 1 gate claim), got ' + claims.length + '\n'); process.exit(1); }
|
|
3548
|
+
NODEOF
|
|
3549
|
+
then
|
|
3550
|
+
_pass "AC1/AC10: record-gate-claim accumulates (prior checks/criteria survive) and --command sets execution.label on the gate claim"
|
|
3551
|
+
else
|
|
3552
|
+
_fail "AC1/AC10: record-gate-claim accumulation/command assertion failed: $(cat "$TMPDIR_EVAL/compose-gate1-assert.err")"
|
|
3553
|
+
fi
|
|
3554
|
+
|
|
3555
|
+
# ─── advance to `plan` step (phase_map.planning → plan) so a SECOND gate claim can target a ──
|
|
3556
|
+
# DIFFERENT expectation than pull-work-gate's single "selected-work" entry ────────────────────
|
|
3557
|
+
if flow_agents_node "$WRITER" advance-state "$COMPOSE_DIR" \
|
|
3558
|
+
--actor compose-actor \
|
|
3559
|
+
--status in_progress \
|
|
3560
|
+
--phase planning \
|
|
3561
|
+
--summary "Advance to plan step for a second, different-expectation gate claim" \
|
|
3562
|
+
--next-action "Continue" \
|
|
3563
|
+
--flow-definition builder.build \
|
|
3564
|
+
--route-back-reason implementation_defect \
|
|
3565
|
+
--timestamp "2026-07-05T09:07:15Z" >"$TMPDIR_EVAL/compose-advance-plan.out" 2>"$TMPDIR_EVAL/compose-advance-plan.err"; then
|
|
3566
|
+
_pass "compose: advance-state moves active step to plan"
|
|
3567
|
+
else
|
|
3568
|
+
_fail "compose: advance-state (to plan) failed: $(cat "$TMPDIR_EVAL/compose-advance-plan.out" "$TMPDIR_EVAL/compose-advance-plan.err")"
|
|
3569
|
+
fi
|
|
3570
|
+
|
|
3571
|
+
# ─── AC1: a SECOND gate claim against a DIFFERENT expectation (plan-gate/implementation-plan) ──
|
|
3572
|
+
# is additive, not a replacement, of the FIRST gate claim (pull-work-gate/selected-work) ───────
|
|
3573
|
+
if flow_agents_node "$WRITER" record-gate-claim "$COMPOSE_DIR" \
|
|
3574
|
+
--actor compose-actor \
|
|
3575
|
+
--status pass \
|
|
3576
|
+
--summary "Implementation plan recorded for compose session" \
|
|
3577
|
+
--expectation implementation-plan \
|
|
3578
|
+
--timestamp "2026-07-05T09:07:30Z" >"$TMPDIR_EVAL/compose-gate2.out" 2>"$TMPDIR_EVAL/compose-gate2.err"; then
|
|
3579
|
+
_pass "compose: second record-gate-claim (different expectation, implementation-plan) succeeds"
|
|
3580
|
+
else
|
|
3581
|
+
_fail "compose: second record-gate-claim failed: $(cat "$TMPDIR_EVAL/compose-gate2.out" "$TMPDIR_EVAL/compose-gate2.err")"
|
|
3582
|
+
fi
|
|
3583
|
+
|
|
3584
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/compose-gate2-assert.err"
|
|
3585
|
+
import { readFileSync } from 'node:fs';
|
|
3586
|
+
const bundle = JSON.parse(readFileSync('${COMPOSE_DIR}/trust.bundle', 'utf8'));
|
|
3587
|
+
const claims = bundle.claims;
|
|
3588
|
+
const gate1 = claims.find((c) => c.metadata && c.metadata.gate_claim && c.metadata.gate_claim.expectation_id === 'selected-work');
|
|
3589
|
+
const gate2 = claims.find((c) => c.metadata && c.metadata.gate_claim && c.metadata.gate_claim.expectation_id === 'implementation-plan');
|
|
3590
|
+
if (!gate1) { process.stderr.write('first gate claim (selected-work) LOST after second gate-claim call (#270 wipe class)\n'); process.exit(1); }
|
|
3591
|
+
if (!gate2) { process.stderr.write('second gate claim (implementation-plan) missing\n'); process.exit(1); }
|
|
3592
|
+
if (gate2.claimType !== 'builder.plan.implementation' || gate2.subjectType !== 'artifact') { process.stderr.write('second gate claim typed incorrectly: ' + gate2.claimType + '/' + gate2.subjectType + '\n'); process.exit(1); }
|
|
3593
|
+
if (claims.length !== 5) { process.stderr.write('expected exactly 5 claims after 2 gate claims (2 checks + 1 acceptance + 2 gate claims), got ' + claims.length + '\n'); process.exit(1); }
|
|
3594
|
+
NODEOF
|
|
3595
|
+
then
|
|
3596
|
+
_pass "AC1: two record-gate-claim calls against DIFFERENT expectations are both additive (neither clobbers the other or prior checks)"
|
|
3597
|
+
else
|
|
3598
|
+
_fail "AC1: two-gate-claim additivity assertion failed: $(cat "$TMPDIR_EVAL/compose-gate2-assert.err")"
|
|
3599
|
+
fi
|
|
3600
|
+
|
|
3601
|
+
# ─── AC2: gate-claim typing survives rebuild after the ACTIVE STEP moves to N+1 ──────────────
|
|
3602
|
+
if flow_agents_node "$WRITER" advance-state "$COMPOSE_DIR" \
|
|
3603
|
+
--actor compose-actor \
|
|
3604
|
+
--status in_progress \
|
|
3605
|
+
--phase goal_fit \
|
|
3606
|
+
--summary "Advance to merge-ready for AC2 rebuild assertion" \
|
|
3607
|
+
--next-action "Continue" \
|
|
3608
|
+
--flow-definition builder.build \
|
|
3609
|
+
--timestamp "2026-07-05T09:08:00Z" >"$TMPDIR_EVAL/compose-advance.out" 2>"$TMPDIR_EVAL/compose-advance.err"; then
|
|
3610
|
+
_pass "compose: advance-state moves active step to merge-ready (N+1 relative to pull-work/plan)"
|
|
3611
|
+
else
|
|
3612
|
+
_fail "compose: advance-state failed: $(cat "$TMPDIR_EVAL/compose-advance.out" "$TMPDIR_EVAL/compose-advance.err")"
|
|
3613
|
+
fi
|
|
3614
|
+
|
|
3615
|
+
# record-critique REBUILDS the bundle (checksFromBundle + writeTrustBundle) while the active
|
|
3616
|
+
# step is merge-ready — the gate claims recorded at pull-work/plan above must NOT be silently
|
|
3617
|
+
# re-typed as merge-ready claims (that re-typing IS the #270(c) defect).
|
|
3618
|
+
if flow_agents_node "$WRITER" record-critique "$COMPOSE_DIR" \
|
|
3619
|
+
--id compose-review \
|
|
3620
|
+
--reviewer compose-reviewer \
|
|
3621
|
+
--verdict pass \
|
|
3622
|
+
--summary "Compose review looks good" \
|
|
3623
|
+
--timestamp "2026-07-05T09:09:00Z" >"$TMPDIR_EVAL/compose-critique.out" 2>"$TMPDIR_EVAL/compose-critique.err"; then
|
|
3624
|
+
_pass "compose: record-critique rebuilds the bundle at the new active step (merge-ready)"
|
|
3625
|
+
else
|
|
3626
|
+
_fail "compose: record-critique failed: $(cat "$TMPDIR_EVAL/compose-critique.out" "$TMPDIR_EVAL/compose-critique.err")"
|
|
3627
|
+
fi
|
|
3628
|
+
|
|
3629
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/compose-ac2-assert.err"
|
|
3630
|
+
import { readFileSync } from 'node:fs';
|
|
3631
|
+
const bundle = JSON.parse(readFileSync('${COMPOSE_DIR}/trust.bundle', 'utf8'));
|
|
3632
|
+
const claims = bundle.claims;
|
|
3633
|
+
const gateClaim1 = claims.find((c) => c.metadata && c.metadata.gate_claim && c.metadata.gate_claim.expectation_id === 'selected-work');
|
|
3634
|
+
const gateClaim2 = claims.find((c) => c.metadata && c.metadata.gate_claim && c.metadata.gate_claim.expectation_id === 'implementation-plan');
|
|
3635
|
+
if (!gateClaim1 || !gateClaim2) { process.stderr.write('a gate claim was LOST after rebuild at a later step\n'); process.exit(1); }
|
|
3636
|
+
if (gateClaim1.claimType !== 'builder.pull-work.selected' || gateClaim1.subjectType !== 'work-item') {
|
|
3637
|
+
process.stderr.write('AC2 REGRESSION: gate claim recorded at pull-work was re-typed to ' + gateClaim1.claimType + '/' + gateClaim1.subjectType + ' after rebuild at merge-ready (the #270(c) defect)\n');
|
|
3638
|
+
process.exit(1);
|
|
3639
|
+
}
|
|
3640
|
+
if (gateClaim2.claimType !== 'builder.plan.implementation' || gateClaim2.subjectType !== 'artifact') {
|
|
3641
|
+
process.stderr.write('AC2 REGRESSION: gate claim recorded at plan was re-typed to ' + gateClaim2.claimType + '/' + gateClaim2.subjectType + ' after rebuild at merge-ready\n');
|
|
3642
|
+
process.exit(1);
|
|
3643
|
+
}
|
|
3644
|
+
if (gateClaim1.metadata.gate_claim.step_id !== 'pull-work') { process.stderr.write('gate_claim.step_id was overwritten to the currently-active step instead of staying pull-work: ' + gateClaim1.metadata.gate_claim.step_id + '\n'); process.exit(1); }
|
|
3645
|
+
if (gateClaim2.metadata.gate_claim.step_id !== 'plan') { process.stderr.write('gate_claim.step_id was overwritten to the currently-active step instead of staying plan: ' + gateClaim2.metadata.gate_claim.step_id + '\n'); process.exit(1); }
|
|
3646
|
+
// Meanwhile a PLAIN check claim (no gate_claim stamp) IS correctly re-typed to the new active step —
|
|
3647
|
+
// this is legitimate, documented behavior (only a stamped gate claim is frozen).
|
|
3648
|
+
const buildClaim = claims.find((c) => c.subjectId && c.subjectId.endsWith('/compose-build-check'));
|
|
3649
|
+
if (!buildClaim || buildClaim.claimType !== 'builder.merge-ready.readiness') { process.stderr.write('plain check claim did not re-type to the new active step as expected (unrelated to AC2, sanity check): ' + (buildClaim && buildClaim.claimType) + '\n'); process.exit(1); }
|
|
3650
|
+
NODEOF
|
|
3651
|
+
then
|
|
3652
|
+
_pass "AC2: gate-claim typing (claimType/subjectType/step_id) survives a rebuild after the active step advances — frozen at record time, not re-derived (#270(a)/(c))"
|
|
3653
|
+
else
|
|
3654
|
+
_fail "AC2: gate-claim-typing-survives-rebuild assertion failed: $(cat "$TMPDIR_EVAL/compose-ac2-assert.err")"
|
|
3655
|
+
fi
|
|
3656
|
+
|
|
3657
|
+
# ─── AC4: compose-three-writer (five writer calls total: evidence x2, gate-claim x2, critique, ──
|
|
3658
|
+
# then record-learning) — all claim families present and correctly typed in the final bundle ──
|
|
3659
|
+
if flow_agents_node "$WRITER" record-learning "$COMPOSE_DIR" \
|
|
3660
|
+
--status learned \
|
|
3661
|
+
--summary "Compose layer round-trips losslessly" \
|
|
3662
|
+
--record-json '{"source_refs":[],"facts":["compose layer round-trips losslessly"],"routing":[],"outcome":"success","correction":{"needed":false,"evidence":"compose eval passed"}}' \
|
|
3663
|
+
--timestamp "2026-07-05T09:10:00Z" >"$TMPDIR_EVAL/compose-learning.out" 2>"$TMPDIR_EVAL/compose-learning.err"; then
|
|
3664
|
+
_pass "compose: record-learning (fourth distinct writer family) succeeds"
|
|
3665
|
+
else
|
|
3666
|
+
_fail "compose: record-learning failed: $(cat "$TMPDIR_EVAL/compose-learning.out" "$TMPDIR_EVAL/compose-learning.err")"
|
|
3667
|
+
fi
|
|
3668
|
+
|
|
3669
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/compose-ac4-assert.err"
|
|
3670
|
+
import { readFileSync } from 'node:fs';
|
|
3671
|
+
const bundle = JSON.parse(readFileSync('${COMPOSE_DIR}/trust.bundle', 'utf8'));
|
|
3672
|
+
const claims = bundle.claims;
|
|
3673
|
+
const origins = claims.map((c) => (c.metadata || {}).origin);
|
|
3674
|
+
const originCounts = origins.reduce((acc, o) => { acc[o] = (acc[o] || 0) + 1; return acc; }, {});
|
|
3675
|
+
// Expect: 2 plain "check" claims (build-check, manual-check) + 2 gate-claim checks (also origin
|
|
3676
|
+
// "check", distinguished by metadata.gate_claim) + 1 "acceptance" + 1 "critique" = 6 total.
|
|
3677
|
+
if (originCounts.check !== 4) { process.stderr.write('expected 4 origin:check claims (2 plain checks + 2 gate claims), got ' + originCounts.check + ' (' + JSON.stringify(originCounts) + ')\n'); process.exit(1); }
|
|
3678
|
+
if (originCounts.acceptance !== 1) { process.stderr.write('expected 1 origin:acceptance claim, got ' + originCounts.acceptance + '\n'); process.exit(1); }
|
|
3679
|
+
if (originCounts.critique !== 1) { process.stderr.write('expected 1 origin:critique claim, got ' + originCounts.critique + '\n'); process.exit(1); }
|
|
3680
|
+
if (claims.length !== 6) { process.stderr.write('expected exactly 6 claims total after the full 6-writer sequence, got ' + claims.length + '\n'); process.exit(1); }
|
|
3681
|
+
const gateClaim1 = claims.find((c) => c.metadata && c.metadata.gate_claim && c.metadata.gate_claim.expectation_id === 'selected-work');
|
|
3682
|
+
const gateClaim2 = claims.find((c) => c.metadata && c.metadata.gate_claim && c.metadata.gate_claim.expectation_id === 'implementation-plan');
|
|
3683
|
+
const manualClaim = claims.find((c) => c.subjectId && c.subjectId.endsWith('/compose-manual-check'));
|
|
3684
|
+
if (!gateClaim1 || gateClaim1.claimType !== 'builder.pull-work.selected') { process.stderr.write('gate claim 1 typing did not survive the FIFTH rebuild (record-learning)\n'); process.exit(1); }
|
|
3685
|
+
if (!gateClaim2 || gateClaim2.claimType !== 'builder.plan.implementation') { process.stderr.write('gate claim 2 typing did not survive the FIFTH rebuild (record-learning)\n'); process.exit(1); }
|
|
3686
|
+
if (!manualClaim || !manualClaim.metadata.waiver) { process.stderr.write('waiver did not survive the fifth writer rebuild\n'); process.exit(1); }
|
|
3687
|
+
NODEOF
|
|
3688
|
+
then
|
|
3689
|
+
_pass "AC4: compose-three-writer (six writer calls total: evidence x2, gate-claim x2, critique, learning) — all claim families present and correctly typed in the final bundle"
|
|
3690
|
+
else
|
|
3691
|
+
_fail "AC4: compose-three-writer assertion failed: $(cat "$TMPDIR_EVAL/compose-ac4-assert.err")"
|
|
3692
|
+
fi
|
|
3693
|
+
|
|
3694
|
+
# ─── AC8: a kind:"command" evidence_refs entry carrying PROSE excerpt is rejected at record ──
|
|
3695
|
+
# time by record-evidence (validateAcceptanceEvidenceRefs), not silently accepted for the ─────
|
|
3696
|
+
# Stop-hook backstop to catch later. ───────────────────────────────────────────────────────────
|
|
3697
|
+
AC8_DIR="$TMPDIR_EVAL/repo/.kontourai/flow-agents/ac8-runnability"
|
|
3698
|
+
mkdir -p "$AC8_DIR"
|
|
3699
|
+
cp "$ARTIFACT_DIR/auto-sidecars--deliver.md" "$AC8_DIR/ac8-runnability--deliver.md"
|
|
3700
|
+
flow_agents_node "$WRITER" init-plan "$AC8_DIR/ac8-runnability--deliver.md" \
|
|
3701
|
+
--source-request "AC8 runnability rejection fixture." \
|
|
3702
|
+
--summary "AC8 runnability rejection fixture." \
|
|
3703
|
+
--next-action "Seed a prose kind:command evidence_ref." \
|
|
3704
|
+
--timestamp "2026-07-05T09:00:00Z" >"$TMPDIR_EVAL/ac8-init.out" 2>"$TMPDIR_EVAL/ac8-init.err"
|
|
3705
|
+
|
|
3706
|
+
# Overwrite acceptance.json's criterion with a kind:"command" ref whose excerpt is prose — the
|
|
3707
|
+
# split-literal convention (see test_validate_source_kit_asset_scope.sh's "self-scan hazard"
|
|
3708
|
+
# comments) is not needed here since this file is not itself scanned for the phrase.
|
|
3709
|
+
node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/ac8-mutate.err"
|
|
3710
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
3711
|
+
const file = '${AC8_DIR}/acceptance.json';
|
|
3712
|
+
const data = JSON.parse(readFileSync(file, 'utf8'));
|
|
3713
|
+
if (!Array.isArray(data.criteria) || data.criteria.length === 0) { process.stderr.write('no criteria to mutate\n'); process.exit(1); }
|
|
3714
|
+
data.criteria[0].evidence_refs = [{ kind: 'command', excerpt: 'Manually confirmed the output looks correct.' }];
|
|
3715
|
+
writeFileSync(file, JSON.stringify(data, null, 2) + '\n');
|
|
3716
|
+
NODEOF
|
|
3717
|
+
|
|
3718
|
+
if flow_agents_node "$WRITER" record-evidence "$AC8_DIR" \
|
|
3719
|
+
--verdict pass \
|
|
3720
|
+
--check-json '{"id":"ac8-check","kind":"test","status":"pass","summary":"AC8 fixture check passed."}' \
|
|
3721
|
+
--timestamp "2026-07-05T09:01:00Z" >"$TMPDIR_EVAL/ac8-evidence.out" 2>"$TMPDIR_EVAL/ac8-evidence.err"; then
|
|
3722
|
+
_fail "AC8 REGRESSION: record-evidence accepted a kind:\"command\" evidence_ref with prose excerpt instead of rejecting it at record time"
|
|
3723
|
+
elif grep -qi "not a runnable shell command" "$TMPDIR_EVAL/ac8-evidence.out" "$TMPDIR_EVAL/ac8-evidence.err"; then
|
|
3724
|
+
_pass "AC8: record-evidence rejects a kind:\"command\" evidence_ref with a prose excerpt at record time, with an actionable error"
|
|
3725
|
+
else
|
|
3726
|
+
_fail "AC8: record-evidence rejected the prose ref but without the expected 'not a runnable shell command' message: $(cat "$TMPDIR_EVAL/ac8-evidence.out" "$TMPDIR_EVAL/ac8-evidence.err")"
|
|
3727
|
+
fi
|
|
3728
|
+
|
|
3729
|
+
# ─── #270 MEDIUM security fix eval: --skip-evidence-ref-runnability-guard is a logged bypass, ──
|
|
3730
|
+
# never silent (mirrors --skip-ownership-guard's existing pattern) — reusing AC8_DIR's already-
|
|
3731
|
+
# mutated prose-in-excerpt fixture (same lockout AC8 above proves is otherwise unconditional).
|
|
3732
|
+
if flow_agents_node "$WRITER" record-evidence "$AC8_DIR" \
|
|
3733
|
+
--verdict pass \
|
|
3734
|
+
--check-json '{"id":"ac8-bypass-check","kind":"test","status":"pass","summary":"AC8 bypass fixture check passed."}' \
|
|
3735
|
+
--skip-evidence-ref-runnability-guard \
|
|
3736
|
+
--timestamp "2026-07-05T09:01:30Z" >"$TMPDIR_EVAL/ac8-bypass.out" 2>"$TMPDIR_EVAL/ac8-bypass.err"; then
|
|
3737
|
+
if grep -qi "evidence-ref runnability guard skipped via --skip-evidence-ref-runnability-guard" "$TMPDIR_EVAL/ac8-bypass.out" "$TMPDIR_EVAL/ac8-bypass.err"; then
|
|
3738
|
+
_pass "#270: --skip-evidence-ref-runnability-guard bypasses the lockout AND logs it loudly (never silent), mirroring --skip-ownership-guard's pattern"
|
|
3739
|
+
else
|
|
3740
|
+
_fail "#270: --skip-evidence-ref-runnability-guard bypassed the guard but did NOT log it (silent bypass regression): $(cat "$TMPDIR_EVAL/ac8-bypass.out" "$TMPDIR_EVAL/ac8-bypass.err")"
|
|
3741
|
+
fi
|
|
3742
|
+
else
|
|
3743
|
+
_fail "#270: --skip-evidence-ref-runnability-guard should have bypassed the lockout, but record-evidence still failed: $(cat "$TMPDIR_EVAL/ac8-bypass.out" "$TMPDIR_EVAL/ac8-bypass.err")"
|
|
3744
|
+
fi
|
|
3745
|
+
|
|
3746
|
+
# ─── #362 AC7: validateAcceptanceEvidenceRefs' ADVISORY (non-fatal) ambiguous-absence-command ──
|
|
3747
|
+
# nudge — a kind:"command" evidence ref whose excerpt is a bare (non-negated, non-count-asserted)
|
|
3748
|
+
# grep/diff invocation gets a stderr advisory suggesting the self-asserting rewrite, but the
|
|
3749
|
+
# record-evidence call is NOT blocked (this is guidance, not enforcement — distinct from the
|
|
3750
|
+
# runnability guard immediately above, which stays fatal/unchanged).
|
|
3751
|
+
AC7_ADV_DIR="$TMPDIR_EVAL/repo/.kontourai/flow-agents/ac7-ambiguous-advisory"
|
|
3752
|
+
mkdir -p "$AC7_ADV_DIR"
|
|
3753
|
+
cp "$ARTIFACT_DIR/auto-sidecars--deliver.md" "$AC7_ADV_DIR/ac7-ambiguous-advisory--deliver.md"
|
|
3754
|
+
flow_agents_node "$WRITER" init-plan "$AC7_ADV_DIR/ac7-ambiguous-advisory--deliver.md" --source-request "AC7 ambiguous-absence-command advisory fixture." --summary "AC7 ambiguous-absence-command advisory fixture." --next-action "Seed a bare-grep kind:command evidence_ref." --timestamp "2026-07-05T09:02:00Z" >"$TMPDIR_EVAL/ac7-init.out" 2>"$TMPDIR_EVAL/ac7-init.err"
|
|
3755
|
+
|
|
3756
|
+
node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/ac7-mutate.err"
|
|
3757
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
3758
|
+
const file = '${AC7_ADV_DIR}/acceptance.json';
|
|
3759
|
+
const data = JSON.parse(readFileSync(file, 'utf8'));
|
|
3760
|
+
if (!Array.isArray(data.criteria) || data.criteria.length === 0) { process.stderr.write('no criteria to mutate\n'); process.exit(1); }
|
|
3761
|
+
data.criteria[0].evidence_refs = [{ kind: 'command', excerpt: "grep -E 'this-pattern-does-not-exist-anywhere' package.json" }];
|
|
3762
|
+
writeFileSync(file, JSON.stringify(data, null, 2) + '\n');
|
|
3763
|
+
NODEOF
|
|
3764
|
+
|
|
3765
|
+
if flow_agents_node "$WRITER" record-evidence "$AC7_ADV_DIR" --verdict pass --check-json '{"id":"ac7-adv-check","kind":"test","status":"pass","summary":"AC7 advisory fixture check passed."}' --timestamp "2026-07-05T09:02:30Z" >"$TMPDIR_EVAL/ac7-adv.out" 2>"$TMPDIR_EVAL/ac7-adv.err"; then
|
|
3766
|
+
_pass "AC7: record-evidence with a bare-grep-shaped kind:\"command\" evidence ref succeeds (advisory is non-fatal, does NOT block the write)"
|
|
3767
|
+
else
|
|
3768
|
+
_fail "AC7 REGRESSION: record-evidence with a bare-grep-shaped evidence ref should succeed (advisory only), but it was rejected: $(cat "$TMPDIR_EVAL/ac7-adv.out" "$TMPDIR_EVAL/ac7-adv.err")"
|
|
3769
|
+
fi
|
|
3770
|
+
|
|
3771
|
+
if grep -qi "advisory" "$TMPDIR_EVAL/ac7-adv.err" && grep -qi "self-asserting\|ambiguous" "$TMPDIR_EVAL/ac7-adv.err"; then
|
|
3772
|
+
_pass "AC7: record-evidence emits an advisory stderr note recommending the self-asserting form for a bare-grep evidence ref"
|
|
3773
|
+
else
|
|
3774
|
+
_fail "AC7: expected advisory stderr note (mentioning 'advisory' and 'self-asserting'/'ambiguous') was missing: $(cat "$TMPDIR_EVAL/ac7-adv.err")"
|
|
3775
|
+
fi
|
|
3776
|
+
|
|
3777
|
+
if [[ -f "$AC7_ADV_DIR/acceptance.json" ]] && node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/ac7-adv-assert.err"
|
|
3778
|
+
import { readFileSync } from 'node:fs';
|
|
3779
|
+
const data = JSON.parse(readFileSync('${AC7_ADV_DIR}/acceptance.json', 'utf8'));
|
|
3780
|
+
const ref = data.criteria[0].evidence_refs[0];
|
|
3781
|
+
if (ref.kind !== 'command') { process.stderr.write('expected the ref to remain kind:"command", got: ' + ref.kind + '\n'); process.exit(1); }
|
|
3782
|
+
if (ref.excerpt !== "grep -E 'this-pattern-does-not-exist-anywhere' package.json") { process.stderr.write('the advisory must NOT alter the ref excerpt: got ' + JSON.stringify(ref.excerpt) + '\n'); process.exit(1); }
|
|
3783
|
+
NODEOF
|
|
3784
|
+
then
|
|
3785
|
+
_pass "AC7: the advisory does not alter or reject the existing evidence ref (excerpt unchanged, still kind:\"command\")"
|
|
3786
|
+
else
|
|
3787
|
+
_fail "AC7: evidence-ref-unchanged assertion failed: $(cat "$TMPDIR_EVAL/ac7-adv-assert.err")"
|
|
3788
|
+
fi
|
|
3789
|
+
|
|
3790
|
+
# Regression guard: a self-asserting form (`! grep ...`, the NATURAL negated form -- not a
|
|
3791
|
+
# count-assertion workaround; isRunnableCommandText now strips a leading `!` before evaluating
|
|
3792
|
+
# runnability, per the coherence fix, so the runnability guard ACCEPTS this excerpt) never
|
|
3793
|
+
# triggers the advisory.
|
|
3794
|
+
AC7_NEG_DIR="$TMPDIR_EVAL/repo/.kontourai/flow-agents/ac7-self-asserting-no-advisory"
|
|
3795
|
+
mkdir -p "$AC7_NEG_DIR"
|
|
3796
|
+
cp "$ARTIFACT_DIR/auto-sidecars--deliver.md" "$AC7_NEG_DIR/ac7-self-asserting-no-advisory--deliver.md"
|
|
3797
|
+
flow_agents_node "$WRITER" init-plan "$AC7_NEG_DIR/ac7-self-asserting-no-advisory--deliver.md" --source-request "AC7 self-asserting-form no-advisory regression fixture." --summary "AC7 self-asserting-form no-advisory regression fixture." --next-action "Seed a negated-grep kind:command evidence_ref." --timestamp "2026-07-05T09:03:00Z" >"$TMPDIR_EVAL/ac7-neg-init.out" 2>"$TMPDIR_EVAL/ac7-neg-init.err"
|
|
3798
|
+
|
|
3799
|
+
node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/ac7-neg-mutate.err"
|
|
3800
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
3801
|
+
const file = '${AC7_NEG_DIR}/acceptance.json';
|
|
3802
|
+
const data = JSON.parse(readFileSync(file, 'utf8'));
|
|
3803
|
+
if (!Array.isArray(data.criteria) || data.criteria.length === 0) { process.stderr.write('no criteria to mutate\n'); process.exit(1); }
|
|
3804
|
+
data.criteria[0].evidence_refs = [{ kind: 'command', excerpt: "! grep -E 'this-pattern-does-not-exist-anywhere' package.json" }];
|
|
3805
|
+
writeFileSync(file, JSON.stringify(data, null, 2) + '\n');
|
|
3806
|
+
NODEOF
|
|
3807
|
+
|
|
3808
|
+
if flow_agents_node "$WRITER" record-evidence "$AC7_NEG_DIR" --verdict pass --check-json '{"id":"ac7-neg-check","kind":"test","status":"pass","summary":"AC7 negation regression fixture check passed."}' --timestamp "2026-07-05T09:03:30Z" >"$TMPDIR_EVAL/ac7-neg.out" 2>"$TMPDIR_EVAL/ac7-neg.err"; then
|
|
3809
|
+
if grep -qi "not a runnable shell command" "$TMPDIR_EVAL/ac7-neg.out" "$TMPDIR_EVAL/ac7-neg.err"; then
|
|
3810
|
+
_fail "REGRESSION: record-evidence rejected a '! grep ...' evidence ref as non-runnable — isRunnableCommandText's leading-'!' strip is not working"
|
|
3811
|
+
else
|
|
3812
|
+
_pass "record-evidence ACCEPTS a '! grep ...' evidence ref as runnable (leading '!' is stripped before evaluation)"
|
|
3813
|
+
fi
|
|
3814
|
+
if grep -qi "advisory" "$TMPDIR_EVAL/ac7-neg.err"; then
|
|
3815
|
+
_fail "regression: a self-asserting '! grep ...' evidence ref should NOT trigger the ambiguous-absence advisory, but it did: $(cat "$TMPDIR_EVAL/ac7-neg.err")"
|
|
3816
|
+
else
|
|
3817
|
+
_pass "regression guard: a self-asserting '! grep ...' evidence ref does not trigger the ambiguous-absence advisory"
|
|
3818
|
+
fi
|
|
3819
|
+
else
|
|
3820
|
+
_fail "regression fixture: record-evidence with a self-asserting negated-grep evidence ref unexpectedly failed: $(cat "$TMPDIR_EVAL/ac7-neg.out" "$TMPDIR_EVAL/ac7-neg.err")"
|
|
3821
|
+
fi
|
|
3822
|
+
|
|
3823
|
+
# ─── #270 MEDIUM fix eval: parseCriterion routes a PROSE "- Evidence:" line through the REAL ──
|
|
3824
|
+
# init-plan parse path (definitionAcceptanceLines -> parseCriterion), not a hand-mutated
|
|
3825
|
+
# acceptance.json — asserting the resulting acceptance.json ref carries the prose in `summary`
|
|
3826
|
+
# and has NO `excerpt` (never validated for runnability, never executed), per the #412 contract
|
|
3827
|
+
# ("prose belongs in ref.summary"). This is a DIRECT eval of parseCriterion's classification
|
|
3828
|
+
# behavior (distinct from AC8 above, which exercises the record-time REJECTION of an
|
|
3829
|
+
# already-mutated prose-in-excerpt ref).
|
|
3830
|
+
PARSE_CRITERION_DIR="$TMPDIR_EVAL/repo/.kontourai/flow-agents/parse-criterion-prose"
|
|
3831
|
+
mkdir -p "$PARSE_CRITERION_DIR"
|
|
3832
|
+
cat > "$PARSE_CRITERION_DIR/parse-criterion-prose--deliver.md" <<'MARKDOWN'
|
|
3833
|
+
# parseCriterion prose Evidence line eval
|
|
3834
|
+
|
|
3835
|
+
status: delivered
|
|
3836
|
+
type: deliver
|
|
3837
|
+
|
|
3838
|
+
## Plan
|
|
3839
|
+
|
|
3840
|
+
Exercise parseCriterion's classification of a prose Evidence line through the real parse path.
|
|
3841
|
+
|
|
3842
|
+
## Definition Of Done
|
|
3843
|
+
|
|
3844
|
+
- **User outcome:** Prose evidence is classified correctly at parse time.
|
|
3845
|
+
- **Scope:** parseCriterion / definitionAcceptanceLines.
|
|
3846
|
+
- **Acceptance criteria:**
|
|
3847
|
+
- [ ] Dashboard reviewed - Evidence: manual review of the dashboard.
|
|
3848
|
+
- **Usefulness checks:**
|
|
3849
|
+
- [ ] User-facing workflow is documented or discoverable
|
|
3850
|
+
- **Stop-short risks:** none
|
|
3851
|
+
- **Durable docs target:** not needed
|
|
3852
|
+
- **Sandbox mode:** local-edit
|
|
3853
|
+
|
|
3854
|
+
## Verification Report
|
|
3855
|
+
|
|
3856
|
+
Build: [NOT_VERIFIED] pending
|
|
3857
|
+
|
|
3858
|
+
### Verdict: NOT_VERIFIED
|
|
3859
|
+
|
|
3860
|
+
## Goal Fit Gate
|
|
3861
|
+
|
|
3862
|
+
- [ ] Original user goal restated
|
|
3863
|
+
|
|
3864
|
+
## Final Acceptance
|
|
3865
|
+
|
|
3866
|
+
- [ ] CI/relevant checks passed or local equivalent recorded
|
|
3867
|
+
MARKDOWN
|
|
3868
|
+
|
|
3869
|
+
if flow_agents_node "$WRITER" init-plan "$PARSE_CRITERION_DIR/parse-criterion-prose--deliver.md" --source-request "parseCriterion prose Evidence line eval." --summary "Exercise the real parse path for a prose Evidence line." --next-action "Assert the resulting acceptance.json ref." --timestamp "2026-07-05T09:00:00Z" >"$TMPDIR_EVAL/parse-criterion-init.out" 2>"$TMPDIR_EVAL/parse-criterion-init.err"; then
|
|
3870
|
+
_pass "parseCriterion eval: init-plan runs against the prose-Evidence-line fixture"
|
|
3871
|
+
else
|
|
3872
|
+
_fail "parseCriterion eval: init-plan failed: $(cat "$TMPDIR_EVAL/parse-criterion-init.out" "$TMPDIR_EVAL/parse-criterion-init.err")"
|
|
3873
|
+
fi
|
|
3874
|
+
|
|
3875
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/parse-criterion-assert.err"
|
|
3876
|
+
import { readFileSync } from 'node:fs';
|
|
3877
|
+
const data = JSON.parse(readFileSync('${PARSE_CRITERION_DIR}/acceptance.json', 'utf8'));
|
|
3878
|
+
const criterion = Array.isArray(data.criteria) ? data.criteria.find((c) => c.description && c.description.includes('Dashboard reviewed')) : null;
|
|
3879
|
+
if (!criterion) { process.stderr.write('no matching criterion found in acceptance.json: ' + JSON.stringify(data.criteria) + '\n'); process.exit(1); }
|
|
3880
|
+
if (!Array.isArray(criterion.evidence_refs) || criterion.evidence_refs.length !== 1) { process.stderr.write('expected exactly one evidence_refs entry, got: ' + JSON.stringify(criterion.evidence_refs) + '\n'); process.exit(1); }
|
|
3881
|
+
const ref = criterion.evidence_refs[0];
|
|
3882
|
+
if (ref.kind !== 'command') { process.stderr.write('expected kind:"command" (still a structured ref; only WHICH field carries the text differs), got: ' + ref.kind + '\n'); process.exit(1); }
|
|
3883
|
+
if (typeof ref.summary !== 'string' || !ref.summary.includes('manual review of the dashboard')) { process.stderr.write('prose text did not land in ref.summary: ' + JSON.stringify(ref) + '\n'); process.exit(1); }
|
|
3884
|
+
if (ref.excerpt !== undefined && ref.excerpt !== '') { process.stderr.write('ref.excerpt should be absent/empty for prose evidence (prose never belongs in excerpt), got: ' + JSON.stringify(ref.excerpt) + '\n'); process.exit(1); }
|
|
3885
|
+
NODEOF
|
|
3886
|
+
then
|
|
3887
|
+
_pass "#270: parseCriterion (via the real init-plan parse path) routes a prose \"- Evidence:\" line into ref.summary with no excerpt, not a hand-mutated acceptance.json"
|
|
3888
|
+
else
|
|
3889
|
+
_fail "#270: parseCriterion prose-routing assertion failed: $(cat "$TMPDIR_EVAL/parse-criterion-assert.err")"
|
|
3890
|
+
fi
|
|
3891
|
+
|
|
3892
|
+
# ─── AC10: record-gate-claim --command with PROSE is rejected at record time ─────────────────
|
|
3893
|
+
if flow_agents_node "$WRITER" record-gate-claim "$COMPOSE_DIR" \
|
|
3894
|
+
--actor compose-actor \
|
|
3895
|
+
--status pass \
|
|
3896
|
+
--summary "Manual review" \
|
|
3897
|
+
--expectation merge-readiness \
|
|
3898
|
+
--command "Manually verified the output looks correct." \
|
|
3899
|
+
--timestamp "2026-07-05T09:11:00Z" >"$TMPDIR_EVAL/compose-gate-prose.out" 2>"$TMPDIR_EVAL/compose-gate-prose.err"; then
|
|
3900
|
+
_fail "AC10 REGRESSION: record-gate-claim --command accepted prose instead of rejecting it at record time"
|
|
3901
|
+
elif grep -qi "not a runnable shell command" "$TMPDIR_EVAL/compose-gate-prose.out" "$TMPDIR_EVAL/compose-gate-prose.err"; then
|
|
3902
|
+
_pass "AC10: record-gate-claim --command rejects a prose value at record time, with an actionable error"
|
|
3903
|
+
else
|
|
3904
|
+
_fail "AC10: record-gate-claim --command prose rejection message was unexpected: $(cat "$TMPDIR_EVAL/compose-gate-prose.out" "$TMPDIR_EVAL/compose-gate-prose.err")"
|
|
3905
|
+
fi
|
|
3906
|
+
|
|
3907
|
+
# ─── AC11: command-backed check claims reject prose; external prose remains valid ─────────────
|
|
3908
|
+
CHECK_PROSE_DIR="$TMPDIR_EVAL/repo/.kontourai/flow-agents/check-prose-guard"
|
|
3909
|
+
mkdir -p "$CHECK_PROSE_DIR"
|
|
3910
|
+
cp "$ARTIFACT_DIR/auto-sidecars--deliver.md" "$CHECK_PROSE_DIR/check-prose-guard--deliver.md"
|
|
3911
|
+
flow_agents_node "$WRITER" init-plan "$CHECK_PROSE_DIR/check-prose-guard--deliver.md" \
|
|
3912
|
+
--source-request "Check command prose guard fixture." \
|
|
3913
|
+
--summary "Check command prose guard fixture." \
|
|
3914
|
+
--next-action "Exercise command check runnability validation." \
|
|
3915
|
+
--timestamp "2026-07-05T09:11:10Z" >"$TMPDIR_EVAL/check-prose-init.out" 2>"$TMPDIR_EVAL/check-prose-init.err"
|
|
3916
|
+
|
|
3917
|
+
if flow_agents_node "$WRITER" record-evidence "$CHECK_PROSE_DIR" \
|
|
3918
|
+
--verdict pass \
|
|
3919
|
+
--check-json '{"id":"prose-command-check","kind":"command","status":"pass","summary":"External CI is green.","command":"External CI ground truth: PR #475 is merged and all checks are green."}' \
|
|
3920
|
+
--timestamp "2026-07-05T09:11:20Z" >"$TMPDIR_EVAL/check-prose-command.out" 2>"$TMPDIR_EVAL/check-prose-command.err"; then
|
|
3921
|
+
_fail "AC11 REGRESSION: record-evidence accepted a kind:\"command\" check whose command text is prose"
|
|
3922
|
+
elif grep -qi "not a runnable shell command" "$TMPDIR_EVAL/check-prose-command.out" "$TMPDIR_EVAL/check-prose-command.err" \
|
|
3923
|
+
&& grep -qi "kind:\"external\"" "$TMPDIR_EVAL/check-prose-command.out" "$TMPDIR_EVAL/check-prose-command.err"; then
|
|
3924
|
+
_pass "AC11: record-evidence rejects prose command-backed check claims with external-attestation remediation"
|
|
3925
|
+
else
|
|
3926
|
+
_fail "AC11: record-evidence prose command rejection message was unexpected: $(cat "$TMPDIR_EVAL/check-prose-command.out" "$TMPDIR_EVAL/check-prose-command.err")"
|
|
3927
|
+
fi
|
|
3928
|
+
|
|
3929
|
+
if flow_agents_node "$WRITER" record-check "$CHECK_PROSE_DIR" \
|
|
3930
|
+
--command "External CI ground truth: PR #475 is merged and all checks are green." \
|
|
3931
|
+
>"$TMPDIR_EVAL/record-check-prose.out" 2>"$TMPDIR_EVAL/record-check-prose.err"; then
|
|
3932
|
+
_fail "AC11 REGRESSION: record-check --command accepted prose instead of rejecting it"
|
|
3933
|
+
elif grep -qi "not a runnable shell command" "$TMPDIR_EVAL/record-check-prose.out" "$TMPDIR_EVAL/record-check-prose.err"; then
|
|
3934
|
+
_pass "AC11: record-check --command rejects prose before it can become execution.label"
|
|
3935
|
+
else
|
|
3936
|
+
_fail "AC11: record-check prose rejection message was unexpected: $(cat "$TMPDIR_EVAL/record-check-prose.out" "$TMPDIR_EVAL/record-check-prose.err")"
|
|
3937
|
+
fi
|
|
3938
|
+
|
|
3939
|
+
for AC494_PROSE in "External CI (green)" "Reconciled after merge; all green"; do
|
|
3940
|
+
AC494_ID="$(printf '%s' "$AC494_PROSE" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g; s/--*/-/g; s/^-//; s/-$//')"
|
|
3941
|
+
if flow_agents_node "$WRITER" record-evidence "$CHECK_PROSE_DIR" \
|
|
3942
|
+
--verdict pass \
|
|
3943
|
+
--check-json "{\"id\":\"ac494-${AC494_ID}\",\"kind\":\"command\",\"status\":\"pass\",\"summary\":\"AC494 prose punctuation fixture.\",\"command\":\"${AC494_PROSE}\"}" \
|
|
3944
|
+
--timestamp "2026-07-05T09:11:24Z" >"$TMPDIR_EVAL/ac494-${AC494_ID}-evidence.out" 2>"$TMPDIR_EVAL/ac494-${AC494_ID}-evidence.err"; then
|
|
3945
|
+
_fail "AC-FIX-2 REGRESSION: record-evidence accepted incidental-punctuation prose as kind:\"command\": ${AC494_PROSE}"
|
|
3946
|
+
elif grep -qi "not a runnable shell command" "$TMPDIR_EVAL/ac494-${AC494_ID}-evidence.out" "$TMPDIR_EVAL/ac494-${AC494_ID}-evidence.err"; then
|
|
3947
|
+
_pass "AC-FIX-2: record-evidence rejects incidental-punctuation prose as kind:\"command\": ${AC494_PROSE}"
|
|
3948
|
+
else
|
|
3949
|
+
_fail "AC-FIX-2: record-evidence rejection for '${AC494_PROSE}' had unexpected output: $(cat "$TMPDIR_EVAL/ac494-${AC494_ID}-evidence.out" "$TMPDIR_EVAL/ac494-${AC494_ID}-evidence.err")"
|
|
3950
|
+
fi
|
|
3951
|
+
|
|
3952
|
+
if flow_agents_node "$WRITER" record-check "$CHECK_PROSE_DIR" \
|
|
3953
|
+
--command "$AC494_PROSE" \
|
|
3954
|
+
>"$TMPDIR_EVAL/ac494-${AC494_ID}-record-check.out" 2>"$TMPDIR_EVAL/ac494-${AC494_ID}-record-check.err"; then
|
|
3955
|
+
_fail "AC-FIX-2 REGRESSION: record-check --command accepted incidental-punctuation prose: ${AC494_PROSE}"
|
|
3956
|
+
elif grep -qi "not a runnable shell command" "$TMPDIR_EVAL/ac494-${AC494_ID}-record-check.out" "$TMPDIR_EVAL/ac494-${AC494_ID}-record-check.err"; then
|
|
3957
|
+
_pass "AC-FIX-2: record-check --command rejects incidental-punctuation prose: ${AC494_PROSE}"
|
|
3958
|
+
else
|
|
3959
|
+
_fail "AC-FIX-2: record-check rejection for '${AC494_PROSE}' had unexpected output: $(cat "$TMPDIR_EVAL/ac494-${AC494_ID}-record-check.out" "$TMPDIR_EVAL/ac494-${AC494_ID}-record-check.err")"
|
|
3960
|
+
fi
|
|
3961
|
+
done
|
|
3962
|
+
|
|
3963
|
+
if node - "$ROOT/scripts/hooks/lib/runnable-command.js" <<'NODEEOF' >"$TMPDIR_EVAL/ac494-runnable-contract.out" 2>"$TMPDIR_EVAL/ac494-runnable-contract.err"
|
|
3964
|
+
const { isRunnableCommandText } = require(process.argv[2]);
|
|
3965
|
+
const cases = [
|
|
3966
|
+
['exit 42', true],
|
|
3967
|
+
['false', true],
|
|
3968
|
+
['true', true],
|
|
3969
|
+
['cd nonexistent', true],
|
|
3970
|
+
['FOO=bar npm test', true],
|
|
3971
|
+
['npm test', true],
|
|
3972
|
+
['./script.sh', true],
|
|
3973
|
+
['test -f somefile', true],
|
|
3974
|
+
['set -e', true],
|
|
3975
|
+
['export FOO=bar', true],
|
|
3976
|
+
['echo hi > out.txt', true],
|
|
3977
|
+
['source ./env.sh', true],
|
|
3978
|
+
[': noop', true],
|
|
3979
|
+
['test coverage improved across the suite', false],
|
|
3980
|
+
['set the stage for the next release', false],
|
|
3981
|
+
['export controls apply to this build', false],
|
|
3982
|
+
['source of truth confirmed by CI', false],
|
|
3983
|
+
['echo of the original design', false],
|
|
3984
|
+
['return on investment was high', false],
|
|
3985
|
+
['true to the original plan', false],
|
|
3986
|
+
['false alarm, everything passed', false],
|
|
3987
|
+
['cd into the details later', false],
|
|
3988
|
+
['Test coverage improved', false],
|
|
3989
|
+
['Set up the environment first', false],
|
|
3990
|
+
['External CI (green)', false],
|
|
3991
|
+
['Reconciled after merge; all green', false],
|
|
3992
|
+
['Reconciled after merge: PR #475 is orchestrator-confirmed MERGED with Trust Reconcile, Source and Static, Runtime and Kit, Integration Coverage, Workflow Contracts, Usage Feedback, and Secret Scan all SUCCESS.', false],
|
|
3993
|
+
['A codex-shaped PostToolUse payload whose only failure signal is the host prose banner is never captured as observedResult:pass', false],
|
|
3994
|
+
];
|
|
3995
|
+
let ok = true;
|
|
3996
|
+
for (const [text, expected] of cases) {
|
|
3997
|
+
const actual = isRunnableCommandText(text);
|
|
3998
|
+
if (actual !== expected) {
|
|
3999
|
+
ok = false;
|
|
4000
|
+
console.error(`${actual ? 'RUNNABLE' : 'PROSE'} != ${expected ? 'RUNNABLE' : 'PROSE'} :: ${text}`);
|
|
4001
|
+
}
|
|
4002
|
+
}
|
|
4003
|
+
process.exit(ok ? 0 : 1);
|
|
4004
|
+
NODEEOF
|
|
4005
|
+
then
|
|
4006
|
+
_pass "AC-FIX-3: shared runnable-command detector classifies shell keyword commands and plain-English prose per contract"
|
|
4007
|
+
else
|
|
4008
|
+
_fail "AC-FIX-3: shared runnable-command detector contract failed: $(cat "$TMPDIR_EVAL/ac494-runnable-contract.err")"
|
|
4009
|
+
fi
|
|
4010
|
+
|
|
4011
|
+
if flow_agents_node "$WRITER" record-evidence "$CHECK_PROSE_DIR" \
|
|
4012
|
+
--verdict pass \
|
|
4013
|
+
--check-json '{"id":"external-ci-attestation","kind":"external","status":"pass","summary":"External CI ground truth: PR #475 is merged and all checks are green."}' \
|
|
4014
|
+
--timestamp "2026-07-05T09:11:30Z" >"$TMPDIR_EVAL/check-prose-external.out" 2>"$TMPDIR_EVAL/check-prose-external.err"; then
|
|
4015
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/check-prose-external-assert.err"
|
|
4016
|
+
import { readFileSync } from 'node:fs';
|
|
4017
|
+
const bundle = JSON.parse(readFileSync('${CHECK_PROSE_DIR}/trust.bundle', 'utf8'));
|
|
4018
|
+
const claim = bundle.claims.find((c) => c.subjectId && c.subjectId.endsWith('/external-ci-attestation'));
|
|
4019
|
+
const evidence = bundle.evidence.find((e) => e.claimId === (claim && claim.id));
|
|
4020
|
+
if (!claim) { process.stderr.write('external attestation claim missing\n'); process.exit(1); }
|
|
4021
|
+
if (claim.claimType !== 'workflow.check.external') { process.stderr.write('expected external claim type, got ' + claim.claimType + '\n'); process.exit(1); }
|
|
4022
|
+
if (evidence && evidence.execution && evidence.execution.label) { process.stderr.write('external attestation must not carry execution.label: ' + JSON.stringify(evidence.execution) + '\n'); process.exit(1); }
|
|
4023
|
+
NODEOF
|
|
4024
|
+
then
|
|
4025
|
+
_pass "AC11: kind:\"external\" prose attestation is accepted and carries no execution.label"
|
|
4026
|
+
else
|
|
4027
|
+
_fail "AC11: external prose attestation bundle shape was wrong: $(cat "$TMPDIR_EVAL/check-prose-external-assert.err")"
|
|
4028
|
+
fi
|
|
4029
|
+
else
|
|
4030
|
+
_fail "AC11: kind:\"external\" prose attestation should be accepted: $(cat "$TMPDIR_EVAL/check-prose-external.out" "$TMPDIR_EVAL/check-prose-external.err")"
|
|
4031
|
+
fi
|
|
4032
|
+
|
|
4033
|
+
# ─── #270 CRITICAL/HIGH fix evals: forged-stamp negative + pre-cluster missing-stamp negative ──
|
|
4034
|
+
# Both mutate a FRESH COPY of the compose session's trust.bundle (never COMPOSE_DIR itself) so
|
|
4035
|
+
# these negative fixtures stay isolated from the compose-layer assertions above.
|
|
4036
|
+
|
|
4037
|
+
# ─── Forged-stamp negative: hand-edit metadata.gate_claim.expectation_id to a nonexistent ─────
|
|
4038
|
+
# expects[] id. Any rebuild (record-critique) must die naming the stamp AND the claim id — never
|
|
4039
|
+
# silently fall through to matchExpectsEntry (the exact #270 CRITICAL/HIGH defect this closes).
|
|
4040
|
+
# Copy the WHOLE compose root (not just COMPOSE_DIR) — current.json / current/<actor>.json live
|
|
4041
|
+
# in the PARENT of the session dir, and resolveActiveFlowStep (hence sessionFlowId/flowRepoRoot in
|
|
4042
|
+
# buildTrustBundle) needs them to resolve the flow definition for stamp validation. Copying only
|
|
4043
|
+
# the session dir would silently make sessionFlowId resolve to null, which changes what's being
|
|
4044
|
+
# tested (the "no flow definition resolvable" edge case) rather than the forged-stamp case.
|
|
4045
|
+
FORGED_ROOT="$TMPDIR_EVAL/forged-stamp-root"
|
|
4046
|
+
cp -r "$COMPOSE_ROOT" "$FORGED_ROOT"
|
|
4047
|
+
FORGED_DIR="$FORGED_ROOT/$COMPOSE_SLUG"
|
|
4048
|
+
|
|
4049
|
+
node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/forged-mutate.err"
|
|
4050
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4051
|
+
const file = '${FORGED_DIR}/trust.bundle';
|
|
4052
|
+
const bundle = JSON.parse(readFileSync(file, 'utf8'));
|
|
4053
|
+
const claim = bundle.claims.find((c) => c.metadata && c.metadata.gate_claim && c.metadata.gate_claim.expectation_id === 'selected-work');
|
|
4054
|
+
if (!claim) { process.stderr.write('setup: selected-work gate claim not found to forge\n'); process.exit(1); }
|
|
4055
|
+
claim.metadata.gate_claim.expectation_id = 'nonexistent-expectation-id';
|
|
4056
|
+
writeFileSync(file, JSON.stringify(bundle, null, 2) + '\n');
|
|
4057
|
+
NODEOF
|
|
4058
|
+
|
|
4059
|
+
if [[ -s "$TMPDIR_EVAL/forged-mutate.err" ]]; then
|
|
4060
|
+
_fail "eval setup: forging metadata.gate_claim.expectation_id failed: $(cat "$TMPDIR_EVAL/forged-mutate.err")"
|
|
4061
|
+
fi
|
|
4062
|
+
|
|
4063
|
+
if flow_agents_node "$WRITER" record-critique "$FORGED_DIR" \
|
|
4064
|
+
--id forged-stamp-review \
|
|
4065
|
+
--reviewer forged-stamp-tester \
|
|
4066
|
+
--verdict pass \
|
|
4067
|
+
--summary "Rebuild attempt against a forged gate_claim stamp" \
|
|
4068
|
+
--timestamp "2026-07-05T09:12:00Z" >"$TMPDIR_EVAL/forged-critique.out" 2>"$TMPDIR_EVAL/forged-critique.err"; then
|
|
4069
|
+
_fail "FORGED-STAMP REGRESSION: record-critique succeeded against a bundle with a forged metadata.gate_claim.expectation_id (silent re-typing fall-through, the #270 CRITICAL/HIGH defect)"
|
|
4070
|
+
elif grep -qi "forged or corrupt" "$TMPDIR_EVAL/forged-critique.out" "$TMPDIR_EVAL/forged-critique.err" \
|
|
4071
|
+
&& grep -q "nonexistent-expectation-id" "$TMPDIR_EVAL/forged-critique.out" "$TMPDIR_EVAL/forged-critique.err"; then
|
|
4072
|
+
_pass "#270 CRITICAL/HIGH: a forged metadata.gate_claim stamp (nonexistent expectation_id) makes any rebuild die loudly, naming the stamp — never silently re-typed"
|
|
4073
|
+
else
|
|
4074
|
+
_fail "#270 forged-stamp rejection message was unexpected: $(cat "$TMPDIR_EVAL/forged-critique.out" "$TMPDIR_EVAL/forged-critique.err")"
|
|
4075
|
+
fi
|
|
4076
|
+
|
|
4077
|
+
# ─── Pre-cluster missing-stamp negative: strip metadata.gate_claim from a gate-claim-SHAPED ────
|
|
4078
|
+
# claim (origin:"check", check_kind:"external", kit-typed claimType) — mimicking the real
|
|
4079
|
+
# kontourai-flow-agents-303 bundle shape (a claim recorded before the gate_claim stamp existed).
|
|
4080
|
+
# Any rebuild must die with the re-record remedy, never silently re-type via matchExpectsEntry.
|
|
4081
|
+
PRECLUSTER_ROOT="$TMPDIR_EVAL/precluster-missing-stamp-root"
|
|
4082
|
+
cp -r "$COMPOSE_ROOT" "$PRECLUSTER_ROOT"
|
|
4083
|
+
PRECLUSTER_DIR="$PRECLUSTER_ROOT/$COMPOSE_SLUG"
|
|
4084
|
+
|
|
4085
|
+
node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/precluster-mutate.err"
|
|
4086
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4087
|
+
const file = '${PRECLUSTER_DIR}/trust.bundle';
|
|
4088
|
+
const bundle = JSON.parse(readFileSync(file, 'utf8'));
|
|
4089
|
+
const claim = bundle.claims.find((c) => c.metadata && c.metadata.gate_claim && c.metadata.gate_claim.expectation_id === 'implementation-plan');
|
|
4090
|
+
if (!claim) { process.stderr.write('setup: implementation-plan gate claim not found to strip\n'); process.exit(1); }
|
|
4091
|
+
delete claim.metadata.gate_claim;
|
|
4092
|
+
writeFileSync(file, JSON.stringify(bundle, null, 2) + '\n');
|
|
4093
|
+
NODEOF
|
|
4094
|
+
|
|
4095
|
+
if [[ -s "$TMPDIR_EVAL/precluster-mutate.err" ]]; then
|
|
4096
|
+
_fail "eval setup: stripping metadata.gate_claim failed: $(cat "$TMPDIR_EVAL/precluster-mutate.err")"
|
|
4097
|
+
fi
|
|
4098
|
+
|
|
4099
|
+
if flow_agents_node "$WRITER" record-critique "$PRECLUSTER_DIR" \
|
|
4100
|
+
--id precluster-review \
|
|
4101
|
+
--reviewer precluster-tester \
|
|
4102
|
+
--verdict pass \
|
|
4103
|
+
--summary "Rebuild attempt against a pre-cluster-270 unstamped gate claim" \
|
|
4104
|
+
--timestamp "2026-07-05T09:13:00Z" >"$TMPDIR_EVAL/precluster-critique.out" 2>"$TMPDIR_EVAL/precluster-critique.err"; then
|
|
4105
|
+
_fail "PRE-CLUSTER-270 REGRESSION: record-critique succeeded against a bundle with an unstamped gate-claim-shaped claim (silent re-typing fall-through)"
|
|
4106
|
+
elif grep -qi "pre-cluster-270 gate claim" "$TMPDIR_EVAL/precluster-critique.out" "$TMPDIR_EVAL/precluster-critique.err" \
|
|
4107
|
+
&& grep -qi "re-record it (record-gate-claim) to regenerate" "$TMPDIR_EVAL/precluster-critique.out" "$TMPDIR_EVAL/precluster-critique.err"; then
|
|
4108
|
+
_pass "#270 pre-cluster-270 missing-stamp: an unstamped gate-claim-shaped claim makes any rebuild die with the re-record remedy — never silently re-typed"
|
|
4109
|
+
else
|
|
4110
|
+
_fail "#270 pre-cluster missing-stamp rejection message was unexpected: $(cat "$TMPDIR_EVAL/precluster-critique.out" "$TMPDIR_EVAL/precluster-critique.err")"
|
|
4111
|
+
fi
|
|
4112
|
+
|
|
4113
|
+
# ─── #270 HIGH fix (iteration 3): reserve the "gate-claim-" check-id namespace ─────────────────
|
|
4114
|
+
# (a) reserved-prefix rejection via record-evidence --check-json (exit + message)
|
|
4115
|
+
if flow_agents_node "$WRITER" record-evidence "$COMPOSE_DIR" \
|
|
4116
|
+
--verdict pass \
|
|
4117
|
+
--check-json '{"id":"gate-claim-caller-supplied","kind":"external","status":"pass","summary":"A caller-chosen id that collides with the reserved gate-claim- prefix"}' \
|
|
4118
|
+
--timestamp "2026-07-05T09:15:30Z" >"$TMPDIR_EVAL/reserved-prefix.out" 2>"$TMPDIR_EVAL/reserved-prefix.err"; then
|
|
4119
|
+
_fail "#270 HIGH REGRESSION: record-evidence --check-json accepted a caller-supplied id starting with the reserved 'gate-claim-' prefix"
|
|
4120
|
+
elif grep -qi 'reserved for record-gate-claim' "$TMPDIR_EVAL/reserved-prefix.out" "$TMPDIR_EVAL/reserved-prefix.err" \
|
|
4121
|
+
&& grep -q 'gate-claim-caller-supplied' "$TMPDIR_EVAL/reserved-prefix.out" "$TMPDIR_EVAL/reserved-prefix.err"; then
|
|
4122
|
+
_pass "#270 HIGH: record-evidence --check-json rejects a caller-supplied id starting with the reserved 'gate-claim-' prefix, naming the offending id"
|
|
4123
|
+
else
|
|
4124
|
+
_fail "#270 HIGH: reserved-prefix rejection message was unexpected: $(cat "$TMPDIR_EVAL/reserved-prefix.out" "$TMPDIR_EVAL/reserved-prefix.err")"
|
|
4125
|
+
fi
|
|
4126
|
+
|
|
4127
|
+
# Confirm record-check's --id is covered by the same guard (a second writer of caller-supplied ids).
|
|
4128
|
+
if flow_agents_node "$WRITER" record-check "$COMPOSE_DIR" --id "gate-claim-record-check-collision" -- true \
|
|
4129
|
+
>"$TMPDIR_EVAL/reserved-prefix-record-check.out" 2>"$TMPDIR_EVAL/reserved-prefix-record-check.err"; then
|
|
4130
|
+
_fail "#270 HIGH REGRESSION: record-check accepted a caller-supplied --id starting with the reserved 'gate-claim-' prefix"
|
|
4131
|
+
elif grep -qi 'reserved for record-gate-claim' "$TMPDIR_EVAL/reserved-prefix-record-check.out" "$TMPDIR_EVAL/reserved-prefix-record-check.err"; then
|
|
4132
|
+
_pass "#270 HIGH: record-check --id is covered by the same reserved-prefix guard as record-evidence"
|
|
4133
|
+
else
|
|
4134
|
+
_fail "#270 HIGH: record-check reserved-prefix rejection message was unexpected: $(cat "$TMPDIR_EVAL/reserved-prefix-record-check.out" "$TMPDIR_EVAL/reserved-prefix-record-check.err")"
|
|
4135
|
+
fi
|
|
4136
|
+
|
|
4137
|
+
# Confirm record-gate-claim's OWN internally-constructed gate-claim-<id> is still accepted — the
|
|
4138
|
+
# guard must reject only CALLER-supplied ids on other writers, never record-gate-claim's own path.
|
|
4139
|
+
# (compose-gate1's earlier record-gate-claim call above already exercises and asserts this; this
|
|
4140
|
+
# is a targeted re-assertion scoped to the reserved-prefix fix itself.)
|
|
4141
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/reserved-prefix-gate-claim-own.err"
|
|
4142
|
+
import { readFileSync } from 'node:fs';
|
|
4143
|
+
const bundle = JSON.parse(readFileSync('${COMPOSE_DIR}/trust.bundle', 'utf8'));
|
|
4144
|
+
const gateClaim = bundle.claims.find((c) => c.subjectId && c.subjectId.endsWith('/gate-claim-selected-work'));
|
|
4145
|
+
if (!gateClaim) { process.stderr.write('record-gate-claims own gate-claim-selected-work id was not found — its own reserved-prefix construction must still succeed\n'); process.exit(1); }
|
|
4146
|
+
if (!gateClaim.metadata || !gateClaim.metadata.gate_claim) { process.stderr.write('record-gate-claims own claim is missing its metadata.gate_claim stamp\n'); process.exit(1); }
|
|
4147
|
+
NODEOF
|
|
4148
|
+
then
|
|
4149
|
+
_pass "#270 HIGH: record-gate-claim's own internally-constructed 'gate-claim-<id>' still succeeds (the guard applies only to caller-supplied ids on OTHER writers)"
|
|
4150
|
+
else
|
|
4151
|
+
_fail "#270 HIGH: record-gate-claim's own id-construction regressed: $(cat "$TMPDIR_EVAL/reserved-prefix-gate-claim-own.err")"
|
|
4152
|
+
fi
|
|
4153
|
+
|
|
4154
|
+
# (b) collision regression scenario end-to-end: the EXACT pre-cluster-false-positive collision —
|
|
4155
|
+
# a caller-chosen record-evidence check id starting with "gate-claim-" that would, absent this
|
|
4156
|
+
# fix, get declared-typed via matchExpectsEntry's P-d fallback (kit-typed, unstamped) and then be
|
|
4157
|
+
# misclassified as an unstamped pre-cluster-270 gate claim on the NEXT rebuild — is now rejected
|
|
4158
|
+
# AT RECORD TIME instead, in a FRESH session (isolated from COMPOSE_DIR) so no rebuild landmine
|
|
4159
|
+
# is ever created.
|
|
4160
|
+
COLLISION_ROOT="$TMPDIR_EVAL/collision-root"
|
|
4161
|
+
COLLISION_SLUG="collision-270"
|
|
4162
|
+
COLLISION_DIR="$COLLISION_ROOT/$COLLISION_SLUG"
|
|
4163
|
+
mkdir -p "$COLLISION_ROOT"
|
|
4164
|
+
|
|
4165
|
+
flow_agents_node "$WRITER" ensure-session \
|
|
4166
|
+
--artifact-root "$COLLISION_ROOT" \
|
|
4167
|
+
--task-slug "$COLLISION_SLUG" \
|
|
4168
|
+
--actor collision-actor \
|
|
4169
|
+
--flow-id builder.build \
|
|
4170
|
+
--title "Collision regression session" \
|
|
4171
|
+
--source-request "Regression: gate-claim- id collision must die at record time, not rebuild time." \
|
|
4172
|
+
--summary "Seed session for the collision regression eval." \
|
|
4173
|
+
--criterion "Collision regression is caught at record time" \
|
|
4174
|
+
--timestamp "2026-07-05T09:16:00Z" >"$TMPDIR_EVAL/collision-seed.out" 2>"$TMPDIR_EVAL/collision-seed.err"
|
|
4175
|
+
|
|
4176
|
+
if flow_agents_node "$WRITER" record-evidence "$COLLISION_DIR" \
|
|
4177
|
+
--verdict pass \
|
|
4178
|
+
--check-json '{"id":"gate-claim-my-custom-check","kind":"external","status":"pass","summary":"A caller-chosen check id colliding with the reserved prefix, at the active pull-work step (kit-typed via matchExpectsEntry fallback)"}' \
|
|
4179
|
+
--timestamp "2026-07-05T09:16:15Z" >"$TMPDIR_EVAL/collision-record.out" 2>"$TMPDIR_EVAL/collision-record.err"; then
|
|
4180
|
+
_fail "#270 HIGH REGRESSION: the collision scenario's record-evidence call succeeded — this recreates the pre-cluster false-positive rebuild landmine"
|
|
4181
|
+
elif grep -qi 'reserved for record-gate-claim' "$TMPDIR_EVAL/collision-record.out" "$TMPDIR_EVAL/collision-record.err" \
|
|
4182
|
+
&& [[ ! -f "$COLLISION_DIR/trust.bundle" || ! $(node --input-type=module -e "
|
|
4183
|
+
import { readFileSync } from 'node:fs';
|
|
4184
|
+
const b = JSON.parse(readFileSync('${COLLISION_DIR}/trust.bundle', 'utf8'));
|
|
4185
|
+
process.stdout.write(String(b.claims.some((c) => c.subjectId && c.subjectId.endsWith('/gate-claim-my-custom-check'))));
|
|
4186
|
+
" 2>/dev/null) == "true" ]]; then
|
|
4187
|
+
_pass "#270 HIGH: the collision scenario is rejected AT RECORD TIME (record-evidence), never lands in trust.bundle, and no rebuild landmine is created"
|
|
4188
|
+
else
|
|
4189
|
+
_fail "#270 HIGH: collision regression assertion failed: $(cat "$TMPDIR_EVAL/collision-record.out" "$TMPDIR_EVAL/collision-record.err")"
|
|
4190
|
+
fi
|
|
4191
|
+
|
|
4192
|
+
# Now prove a REBUILD against the (empty/unaffected) collision session succeeds cleanly — there is
|
|
4193
|
+
# no landmine left behind by the rejected record-evidence call above.
|
|
4194
|
+
if flow_agents_node "$WRITER" record-critique "$COLLISION_DIR" \
|
|
4195
|
+
--id collision-rebuild-review \
|
|
4196
|
+
--reviewer collision-tester \
|
|
4197
|
+
--verdict pass \
|
|
4198
|
+
--summary "Rebuild after the rejected collision write — must succeed, no landmine" \
|
|
4199
|
+
--timestamp "2026-07-05T09:16:30Z" >"$TMPDIR_EVAL/collision-rebuild.out" 2>"$TMPDIR_EVAL/collision-rebuild.err"; then
|
|
4200
|
+
_pass "#270 HIGH: a rebuild (record-critique) after the rejected collision write succeeds cleanly — no rebuild-time landmine"
|
|
4201
|
+
else
|
|
4202
|
+
_fail "#270 HIGH REGRESSION: rebuild after the rejected collision write unexpectedly failed: $(cat "$TMPDIR_EVAL/collision-rebuild.out" "$TMPDIR_EVAL/collision-rebuild.err")"
|
|
4203
|
+
fi
|
|
4204
|
+
|
|
4205
|
+
# ─── #270 MEDIUM fix (iteration 3): unresolvable FlowDefinition dies with the dedicated ────────
|
|
4206
|
+
# "cannot be loaded" message, never "forged or corrupt". Create a REAL stamped gate claim first
|
|
4207
|
+
# (real flow, real record-gate-claim), then break resolution by pointing active_flow_id at a
|
|
4208
|
+
# bogus flow id that cannot resolve under kits/ (mirrors the re-reviewer's approach of pointing
|
|
4209
|
+
# at an unresolvable kits/ path), in an ISOLATED fixture copied from a fresh session.
|
|
4210
|
+
UNRESOLVABLE_ROOT="$TMPDIR_EVAL/unresolvable-root"
|
|
4211
|
+
UNRESOLVABLE_SLUG="unresolvable-270"
|
|
4212
|
+
UNRESOLVABLE_DIR="$UNRESOLVABLE_ROOT/$UNRESOLVABLE_SLUG"
|
|
4213
|
+
mkdir -p "$UNRESOLVABLE_ROOT"
|
|
4214
|
+
|
|
4215
|
+
flow_agents_node "$WRITER" ensure-session \
|
|
4216
|
+
--artifact-root "$UNRESOLVABLE_ROOT" \
|
|
4217
|
+
--task-slug "$UNRESOLVABLE_SLUG" \
|
|
4218
|
+
--actor unresolvable-actor \
|
|
4219
|
+
--flow-id builder.build \
|
|
4220
|
+
--title "Unresolvable FlowDefinition regression" \
|
|
4221
|
+
--source-request "Regression: an unresolvable FlowDefinition must die with the dedicated cannot-be-loaded message." \
|
|
4222
|
+
--summary "Seed session with a REAL flow so a real stamped gate claim can be recorded." \
|
|
4223
|
+
--criterion "Unresolvable FlowDefinition dies with the dedicated message" \
|
|
4224
|
+
--timestamp "2026-07-05T09:17:00Z" >"$TMPDIR_EVAL/unresolvable-seed.out" 2>"$TMPDIR_EVAL/unresolvable-seed.err"
|
|
4225
|
+
|
|
4226
|
+
if flow_agents_node "$WRITER" record-gate-claim "$UNRESOLVABLE_DIR" \
|
|
4227
|
+
--actor unresolvable-actor \
|
|
4228
|
+
--status pass \
|
|
4229
|
+
--summary "Work item selected for unresolvable-flow-definition regression" \
|
|
4230
|
+
--timestamp "2026-07-05T09:17:15Z" >"$TMPDIR_EVAL/unresolvable-gate-claim.out" 2>"$TMPDIR_EVAL/unresolvable-gate-claim.err"; then
|
|
4231
|
+
_pass "#270 MEDIUM setup: real stamped gate claim recorded before breaking FlowDefinition resolution"
|
|
4232
|
+
else
|
|
4233
|
+
_fail "#270 MEDIUM setup: record-gate-claim (real flow) failed: $(cat "$TMPDIR_EVAL/unresolvable-gate-claim.out" "$TMPDIR_EVAL/unresolvable-gate-claim.err")"
|
|
4234
|
+
fi
|
|
4235
|
+
|
|
4236
|
+
# Break resolution: point BOTH the legacy and per-actor current-pointer files at a flow id with
|
|
4237
|
+
# no corresponding kits/<kit>/flows/<name>.flow.json file.
|
|
4238
|
+
node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/unresolvable-mutate.err"
|
|
4239
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4240
|
+
for (const f of ['${UNRESOLVABLE_ROOT}/current.json', '${UNRESOLVABLE_ROOT}/current/unresolvable-actor.json']) {
|
|
4241
|
+
const d = JSON.parse(readFileSync(f, 'utf8'));
|
|
4242
|
+
d.active_flow_id = 'builder.bogus-nonexistent-flow-270';
|
|
4243
|
+
writeFileSync(f, JSON.stringify(d, null, 2) + '\n');
|
|
4244
|
+
}
|
|
4245
|
+
NODEOF
|
|
4246
|
+
|
|
4247
|
+
if [[ -s "$TMPDIR_EVAL/unresolvable-mutate.err" ]]; then
|
|
4248
|
+
_fail "eval setup: pointing active_flow_id at a bogus flow id failed: $(cat "$TMPDIR_EVAL/unresolvable-mutate.err")"
|
|
4249
|
+
fi
|
|
4250
|
+
|
|
4251
|
+
if flow_agents_node "$WRITER" record-critique "$UNRESOLVABLE_DIR" \
|
|
4252
|
+
--id unresolvable-review \
|
|
4253
|
+
--reviewer unresolvable-tester \
|
|
4254
|
+
--verdict pass \
|
|
4255
|
+
--summary "Rebuild attempt against an unresolvable FlowDefinition" \
|
|
4256
|
+
--timestamp "2026-07-05T09:17:30Z" >"$TMPDIR_EVAL/unresolvable-critique.out" 2>"$TMPDIR_EVAL/unresolvable-critique.err"; then
|
|
4257
|
+
_fail "#270 MEDIUM REGRESSION: record-critique succeeded against a stamped gate claim whose FlowDefinition cannot be loaded"
|
|
4258
|
+
elif grep -qi "cannot be loaded" "$TMPDIR_EVAL/unresolvable-critique.out" "$TMPDIR_EVAL/unresolvable-critique.err" \
|
|
4259
|
+
&& grep -qi "cannot validate the gate-claim stamp" "$TMPDIR_EVAL/unresolvable-critique.out" "$TMPDIR_EVAL/unresolvable-critique.err" \
|
|
4260
|
+
&& ! grep -qi "forged or corrupt" "$TMPDIR_EVAL/unresolvable-critique.out" "$TMPDIR_EVAL/unresolvable-critique.err"; then
|
|
4261
|
+
_pass "#270 MEDIUM: an unresolvable FlowDefinition dies with the dedicated 'cannot be loaded' message, naming the flow id — NEVER the 'forged or corrupt' message"
|
|
4262
|
+
else
|
|
4263
|
+
_fail "#270 MEDIUM: unresolvable-FlowDefinition rejection message was unexpected: $(cat "$TMPDIR_EVAL/unresolvable-critique.out" "$TMPDIR_EVAL/unresolvable-critique.err")"
|
|
4264
|
+
fi
|
|
4265
|
+
|
|
4266
|
+
# ─── Mutation test (#270 HIGH reserved-prefix guard): temporarily disable normalizeCheck's ──────
|
|
4267
|
+
# reserved-"gate-claim-"-prefix rejection in a SCRATCH COPY of the compiled build/ output,
|
|
4268
|
+
# confirm the reserved-prefix fixture above now SUCCEEDS against that mutated binary (eval "goes
|
|
4269
|
+
# red" without the guard), then restore the original compiled file immediately. Proves the eval
|
|
4270
|
+
# is actually exercising this specific guard, not passing vacuously for an unrelated reason.
|
|
4271
|
+
DIST_SIDECAR="$ROOT/build/src/cli/workflow-sidecar.js"
|
|
4272
|
+
RESERVED_PREFIX_SCRATCH="$TMPDIR_EVAL/reserved-prefix-mutation-scratch"
|
|
4273
|
+
mkdir -p "$RESERVED_PREFIX_SCRATCH"
|
|
4274
|
+
|
|
4275
|
+
if [[ -f "$DIST_SIDECAR" ]]; then
|
|
4276
|
+
cp "$DIST_SIDECAR" "$RESERVED_PREFIX_SCRATCH/workflow-sidecar.orig.js"
|
|
4277
|
+
node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/reserved-prefix-mutation-patch.err"
|
|
4278
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4279
|
+
const file = '${RESERVED_PREFIX_SCRATCH}/workflow-sidecar.orig.js';
|
|
4280
|
+
let src = readFileSync(file, 'utf8');
|
|
4281
|
+
const needle = 'if (!allowGateClaimPrefix && typeof check.id === "string" && check.id.startsWith("gate-claim-")) {';
|
|
4282
|
+
if (!src.includes(needle)) { process.stderr.write('mutation: reserved-prefix guard text not found — source pattern drifted, cannot mutation-test\n'); process.exit(1); }
|
|
4283
|
+
src = src.split(needle).join('if (false) {');
|
|
4284
|
+
writeFileSync('${RESERVED_PREFIX_SCRATCH}/workflow-sidecar.mutated.js', src);
|
|
4285
|
+
NODEOF
|
|
4286
|
+
|
|
4287
|
+
if [[ -s "$TMPDIR_EVAL/reserved-prefix-mutation-patch.err" ]]; then
|
|
4288
|
+
_fail "mutation-test setup failed (reserved-prefix guard source pattern did not match compiled output): $(cat "$TMPDIR_EVAL/reserved-prefix-mutation-patch.err")"
|
|
4289
|
+
else
|
|
4290
|
+
if node --check "$RESERVED_PREFIX_SCRATCH/workflow-sidecar.mutated.js" 2>"$TMPDIR_EVAL/reserved-prefix-mutation-syntax.err"; then
|
|
4291
|
+
cp "$RESERVED_PREFIX_SCRATCH/workflow-sidecar.mutated.js" "$DIST_SIDECAR"
|
|
4292
|
+
|
|
4293
|
+
if flow_agents_node "$WRITER" record-evidence "$COMPOSE_DIR" \
|
|
4294
|
+
--verdict pass \
|
|
4295
|
+
--check-json '{"id":"gate-claim-mutation-test","kind":"external","status":"pass","summary":"Mutation-test: reserved-prefix guard disabled, must go red"}' \
|
|
4296
|
+
--timestamp "2026-07-05T09:17:45Z" >"$TMPDIR_EVAL/reserved-prefix-mutation.out" 2>"$TMPDIR_EVAL/reserved-prefix-mutation.err"; then
|
|
4297
|
+
_pass "mutation-test: with the reserved-prefix guard neutered, a caller-supplied 'gate-claim-' id now SUCCEEDS (eval correctly goes red without the guard, proving the eval exercises it)"
|
|
4298
|
+
else
|
|
4299
|
+
_fail "mutation-test: reserved-prefix fixture still rejected even with the guard neutered — the eval may not be exercising the intended guard: $(cat "$TMPDIR_EVAL/reserved-prefix-mutation.out" "$TMPDIR_EVAL/reserved-prefix-mutation.err")"
|
|
4300
|
+
fi
|
|
4301
|
+
else
|
|
4302
|
+
_fail "mutation-test setup: mutated workflow-sidecar.js (reserved-prefix guard) failed a syntax check, refusing to run it: $(cat "$TMPDIR_EVAL/reserved-prefix-mutation-syntax.err")"
|
|
4303
|
+
fi
|
|
4304
|
+
|
|
4305
|
+
# Restore the real compiled guard immediately — never leave the mutated binary in place — and
|
|
4306
|
+
# re-run the reserved-prefix negative to confirm the restored binary rejects it again.
|
|
4307
|
+
cp "$RESERVED_PREFIX_SCRATCH/workflow-sidecar.orig.js" "$DIST_SIDECAR"
|
|
4308
|
+
if flow_agents_node "$WRITER" record-evidence "$COMPOSE_DIR" \
|
|
4309
|
+
--verdict pass \
|
|
4310
|
+
--check-json '{"id":"gate-claim-restore-check","kind":"external","status":"pass","summary":"Restore check: reserved-prefix guard must be back after mutation-test cleanup"}' \
|
|
4311
|
+
--timestamp "2026-07-05T09:18:00Z" >"$TMPDIR_EVAL/reserved-prefix-restore.out" 2>"$TMPDIR_EVAL/reserved-prefix-restore.err"; then
|
|
4312
|
+
_fail "mutation-test cleanup REGRESSION: the reserved-prefix guard did not come back after restoring the original compiled file"
|
|
4313
|
+
else
|
|
4314
|
+
_pass "mutation-test cleanup: the real compiled reserved-prefix guard is restored and rejects a caller-supplied 'gate-claim-' id again"
|
|
4315
|
+
fi
|
|
4316
|
+
fi
|
|
4317
|
+
else
|
|
4318
|
+
_fail "mutation-test setup: could not locate the compiled build/src/cli/workflow-sidecar.js to mutate for the reserved-prefix guard (ran 'npm run build' first?)"
|
|
4319
|
+
fi
|
|
4320
|
+
|
|
4321
|
+
|
|
4322
|
+
# ─── #270 follow-up fix (publish-preflight, iteration 5): reserved-prefix rejection applies ────
|
|
4323
|
+
# ONLY to NEW mints — a caller-supplied id that ALREADY EXISTS as a check claim id in the
|
|
4324
|
+
# session's CURRENT trust.bundle is a CORRECTION (supersession), not a new mint, and must be
|
|
4325
|
+
# allowed even when it starts with the reserved "gate-claim-" prefix. Without this exemption a
|
|
4326
|
+
# mis-recorded gate-claim-* check (e.g. one mistakenly recorded kind:"test" with no manifest
|
|
4327
|
+
# label, exactly the real kontourai-flow-agents-270 wedge this closes) can NEVER be corrected:
|
|
4328
|
+
# every attempt to re-record that exact id — the only way to supersede/fix it — is itself
|
|
4329
|
+
# rejected by the guard, permanently wedging that id and blocking publish-preflight forever.
|
|
4330
|
+
CORRECTION_ROOT="$TMPDIR_EVAL/correction-root"
|
|
4331
|
+
CORRECTION_SLUG="correction-270"
|
|
4332
|
+
CORRECTION_DIR="$CORRECTION_ROOT/$CORRECTION_SLUG"
|
|
4333
|
+
mkdir -p "$CORRECTION_ROOT"
|
|
4334
|
+
|
|
4335
|
+
# A dedicated, FRESH session (never copied from COMPOSE_ROOT/COMPOSE_DIR) — COMPOSE_DIR's bundle
|
|
4336
|
+
# already carries the reserved-prefix mutation-test's own transient (deliberately unstamped,
|
|
4337
|
+
# guard-neutered) 'gate-claim-mutation-test' claim by this point in the script; copying it here
|
|
4338
|
+
# would poison this fixture with an unrelated pre-cluster-270 unstamped-claim landmine that has
|
|
4339
|
+
# nothing to do with the correction-path behavior under test.
|
|
4340
|
+
flow_agents_node "$WRITER" ensure-session \
|
|
4341
|
+
--artifact-root "$CORRECTION_ROOT" \
|
|
4342
|
+
--task-slug "$CORRECTION_SLUG" \
|
|
4343
|
+
--actor correction-actor \
|
|
4344
|
+
--flow-id builder.build \
|
|
4345
|
+
--title "Correction-path session" \
|
|
4346
|
+
--source-request "Regression: a mis-recorded gate-claim-* id (kind test, no manifest label) must be correctable via supersession, not permanently wedged." \
|
|
4347
|
+
--summary "Seed session for the reserved-prefix existing-id correction eval." \
|
|
4348
|
+
--criterion "Mis-recorded gate-claim-* ids are correctable via same-id supersession" \
|
|
4349
|
+
--timestamp "2026-07-05T09:10:45Z" >"$TMPDIR_EVAL/correction-seed.out" 2>"$TMPDIR_EVAL/correction-seed.err"
|
|
4350
|
+
|
|
4351
|
+
# A real record-gate-claim call creates the FIRST trust.bundle write (ensure-session alone does
|
|
4352
|
+
# not write trust.bundle yet) — a real, properly-stamped gate claim at pull-work-gate/selected-work,
|
|
4353
|
+
# exactly the compose-layer pattern above, so this session has a normal, valid bundle to seed the
|
|
4354
|
+
# mis-recorded claim alongside (never the sole content of the bundle).
|
|
4355
|
+
flow_agents_node "$WRITER" record-gate-claim "$CORRECTION_DIR" \
|
|
4356
|
+
--actor correction-actor \
|
|
4357
|
+
--status pass \
|
|
4358
|
+
--summary "Work item selected for correction-path session" \
|
|
4359
|
+
--timestamp "2026-07-05T09:10:50Z" >"$TMPDIR_EVAL/correction-gate-seed.out" 2>"$TMPDIR_EVAL/correction-gate-seed.err"
|
|
4360
|
+
|
|
4361
|
+
if [[ ! -f "$CORRECTION_DIR/trust.bundle" ]]; then
|
|
4362
|
+
_fail "eval setup: record-gate-claim did not create trust.bundle for the correction-path session: $(cat "$TMPDIR_EVAL/correction-gate-seed.out" "$TMPDIR_EVAL/correction-gate-seed.err")"
|
|
4363
|
+
fi
|
|
4364
|
+
|
|
4365
|
+
# Seed the EXACT real wedge shape by hand-editing a copy of trust.bundle (the current, already-
|
|
4366
|
+
# fixed binary can no longer MINT a fresh gate-claim-* id via record-evidence/record-check, so a
|
|
4367
|
+
# hand-constructed claim is how a pre-fix mis-recording is replicated here — same idiom the
|
|
4368
|
+
# forged-stamp/pre-cluster-missing-stamp negatives above already use for defect-class shapes the
|
|
4369
|
+
# CLI itself must never be able to produce). Mirrors the real wedged claim byte-for-byte in
|
|
4370
|
+
# shape: origin:"check", check_kind:"test" (WS8's classifyEvidence always maps kind:"test" to
|
|
4371
|
+
# evidenceType:"test_output" regardless of a command), no execution.label on its evidence entry
|
|
4372
|
+
# (no manifest-matchable command — the "without manifest labels" defect) — exactly what makes
|
|
4373
|
+
# reconcile-preflight's sessionLocalShapeIssues/noEvidenceCommandIssues path a divergence
|
|
4374
|
+
# ('test_output-unreconciled') and blocks publish.
|
|
4375
|
+
CORRECTION_CHECK_ID="gate-claim-accumulates"
|
|
4376
|
+
node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/correction-seed-mutate.err"
|
|
4377
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4378
|
+
const file = '${CORRECTION_DIR}/trust.bundle';
|
|
4379
|
+
const bundle = JSON.parse(readFileSync(file, 'utf8'));
|
|
4380
|
+
const ts = '2026-07-05T09:11:00Z';
|
|
4381
|
+
const claimId = 'correction-270-${CORRECTION_CHECK_ID}';
|
|
4382
|
+
bundle.claims.push({
|
|
4383
|
+
id: claimId,
|
|
4384
|
+
subjectType: 'change',
|
|
4385
|
+
subjectId: '${CORRECTION_SLUG}/${CORRECTION_CHECK_ID}',
|
|
4386
|
+
facet: 'flow-agents.workflow',
|
|
4387
|
+
claimType: 'builder.execute.scope',
|
|
4388
|
+
fieldOrBehavior: 'Mis-recorded (pre-fix) gate claim: kind test, no manifest label — the exact real wedge shape.',
|
|
4389
|
+
value: 'pass',
|
|
4390
|
+
createdAt: ts,
|
|
4391
|
+
updatedAt: ts,
|
|
4392
|
+
impactLevel: 'high',
|
|
4393
|
+
verificationPolicyId: 'policy:builder.execute.scope:test_output',
|
|
4394
|
+
metadata: { origin: 'check', check_kind: 'test' },
|
|
4395
|
+
status: 'verified',
|
|
4396
|
+
});
|
|
4397
|
+
bundle.evidence.push({
|
|
4398
|
+
id: 'ev:' + claimId,
|
|
4399
|
+
claimId,
|
|
4400
|
+
evidenceType: 'test_output',
|
|
4401
|
+
method: 'validation',
|
|
4402
|
+
sourceRef: '${CORRECTION_SLUG}/evidence.json',
|
|
4403
|
+
excerptOrSummary: 'Mis-recorded (pre-fix) gate claim: kind test, no manifest label.',
|
|
4404
|
+
observedAt: ts,
|
|
4405
|
+
collectedBy: 'flow-agents/workflow-sidecar',
|
|
4406
|
+
passing: true,
|
|
4407
|
+
});
|
|
4408
|
+
writeFileSync(file, JSON.stringify(bundle, null, 2) + '\n');
|
|
4409
|
+
NODEOF
|
|
4410
|
+
|
|
4411
|
+
if [[ -s "$TMPDIR_EVAL/correction-seed-mutate.err" ]]; then
|
|
4412
|
+
_fail "eval setup: seeding the mis-recorded gate-claim-* (kind:test, no manifest label) wedge claim failed: $(cat "$TMPDIR_EVAL/correction-seed-mutate.err")"
|
|
4413
|
+
fi
|
|
4414
|
+
|
|
4415
|
+
# Sanity: reconcile-preflight FAILS against the wedged bundle (the real defect this closes) —
|
|
4416
|
+
# an unreconciled test_output claim with no manifest-matchable command is a divergence.
|
|
4417
|
+
if flow_agents_node "$WRITER" reconcile-preflight "$CORRECTION_DIR" --repo-root "$ROOT" \
|
|
4418
|
+
>"$TMPDIR_EVAL/correction-preflight-before.out" 2>"$TMPDIR_EVAL/correction-preflight-before.err"; then
|
|
4419
|
+
_fail "eval setup REGRESSION: reconcile-preflight unexpectedly PASSED against the seeded wedge claim (kind:test, no manifest label) — the wedge fixture is not actually reproducing the real defect"
|
|
4420
|
+
else
|
|
4421
|
+
_pass "eval setup: reconcile-preflight FAILS against the seeded wedge claim (mis-recorded kind:test, no manifest label) — reproduces the real publish-preflight defect before the correction"
|
|
4422
|
+
fi
|
|
4423
|
+
|
|
4424
|
+
# ─── Eval 1 (correction path): record-evidence --check-json with the SAME id, kind "policy" ────
|
|
4425
|
+
# (no command) — must SUCCEED and supersede the mis-recorded claim, even though the id starts
|
|
4426
|
+
# with the reserved "gate-claim-" prefix, because that id ALREADY EXISTS in the bundle.
|
|
4427
|
+
if flow_agents_node "$WRITER" record-evidence "$CORRECTION_DIR" \
|
|
4428
|
+
--verdict pass \
|
|
4429
|
+
--check-json "{\"id\":\"${CORRECTION_CHECK_ID}\",\"kind\":\"policy\",\"status\":\"pass\",\"summary\":\"Correction: re-recorded as a policy claim, superseding the mis-recorded kind:test entry.\"}" \
|
|
4430
|
+
--timestamp "2026-07-05T09:11:15Z" >"$TMPDIR_EVAL/correction-record.out" 2>"$TMPDIR_EVAL/correction-record.err"; then
|
|
4431
|
+
_pass "correction path: record-evidence supersedes an EXISTING gate-claim-* id (kind test -> policy) instead of rejecting it as a new mint"
|
|
4432
|
+
else
|
|
4433
|
+
_fail "correction path REGRESSION: record-evidence rejected supersession of an EXISTING gate-claim-* id: $(cat "$TMPDIR_EVAL/correction-record.out" "$TMPDIR_EVAL/correction-record.err")"
|
|
4434
|
+
fi
|
|
4435
|
+
|
|
4436
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/correction-assert.err"
|
|
4437
|
+
import { readFileSync } from 'node:fs';
|
|
4438
|
+
const bundle = JSON.parse(readFileSync('${CORRECTION_DIR}/trust.bundle', 'utf8'));
|
|
4439
|
+
const claims = bundle.claims.filter((c) => c.subjectId && c.subjectId.endsWith('/${CORRECTION_CHECK_ID}'));
|
|
4440
|
+
if (claims.length !== 1) { process.stderr.write('expected exactly ONE claim for ${CORRECTION_CHECK_ID} after supersession (same-id resupply must replace, not duplicate), found ' + claims.length + '\n'); process.exit(1); }
|
|
4441
|
+
const claim = claims[0];
|
|
4442
|
+
if (!claim.metadata || claim.metadata.check_kind !== 'policy') { process.stderr.write('claim.metadata.check_kind is not policy after the correction: ' + JSON.stringify(claim.metadata) + '\n'); process.exit(1); }
|
|
4443
|
+
NODEOF
|
|
4444
|
+
then
|
|
4445
|
+
_pass "correction path: the bundle's claim for gate-claim-accumulates now has check_kind:policy (superseded, not duplicated)"
|
|
4446
|
+
else
|
|
4447
|
+
_fail "correction path assertion failed: $(cat "$TMPDIR_EVAL/correction-assert.err")"
|
|
4448
|
+
fi
|
|
4449
|
+
|
|
4450
|
+
# Assert reconcile-preflight now PASSES (or at least no longer reports the corrected claim as a
|
|
4451
|
+
# divergence) — the correction is what publish-preflight needed to unwedge.
|
|
4452
|
+
if flow_agents_node "$WRITER" reconcile-preflight "$CORRECTION_DIR" --repo-root "$ROOT" \
|
|
4453
|
+
>"$TMPDIR_EVAL/correction-preflight-after.out" 2>"$TMPDIR_EVAL/correction-preflight-after.err"; then
|
|
4454
|
+
_pass "correction path: reconcile-preflight now PASSES after the correction (policy claim is session-local, not an unreconciled test_output divergence)"
|
|
4455
|
+
else
|
|
4456
|
+
_fail "correction path: reconcile-preflight still fails after the correction: $(cat "$TMPDIR_EVAL/correction-preflight-after.out" "$TMPDIR_EVAL/correction-preflight-after.err")"
|
|
4457
|
+
fi
|
|
4458
|
+
|
|
4459
|
+
# ─── Eval 2 (new-mint rejection still enforced): a NOVEL gate-claim-* id (never seen in this ───
|
|
4460
|
+
# session's bundle) is still rejected exactly as before — the exemption above must be scoped
|
|
4461
|
+
# strictly to ids that already exist, never a blanket bypass.
|
|
4462
|
+
if flow_agents_node "$WRITER" record-evidence "$CORRECTION_DIR" \
|
|
4463
|
+
--verdict pass \
|
|
4464
|
+
--check-json '{"id":"gate-claim-novel-never-seen-before","kind":"policy","status":"pass","summary":"A brand-new gate-claim- id that has never existed in this bundle before."}' \
|
|
4465
|
+
--timestamp "2026-07-05T09:11:30Z" >"$TMPDIR_EVAL/correction-novel.out" 2>"$TMPDIR_EVAL/correction-novel.err"; then
|
|
4466
|
+
_fail "new-mint REGRESSION: record-evidence accepted a NOVEL gate-claim-* id that never existed in the bundle — the existing-id exemption must not apply to brand-new mints"
|
|
4467
|
+
elif grep -qi 'reserved for record-gate-claim' "$TMPDIR_EVAL/correction-novel.out" "$TMPDIR_EVAL/correction-novel.err" \
|
|
4468
|
+
&& grep -q 'gate-claim-novel-never-seen-before' "$TMPDIR_EVAL/correction-novel.out" "$TMPDIR_EVAL/correction-novel.err"; then
|
|
4469
|
+
_pass "new-mint rejection: a NOVEL gate-claim-* id (not already present) is still rejected with the reserved-prefix message"
|
|
4470
|
+
else
|
|
4471
|
+
_fail "new-mint rejection message was unexpected: $(cat "$TMPDIR_EVAL/correction-novel.out" "$TMPDIR_EVAL/correction-novel.err")"
|
|
4472
|
+
fi
|
|
4473
|
+
|
|
4474
|
+
# ─── Eval 3 (#270 CRITICAL, iteration 6 — the reviewer's exact repro): superseding a REAL, ─────
|
|
4475
|
+
# STAMPED gate claim via record-evidence --check-json must DIE, and the stamp must survive
|
|
4476
|
+
# UNTOUCHED. The iteration-5 exemption treated "id already exists in the bundle" as sufficient
|
|
4477
|
+
# for supersession, which also exempted overwriting a live, properly-stamped record-gate-claim
|
|
4478
|
+
# output — silently destroying its metadata.gate_claim stamp, and the caller fully controls
|
|
4479
|
+
# check_kind so the replacement claim can be shaped to evade gateClaimShapeUnstampedId's detector
|
|
4480
|
+
# (which requires check_kind==="external"). This eval reproduces that exact attack end-to-end
|
|
4481
|
+
# against the CURRENT (narrowed) binary and asserts it now dies, with the stamp intact afterward.
|
|
4482
|
+
STAMPED_ROOT="$TMPDIR_EVAL/stamped-claim-root"
|
|
4483
|
+
STAMPED_SLUG="stamped-claim-270"
|
|
4484
|
+
STAMPED_DIR="$STAMPED_ROOT/$STAMPED_SLUG"
|
|
4485
|
+
mkdir -p "$STAMPED_ROOT"
|
|
4486
|
+
|
|
4487
|
+
flow_agents_node "$WRITER" ensure-session --artifact-root "$STAMPED_ROOT" --task-slug "$STAMPED_SLUG" --actor stamped-claim-actor --flow-id builder.build --title "Stamped gate-claim supersession attack session" --source-request "Regression: a REAL, stamped gate claim must never be supersedable via record-evidence/record-check/dogfood-pass." --summary "Seed session for the stamped-claim supersession negative." --criterion "A stamped gate claim id cannot be superseded by record-evidence" --timestamp "2026-07-05T09:12:30Z" >"$TMPDIR_EVAL/stamped-claim-seed.out" 2>"$TMPDIR_EVAL/stamped-claim-seed.err"
|
|
4488
|
+
|
|
4489
|
+
# Real record-gate-claim call — a genuine, properly-stamped gate claim (metadata.gate_claim is
|
|
4490
|
+
# stamped by buildTrustBundle itself, never hand-constructed).
|
|
4491
|
+
flow_agents_node "$WRITER" record-gate-claim "$STAMPED_DIR" --actor stamped-claim-actor --status pass --summary "Work item selected for stamped-claim supersession session" --timestamp "2026-07-05T09:12:35Z" >"$TMPDIR_EVAL/stamped-claim-gate.out" 2>"$TMPDIR_EVAL/stamped-claim-gate.err"
|
|
4492
|
+
|
|
4493
|
+
if [[ ! -f "$STAMPED_DIR/trust.bundle" ]]; then
|
|
4494
|
+
_fail "eval setup: record-gate-claim did not create trust.bundle for the stamped-claim session: $(cat "$TMPDIR_EVAL/stamped-claim-gate.out" "$TMPDIR_EVAL/stamped-claim-gate.err")"
|
|
4495
|
+
fi
|
|
4496
|
+
|
|
4497
|
+
STAMPED_CHECK_ID="gate-claim-selected-work"
|
|
4498
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/stamped-claim-preassert.err"
|
|
4499
|
+
import { readFileSync } from 'node:fs';
|
|
4500
|
+
const bundle = JSON.parse(readFileSync('${STAMPED_DIR}/trust.bundle', 'utf8'));
|
|
4501
|
+
const claim = bundle.claims.find((c) => c.subjectId && c.subjectId.endsWith('/${STAMPED_CHECK_ID}'));
|
|
4502
|
+
if (!claim) { process.stderr.write('eval setup: no claim found for ${STAMPED_CHECK_ID} after record-gate-claim\n'); process.exit(1); }
|
|
4503
|
+
if (!claim.metadata || !claim.metadata.gate_claim || typeof claim.metadata.gate_claim !== 'object') { process.stderr.write('eval setup: record-gate-claim did NOT stamp metadata.gate_claim — cannot exercise this eval: ' + JSON.stringify(claim.metadata) + '\n'); process.exit(1); }
|
|
4504
|
+
NODEOF
|
|
4505
|
+
then
|
|
4506
|
+
_pass "eval setup: record-gate-claim produced a REAL, stamped claim for ${STAMPED_CHECK_ID} (metadata.gate_claim present)"
|
|
4507
|
+
else
|
|
4508
|
+
_fail "eval setup: stamped-claim precondition failed: $(cat "$TMPDIR_EVAL/stamped-claim-preassert.err")"
|
|
4509
|
+
fi
|
|
4510
|
+
|
|
4511
|
+
cp "$STAMPED_DIR/trust.bundle" "$TMPDIR_EVAL/stamped-claim-trust-bundle.before"
|
|
4512
|
+
|
|
4513
|
+
# The attack: record-evidence --check-json with the SAME id, caller-chosen kind:"policy" — must
|
|
4514
|
+
# now DIE (the narrowed iteration-6 rule), not silently supersede and destroy the stamp.
|
|
4515
|
+
if flow_agents_node "$WRITER" record-evidence "$STAMPED_DIR" --verdict pass --check-json "{\"id\":\"${STAMPED_CHECK_ID}\",\"kind\":\"policy\",\"status\":\"pass\",\"summary\":\"Attack: attempt to supersede a live stamped gate claim via record-evidence.\"}" --timestamp "2026-07-05T09:12:45Z" >"$TMPDIR_EVAL/stamped-claim-attack.out" 2>"$TMPDIR_EVAL/stamped-claim-attack.err"; then
|
|
4516
|
+
_fail "#270 CRITICAL REGRESSION: record-evidence --check-json SUCCEEDED in superseding a live, stamped gate claim (${STAMPED_CHECK_ID}) — the narrowed guard did not fire"
|
|
4517
|
+
elif grep -qi 'live, properly-stamped gate claim' "$TMPDIR_EVAL/stamped-claim-attack.out" "$TMPDIR_EVAL/stamped-claim-attack.err" && grep -q "${STAMPED_CHECK_ID}" "$TMPDIR_EVAL/stamped-claim-attack.out" "$TMPDIR_EVAL/stamped-claim-attack.err" && grep -qi 'only record-gate-claim' "$TMPDIR_EVAL/stamped-claim-attack.out" "$TMPDIR_EVAL/stamped-claim-attack.err"; then
|
|
4518
|
+
_pass "#270 CRITICAL: record-evidence --check-json dies when attempting to supersede a live, stamped gate claim, naming the id and pointing at record-gate-claim as the only legitimate path"
|
|
4519
|
+
else
|
|
4520
|
+
_fail "#270 CRITICAL: stamped-claim supersession rejection message was unexpected: $(cat "$TMPDIR_EVAL/stamped-claim-attack.out" "$TMPDIR_EVAL/stamped-claim-attack.err")"
|
|
4521
|
+
fi
|
|
4522
|
+
|
|
4523
|
+
# Assert the bundle is BYTE-IDENTICAL to before the attack — the die must happen before ANY
|
|
4524
|
+
# write, so the stamp (and everything else) survives completely untouched.
|
|
4525
|
+
if cmp -s "$TMPDIR_EVAL/stamped-claim-trust-bundle.before" "$STAMPED_DIR/trust.bundle"; then
|
|
4526
|
+
_pass "#270 CRITICAL: trust.bundle is byte-identical after the rejected attack — the die is fail-closed BEFORE any write"
|
|
4527
|
+
else
|
|
4528
|
+
_fail "#270 CRITICAL REGRESSION: trust.bundle changed after the rejected supersession attempt — the die is not fail-closed before writing"
|
|
4529
|
+
fi
|
|
4530
|
+
|
|
4531
|
+
# Semantic re-assertion: metadata.gate_claim is still present and unchanged (belt-and-suspenders
|
|
4532
|
+
# on top of the byte-identical check above).
|
|
4533
|
+
if node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/stamped-claim-postassert.err"
|
|
4534
|
+
import { readFileSync } from 'node:fs';
|
|
4535
|
+
const before = JSON.parse(readFileSync('${TMPDIR_EVAL}/stamped-claim-trust-bundle.before', 'utf8'));
|
|
4536
|
+
const after = JSON.parse(readFileSync('${STAMPED_DIR}/trust.bundle', 'utf8'));
|
|
4537
|
+
const claimBefore = before.claims.find((c) => c.subjectId && c.subjectId.endsWith('/${STAMPED_CHECK_ID}'));
|
|
4538
|
+
const claimAfter = after.claims.find((c) => c.subjectId && c.subjectId.endsWith('/${STAMPED_CHECK_ID}'));
|
|
4539
|
+
if (!claimAfter || !claimAfter.metadata || !claimAfter.metadata.gate_claim) { process.stderr.write('metadata.gate_claim is MISSING after the attack attempt — the stamp was destroyed\n'); process.exit(1); }
|
|
4540
|
+
if (JSON.stringify(claimAfter.metadata.gate_claim) !== JSON.stringify(claimBefore.metadata.gate_claim)) { process.stderr.write('metadata.gate_claim CHANGED after the attack attempt: before=' + JSON.stringify(claimBefore.metadata.gate_claim) + ' after=' + JSON.stringify(claimAfter.metadata.gate_claim) + '\n'); process.exit(1); }
|
|
4541
|
+
if (claimAfter.metadata.check_kind === 'policy') { process.stderr.write('claim.metadata.check_kind was overwritten to policy — the attack partially succeeded\n'); process.exit(1); }
|
|
4542
|
+
NODEOF
|
|
4543
|
+
then
|
|
4544
|
+
_pass "#270 CRITICAL: metadata.gate_claim stamp is semantically intact (present, unchanged) after the rejected attack"
|
|
4545
|
+
else
|
|
4546
|
+
_fail "#270 CRITICAL: stamp-intact semantic assertion failed: $(cat "$TMPDIR_EVAL/stamped-claim-postassert.err")"
|
|
4547
|
+
fi
|
|
4548
|
+
|
|
4549
|
+
# ─── Mutation test (existing-id exists-check, iteration 5, needle updated iteration 6): neuter ──
|
|
4550
|
+
# the EXISTING-ID LOOKUP in a scratch copy of the compiled build/ output (force
|
|
4551
|
+
# existingHasStamp to always be `undefined`, i.e. always treated as "not already present") — the
|
|
4552
|
+
# correction eval above must go RED (the mis-recorded, unstamped wedge id becomes uncorrectable
|
|
4553
|
+
# again, since `undefined` now routes to the new-mint die), proving the eval actually exercises
|
|
4554
|
+
# the exists-check, not passing vacuously.
|
|
4555
|
+
EXISTS_CHECK_SCRATCH="$TMPDIR_EVAL/exists-check-mutation-scratch"
|
|
4556
|
+
mkdir -p "$EXISTS_CHECK_SCRATCH"
|
|
4557
|
+
|
|
4558
|
+
if [[ -f "$DIST_SIDECAR" ]]; then
|
|
4559
|
+
cp "$DIST_SIDECAR" "$EXISTS_CHECK_SCRATCH/workflow-sidecar.orig.js"
|
|
4560
|
+
node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/exists-check-neuter-allowance.err"
|
|
4561
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4562
|
+
const file = '${EXISTS_CHECK_SCRATCH}/workflow-sidecar.orig.js';
|
|
4563
|
+
let src = readFileSync(file, 'utf8');
|
|
4564
|
+
const needle = 'const existingHasStamp = existingCheckStampById?.get(check.id);';
|
|
4565
|
+
if (!src.includes(needle)) { process.stderr.write('mutation: existing-id stamp-map lookup text not found — source pattern drifted, cannot mutation-test\n'); process.exit(1); }
|
|
4566
|
+
src = src.split(needle).join('const existingHasStamp = undefined;');
|
|
4567
|
+
writeFileSync('${EXISTS_CHECK_SCRATCH}/workflow-sidecar.mutated-neuter-allowance.js', src);
|
|
4568
|
+
NODEOF
|
|
4569
|
+
|
|
4570
|
+
if [[ -s "$TMPDIR_EVAL/exists-check-neuter-allowance.err" ]]; then
|
|
4571
|
+
_fail "mutation-test setup failed (existing-id allowance source pattern did not match compiled output): $(cat "$TMPDIR_EVAL/exists-check-neuter-allowance.err")"
|
|
4572
|
+
else
|
|
4573
|
+
if node --check "$EXISTS_CHECK_SCRATCH/workflow-sidecar.mutated-neuter-allowance.js" 2>"$TMPDIR_EVAL/exists-check-neuter-allowance-syntax.err"; then
|
|
4574
|
+
cp "$EXISTS_CHECK_SCRATCH/workflow-sidecar.mutated-neuter-allowance.js" "$DIST_SIDECAR"
|
|
4575
|
+
|
|
4576
|
+
MUTATION_CORRECTION_ROOT="$TMPDIR_EVAL/mutation-correction-root"
|
|
4577
|
+
cp -r "$CORRECTION_ROOT" "$MUTATION_CORRECTION_ROOT"
|
|
4578
|
+
MUTATION_CORRECTION_DIR="$MUTATION_CORRECTION_ROOT/$CORRECTION_SLUG"
|
|
4579
|
+
|
|
4580
|
+
if flow_agents_node "$WRITER" record-evidence "$MUTATION_CORRECTION_DIR" \
|
|
4581
|
+
--verdict pass \
|
|
4582
|
+
--check-json "{\"id\":\"${CORRECTION_CHECK_ID}\",\"kind\":\"policy\",\"status\":\"pass\",\"summary\":\"Correction attempt with the existing-id allowance neutered — must go red.\"}" \
|
|
4583
|
+
--timestamp "2026-07-05T09:11:45Z" >"$TMPDIR_EVAL/exists-check-neuter-allowance-run.out" 2>"$TMPDIR_EVAL/exists-check-neuter-allowance-run.err"; then
|
|
4584
|
+
_fail "mutation-test: with the existing-id allowance neutered, the correction still SUCCEEDED — the eval may not be exercising the intended exists-check"
|
|
4585
|
+
else
|
|
4586
|
+
_pass "mutation-test: with the existing-id allowance neutered, the SAME correction that succeeded above now FAILS (eval correctly goes red without the exists-check, proving it exercises the exemption)"
|
|
4587
|
+
fi
|
|
4588
|
+
else
|
|
4589
|
+
_fail "mutation-test setup: mutated workflow-sidecar.js (existing-id allowance) failed a syntax check, refusing to run it: $(cat "$TMPDIR_EVAL/exists-check-neuter-allowance-syntax.err")"
|
|
4590
|
+
fi
|
|
4591
|
+
|
|
4592
|
+
# Restore the real compiled guard immediately.
|
|
4593
|
+
cp "$EXISTS_CHECK_SCRATCH/workflow-sidecar.orig.js" "$DIST_SIDECAR"
|
|
4594
|
+
fi
|
|
4595
|
+
else
|
|
4596
|
+
_fail "mutation-test setup: could not locate the compiled build/src/cli/workflow-sidecar.js to mutate for the existing-id allowance (ran 'npm run build' first?)"
|
|
4597
|
+
fi
|
|
4598
|
+
|
|
4599
|
+
# ─── Mutation test (new-mint rejection, iteration 5): neuter the reserved-prefix rejection ─────
|
|
4600
|
+
# ITSELF (force the whole `if` to false, same technique as the original reserved-prefix mutation
|
|
4601
|
+
# test below but re-run here against the NOVEL-id eval) — Eval 2 (new-mint rejection) must go
|
|
4602
|
+
# RED, proving that eval still exercises the underlying rejection and is not vacuously green
|
|
4603
|
+
# just because the existing-id exemption happens to be present.
|
|
4604
|
+
if [[ -f "$DIST_SIDECAR" ]]; then
|
|
4605
|
+
cp "$DIST_SIDECAR" "$EXISTS_CHECK_SCRATCH/workflow-sidecar.orig2.js"
|
|
4606
|
+
node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/exists-check-neuter-rejection.err"
|
|
4607
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4608
|
+
const file = '${EXISTS_CHECK_SCRATCH}/workflow-sidecar.orig2.js';
|
|
4609
|
+
let src = readFileSync(file, 'utf8');
|
|
4610
|
+
const needle = 'if (!allowGateClaimPrefix && typeof check.id === "string" && check.id.startsWith("gate-claim-")) {';
|
|
4611
|
+
if (!src.includes(needle)) { process.stderr.write('mutation: reserved-prefix guard text not found — source pattern drifted, cannot mutation-test\n'); process.exit(1); }
|
|
4612
|
+
src = src.split(needle).join('if (false) {');
|
|
4613
|
+
writeFileSync('${EXISTS_CHECK_SCRATCH}/workflow-sidecar.mutated-neuter-rejection.js', src);
|
|
4614
|
+
NODEOF
|
|
4615
|
+
|
|
4616
|
+
if [[ -s "$TMPDIR_EVAL/exists-check-neuter-rejection.err" ]]; then
|
|
4617
|
+
_fail "mutation-test setup failed (reserved-prefix guard source pattern did not match compiled output for the new-mint mutation test): $(cat "$TMPDIR_EVAL/exists-check-neuter-rejection.err")"
|
|
4618
|
+
else
|
|
4619
|
+
if node --check "$EXISTS_CHECK_SCRATCH/workflow-sidecar.mutated-neuter-rejection.js" 2>"$TMPDIR_EVAL/exists-check-neuter-rejection-syntax.err"; then
|
|
4620
|
+
cp "$EXISTS_CHECK_SCRATCH/workflow-sidecar.mutated-neuter-rejection.js" "$DIST_SIDECAR"
|
|
4621
|
+
|
|
4622
|
+
if flow_agents_node "$WRITER" record-evidence "$CORRECTION_DIR" \
|
|
4623
|
+
--verdict pass \
|
|
4624
|
+
--check-json '{"id":"gate-claim-novel-mutation-red-check","kind":"policy","status":"pass","summary":"New-mint attempt with the reserved-prefix rejection neutered — must go red."}' \
|
|
4625
|
+
--timestamp "2026-07-05T09:12:00Z" >"$TMPDIR_EVAL/exists-check-neuter-rejection-run.out" 2>"$TMPDIR_EVAL/exists-check-neuter-rejection-run.err"; then
|
|
4626
|
+
_pass "mutation-test: with the reserved-prefix rejection neutered, a NOVEL gate-claim-* id now SUCCEEDS (eval correctly goes red without the rejection, proving Eval 2 exercises it)"
|
|
4627
|
+
else
|
|
4628
|
+
_fail "mutation-test: NOVEL gate-claim-* id still rejected even with the rejection neutered — Eval 2 may not be exercising the intended rejection: $(cat "$TMPDIR_EVAL/exists-check-neuter-rejection-run.out" "$TMPDIR_EVAL/exists-check-neuter-rejection-run.err")"
|
|
4629
|
+
fi
|
|
4630
|
+
else
|
|
4631
|
+
_fail "mutation-test setup: mutated workflow-sidecar.js (reserved-prefix rejection) failed a syntax check, refusing to run it: $(cat "$TMPDIR_EVAL/exists-check-neuter-rejection-syntax.err")"
|
|
4632
|
+
fi
|
|
4633
|
+
|
|
4634
|
+
# Restore the real compiled guard immediately, and re-run BOTH evals to confirm green again.
|
|
4635
|
+
cp "$EXISTS_CHECK_SCRATCH/workflow-sidecar.orig2.js" "$DIST_SIDECAR"
|
|
4636
|
+
if flow_agents_node "$WRITER" record-evidence "$CORRECTION_DIR" \
|
|
4637
|
+
--verdict pass \
|
|
4638
|
+
--check-json '{"id":"gate-claim-novel-restore-check","kind":"policy","status":"pass","summary":"Restore check: new-mint rejection must be back after mutation-test cleanup."}' \
|
|
4639
|
+
--timestamp "2026-07-05T09:12:15Z" >"$TMPDIR_EVAL/exists-check-restore-novel.out" 2>"$TMPDIR_EVAL/exists-check-restore-novel.err"; then
|
|
4640
|
+
_fail "mutation-test cleanup REGRESSION: the new-mint reserved-prefix rejection did not come back after restoring the original compiled file"
|
|
4641
|
+
else
|
|
4642
|
+
_pass "mutation-test cleanup: the real compiled new-mint rejection is restored and rejects a caller-supplied 'gate-claim-' id again"
|
|
4643
|
+
fi
|
|
4644
|
+
fi
|
|
4645
|
+
else
|
|
4646
|
+
_fail "mutation-test setup: could not locate the compiled build/src/cli/workflow-sidecar.js to mutate for the new-mint rejection (ran 'npm run build' first?)"
|
|
4647
|
+
fi
|
|
4648
|
+
|
|
4649
|
+
# ─── Mutation test (#270 CRITICAL, iteration 6): neuter the STAMPED-CLAIM GUARD itself (force ──
|
|
4650
|
+
# `existingHasStamp === true` to never be treated as stamped) in a scratch copy of the compiled
|
|
4651
|
+
# build/ output — the Eval 3 negative above (stamped-claim supersession must die) must go RED
|
|
4652
|
+
# (the attack SUCCEEDS against the mutated binary), proving Eval 3 actually exercises the new
|
|
4653
|
+
# narrowed guard and is not vacuously green for an unrelated reason.
|
|
4654
|
+
STAMPED_GUARD_SCRATCH="$TMPDIR_EVAL/stamped-guard-mutation-scratch"
|
|
4655
|
+
mkdir -p "$STAMPED_GUARD_SCRATCH"
|
|
4656
|
+
|
|
4657
|
+
if [[ -f "$DIST_SIDECAR" ]]; then
|
|
4658
|
+
cp "$DIST_SIDECAR" "$STAMPED_GUARD_SCRATCH/workflow-sidecar.orig.js"
|
|
4659
|
+
node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/stamped-guard-neuter.err"
|
|
4660
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4661
|
+
const file = '${STAMPED_GUARD_SCRATCH}/workflow-sidecar.orig.js';
|
|
4662
|
+
let src = readFileSync(file, 'utf8');
|
|
4663
|
+
const needle = 'if (existingHasStamp === true)';
|
|
4664
|
+
if (!src.includes(needle)) { process.stderr.write('mutation: stamped-claim guard text not found — source pattern drifted, cannot mutation-test\n'); process.exit(1); }
|
|
4665
|
+
src = src.split(needle).join('if (false)');
|
|
4666
|
+
writeFileSync('${STAMPED_GUARD_SCRATCH}/workflow-sidecar.mutated-neuter-stamp-guard.js', src);
|
|
4667
|
+
NODEOF
|
|
4668
|
+
|
|
4669
|
+
if [[ -s "$TMPDIR_EVAL/stamped-guard-neuter.err" ]]; then
|
|
4670
|
+
_fail "mutation-test setup failed (stamped-claim guard source pattern did not match compiled output): $(cat "$TMPDIR_EVAL/stamped-guard-neuter.err")"
|
|
4671
|
+
else
|
|
4672
|
+
if node --check "$STAMPED_GUARD_SCRATCH/workflow-sidecar.mutated-neuter-stamp-guard.js" 2>"$TMPDIR_EVAL/stamped-guard-neuter-syntax.err"; then
|
|
4673
|
+
cp "$STAMPED_GUARD_SCRATCH/workflow-sidecar.mutated-neuter-stamp-guard.js" "$DIST_SIDECAR"
|
|
4674
|
+
|
|
4675
|
+
MUTATION_STAMPED_ROOT="$TMPDIR_EVAL/mutation-stamped-root"
|
|
4676
|
+
cp -r "$STAMPED_ROOT" "$MUTATION_STAMPED_ROOT"
|
|
4677
|
+
MUTATION_STAMPED_DIR="$MUTATION_STAMPED_ROOT/$STAMPED_SLUG"
|
|
4678
|
+
|
|
4679
|
+
if flow_agents_node "$WRITER" record-evidence "$MUTATION_STAMPED_DIR" --verdict pass --check-json "{\"id\":\"${STAMPED_CHECK_ID}\",\"kind\":\"policy\",\"status\":\"pass\",\"summary\":\"Mutation-test: stamped-claim guard disabled, attack must now succeed.\"}" --timestamp "2026-07-05T09:13:00Z" >"$TMPDIR_EVAL/stamped-guard-neuter-run.out" 2>"$TMPDIR_EVAL/stamped-guard-neuter-run.err"; then
|
|
4680
|
+
_pass "mutation-test: with the stamped-claim guard neutered, superseding a live stamped gate claim now SUCCEEDS (eval correctly goes red without the guard, proving Eval 3 exercises it)"
|
|
4681
|
+
else
|
|
4682
|
+
_fail "mutation-test: stamped-claim supersession attack still rejected even with the guard neutered — Eval 3 may not be exercising the intended guard: $(cat "$TMPDIR_EVAL/stamped-guard-neuter-run.out" "$TMPDIR_EVAL/stamped-guard-neuter-run.err")"
|
|
4683
|
+
fi
|
|
4684
|
+
else
|
|
4685
|
+
_fail "mutation-test setup: mutated workflow-sidecar.js (stamped-claim guard) failed a syntax check, refusing to run it: $(cat "$TMPDIR_EVAL/stamped-guard-neuter-syntax.err")"
|
|
4686
|
+
fi
|
|
4687
|
+
|
|
4688
|
+
# Restore the real compiled guard immediately, and re-run the negative to confirm green again.
|
|
4689
|
+
cp "$STAMPED_GUARD_SCRATCH/workflow-sidecar.orig.js" "$DIST_SIDECAR"
|
|
4690
|
+
if flow_agents_node "$WRITER" record-evidence "$STAMPED_DIR" --verdict pass --check-json "{\"id\":\"${STAMPED_CHECK_ID}\",\"kind\":\"policy\",\"status\":\"pass\",\"summary\":\"Restore check: stamped-claim guard must be back after mutation-test cleanup.\"}" --timestamp "2026-07-05T09:13:15Z" >"$TMPDIR_EVAL/stamped-guard-restore.out" 2>"$TMPDIR_EVAL/stamped-guard-restore.err"; then
|
|
4691
|
+
_fail "mutation-test cleanup REGRESSION: the stamped-claim guard did not come back after restoring the original compiled file"
|
|
4692
|
+
else
|
|
4693
|
+
_pass "mutation-test cleanup: the real compiled stamped-claim guard is restored and rejects supersession of a live stamped gate claim again"
|
|
4694
|
+
fi
|
|
4695
|
+
fi
|
|
4696
|
+
else
|
|
4697
|
+
_fail "mutation-test setup: could not locate the compiled build/src/cli/workflow-sidecar.js to mutate for the stamped-claim guard (ran 'npm run build' first?)"
|
|
4698
|
+
fi
|
|
4699
|
+
|
|
4700
|
+
# ─── Mutation test (forged-stamp guard): temporarily disable assertStampedGateClaimValid's ────
|
|
4701
|
+
# die() in a SCRATCH COPY of the compiled build/ output, confirm the forged-stamp fixture above
|
|
4702
|
+
# now SUCCEEDS against that mutated binary (eval "goes red" — i.e. would no longer catch the
|
|
4703
|
+
# defect — without the guard), then restore the original compiled file immediately. Proves the
|
|
4704
|
+
# eval is actually exercising this guard, not passing vacuously for an unrelated reason.
|
|
4705
|
+
MUTATION_SCRATCH="$TMPDIR_EVAL/mutation-scratch"
|
|
4706
|
+
mkdir -p "$MUTATION_SCRATCH"
|
|
4707
|
+
|
|
4708
|
+
if [[ -f "$DIST_SIDECAR" ]]; then
|
|
4709
|
+
cp "$DIST_SIDECAR" "$MUTATION_SCRATCH/workflow-sidecar.orig.js"
|
|
4710
|
+
node --input-type=module <<NODEOF 2>"$TMPDIR_EVAL/mutation-patch.err"
|
|
4711
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
4712
|
+
const file = '${MUTATION_SCRATCH}/workflow-sidecar.orig.js';
|
|
4713
|
+
let src = readFileSync(file, 'utf8');
|
|
4714
|
+
// Neuter the stamp-tuple-mismatch check (assertStampedGateClaimValid's SECOND die call — the
|
|
4715
|
+
// mismatch guard, not the no-flow-definition guard) by short-circuiting 'const match = ...' to
|
|
4716
|
+
// an unconditional 'const match = true;' via a simple, exact single-line string replacement (no
|
|
4717
|
+
// regex, no unbalanced parens) so the mutated file stays syntactically valid.
|
|
4718
|
+
const matchNeedle = 'const match = allExpects.some((entry) => entry.gateExpects.some((exp) => exp.id === stamp.expectationId\n && exp.bundle_claim.claimType === stamp.claimType\n && exp.bundle_claim.subjectType === stamp.subjectType\n && (stamp.stepId === null || stamp.stepId === entry.stepId)));';
|
|
4719
|
+
if (!src.includes(matchNeedle)) { process.stderr.write('mutation: assertStampedGateClaimValid match-check text not found — source pattern drifted, cannot mutation-test\n'); process.exit(1); }
|
|
4720
|
+
src = src.split(matchNeedle).join('const match = true;');
|
|
4721
|
+
// Neuter the pre-cluster-270 unstamped-guard's if() condition the same way — exact single-line
|
|
4722
|
+
// string replacement, no regex.
|
|
4723
|
+
const unstampedNeedle = 'if (gateClaimShapeUnstampedClaimId) {';
|
|
4724
|
+
if (!src.includes(unstampedNeedle)) { process.stderr.write('mutation: gateClaimShapeUnstampedClaimId guard text not found — source pattern drifted\n'); process.exit(1); }
|
|
4725
|
+
src = src.split(unstampedNeedle).join('if (false && gateClaimShapeUnstampedClaimId) {');
|
|
4726
|
+
writeFileSync('${MUTATION_SCRATCH}/workflow-sidecar.mutated.js', src);
|
|
4727
|
+
NODEOF
|
|
4728
|
+
|
|
4729
|
+
if [[ -s "$TMPDIR_EVAL/mutation-patch.err" ]]; then
|
|
4730
|
+
_fail "mutation-test setup failed (guard source pattern did not match compiled output): $(cat "$TMPDIR_EVAL/mutation-patch.err")"
|
|
4731
|
+
else
|
|
4732
|
+
# Syntax-check the mutated file before swapping it in — a broken mutation must not corrupt
|
|
4733
|
+
# the eval run itself.
|
|
4734
|
+
if node --check "$MUTATION_SCRATCH/workflow-sidecar.mutated.js" 2>"$TMPDIR_EVAL/mutation-syntax.err"; then
|
|
4735
|
+
cp "$MUTATION_SCRATCH/workflow-sidecar.mutated.js" "$DIST_SIDECAR"
|
|
4736
|
+
# Copy the WHOLE root again (see the FORGED_ROOT/PRECLUSTER_ROOT comment above) — same
|
|
4737
|
+
# current.json/current/<actor>.json resolution requirement applies here.
|
|
4738
|
+
MUTATION_FORGED_ROOT="$TMPDIR_EVAL/mutation-forged-root"
|
|
4739
|
+
cp -r "$FORGED_ROOT" "$MUTATION_FORGED_ROOT"
|
|
4740
|
+
MUTATION_FORGED_DIR="$MUTATION_FORGED_ROOT/$COMPOSE_SLUG"
|
|
4741
|
+
MUTATION_PRECLUSTER_ROOT="$TMPDIR_EVAL/mutation-precluster-root"
|
|
4742
|
+
cp -r "$PRECLUSTER_ROOT" "$MUTATION_PRECLUSTER_ROOT"
|
|
4743
|
+
MUTATION_PRECLUSTER_DIR="$MUTATION_PRECLUSTER_ROOT/$COMPOSE_SLUG"
|
|
4744
|
+
|
|
4745
|
+
if flow_agents_node "$WRITER" record-critique "$MUTATION_FORGED_DIR" \
|
|
4746
|
+
--id mutation-forged-review --reviewer mutation-tester --verdict pass \
|
|
4747
|
+
--summary "Mutation-test: forged-stamp guard disabled, must go red" \
|
|
4748
|
+
--timestamp "2026-07-05T09:14:00Z" >"$TMPDIR_EVAL/mutation-forged.out" 2>"$TMPDIR_EVAL/mutation-forged.err"; then
|
|
4749
|
+
_pass "mutation-test: with the stamp-mismatch guard neutered, the forged-stamp fixture now SUCCEEDS (eval correctly goes red without the guard, proving the guard is what the eval exercises)"
|
|
4750
|
+
else
|
|
4751
|
+
_fail "mutation-test: forged-stamp fixture still failed even with the guard neutered — the eval may not be exercising the intended guard: $(cat "$TMPDIR_EVAL/mutation-forged.out" "$TMPDIR_EVAL/mutation-forged.err")"
|
|
4752
|
+
fi
|
|
4753
|
+
|
|
4754
|
+
if flow_agents_node "$WRITER" record-critique "$MUTATION_PRECLUSTER_DIR" \
|
|
4755
|
+
--id mutation-precluster-review --reviewer mutation-tester --verdict pass \
|
|
4756
|
+
--summary "Mutation-test: pre-cluster-270 guard disabled, must go red" \
|
|
4757
|
+
--timestamp "2026-07-05T09:14:30Z" >"$TMPDIR_EVAL/mutation-precluster.out" 2>"$TMPDIR_EVAL/mutation-precluster.err"; then
|
|
4758
|
+
_pass "mutation-test: with the pre-cluster-270 unstamped guard neutered, the missing-stamp fixture now SUCCEEDS (eval correctly goes red without the guard)"
|
|
4759
|
+
else
|
|
4760
|
+
_fail "mutation-test: pre-cluster missing-stamp fixture still failed even with the guard neutered: $(cat "$TMPDIR_EVAL/mutation-precluster.out" "$TMPDIR_EVAL/mutation-precluster.err")"
|
|
4761
|
+
fi
|
|
4762
|
+
else
|
|
4763
|
+
_fail "mutation-test setup: mutated workflow-sidecar.js failed a syntax check, refusing to run it: $(cat "$TMPDIR_EVAL/mutation-syntax.err")"
|
|
4764
|
+
fi
|
|
4765
|
+
|
|
4766
|
+
# Restore the real compiled guard immediately — never leave the mutated binary in place, and
|
|
4767
|
+
# re-run BOTH negatives to confirm the restored binary rejects them again (guard is back).
|
|
4768
|
+
cp "$MUTATION_SCRATCH/workflow-sidecar.orig.js" "$DIST_SIDECAR"
|
|
4769
|
+
RESTORE_FORGED_ROOT="$TMPDIR_EVAL/restore-check-forged-root"
|
|
4770
|
+
cp -r "$FORGED_ROOT" "$RESTORE_FORGED_ROOT"
|
|
4771
|
+
RESTORE_FORGED_DIR="$RESTORE_FORGED_ROOT/$COMPOSE_SLUG"
|
|
4772
|
+
if flow_agents_node "$WRITER" record-critique "$RESTORE_FORGED_DIR" \
|
|
4773
|
+
--id restore-check-review --reviewer restore-check-tester --verdict pass \
|
|
4774
|
+
--summary "Restore check: forged-stamp guard must be back after mutation-test cleanup" \
|
|
4775
|
+
--timestamp "2026-07-05T09:15:00Z" >"$TMPDIR_EVAL/mutation-restore.out" 2>"$TMPDIR_EVAL/mutation-restore.err"; then
|
|
4776
|
+
_fail "mutation-test cleanup REGRESSION: the forged-stamp guard did not come back after restoring the original compiled file"
|
|
4777
|
+
else
|
|
4778
|
+
_pass "mutation-test cleanup: the real compiled guard is restored and rejects the (already-forged) fixture again"
|
|
4779
|
+
fi
|
|
4780
|
+
fi
|
|
4781
|
+
else
|
|
4782
|
+
_fail "mutation-test setup: could not locate the compiled build/src/cli/workflow-sidecar.js to mutate (ran 'npm run build' first?)"
|
|
4783
|
+
fi
|
|
4784
|
+
|
|
4785
|
+
|
|
3391
4786
|
|
|
3392
4787
|
if [[ "$errors" -eq 0 ]]; then
|
|
3393
4788
|
echo "Workflow sidecar writer integration passed."
|