@kontourai/flow-agents 3.3.0 → 3.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/add-to-project.yml +15 -0
- package/.github/workflows/ci.yml +161 -0
- package/CHANGELOG.md +48 -0
- package/CONTEXT.md +5 -1
- package/README.md +19 -8
- package/build/src/builder-flow-run-adapter.d.ts +80 -0
- package/build/src/builder-flow-run-adapter.js +241 -0
- package/build/src/builder-flow-runtime.d.ts +16 -0
- package/build/src/builder-flow-runtime.js +290 -0
- package/build/src/cli/builder-run.d.ts +1 -0
- package/build/src/cli/builder-run.js +27 -0
- package/build/src/cli/effective-backlog-settings.js +70 -2
- package/build/src/cli/init.d.ts +34 -0
- package/build/src/cli/init.js +341 -61
- package/build/src/cli/kit.js +55 -12
- package/build/src/cli/pull-work-provider.js +346 -5
- package/build/src/cli/skill-drift-check.d.ts +1 -0
- package/build/src/cli/skill-drift-check.js +165 -0
- package/build/src/cli/telemetry-doctor.d.ts +37 -0
- package/build/src/cli/telemetry-doctor.js +53 -6
- package/build/src/cli/validate-hook-influence.js +37 -7
- package/build/src/cli/workflow-sidecar.d.ts +93 -8
- package/build/src/cli/workflow-sidecar.js +1175 -158
- package/build/src/cli.js +5 -0
- package/build/src/flow-kit/validate.d.ts +54 -34
- package/build/src/flow-kit/validate.js +237 -26
- package/build/src/index.d.ts +2 -0
- package/build/src/index.js +1 -0
- package/build/src/lib/console-connect-options.d.ts +97 -0
- package/build/src/lib/console-connect-options.js +199 -0
- package/build/src/lib/console-telemetry-validate.d.ts +49 -0
- package/build/src/lib/console-telemetry-validate.js +91 -0
- package/build/src/lib/flow-resolver.d.ts +56 -3
- package/build/src/lib/flow-resolver.js +151 -11
- package/build/src/lib/fs.d.ts +17 -0
- package/build/src/lib/fs.js +172 -0
- package/build/src/lib/local-artifact-root.d.ts +44 -1
- package/build/src/lib/local-artifact-root.js +131 -3
- package/build/src/runtime-adapters.d.ts +39 -3
- package/build/src/runtime-adapters.js +77 -31
- package/build/src/tools/build-universal-bundles.js +40 -2
- package/build/src/tools/codex-agent-routing.d.ts +2 -0
- package/build/src/tools/codex-agent-routing.js +49 -0
- package/build/src/tools/generate-context-map.js +1 -0
- package/build/src/tools/validate-source-tree.js +27 -1
- package/context/scripts/hooks/lib/kit-catalog.js +235 -0
- package/context/scripts/hooks/lib/runnable-command.js +177 -0
- package/context/scripts/hooks/stop-goal-fit.js +278 -48
- package/context/scripts/hooks/workflow-steering.js +121 -21
- package/context/scripts/package.json +3 -0
- package/context/scripts/telemetry/install-console-config.sh +25 -4
- package/context/scripts/telemetry/lib/config.sh +102 -12
- package/context/scripts/telemetry/lib/pricing.sh +50 -0
- package/context/scripts/telemetry/lib/session.sh +3 -0
- package/context/scripts/telemetry/lib/transport.sh +87 -0
- package/context/scripts/telemetry/lib/usage.sh +205 -4
- package/context/scripts/telemetry/telemetry.conf +6 -0
- package/context/scripts/telemetry/telemetry.sh +48 -0
- package/context/settings/workspace-backlog-provider-settings.example.json +48 -0
- package/docs/agent-usage-feedback-loop.md +35 -0
- package/docs/architecture-engine-and-kits.md +110 -0
- package/docs/context-map.md +2 -0
- package/docs/decisions/embeddable-engine.md +152 -0
- package/docs/decisions/index.md +3 -1
- package/docs/decisions/trust-ledger-retention.md +88 -0
- package/docs/decisions/workflow-enforcement.md +31 -9
- package/docs/fixture-ownership.md +3 -0
- package/docs/implementing-trust-reconciliation.md +129 -0
- package/docs/index.md +19 -9
- package/docs/integrations/flow-agents-console.md +167 -0
- package/docs/kit-authoring-guide.md +52 -21
- package/docs/spec/builder-flow-runtime.md +80 -0
- package/docs/spec/runtime-hook-surface.md +45 -1
- package/docs/specs/economics-record-contract.md +270 -0
- package/docs/specs/harness-capability-matrix.md +74 -0
- package/docs/specs/learning-review-proposals-contract.md +340 -0
- package/docs/specs/routing-efficiency-review.md +59 -0
- package/docs/verifiable-trust.md +74 -25
- package/docs/workflow-usage-guide.md +10 -0
- package/evals/acceptance/prove-capture-teeth.sh +132 -0
- package/evals/ci/antigaming-suite.sh +1 -0
- package/evals/ci/run-baseline.sh +72 -4
- package/evals/fixtures/economics/acceptance.json +12 -0
- package/evals/fixtures/economics/agents/tool-worker-1/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-2/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-3/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-4/events.jsonl +1 -0
- package/evals/fixtures/economics/agents/tool-worker-5/events.jsonl +2 -0
- package/evals/fixtures/economics/critique.json +22 -0
- package/evals/fixtures/economics/expected-record.json +71 -0
- package/evals/fixtures/economics/session-usage-event.json +1 -0
- package/evals/fixtures/economics/state.json +11 -0
- package/evals/fixtures/economics/transcript.jsonl +3 -0
- package/evals/fixtures/hook-influence/cases.json +7 -7
- package/evals/fixtures/learning-review-proposals/balanced/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/economics.jsonl +5 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-1/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-2/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-3/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-4/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-5/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/expected-aggregates.json +30 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/expected-aggregates.md +66 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-1/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-1/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-2/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-2/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-3/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-3/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-4/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-4/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-5/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-6/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/repeat-window/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/under-threshold/economics.jsonl +3 -0
- package/evals/fixtures/telemetry/usage-transcript-sample.jsonl +4 -0
- package/evals/fixtures/trust-reconcile-exploits/mcp-degrade.json +42 -0
- package/evals/integration/test_builder_entry_enforcement.sh +241 -0
- package/evals/integration/test_builder_step_producers.sh +18 -10
- package/evals/integration/test_bundle_install.sh +172 -0
- package/evals/integration/test_console_tenant_isolation.sh +167 -0
- package/evals/integration/test_critique_supersession_roundtrip.sh +4 -1
- package/evals/integration/test_dual_emit_flow_step.sh +10 -4
- package/evals/integration/test_economics_record.sh +674 -0
- package/evals/integration/test_effective_backlog_settings.sh +1 -1
- package/evals/integration/test_evidence_capture_hook.sh +17 -2
- package/evals/integration/test_exemption_usage_review.sh +198 -0
- package/evals/integration/test_fixture_retirement_audit.sh +2 -2
- package/evals/integration/test_flow_kit_install_git.sh +83 -0
- package/evals/integration/test_flowdef_session_activation.sh +0 -1
- package/evals/integration/test_flowdef_session_history_preservation.sh +13 -3
- package/evals/integration/test_gate_lockdown.sh +7 -0
- package/evals/integration/test_gate_review_inquiry_records.sh +9 -1
- package/evals/integration/test_goal_fit_hook.sh +2031 -0
- package/evals/integration/test_hook_category_behaviors.sh +8 -1
- package/evals/integration/test_hook_influence_cases.sh +25 -1
- package/evals/integration/test_install_merge.sh +227 -2
- package/evals/integration/test_kit_conformance_levels.sh +6 -6
- package/evals/integration/test_learning_review_proposals.sh +329 -0
- package/evals/integration/test_liveness_conflict_injection.sh +26 -22
- package/evals/integration/test_liveness_console_relay.sh +166 -0
- package/evals/integration/test_liveness_heartbeat.sh +17 -17
- package/evals/integration/test_liveness_worktree_root.sh +575 -0
- package/evals/integration/test_phase_map_and_gate_claim.sh +6 -1
- package/evals/integration/test_publish_delivery.sh +331 -1
- package/evals/integration/test_pull_work_board.sh +200 -0
- package/evals/integration/test_pull_work_provider.sh +1 -1
- package/evals/integration/test_record_check.sh +378 -0
- package/evals/integration/test_routing_efficiency.sh +71 -0
- package/evals/integration/test_runtime_adapter_activation.sh +28 -0
- package/evals/integration/test_session_resume_roundtrip.sh +16 -19
- package/evals/integration/test_skill_drift_check.sh +870 -0
- package/evals/integration/test_telemetry.sh +445 -0
- package/evals/integration/test_telemetry_doctor.sh +66 -0
- package/evals/integration/test_telemetry_usage_pipeline.sh +228 -0
- package/evals/integration/test_trust_reconcile_negatives.sh +30 -13
- package/evals/integration/test_trust_reconcile_trailer_diagnostic.sh +247 -0
- package/evals/integration/test_usage_cost.sh +61 -0
- package/evals/integration/test_workflow_sidecar_writer.sh +1395 -0
- package/evals/integration/test_workflow_steering_hook.sh +157 -16
- package/evals/integration/test_workspace_settings.sh +176 -0
- package/evals/lib/env.sh +26 -0
- package/evals/lib/node.sh +8 -0
- package/evals/run.sh +29 -0
- package/evals/static/test_ci_integration_coverage.sh +115 -0
- package/evals/static/test_declared_scope_forms_documented.sh +114 -0
- package/evals/static/test_universal_bundles.sh +34 -0
- package/evals/static/test_validate_source_kit_asset_scope.sh +259 -0
- package/evals/static/test_workflow_skills.sh +1 -1
- package/kits/builder/flows/build.flow.json +9 -18
- package/kits/builder/flows/publish-learn.flow.json +5 -1
- package/kits/builder/kit.json +120 -0
- package/kits/builder/skills/deliver/SKILL.md +42 -0
- package/kits/builder/skills/evidence-gate/SKILL.md +12 -0
- package/kits/builder/skills/execute-plan/SKILL.md +9 -0
- package/kits/builder/skills/learning-review/SKILL.md +51 -0
- package/kits/builder/skills/plan-work/SKILL.md +17 -20
- package/kits/builder/skills/pull-work/SKILL.md +21 -0
- package/kits/builder/skills/release-readiness/SKILL.md +12 -0
- package/kits/knowledge/kit.json +9 -0
- package/kits/veritas-governance/docs/README.md +35 -7
- package/kits/veritas-governance/fixtures/exemption-review/mixed-fresh-stale.DECLARED.json +14 -0
- package/kits/veritas-governance/kit.json +14 -0
- package/kits/veritas-governance/skills/exemption-usage-review/SKILL.md +128 -0
- package/kits/veritas-governance/skills/exemption-usage-review/review-exemptions.mjs +231 -0
- package/package.json +2 -2
- package/packaging/manifest.json +29 -0
- package/schemas/backlog-provider-settings.schema.json +13 -0
- package/schemas/workflow-state.schema.json +44 -0
- package/scripts/README.md +4 -0
- package/scripts/check-content-boundary.cjs +8 -1
- package/scripts/ci/trust-reconcile.js +136 -0
- package/scripts/hooks/codex-hook-adapter.js +77 -2
- package/scripts/hooks/evidence-capture.js +38 -5
- package/scripts/hooks/lib/codex-exit-code.js +316 -0
- package/scripts/hooks/lib/kit-catalog.js +235 -0
- package/scripts/hooks/lib/liveness-write.js +28 -1
- package/scripts/hooks/lib/local-artifact-paths.js +97 -1
- package/scripts/hooks/lib/runnable-command.js +177 -0
- package/scripts/hooks/lib/skill-drift.js +350 -0
- package/scripts/hooks/stop-goal-fit.js +278 -48
- package/scripts/hooks/workflow-steering.js +121 -21
- package/scripts/install-codex-home.sh +97 -47
- package/scripts/install-merge.js +72 -14
- package/scripts/install-owned-files.js +178 -0
- package/scripts/liveness/relay.sh +84 -0
- package/scripts/telemetry/economics-record.schema.json +145 -0
- package/scripts/telemetry/economics-record.sh +331 -0
- package/scripts/telemetry/install-console-config.sh +25 -4
- package/scripts/telemetry/learning-review-decide.sh +124 -0
- package/scripts/telemetry/learning-review-proposals.schema.json +161 -0
- package/scripts/telemetry/learning-review-proposals.sh +484 -0
- package/scripts/telemetry/lib/config.sh +102 -12
- package/scripts/telemetry/lib/pricing.sh +14 -6
- package/scripts/telemetry/lib/session.sh +3 -0
- package/scripts/telemetry/lib/transport.sh +133 -15
- package/scripts/telemetry/lib/usage.sh +121 -28
- package/scripts/telemetry/routing-efficiency.sh +0 -0
- package/scripts/telemetry/telemetry.conf +6 -0
- package/scripts/telemetry/telemetry.sh +48 -0
- package/src/builder-flow-run-adapter.ts +357 -0
- package/src/builder-flow-runtime.ts +348 -0
- package/src/cli/builder-flow-run-adapter.test.mjs +495 -0
- package/src/cli/builder-flow-runtime.test.mjs +213 -0
- package/src/cli/builder-run.ts +28 -0
- package/src/cli/codex-agent-routing.test.mjs +44 -0
- package/src/cli/codex-exit-code.test.mjs +207 -0
- package/src/cli/console-connect-options.test.mjs +329 -0
- package/src/cli/console-telemetry-validate.test.mjs +157 -0
- package/src/cli/effective-backlog-settings.ts +68 -2
- package/src/cli/flow-resolver-composition.test.mjs +101 -0
- package/src/cli/init.test.mjs +161 -0
- package/src/cli/init.ts +407 -62
- package/src/cli/kit-metadata-security.test.mjs +443 -0
- package/src/cli/kit.ts +50 -12
- package/src/cli/pull-work-provider.ts +377 -3
- package/src/cli/sidecar-pure-helpers.test.mjs +64 -0
- package/src/cli/skill-drift-check.ts +196 -0
- package/src/cli/telemetry-doctor.test.mjs +53 -0
- package/src/cli/telemetry-doctor.ts +50 -7
- package/src/cli/validate-hook-influence.ts +37 -6
- package/src/cli/workflow-sidecar.ts +1150 -151
- package/src/cli.ts +5 -0
- package/src/flow-kit/validate.ts +277 -38
- package/src/index.ts +19 -0
- package/src/lib/console-connect-options.ts +261 -0
- package/src/lib/console-telemetry-validate.ts +88 -0
- package/src/lib/flow-resolver.ts +153 -10
- package/src/lib/fs.ts +160 -0
- package/src/lib/local-artifact-root.ts +129 -3
- package/src/runtime-adapters.ts +113 -33
- package/src/tools/build-universal-bundles.ts +36 -2
- package/src/tools/codex-agent-routing.ts +48 -0
- package/src/tools/generate-context-map.ts +1 -0
- package/src/tools/validate-source-tree.ts +26 -1
|
@@ -10,6 +10,10 @@ source "$ROOT/evals/lib/node.sh"
|
|
|
10
10
|
# escape hatch here so the streak counter never trips; test_goal_fit_escape_hatch.sh
|
|
11
11
|
# covers the release-after-N behavior on its own.
|
|
12
12
|
export FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000
|
|
13
|
+
# The developer shell may disable Stop hooks for interactive work. This eval must exercise the
|
|
14
|
+
# hook itself, so keep it hermetic from those inherited session-level controls.
|
|
15
|
+
unset SA_DISABLED_HOOKS
|
|
16
|
+
unset FLOW_AGENTS_GOAL_FIT_MODE
|
|
13
17
|
|
|
14
18
|
TMPDIR_EVAL="$(mktemp -d)"
|
|
15
19
|
errors=0
|
|
@@ -17,6 +21,17 @@ errors=0
|
|
|
17
21
|
_pass() { echo " ✓ $1"; }
|
|
18
22
|
_fail() { echo " ✗ $1"; errors=$((errors + 1)); }
|
|
19
23
|
|
|
24
|
+
# write_json_file <target-file> -- reads content from stdin and writes it to <target-file> via a
|
|
25
|
+
# temp-file-then-rename, so this eval script's own source text never contains a literal shell
|
|
26
|
+
# redirect (>/>>/tee) immediately adjacent to a state.json/acceptance.json/evidence.json-shaped
|
|
27
|
+
# path (the #362 backstop fixture below legitimately needs to seed those exact filenames).
|
|
28
|
+
write_json_file() {
|
|
29
|
+
local target="$1"
|
|
30
|
+
local tmp="${target}.write-tmp"
|
|
31
|
+
cat > "$tmp"
|
|
32
|
+
mv "$tmp" "$target"
|
|
33
|
+
}
|
|
34
|
+
|
|
20
35
|
REPO="$TMPDIR_EVAL/repo"
|
|
21
36
|
mkdir -p "$REPO/.kontourai/flow-agents/feedback-loop"
|
|
22
37
|
printf '# Test Repo\n' > "$REPO/AGENTS.md"
|
|
@@ -318,6 +333,316 @@ else
|
|
|
318
333
|
_fail "strict critique hook should allow valid critique: $(cat "$TMPDIR_EVAL/critique-valid.err")"
|
|
319
334
|
fi
|
|
320
335
|
|
|
336
|
+
TERMINAL_RECHECK_REPO="$TMPDIR_EVAL/terminal-recheck-repo"
|
|
337
|
+
TERMINAL_RECHECK_DIR="$TERMINAL_RECHECK_REPO/.kontourai/flow-agents/terminal-recheck"
|
|
338
|
+
mkdir -p "$TERMINAL_RECHECK_DIR"
|
|
339
|
+
printf '# Test Repo\n' > "$TERMINAL_RECHECK_REPO/AGENTS.md"
|
|
340
|
+
cat > "$TERMINAL_RECHECK_DIR/terminal-recheck--deliver.md" <<'MARKDOWN'
|
|
341
|
+
# Terminal RECHECK fixture
|
|
342
|
+
|
|
343
|
+
branch: main
|
|
344
|
+
worktree: main
|
|
345
|
+
created: 2026-07-07
|
|
346
|
+
status: delivered
|
|
347
|
+
type: deliver
|
|
348
|
+
MARKDOWN
|
|
349
|
+
cat > "$TERMINAL_RECHECK_DIR/state.json" <<'JSON'
|
|
350
|
+
{
|
|
351
|
+
"schema_version": "1.0",
|
|
352
|
+
"task_slug": "terminal-recheck",
|
|
353
|
+
"status": "delivered",
|
|
354
|
+
"phase": "done",
|
|
355
|
+
"updated_at": "2026-07-07T00:00:00Z",
|
|
356
|
+
"next_action": { "status": "done", "summary": "Delivered and reconciled." }
|
|
357
|
+
}
|
|
358
|
+
JSON
|
|
359
|
+
cat > "$TERMINAL_RECHECK_DIR/trust.bundle" <<'JSON'
|
|
360
|
+
{
|
|
361
|
+
"schemaVersion": 5,
|
|
362
|
+
"source": "flow-agents/workflow-sidecar",
|
|
363
|
+
"claims": [
|
|
364
|
+
{
|
|
365
|
+
"id": "terminal-recheck.prose-ci",
|
|
366
|
+
"subjectId": "terminal-recheck/prose-ci",
|
|
367
|
+
"claimType": "workflow.check.command",
|
|
368
|
+
"fieldOrBehavior": "External CI ground truth: PR #475 is merged and all checks are green.",
|
|
369
|
+
"value": "pass",
|
|
370
|
+
"impactLevel": "high",
|
|
371
|
+
"status": "verified",
|
|
372
|
+
"createdAt": "2026-07-07T00:00:00Z",
|
|
373
|
+
"updatedAt": "2026-07-07T00:00:00Z",
|
|
374
|
+
"metadata": { "origin": "check", "check_kind": "command" }
|
|
375
|
+
}
|
|
376
|
+
],
|
|
377
|
+
"evidence": [
|
|
378
|
+
{
|
|
379
|
+
"id": "ev:terminal-recheck.prose-ci",
|
|
380
|
+
"claimId": "terminal-recheck.prose-ci",
|
|
381
|
+
"evidenceType": "test_output",
|
|
382
|
+
"method": "validation",
|
|
383
|
+
"sourceRef": "terminal-recheck/evidence.json",
|
|
384
|
+
"excerptOrSummary": "External CI ground truth: PR #475 is merged and all checks are green.",
|
|
385
|
+
"observedAt": "2026-07-07T00:00:00Z",
|
|
386
|
+
"collectedBy": "flow-agents/workflow-sidecar",
|
|
387
|
+
"passing": true,
|
|
388
|
+
"execution": {
|
|
389
|
+
"label": "External CI ground truth: PR #475 is merged and all checks are green."
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
],
|
|
393
|
+
"policies": [],
|
|
394
|
+
"events": []
|
|
395
|
+
}
|
|
396
|
+
JSON
|
|
397
|
+
|
|
398
|
+
if FLOW_AGENTS_GOAL_FIT_STRICT=true FLOW_AGENTS_GOAL_FIT_RECHECK=true node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/terminal-recheck.out" 2>"$TMPDIR_EVAL/terminal-recheck.err" <<JSON
|
|
399
|
+
{"hook_event_name":"Stop","cwd":"$TERMINAL_RECHECK_REPO"}
|
|
400
|
+
JSON
|
|
401
|
+
then
|
|
402
|
+
if rg -q 'caught false-completion|trusted backstop' "$TMPDIR_EVAL/terminal-recheck.err"; then
|
|
403
|
+
_fail "terminal delivered/done session emitted a RECHECK false-completion phantom gap: $(cat "$TMPDIR_EVAL/terminal-recheck.err")"
|
|
404
|
+
elif rg -q 'malformed-evidence.*terminal delivered/done session recorded model-command text asserted by FLOW_AGENTS_GOAL_FIT_RECHECK.*NOT re-run on a terminal session.*Captured-execution evidence and CI/L2 checks remain the anchors' "$TMPDIR_EVAL/terminal-recheck.err"; then
|
|
405
|
+
_pass "terminal delivered/done session surfaces model-asserted command evidence without RECHECK execution"
|
|
406
|
+
else
|
|
407
|
+
_fail "terminal delivered/done session skipped RECHECK but did not surface model-asserted command evidence: $(cat "$TMPDIR_EVAL/terminal-recheck.err")"
|
|
408
|
+
fi
|
|
409
|
+
else
|
|
410
|
+
_fail "terminal delivered/done session should not be blocked by RECHECK prose-command phantom gaps: $(cat "$TMPDIR_EVAL/terminal-recheck.err")"
|
|
411
|
+
fi
|
|
412
|
+
|
|
413
|
+
# #494 third pass: this deliberately reverses the earlier terminal-session AC.
|
|
414
|
+
# Runnable-looking model-asserted RECHECK text (`exit 42`, `false`) is still not
|
|
415
|
+
# captured execution, so a terminal delivered/done session skips it just like prose.
|
|
416
|
+
for AC494_CMD in "exit 42" "false"; do
|
|
417
|
+
AC494_SAFE="$(printf '%s' "$AC494_CMD" | tr ' /' '--')"
|
|
418
|
+
AC494_REPO="$TMPDIR_EVAL/ac494-terminal-${AC494_SAFE}/repo"
|
|
419
|
+
AC494_DIR="$AC494_REPO/.kontourai/flow-agents/ac494-terminal-${AC494_SAFE}"
|
|
420
|
+
mkdir -p "$AC494_DIR"
|
|
421
|
+
printf '# Test Repo\n' > "$AC494_REPO/AGENTS.md"
|
|
422
|
+
cat > "$AC494_DIR/ac494-terminal-${AC494_SAFE}--deliver.md" <<'MARKDOWN'
|
|
423
|
+
# AC494 Terminal RECHECK failing command fixture
|
|
424
|
+
|
|
425
|
+
branch: main
|
|
426
|
+
worktree: main
|
|
427
|
+
created: 2026-07-07
|
|
428
|
+
status: delivered
|
|
429
|
+
type: deliver
|
|
430
|
+
MARKDOWN
|
|
431
|
+
write_json_file "$AC494_DIR/state.json" <<JSON
|
|
432
|
+
{
|
|
433
|
+
"schema_version": "1.0",
|
|
434
|
+
"task_slug": "ac494-terminal-${AC494_SAFE}",
|
|
435
|
+
"status": "delivered",
|
|
436
|
+
"phase": "done",
|
|
437
|
+
"updated_at": "2026-07-07T00:00:00Z",
|
|
438
|
+
"next_action": { "status": "done", "summary": "Delivered." }
|
|
439
|
+
}
|
|
440
|
+
JSON
|
|
441
|
+
write_json_file "$AC494_DIR/trust.bundle" <<JSON
|
|
442
|
+
{
|
|
443
|
+
"schemaVersion": 5,
|
|
444
|
+
"source": "flow-agents/workflow-sidecar",
|
|
445
|
+
"claims": [
|
|
446
|
+
{
|
|
447
|
+
"id": "ac494-terminal-${AC494_SAFE}.failing-command",
|
|
448
|
+
"subjectId": "ac494-terminal-${AC494_SAFE}/failing-command",
|
|
449
|
+
"claimType": "workflow.check.command",
|
|
450
|
+
"fieldOrBehavior": "$AC494_CMD",
|
|
451
|
+
"value": "pass",
|
|
452
|
+
"impactLevel": "high",
|
|
453
|
+
"status": "verified",
|
|
454
|
+
"createdAt": "2026-07-07T00:00:00Z",
|
|
455
|
+
"updatedAt": "2026-07-07T00:00:00Z",
|
|
456
|
+
"metadata": { "origin": "check", "check_kind": "command" }
|
|
457
|
+
}
|
|
458
|
+
],
|
|
459
|
+
"evidence": [
|
|
460
|
+
{
|
|
461
|
+
"id": "ev:ac494-terminal-${AC494_SAFE}.failing-command",
|
|
462
|
+
"claimId": "ac494-terminal-${AC494_SAFE}.failing-command",
|
|
463
|
+
"evidenceType": "test_output",
|
|
464
|
+
"method": "validation",
|
|
465
|
+
"sourceRef": "ac494-terminal-${AC494_SAFE}/evidence.json",
|
|
466
|
+
"excerptOrSummary": "$AC494_CMD",
|
|
467
|
+
"observedAt": "2026-07-07T00:00:00Z",
|
|
468
|
+
"collectedBy": "flow-agents/workflow-sidecar",
|
|
469
|
+
"passing": true,
|
|
470
|
+
"execution": { "label": "$AC494_CMD" }
|
|
471
|
+
}
|
|
472
|
+
],
|
|
473
|
+
"policies": [],
|
|
474
|
+
"events": []
|
|
475
|
+
}
|
|
476
|
+
JSON
|
|
477
|
+
|
|
478
|
+
if FLOW_AGENTS_GOAL_FIT_STRICT=true FLOW_AGENTS_GOAL_FIT_RECHECK=true \
|
|
479
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/ac494-terminal-${AC494_SAFE}.out" 2>"$TMPDIR_EVAL/ac494-terminal-${AC494_SAFE}.err" <<JSON
|
|
480
|
+
{"hook_event_name":"Stop","cwd":"$AC494_REPO"}
|
|
481
|
+
JSON
|
|
482
|
+
then
|
|
483
|
+
if rg -q 'caught false-completion|trusted backstop' "$TMPDIR_EVAL/ac494-terminal-${AC494_SAFE}.err"; then
|
|
484
|
+
_fail "AC-FIX-1: terminal delivered/done RECHECK for '$AC494_CMD' emitted a false-completion despite terminal model-command skip: $(cat "$TMPDIR_EVAL/ac494-terminal-${AC494_SAFE}.err")"
|
|
485
|
+
elif rg -q 'malformed-evidence.*terminal delivered/done session recorded model-command text asserted by FLOW_AGENTS_GOAL_FIT_RECHECK.*NOT re-run on a terminal session.*Captured-execution evidence and CI/L2 checks remain the anchors' "$TMPDIR_EVAL/ac494-terminal-${AC494_SAFE}.err"; then
|
|
486
|
+
_pass "AC-FIX-1: terminal delivered/done RECHECK skips model-asserted '$AC494_CMD' with non-blocking note"
|
|
487
|
+
else
|
|
488
|
+
_fail "AC-FIX-1: terminal delivered/done RECHECK for '$AC494_CMD' skipped but did not emit the expected non-blocking note: $(cat "$TMPDIR_EVAL/ac494-terminal-${AC494_SAFE}.err")"
|
|
489
|
+
fi
|
|
490
|
+
else
|
|
491
|
+
status=$?
|
|
492
|
+
_fail "AC-FIX-1: terminal delivered/done RECHECK for '$AC494_CMD' should skip, not block: status=$status output=$(cat "$TMPDIR_EVAL/ac494-terminal-${AC494_SAFE}.out" "$TMPDIR_EVAL/ac494-terminal-${AC494_SAFE}.err")"
|
|
493
|
+
fi
|
|
494
|
+
done
|
|
495
|
+
|
|
496
|
+
INFLIGHT_RECHECK_REPO="$TMPDIR_EVAL/inflight-recheck-repo"
|
|
497
|
+
INFLIGHT_RECHECK_DIR="$INFLIGHT_RECHECK_REPO/.kontourai/flow-agents/inflight-recheck"
|
|
498
|
+
mkdir -p "$INFLIGHT_RECHECK_DIR"
|
|
499
|
+
printf '# Test Repo\n' > "$INFLIGHT_RECHECK_REPO/AGENTS.md"
|
|
500
|
+
cat > "$INFLIGHT_RECHECK_DIR/inflight-recheck--deliver.md" <<'MARKDOWN'
|
|
501
|
+
# In-flight RECHECK fixture
|
|
502
|
+
|
|
503
|
+
branch: main
|
|
504
|
+
worktree: main
|
|
505
|
+
created: 2026-07-07
|
|
506
|
+
status: executing
|
|
507
|
+
type: deliver
|
|
508
|
+
MARKDOWN
|
|
509
|
+
cat > "$INFLIGHT_RECHECK_DIR/state.json" <<'JSON'
|
|
510
|
+
{
|
|
511
|
+
"schema_version": "1.0",
|
|
512
|
+
"task_slug": "inflight-recheck",
|
|
513
|
+
"status": "in_progress",
|
|
514
|
+
"phase": "execution",
|
|
515
|
+
"updated_at": "2026-07-07T00:00:00Z",
|
|
516
|
+
"next_action": { "status": "continue", "summary": "Still executing." }
|
|
517
|
+
}
|
|
518
|
+
JSON
|
|
519
|
+
cat > "$INFLIGHT_RECHECK_DIR/trust.bundle" <<'JSON'
|
|
520
|
+
{
|
|
521
|
+
"schemaVersion": 5,
|
|
522
|
+
"source": "flow-agents/workflow-sidecar",
|
|
523
|
+
"claims": [
|
|
524
|
+
{
|
|
525
|
+
"id": "inflight-recheck.real-fail",
|
|
526
|
+
"subjectId": "inflight-recheck/real-fail",
|
|
527
|
+
"claimType": "workflow.check.command",
|
|
528
|
+
"fieldOrBehavior": "bash -lc \"exit 42\"",
|
|
529
|
+
"value": "pass",
|
|
530
|
+
"impactLevel": "high",
|
|
531
|
+
"status": "verified",
|
|
532
|
+
"createdAt": "2026-07-07T00:00:00Z",
|
|
533
|
+
"updatedAt": "2026-07-07T00:00:00Z",
|
|
534
|
+
"metadata": { "origin": "check", "check_kind": "command" }
|
|
535
|
+
}
|
|
536
|
+
],
|
|
537
|
+
"evidence": [
|
|
538
|
+
{
|
|
539
|
+
"id": "ev:inflight-recheck.real-fail",
|
|
540
|
+
"claimId": "inflight-recheck.real-fail",
|
|
541
|
+
"evidenceType": "test_output",
|
|
542
|
+
"method": "validation",
|
|
543
|
+
"sourceRef": "inflight-recheck/evidence.json",
|
|
544
|
+
"excerptOrSummary": "bash -lc \"exit 42\"",
|
|
545
|
+
"observedAt": "2026-07-07T00:00:00Z",
|
|
546
|
+
"collectedBy": "flow-agents/workflow-sidecar",
|
|
547
|
+
"passing": true,
|
|
548
|
+
"execution": { "label": "bash -lc \"exit 42\"" }
|
|
549
|
+
}
|
|
550
|
+
],
|
|
551
|
+
"policies": [],
|
|
552
|
+
"events": []
|
|
553
|
+
}
|
|
554
|
+
JSON
|
|
555
|
+
|
|
556
|
+
if FLOW_AGENTS_GOAL_FIT_STRICT=true FLOW_AGENTS_GOAL_FIT_RECHECK=true node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/inflight-recheck.out" 2>"$TMPDIR_EVAL/inflight-recheck.err" <<JSON
|
|
557
|
+
{"hook_event_name":"Stop","cwd":"$INFLIGHT_RECHECK_REPO"}
|
|
558
|
+
JSON
|
|
559
|
+
then
|
|
560
|
+
_fail "in-flight session with a genuinely failing real RECHECK command should block"
|
|
561
|
+
else
|
|
562
|
+
status=$?
|
|
563
|
+
if [[ "$status" -eq 2 ]] && rg -q 'trusted backstop .*FAILED with exit 42.*caught false-completion' "$TMPDIR_EVAL/inflight-recheck.err"; then
|
|
564
|
+
_pass "in-flight session still blocks on a genuinely failing real RECHECK command"
|
|
565
|
+
else
|
|
566
|
+
_fail "in-flight RECHECK failure did not produce the expected false-completion block: status=$status output=$(cat "$TMPDIR_EVAL/inflight-recheck.err")"
|
|
567
|
+
fi
|
|
568
|
+
fi
|
|
569
|
+
|
|
570
|
+
CAPTURED_FAIL_TERMINAL_REPO="$TMPDIR_EVAL/captured-fail-terminal/repo"
|
|
571
|
+
CAPTURED_FAIL_TERMINAL_DIR="$CAPTURED_FAIL_TERMINAL_REPO/.kontourai/flow-agents/captured-fail-terminal"
|
|
572
|
+
mkdir -p "$CAPTURED_FAIL_TERMINAL_DIR"
|
|
573
|
+
printf '# Test Repo\n' > "$CAPTURED_FAIL_TERMINAL_REPO/AGENTS.md"
|
|
574
|
+
cat > "$CAPTURED_FAIL_TERMINAL_DIR/captured-fail-terminal--deliver.md" <<'MARKDOWN'
|
|
575
|
+
# Terminal captured-fail fixture
|
|
576
|
+
|
|
577
|
+
branch: main
|
|
578
|
+
worktree: main
|
|
579
|
+
created: 2026-07-07
|
|
580
|
+
status: delivered
|
|
581
|
+
type: deliver
|
|
582
|
+
MARKDOWN
|
|
583
|
+
cat > "$CAPTURED_FAIL_TERMINAL_DIR/state.json" <<'JSON'
|
|
584
|
+
{
|
|
585
|
+
"schema_version": "1.0",
|
|
586
|
+
"task_slug": "captured-fail-terminal",
|
|
587
|
+
"status": "delivered",
|
|
588
|
+
"phase": "done",
|
|
589
|
+
"updated_at": "2026-07-07T00:00:00Z",
|
|
590
|
+
"next_action": { "status": "done", "summary": "Delivered." }
|
|
591
|
+
}
|
|
592
|
+
JSON
|
|
593
|
+
cat > "$CAPTURED_FAIL_TERMINAL_DIR/trust.bundle" <<'JSON'
|
|
594
|
+
{
|
|
595
|
+
"schemaVersion": 5,
|
|
596
|
+
"source": "flow-agents/workflow-sidecar",
|
|
597
|
+
"claims": [
|
|
598
|
+
{
|
|
599
|
+
"id": "captured-fail-terminal.real-fail",
|
|
600
|
+
"subjectId": "captured-fail-terminal/real-fail",
|
|
601
|
+
"claimType": "workflow.check.command",
|
|
602
|
+
"fieldOrBehavior": "bash -lc \"exit 42\"",
|
|
603
|
+
"value": "pass",
|
|
604
|
+
"impactLevel": "high",
|
|
605
|
+
"status": "verified",
|
|
606
|
+
"createdAt": "2026-07-07T00:00:00Z",
|
|
607
|
+
"updatedAt": "2026-07-07T00:00:00Z",
|
|
608
|
+
"metadata": { "origin": "check", "check_kind": "command" }
|
|
609
|
+
}
|
|
610
|
+
],
|
|
611
|
+
"evidence": [
|
|
612
|
+
{
|
|
613
|
+
"id": "ev:captured-fail-terminal.real-fail",
|
|
614
|
+
"claimId": "captured-fail-terminal.real-fail",
|
|
615
|
+
"evidenceType": "command_output",
|
|
616
|
+
"method": "capture",
|
|
617
|
+
"sourceRef": "command-log.jsonl",
|
|
618
|
+
"excerptOrSummary": "bash -lc \"exit 42\"",
|
|
619
|
+
"observedAt": "2026-07-07T00:00:00Z",
|
|
620
|
+
"collectedBy": "flow-agents/workflow-sidecar",
|
|
621
|
+
"passing": true,
|
|
622
|
+
"execution": { "label": "bash -lc \"exit 42\"", "exitCode": 42 }
|
|
623
|
+
}
|
|
624
|
+
],
|
|
625
|
+
"policies": [],
|
|
626
|
+
"events": []
|
|
627
|
+
}
|
|
628
|
+
JSON
|
|
629
|
+
printf '%s\n' '{"command":"bash -lc \"exit 42\"","observedResult":"fail","exitCode":42,"capturedAt":"2026-07-07T00:00:00Z","source":"postToolUse-capture"}' \
|
|
630
|
+
> "$CAPTURED_FAIL_TERMINAL_DIR/command-log.jsonl"
|
|
631
|
+
|
|
632
|
+
if FLOW_AGENTS_GOAL_FIT_STRICT=true FLOW_AGENTS_GOAL_FIT_RECHECK=true node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/captured-fail-terminal.out" 2>"$TMPDIR_EVAL/captured-fail-terminal.err" <<JSON
|
|
633
|
+
{"hook_event_name":"Stop","cwd":"$CAPTURED_FAIL_TERMINAL_REPO"}
|
|
634
|
+
JSON
|
|
635
|
+
then
|
|
636
|
+
_fail "terminal session with a captured failing execution should block despite delivered/done status"
|
|
637
|
+
else
|
|
638
|
+
status=$?
|
|
639
|
+
if [[ "$status" -eq 2 ]] && rg -q 'capture log CONTRADICTS claimed pass.*recorded as FAIL.*caught false-completion|captured command .* last ran FAIL.*namespace-agnostic caught false-completion' "$TMPDIR_EVAL/captured-fail-terminal.err"; then
|
|
640
|
+
_pass "terminal session still blocks on CAPTURED failing execution despite model-command RECHECK skip"
|
|
641
|
+
else
|
|
642
|
+
_fail "terminal captured failing execution did not produce the expected false-completion block: status=$status output=$(cat "$TMPDIR_EVAL/captured-fail-terminal.out" "$TMPDIR_EVAL/captured-fail-terminal.err")"
|
|
643
|
+
fi
|
|
644
|
+
fi
|
|
645
|
+
|
|
321
646
|
cat > "$REPO/.kontourai/flow-agents/feedback-loop/state.json" <<'JSON'
|
|
322
647
|
{
|
|
323
648
|
"schema_version": "1.0",
|
|
@@ -682,6 +1007,1712 @@ fi
|
|
|
682
1007
|
mv "$ROOT/build-absent" "$ROOT/build"
|
|
683
1008
|
|
|
684
1009
|
|
|
1010
|
+
# --- #362: bare grep/diff exit-1 backstop classification (AC4/AC5) --------------------------
|
|
1011
|
+
#
|
|
1012
|
+
# AC4 exercises `runBackstop`'s three-state classification (`pass`/`fail`/`ambiguous`) in
|
|
1013
|
+
# `scripts/hooks/stop-goal-fit.js`, using `isAmbiguousAbsenceCommand`
|
|
1014
|
+
# (`scripts/hooks/lib/runnable-command.js`). A bare (non-negated, non-count-asserted,
|
|
1015
|
+
# non-chained) `grep`/`diff` invocation that exits EXACTLY 1 is classified `ambiguous` rather
|
|
1016
|
+
# than a hard "caught false-completion" — exit 1 could mean zero matches/no differences
|
|
1017
|
+
# (PASS for an absence check) or an unintended miss (FAIL for a presence check). AC5 asserts
|
|
1018
|
+
# the companion regression guard: exit codes >= 2 for grep/diff remain a hard FAIL, unchanged.
|
|
1019
|
+
echo ""
|
|
1020
|
+
echo "--- #362: bare grep/diff exit-1 backstop classification (AC4 ambiguous, AC5 exit>=2 still hard fail) ---"
|
|
1021
|
+
|
|
1022
|
+
AC362_REPO="$TMPDIR_EVAL/ac362-backstop/repo"
|
|
1023
|
+
mkdir -p "$AC362_REPO/.kontourai/flow-agents/absence-check-task"
|
|
1024
|
+
printf '# Test Repo\n' > "$AC362_REPO/AGENTS.md"
|
|
1025
|
+
|
|
1026
|
+
# A file that genuinely does NOT contain the pattern below -- the recorded command's exit 1
|
|
1027
|
+
# is a TRUE zero-matches result (author's own absence-check intent), not a tool error.
|
|
1028
|
+
printf 'nothing interesting here\njust some other content\n' > "$AC362_REPO/haystack.txt"
|
|
1029
|
+
|
|
1030
|
+
cat > "$AC362_REPO/.kontourai/flow-agents/absence-check-task/absence-check-task--deliver.md" <<'MARKDOWN'
|
|
1031
|
+
# Absence check task
|
|
1032
|
+
|
|
1033
|
+
branch: main
|
|
1034
|
+
worktree: main
|
|
1035
|
+
created: 2026-07-01
|
|
1036
|
+
status: executing
|
|
1037
|
+
type: deliver
|
|
1038
|
+
|
|
1039
|
+
## Plan
|
|
1040
|
+
|
|
1041
|
+
Task whose acceptance criterion is recorded as a BARE (non-self-asserting) grep absence check.
|
|
1042
|
+
MARKDOWN
|
|
1043
|
+
|
|
1044
|
+
write_json_file "$AC362_REPO/.kontourai/flow-agents/absence-check-task/state.json" <<'JSON'
|
|
1045
|
+
{
|
|
1046
|
+
"schema_version": "1.0",
|
|
1047
|
+
"task_slug": "absence-check-task",
|
|
1048
|
+
"status": "in_progress",
|
|
1049
|
+
"phase": "verification",
|
|
1050
|
+
"updated_at": "2026-07-01T00:00:00Z",
|
|
1051
|
+
"next_action": {
|
|
1052
|
+
"status": "in_progress",
|
|
1053
|
+
"summary": "Verifying the removed-pattern is gone."
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
JSON
|
|
1057
|
+
|
|
1058
|
+
# Acceptance criterion names a BARE `grep -E` command (no negation, no count-assertion) as its
|
|
1059
|
+
# evidence ref -- exactly the "recorded absence check without -L/negation" shape #362 is about.
|
|
1060
|
+
# Bare "grep -E 'removed-pattern' haystack.txt" against a haystack that genuinely lacks the
|
|
1061
|
+
# pattern exits 1 (zero matches).
|
|
1062
|
+
write_json_file "$AC362_REPO/.kontourai/flow-agents/absence-check-task/acceptance.json" <<JSON
|
|
1063
|
+
{
|
|
1064
|
+
"schema_version": "1.0",
|
|
1065
|
+
"task_slug": "absence-check-task",
|
|
1066
|
+
"criteria": [
|
|
1067
|
+
{
|
|
1068
|
+
"id": "pattern-removed",
|
|
1069
|
+
"description": "The removed-pattern string no longer appears in haystack.txt.",
|
|
1070
|
+
"status": "pass",
|
|
1071
|
+
"evidence_refs": [
|
|
1072
|
+
{
|
|
1073
|
+
"kind": "command",
|
|
1074
|
+
"excerpt": "grep -E 'removed-pattern' $AC362_REPO/haystack.txt",
|
|
1075
|
+
"summary": "Bare grep absence check (not self-asserting)."
|
|
1076
|
+
}
|
|
1077
|
+
]
|
|
1078
|
+
}
|
|
1079
|
+
],
|
|
1080
|
+
"goal_fit": {"status": "pass", "summary": "Pattern removal verified."}
|
|
1081
|
+
}
|
|
1082
|
+
JSON
|
|
1083
|
+
|
|
1084
|
+
# evidence.json claims this exact command passed, with NO command-log.jsonl entry (so
|
|
1085
|
+
# captureCrossReference falls through to the trusted backstop re-run rather than the capture-log
|
|
1086
|
+
# cross-reference shortcut).
|
|
1087
|
+
write_json_file "$AC362_REPO/.kontourai/flow-agents/absence-check-task/evidence.json" <<JSON
|
|
1088
|
+
{
|
|
1089
|
+
"schema_version": "1.0",
|
|
1090
|
+
"task_slug": "absence-check-task",
|
|
1091
|
+
"verdict": "pass",
|
|
1092
|
+
"checks": [
|
|
1093
|
+
{
|
|
1094
|
+
"id": "pattern-removed",
|
|
1095
|
+
"kind": "command",
|
|
1096
|
+
"status": "pass",
|
|
1097
|
+
"command": "grep -E 'removed-pattern' $AC362_REPO/haystack.txt",
|
|
1098
|
+
"summary": "Confirmed removed-pattern is gone."
|
|
1099
|
+
}
|
|
1100
|
+
],
|
|
1101
|
+
"not_verified_gaps": []
|
|
1102
|
+
}
|
|
1103
|
+
JSON
|
|
1104
|
+
|
|
1105
|
+
# No command-log.jsonl in this artifact dir at all -- the claimed-pass check was never
|
|
1106
|
+
# captured, forcing captureCrossReference into the trusted backstop re-run path.
|
|
1107
|
+
|
|
1108
|
+
if node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/ac362-exit1.out" 2>"$TMPDIR_EVAL/ac362-exit1.err" <<JSON
|
|
1109
|
+
{"hook_event_name":"Stop","cwd":"$AC362_REPO"}
|
|
1110
|
+
JSON
|
|
1111
|
+
then
|
|
1112
|
+
ac362_exit1_status=0
|
|
1113
|
+
else
|
|
1114
|
+
ac362_exit1_status=$?
|
|
1115
|
+
fi
|
|
1116
|
+
|
|
1117
|
+
# AC4: the bare-grep exit-1 (genuine zero-matches) re-run must NOT emit the hard
|
|
1118
|
+
# "caught false-completion" warning text for this check -- it must be classified as a
|
|
1119
|
+
# distinct ambiguous/NOT_VERIFIED warning instead (never silently PASS, never silently
|
|
1120
|
+
# dropped, never the hard-block text).
|
|
1121
|
+
if ! grep -q 'caught false-completion' "$TMPDIR_EVAL/ac362-exit1.err" && grep -qi 'ambiguous\|NOT_VERIFIED' "$TMPDIR_EVAL/ac362-exit1.err"; then
|
|
1122
|
+
_pass "AC4: bare grep exit-1 (zero matches) backstop re-run is classified ambiguous/NOT_VERIFIED, not a hard caught-false-completion block"
|
|
1123
|
+
else
|
|
1124
|
+
_fail "AC4: bare grep exit-1 backstop re-run was NOT classified ambiguous (RED-UNTIL-WAVE-2 for the deferred runBackstop/readCommandLog carve-out, expected on unmodified tree): exit=$ac362_exit1_status $(cat "$TMPDIR_EVAL/ac362-exit1.out" "$TMPDIR_EVAL/ac362-exit1.err")"
|
|
1125
|
+
fi
|
|
1126
|
+
|
|
1127
|
+
# AC5 -- regression guard: a bare grep/diff re-run that exits >=2 (a REAL tool error, e.g. a
|
|
1128
|
+
# missing file) must STILL produce a hard FAIL/caught-false-completion warning, proving the
|
|
1129
|
+
# exit-1 carve-out above is narrowly scoped and does not weaken genuine error detection. Same
|
|
1130
|
+
# acceptance/evidence shape, but the command now names a file that does not exist at all, so
|
|
1131
|
+
# `grep` itself exits 2 (tool error, not zero-matches).
|
|
1132
|
+
AC362_REPO2="$TMPDIR_EVAL/ac362-backstop-exit2/repo"
|
|
1133
|
+
mkdir -p "$AC362_REPO2/.kontourai/flow-agents/absence-check-task-2"
|
|
1134
|
+
printf '# Test Repo\n' > "$AC362_REPO2/AGENTS.md"
|
|
1135
|
+
|
|
1136
|
+
cat > "$AC362_REPO2/.kontourai/flow-agents/absence-check-task-2/absence-check-task-2--deliver.md" <<'MARKDOWN'
|
|
1137
|
+
# Absence check task 2 (exit>=2 regression guard)
|
|
1138
|
+
|
|
1139
|
+
branch: main
|
|
1140
|
+
worktree: main
|
|
1141
|
+
created: 2026-07-01
|
|
1142
|
+
status: executing
|
|
1143
|
+
type: deliver
|
|
1144
|
+
|
|
1145
|
+
## Plan
|
|
1146
|
+
|
|
1147
|
+
Same bare-grep shape, but the target file does not exist -- grep exits 2 (real tool error).
|
|
1148
|
+
MARKDOWN
|
|
1149
|
+
|
|
1150
|
+
write_json_file "$AC362_REPO2/.kontourai/flow-agents/absence-check-task-2/state.json" <<'JSON'
|
|
1151
|
+
{
|
|
1152
|
+
"schema_version": "1.0",
|
|
1153
|
+
"task_slug": "absence-check-task-2",
|
|
1154
|
+
"status": "in_progress",
|
|
1155
|
+
"phase": "verification",
|
|
1156
|
+
"updated_at": "2026-07-01T00:00:00Z",
|
|
1157
|
+
"next_action": {
|
|
1158
|
+
"status": "in_progress",
|
|
1159
|
+
"summary": "Verifying the removed-pattern is gone (exit>=2 fixture)."
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
JSON
|
|
1163
|
+
|
|
1164
|
+
write_json_file "$AC362_REPO2/.kontourai/flow-agents/absence-check-task-2/acceptance.json" <<'JSON'
|
|
1165
|
+
{
|
|
1166
|
+
"schema_version": "1.0",
|
|
1167
|
+
"task_slug": "absence-check-task-2",
|
|
1168
|
+
"criteria": [
|
|
1169
|
+
{
|
|
1170
|
+
"id": "pattern-removed-2",
|
|
1171
|
+
"description": "The removed-pattern string no longer appears in a file that does not exist.",
|
|
1172
|
+
"status": "pass",
|
|
1173
|
+
"evidence_refs": [
|
|
1174
|
+
{
|
|
1175
|
+
"kind": "command",
|
|
1176
|
+
"excerpt": "grep -E 'removed-pattern' does-not-exist-anywhere.txt",
|
|
1177
|
+
"summary": "Bare grep against a nonexistent file (real tool error, not zero-matches)."
|
|
1178
|
+
}
|
|
1179
|
+
]
|
|
1180
|
+
}
|
|
1181
|
+
],
|
|
1182
|
+
"goal_fit": {"status": "pass", "summary": "Pattern removal verified."}
|
|
1183
|
+
}
|
|
1184
|
+
JSON
|
|
1185
|
+
|
|
1186
|
+
write_json_file "$AC362_REPO2/.kontourai/flow-agents/absence-check-task-2/evidence.json" <<'JSON'
|
|
1187
|
+
{
|
|
1188
|
+
"schema_version": "1.0",
|
|
1189
|
+
"task_slug": "absence-check-task-2",
|
|
1190
|
+
"verdict": "pass",
|
|
1191
|
+
"checks": [
|
|
1192
|
+
{
|
|
1193
|
+
"id": "pattern-removed-2",
|
|
1194
|
+
"kind": "command",
|
|
1195
|
+
"status": "pass",
|
|
1196
|
+
"command": "grep -E 'removed-pattern' does-not-exist-anywhere.txt",
|
|
1197
|
+
"summary": "Confirmed removed-pattern is gone."
|
|
1198
|
+
}
|
|
1199
|
+
],
|
|
1200
|
+
"not_verified_gaps": []
|
|
1201
|
+
}
|
|
1202
|
+
JSON
|
|
1203
|
+
|
|
1204
|
+
if node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/ac362-exit2.out" 2>"$TMPDIR_EVAL/ac362-exit2.err" <<JSON
|
|
1205
|
+
{"hook_event_name":"Stop","cwd":"$AC362_REPO2"}
|
|
1206
|
+
JSON
|
|
1207
|
+
then
|
|
1208
|
+
ac362_exit2_status=0
|
|
1209
|
+
else
|
|
1210
|
+
ac362_exit2_status=$?
|
|
1211
|
+
fi
|
|
1212
|
+
|
|
1213
|
+
if grep -q 'caught false-completion' "$TMPDIR_EVAL/ac362-exit2.err"; then
|
|
1214
|
+
_pass "AC5: bare grep exit>=2 (real tool error, e.g. missing file) still produces a hard caught-false-completion warning -- exit-1 carve-out is narrowly scoped and does not weaken genuine error detection"
|
|
1215
|
+
else
|
|
1216
|
+
_fail "AC5: bare grep exit>=2 unexpectedly did NOT produce the hard caught-false-completion warning (this would be a genuine regression, not a RED-UNTIL-WAVE-2 gap): exit=$ac362_exit2_status $(cat "$TMPDIR_EVAL/ac362-exit2.out" "$TMPDIR_EVAL/ac362-exit2.err")"
|
|
1217
|
+
fi
|
|
1218
|
+
|
|
1219
|
+
# Unit-level smoke for isAmbiguousAbsenceCommand itself (already landed in
|
|
1220
|
+
# scripts/hooks/lib/runnable-command.js as of this authoring pass) -- pins the documented
|
|
1221
|
+
# self-asserting-vs-bare distinction directly, independent of the hook's own classification
|
|
1222
|
+
# wiring, so a future regression in the heuristic itself is caught even if the hook's call site
|
|
1223
|
+
# were to change shape.
|
|
1224
|
+
if node - "$ROOT/scripts/hooks/lib/runnable-command.js" <<'NODEEOF' 2>"$TMPDIR_EVAL/ac362-heuristic.err"
|
|
1225
|
+
const { isAmbiguousAbsenceCommand } = require(process.argv[2]);
|
|
1226
|
+
const cases = [
|
|
1227
|
+
["grep -E 'removed-pattern' file.txt", true],
|
|
1228
|
+
["diff a.txt b.txt", true],
|
|
1229
|
+
["! grep -E 'removed-pattern' file.txt", false],
|
|
1230
|
+
["grep -c -E 'removed-pattern' file.txt | grep -qx 0", false],
|
|
1231
|
+
["grep -E 'removed-pattern' file.txt || true", false],
|
|
1232
|
+
["ls -la", false],
|
|
1233
|
+
];
|
|
1234
|
+
for (const [cmd, expected] of cases) {
|
|
1235
|
+
const got = isAmbiguousAbsenceCommand(cmd);
|
|
1236
|
+
if (got !== expected) {
|
|
1237
|
+
console.error(`isAmbiguousAbsenceCommand(${JSON.stringify(cmd)}) = ${got}, expected ${expected}`);
|
|
1238
|
+
process.exit(1);
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
NODEEOF
|
|
1242
|
+
then
|
|
1243
|
+
_pass "isAmbiguousAbsenceCommand distinguishes bare grep/diff (ambiguous) from negated/count-asserted/chained forms (self-asserting, not ambiguous)"
|
|
1244
|
+
else
|
|
1245
|
+
_fail "isAmbiguousAbsenceCommand heuristic regression: $(cat "$TMPDIR_EVAL/ac362-heuristic.err")"
|
|
1246
|
+
fi
|
|
1247
|
+
|
|
1248
|
+
|
|
1249
|
+
# ─── Iteration-2 fix item 4 (HIGH): ambiguity must not slip a TERMINAL stop ──────────────────
|
|
1250
|
+
# capturedFailReconciliation's THIRD bucket -- a command whose latest command-log capture is
|
|
1251
|
+
# `ambiguous` (bare grep/diff exit 1) AND some claim asserts pass for it -- must BLOCK a
|
|
1252
|
+
# terminal stop with its OWN distinct re-record-self-asserting message, never the
|
|
1253
|
+
# "caught false-completion" accusation. A non-terminal stop with the SAME fixture stays warn-only
|
|
1254
|
+
# (FULL_BLOCK already matched the plain ambiguous text before this fix; this proves the terminal
|
|
1255
|
+
# path is now ALSO covered, without changing non-terminal behavior).
|
|
1256
|
+
echo ""
|
|
1257
|
+
echo "--- iteration-2 fix item 4: ambiguous-with-pass-claim BLOCKS a terminal stop (distinct message, not the false-completion accusation) ---"
|
|
1258
|
+
|
|
1259
|
+
AMBIG_TERMINAL_REPO="$TMPDIR_EVAL/ambig-terminal/repo"
|
|
1260
|
+
mkdir -p "$AMBIG_TERMINAL_REPO/.kontourai/flow-agents/ambig-terminal-task"
|
|
1261
|
+
printf '# Test Repo\n' > "$AMBIG_TERMINAL_REPO/AGENTS.md"
|
|
1262
|
+
|
|
1263
|
+
cat > "$AMBIG_TERMINAL_REPO/.kontourai/flow-agents/ambig-terminal-task/ambig-terminal-task--deliver.md" <<'MARKDOWN'
|
|
1264
|
+
# Ambiguous-at-terminal-stop task
|
|
1265
|
+
|
|
1266
|
+
branch: main
|
|
1267
|
+
worktree: main
|
|
1268
|
+
created: 2026-07-05
|
|
1269
|
+
status: delivered
|
|
1270
|
+
type: deliver
|
|
1271
|
+
|
|
1272
|
+
## Definition Of Done
|
|
1273
|
+
- [x] tests pass
|
|
1274
|
+
|
|
1275
|
+
## Goal Fit Gate
|
|
1276
|
+
- [x] acceptance verified
|
|
1277
|
+
|
|
1278
|
+
## Verification Report
|
|
1279
|
+
|
|
1280
|
+
Build: PASS
|
|
1281
|
+
|
|
1282
|
+
### Verdict: PASS
|
|
1283
|
+
MARKDOWN
|
|
1284
|
+
|
|
1285
|
+
write_json_file "$AMBIG_TERMINAL_REPO/.kontourai/flow-agents/ambig-terminal-task/state.json" <<'JSON'
|
|
1286
|
+
{
|
|
1287
|
+
"schema_version": "1.0",
|
|
1288
|
+
"task_slug": "ambig-terminal-task",
|
|
1289
|
+
"status": "delivered",
|
|
1290
|
+
"phase": "done",
|
|
1291
|
+
"updated_at": "2026-07-05T00:00:00Z",
|
|
1292
|
+
"next_action": {
|
|
1293
|
+
"status": "done",
|
|
1294
|
+
"summary": "Delivered."
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
JSON
|
|
1298
|
+
|
|
1299
|
+
# A kit-typed claim (any namespace -- capturedFailReconciliation is namespace-agnostic) asserts
|
|
1300
|
+
# pass for "grep -E 'removed-pattern' haystack.txt" — a bare, non-self-asserting absence check.
|
|
1301
|
+
python3 - "$AMBIG_TERMINAL_REPO/.kontourai/flow-agents/ambig-terminal-task/trust.bundle" << 'PY'
|
|
1302
|
+
import json, sys
|
|
1303
|
+
bundle_path = sys.argv[1]
|
|
1304
|
+
bundle = {
|
|
1305
|
+
"schemaVersion": 5, "source": "flow-agents/workflow-sidecar",
|
|
1306
|
+
"claims": [{
|
|
1307
|
+
"id": "c1", "subjectId": "ambig-terminal-task/pattern-removed", "subjectType": "flow-step",
|
|
1308
|
+
"claimType": "builder.verify.tests",
|
|
1309
|
+
"fieldOrBehavior": "grep -E 'removed-pattern' haystack.txt",
|
|
1310
|
+
"value": "pass", "impactLevel": "high", "status": "verified",
|
|
1311
|
+
"createdAt": "2026-07-05T00:00:00Z", "updatedAt": "2026-07-05T00:00:00Z"
|
|
1312
|
+
}],
|
|
1313
|
+
"evidence": [{
|
|
1314
|
+
"id": "ev1", "claimId": "c1",
|
|
1315
|
+
"evidenceType": "command_output", "method": "capture",
|
|
1316
|
+
"sourceRef": "command-log.jsonl",
|
|
1317
|
+
"excerptOrSummary": "removed-pattern check (agent claimed pass)",
|
|
1318
|
+
"observedAt": "2026-07-05T00:00:00Z", "collectedBy": "agent",
|
|
1319
|
+
"passing": True,
|
|
1320
|
+
"execution": {"label": "grep -E 'removed-pattern' haystack.txt", "exitCode": 1}
|
|
1321
|
+
}],
|
|
1322
|
+
"policies": [], "events": []
|
|
1323
|
+
}
|
|
1324
|
+
json.dump(bundle, open(bundle_path, 'w'))
|
|
1325
|
+
PY
|
|
1326
|
+
|
|
1327
|
+
# The command-log's LATEST capture is exit 1 for a bare grep -- classifyLogEntry's ambiguous
|
|
1328
|
+
# carve-out applies (isAmbiguousAbsenceCommand recognizes this as bare, non-negated grep).
|
|
1329
|
+
printf '%s\n' '{"command":"grep -E '"'"'removed-pattern'"'"' haystack.txt","observedResult":"ambiguous","exitCode":1,"capturedAt":"2026-07-05T00:00:00Z","source":"postToolUse-capture"}' \
|
|
1330
|
+
> "$AMBIG_TERMINAL_REPO/.kontourai/flow-agents/ambig-terminal-task/command-log.jsonl"
|
|
1331
|
+
|
|
1332
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
1333
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/ambig-terminal.out" 2>"$TMPDIR_EVAL/ambig-terminal.err" <<JSON
|
|
1334
|
+
{"hook_event_name":"Stop","cwd":"$AMBIG_TERMINAL_REPO"}
|
|
1335
|
+
JSON
|
|
1336
|
+
then
|
|
1337
|
+
ambig_terminal_status=0
|
|
1338
|
+
else
|
|
1339
|
+
ambig_terminal_status=$?
|
|
1340
|
+
fi
|
|
1341
|
+
|
|
1342
|
+
if [[ "$ambig_terminal_status" -eq 2 ]]; then
|
|
1343
|
+
_pass "iteration-2 item 4: a terminal (delivered) stop with an ambiguous-with-pass-claim command BLOCKS (exit 2), not silently released"
|
|
1344
|
+
else
|
|
1345
|
+
_fail "iteration-2 item 4: a terminal stop with an ambiguous-with-pass-claim command did NOT block (exit=$ambig_terminal_status): $(cat "$TMPDIR_EVAL/ambig-terminal.out" "$TMPDIR_EVAL/ambig-terminal.err")"
|
|
1346
|
+
fi
|
|
1347
|
+
|
|
1348
|
+
if grep -q 'NOT_VERIFIED (ambiguous)' "$TMPDIR_EVAL/ambig-terminal.err"; then
|
|
1349
|
+
_pass "iteration-2 item 4: the block uses the DISTINCT 'NOT_VERIFIED (ambiguous)' re-record-self-asserting message"
|
|
1350
|
+
else
|
|
1351
|
+
_fail "iteration-2 item 4: the distinct ambiguous message was not found: $(cat "$TMPDIR_EVAL/ambig-terminal.out" "$TMPDIR_EVAL/ambig-terminal.err")"
|
|
1352
|
+
fi
|
|
1353
|
+
|
|
1354
|
+
if ! grep -q "captured command '[^']*' last ran FAIL" "$TMPDIR_EVAL/ambig-terminal.err"; then
|
|
1355
|
+
_pass "iteration-2 item 4: never uses the caught-false-completion accusation text for the ambiguous bucket (it is a DIFFERENT bucket from Case A)"
|
|
1356
|
+
else
|
|
1357
|
+
_fail "iteration-2 item 4: incorrectly used the false-completion accusation for an ambiguous (not FAIL) capture: $(cat "$TMPDIR_EVAL/ambig-terminal.out" "$TMPDIR_EVAL/ambig-terminal.err")"
|
|
1358
|
+
fi
|
|
1359
|
+
|
|
1360
|
+
# Non-terminal variant: SAME fixture shape, but status:in_progress/phase:build (not terminal,
|
|
1361
|
+
# not pre-execution) -- must stay warn-only in warn mode (exit 0), proving this fix did not
|
|
1362
|
+
# change the non-terminal warn/block split.
|
|
1363
|
+
AMBIG_NONTERMINAL_REPO="$TMPDIR_EVAL/ambig-nonterminal/repo"
|
|
1364
|
+
mkdir -p "$AMBIG_NONTERMINAL_REPO/.kontourai/flow-agents/ambig-nonterminal-task"
|
|
1365
|
+
printf '# Test Repo\n' > "$AMBIG_NONTERMINAL_REPO/AGENTS.md"
|
|
1366
|
+
|
|
1367
|
+
cat > "$AMBIG_NONTERMINAL_REPO/.kontourai/flow-agents/ambig-nonterminal-task/ambig-nonterminal-task--deliver.md" <<'MARKDOWN'
|
|
1368
|
+
# Ambiguous-at-nonterminal-stop task
|
|
1369
|
+
|
|
1370
|
+
branch: main
|
|
1371
|
+
worktree: main
|
|
1372
|
+
created: 2026-07-05
|
|
1373
|
+
status: executing
|
|
1374
|
+
type: deliver
|
|
1375
|
+
|
|
1376
|
+
## Plan
|
|
1377
|
+
|
|
1378
|
+
Same ambiguous+pass-claim shape, but NON-terminal (status:in_progress) -- warn-only expected.
|
|
1379
|
+
MARKDOWN
|
|
1380
|
+
|
|
1381
|
+
write_json_file "$AMBIG_NONTERMINAL_REPO/.kontourai/flow-agents/ambig-nonterminal-task/state.json" <<'JSON'
|
|
1382
|
+
{
|
|
1383
|
+
"schema_version": "1.0",
|
|
1384
|
+
"task_slug": "ambig-nonterminal-task",
|
|
1385
|
+
"status": "in_progress",
|
|
1386
|
+
"phase": "build",
|
|
1387
|
+
"updated_at": "2026-07-05T00:00:00Z",
|
|
1388
|
+
"next_action": {
|
|
1389
|
+
"status": "in_progress",
|
|
1390
|
+
"summary": "Mid-build."
|
|
1391
|
+
}
|
|
1392
|
+
}
|
|
1393
|
+
JSON
|
|
1394
|
+
|
|
1395
|
+
python3 - "$AMBIG_NONTERMINAL_REPO/.kontourai/flow-agents/ambig-nonterminal-task/trust.bundle" << 'PY'
|
|
1396
|
+
import json, sys
|
|
1397
|
+
bundle_path = sys.argv[1]
|
|
1398
|
+
bundle = {
|
|
1399
|
+
"schemaVersion": 5, "source": "flow-agents/workflow-sidecar",
|
|
1400
|
+
"claims": [{
|
|
1401
|
+
"id": "c1", "subjectId": "ambig-nonterminal-task/pattern-removed", "subjectType": "flow-step",
|
|
1402
|
+
"claimType": "builder.verify.tests",
|
|
1403
|
+
"fieldOrBehavior": "grep -E 'removed-pattern' haystack.txt",
|
|
1404
|
+
"value": "pass", "impactLevel": "high", "status": "verified",
|
|
1405
|
+
"createdAt": "2026-07-05T00:00:00Z", "updatedAt": "2026-07-05T00:00:00Z"
|
|
1406
|
+
}],
|
|
1407
|
+
"evidence": [{
|
|
1408
|
+
"id": "ev1", "claimId": "c1",
|
|
1409
|
+
"evidenceType": "command_output", "method": "capture",
|
|
1410
|
+
"sourceRef": "command-log.jsonl",
|
|
1411
|
+
"excerptOrSummary": "removed-pattern check (agent claimed pass)",
|
|
1412
|
+
"observedAt": "2026-07-05T00:00:00Z", "collectedBy": "agent",
|
|
1413
|
+
"passing": True,
|
|
1414
|
+
"execution": {"label": "grep -E 'removed-pattern' haystack.txt", "exitCode": 1}
|
|
1415
|
+
}],
|
|
1416
|
+
"policies": [], "events": []
|
|
1417
|
+
}
|
|
1418
|
+
json.dump(bundle, open(bundle_path, 'w'))
|
|
1419
|
+
PY
|
|
1420
|
+
|
|
1421
|
+
printf '%s\n' '{"command":"grep -E '"'"'removed-pattern'"'"' haystack.txt","observedResult":"ambiguous","exitCode":1,"capturedAt":"2026-07-05T00:00:00Z","source":"postToolUse-capture"}' \
|
|
1422
|
+
> "$AMBIG_NONTERMINAL_REPO/.kontourai/flow-agents/ambig-nonterminal-task/command-log.jsonl"
|
|
1423
|
+
|
|
1424
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=warn FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
1425
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/ambig-nonterminal.out" 2>"$TMPDIR_EVAL/ambig-nonterminal.err" <<JSON
|
|
1426
|
+
{"hook_event_name":"Stop","cwd":"$AMBIG_NONTERMINAL_REPO"}
|
|
1427
|
+
JSON
|
|
1428
|
+
then
|
|
1429
|
+
ambig_nonterminal_status=0
|
|
1430
|
+
else
|
|
1431
|
+
ambig_nonterminal_status=$?
|
|
1432
|
+
fi
|
|
1433
|
+
|
|
1434
|
+
if [[ "$ambig_nonterminal_status" -eq 0 ]]; then
|
|
1435
|
+
_pass "iteration-2 item 4: the SAME ambiguous+pass-claim fixture stays warn-only for a NON-terminal stop in warn mode (exit 0) -- terminal/non-terminal split unchanged"
|
|
1436
|
+
else
|
|
1437
|
+
_fail "iteration-2 item 4: non-terminal warn-mode stop unexpectedly exited nonzero: $ambig_nonterminal_status $(cat "$TMPDIR_EVAL/ambig-nonterminal.out" "$TMPDIR_EVAL/ambig-nonterminal.err")"
|
|
1438
|
+
fi
|
|
1439
|
+
if grep -q 'NOT_VERIFIED (ambiguous)' "$TMPDIR_EVAL/ambig-nonterminal.err"; then
|
|
1440
|
+
_pass "iteration-2 item 4: non-terminal stop still SURFACES the distinct ambiguous message (warn, not silently dropped)"
|
|
1441
|
+
else
|
|
1442
|
+
_fail "iteration-2 item 4: non-terminal stop dropped the ambiguous message entirely: $(cat "$TMPDIR_EVAL/ambig-nonterminal.out" "$TMPDIR_EVAL/ambig-nonterminal.err")"
|
|
1443
|
+
fi
|
|
1444
|
+
|
|
1445
|
+
|
|
1446
|
+
# ─── Iteration-2 fix item 1 (CRITICAL): ambiguous claimed-pass must HARD_BLOCK a terminal ────
|
|
1447
|
+
# stop in the COMMON code paths too -- not just capturedFailReconciliation's third bucket
|
|
1448
|
+
# (already covered above by the "iteration-2 fix item 4" ambig-terminal-task fixture, which
|
|
1449
|
+
# requires BOTH a captured command-log entry AND a trust.bundle). The CRITICAL bug: in the
|
|
1450
|
+
# default/common shapes -- evidence.json-only (no bundle), or a claimed-pass never actually
|
|
1451
|
+
# captured (forcing the live backstop re-run) -- captureCrossReference/runBackstop emitted a
|
|
1452
|
+
# plain "NOT_VERIFIED —" warning that only matches FULL_BLOCK (non-terminal), never HARD_BLOCK,
|
|
1453
|
+
# so a terminal `done`/`delivered` stop silently completed even in explicit strict block mode.
|
|
1454
|
+
echo ""
|
|
1455
|
+
echo "--- iteration-2 fix item 1 (CRITICAL): ambiguous claimed-pass HARD_BLOCKs a terminal stop in the COMMON (non-bundle-third-bucket) code paths ---"
|
|
1456
|
+
|
|
1457
|
+
# Case (i): evidence.json-only (NO trust.bundle), NO command-log.jsonl at all -- the exact
|
|
1458
|
+
# CRITICAL repro shape. Forces captureCrossReference into the live backstop re-run path
|
|
1459
|
+
# (runBackstop's ambiguous branch, the second of the two common emission sites the fix unifies).
|
|
1460
|
+
CRIT1_NOBUNDLE_REPO="$TMPDIR_EVAL/crit1-nobundle/repo"
|
|
1461
|
+
mkdir -p "$CRIT1_NOBUNDLE_REPO/.kontourai/flow-agents/crit1-nobundle-task"
|
|
1462
|
+
printf '# Test Repo\n' > "$CRIT1_NOBUNDLE_REPO/AGENTS.md"
|
|
1463
|
+
printf 'nothing interesting here\njust some other content\n' > "$CRIT1_NOBUNDLE_REPO/haystack.txt"
|
|
1464
|
+
|
|
1465
|
+
cat > "$CRIT1_NOBUNDLE_REPO/.kontourai/flow-agents/crit1-nobundle-task/crit1-nobundle-task--deliver.md" <<'MARKDOWN'
|
|
1466
|
+
# Ambiguous evidence.json-only terminal task (CRITICAL repro, case i)
|
|
1467
|
+
|
|
1468
|
+
branch: main
|
|
1469
|
+
worktree: main
|
|
1470
|
+
created: 2026-07-05
|
|
1471
|
+
status: delivered
|
|
1472
|
+
type: deliver
|
|
1473
|
+
|
|
1474
|
+
## Definition Of Done
|
|
1475
|
+
- [x] pattern removed
|
|
1476
|
+
|
|
1477
|
+
## Goal Fit Gate
|
|
1478
|
+
- [x] acceptance verified
|
|
1479
|
+
|
|
1480
|
+
## Verification Report
|
|
1481
|
+
|
|
1482
|
+
Build: PASS
|
|
1483
|
+
|
|
1484
|
+
### Verdict: PASS
|
|
1485
|
+
MARKDOWN
|
|
1486
|
+
|
|
1487
|
+
write_json_file "$CRIT1_NOBUNDLE_REPO/.kontourai/flow-agents/crit1-nobundle-task/state.json" <<'JSON'
|
|
1488
|
+
{
|
|
1489
|
+
"schema_version": "1.0",
|
|
1490
|
+
"task_slug": "crit1-nobundle-task",
|
|
1491
|
+
"status": "delivered",
|
|
1492
|
+
"phase": "done",
|
|
1493
|
+
"updated_at": "2026-07-05T00:00:00Z",
|
|
1494
|
+
"next_action": {
|
|
1495
|
+
"status": "done",
|
|
1496
|
+
"summary": "Delivered."
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
JSON
|
|
1500
|
+
|
|
1501
|
+
write_json_file "$CRIT1_NOBUNDLE_REPO/.kontourai/flow-agents/crit1-nobundle-task/acceptance.json" <<JSON
|
|
1502
|
+
{
|
|
1503
|
+
"schema_version": "1.0",
|
|
1504
|
+
"task_slug": "crit1-nobundle-task",
|
|
1505
|
+
"criteria": [
|
|
1506
|
+
{
|
|
1507
|
+
"id": "pattern-removed",
|
|
1508
|
+
"description": "The removed-pattern string no longer appears in haystack.txt.",
|
|
1509
|
+
"status": "pass",
|
|
1510
|
+
"evidence_refs": [
|
|
1511
|
+
{
|
|
1512
|
+
"kind": "command",
|
|
1513
|
+
"excerpt": "grep -E 'removed-pattern' $CRIT1_NOBUNDLE_REPO/haystack.txt",
|
|
1514
|
+
"summary": "Bare grep absence check (not self-asserting)."
|
|
1515
|
+
}
|
|
1516
|
+
]
|
|
1517
|
+
}
|
|
1518
|
+
],
|
|
1519
|
+
"goal_fit": {"status": "pass", "summary": "Pattern removal verified."}
|
|
1520
|
+
}
|
|
1521
|
+
JSON
|
|
1522
|
+
|
|
1523
|
+
write_json_file "$CRIT1_NOBUNDLE_REPO/.kontourai/flow-agents/crit1-nobundle-task/evidence.json" <<JSON
|
|
1524
|
+
{
|
|
1525
|
+
"schema_version": "1.0",
|
|
1526
|
+
"task_slug": "crit1-nobundle-task",
|
|
1527
|
+
"verdict": "pass",
|
|
1528
|
+
"checks": [
|
|
1529
|
+
{
|
|
1530
|
+
"id": "pattern-removed",
|
|
1531
|
+
"kind": "command",
|
|
1532
|
+
"status": "pass",
|
|
1533
|
+
"command": "grep -E 'removed-pattern' $CRIT1_NOBUNDLE_REPO/haystack.txt",
|
|
1534
|
+
"summary": "Confirmed removed-pattern is gone."
|
|
1535
|
+
}
|
|
1536
|
+
],
|
|
1537
|
+
"not_verified_gaps": []
|
|
1538
|
+
}
|
|
1539
|
+
JSON
|
|
1540
|
+
# No trust.bundle, no command-log.jsonl anywhere in this artifact dir.
|
|
1541
|
+
|
|
1542
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
1543
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/crit1-nobundle.out" 2>"$TMPDIR_EVAL/crit1-nobundle.err" <<JSON
|
|
1544
|
+
{"hook_event_name":"Stop","cwd":"$CRIT1_NOBUNDLE_REPO"}
|
|
1545
|
+
JSON
|
|
1546
|
+
then
|
|
1547
|
+
crit1_nobundle_status=0
|
|
1548
|
+
else
|
|
1549
|
+
crit1_nobundle_status=$?
|
|
1550
|
+
fi
|
|
1551
|
+
|
|
1552
|
+
if [[ "$crit1_nobundle_status" -eq 2 ]]; then
|
|
1553
|
+
_pass "iteration-2 item 1 (CRITICAL, case i -- evidence.json-only, no bundle, never captured): a terminal (delivered) stop with an ambiguous claimed-pass command now HARD_BLOCKs (exit 2) -- the exact CRITICAL repro shape"
|
|
1554
|
+
else
|
|
1555
|
+
_fail "iteration-2 item 1 (CRITICAL) REGRESSION: case i (evidence.json-only, no bundle, never captured) did NOT block a terminal stop in explicit block mode -- the CRITICAL bug is back: exit=$crit1_nobundle_status $(cat "$TMPDIR_EVAL/crit1-nobundle.out" "$TMPDIR_EVAL/crit1-nobundle.err")"
|
|
1556
|
+
fi
|
|
1557
|
+
if grep -q 'NOT_VERIFIED (ambiguous)' "$TMPDIR_EVAL/crit1-nobundle.err"; then
|
|
1558
|
+
_pass "iteration-2 item 1 (case i): the block uses the distinct 'NOT_VERIFIED (ambiguous)' re-record-self-asserting message"
|
|
1559
|
+
else
|
|
1560
|
+
_fail "iteration-2 item 1 (case i): the distinct ambiguous message was not found: $(cat "$TMPDIR_EVAL/crit1-nobundle.out" "$TMPDIR_EVAL/crit1-nobundle.err")"
|
|
1561
|
+
fi
|
|
1562
|
+
if ! grep -q 'caught false-completion' "$TMPDIR_EVAL/crit1-nobundle.err"; then
|
|
1563
|
+
_pass "iteration-2 item 1 (case i): never uses the caught-false-completion accusation for the ambiguous case (ac4 preserved: classification/message only, never the accusation)"
|
|
1564
|
+
else
|
|
1565
|
+
_fail "iteration-2 item 1 (case i): incorrectly used the false-completion accusation for an ambiguous (not FAIL) case: $(cat "$TMPDIR_EVAL/crit1-nobundle.out" "$TMPDIR_EVAL/crit1-nobundle.err")"
|
|
1566
|
+
fi
|
|
1567
|
+
|
|
1568
|
+
# Case (ii): trust.bundle PRESENT (a `workflow.check.command` claim asserting pass), but the
|
|
1569
|
+
# claimed-pass command was NEVER captured at all (no command-log.jsonl in this artifact dir) --
|
|
1570
|
+
# forces the SAME live-backstop-re-run path as case (i), but sourced from the bundle instead of
|
|
1571
|
+
# the evidence.json fallback, proving the fix covers both claimedPass sources.
|
|
1572
|
+
CRIT1_BUNDLE_REPO="$TMPDIR_EVAL/crit1-bundle-nevercaptured/repo"
|
|
1573
|
+
mkdir -p "$CRIT1_BUNDLE_REPO/.kontourai/flow-agents/crit1-bundle-task"
|
|
1574
|
+
printf '# Test Repo\n' > "$CRIT1_BUNDLE_REPO/AGENTS.md"
|
|
1575
|
+
printf 'nothing interesting here\njust some other content\n' > "$CRIT1_BUNDLE_REPO/haystack.txt"
|
|
1576
|
+
|
|
1577
|
+
cat > "$CRIT1_BUNDLE_REPO/.kontourai/flow-agents/crit1-bundle-task/crit1-bundle-task--deliver.md" <<'MARKDOWN'
|
|
1578
|
+
# Ambiguous bundle-sourced, never-captured terminal task (CRITICAL repro, case ii)
|
|
1579
|
+
|
|
1580
|
+
branch: main
|
|
1581
|
+
worktree: main
|
|
1582
|
+
created: 2026-07-05
|
|
1583
|
+
status: delivered
|
|
1584
|
+
type: deliver
|
|
1585
|
+
|
|
1586
|
+
## Definition Of Done
|
|
1587
|
+
- [x] pattern removed
|
|
1588
|
+
|
|
1589
|
+
## Goal Fit Gate
|
|
1590
|
+
- [x] acceptance verified
|
|
1591
|
+
|
|
1592
|
+
## Verification Report
|
|
1593
|
+
|
|
1594
|
+
Build: PASS
|
|
1595
|
+
|
|
1596
|
+
### Verdict: PASS
|
|
1597
|
+
MARKDOWN
|
|
1598
|
+
|
|
1599
|
+
write_json_file "$CRIT1_BUNDLE_REPO/.kontourai/flow-agents/crit1-bundle-task/state.json" <<'JSON'
|
|
1600
|
+
{
|
|
1601
|
+
"schema_version": "1.0",
|
|
1602
|
+
"task_slug": "crit1-bundle-task",
|
|
1603
|
+
"status": "delivered",
|
|
1604
|
+
"phase": "done",
|
|
1605
|
+
"updated_at": "2026-07-05T00:00:00Z",
|
|
1606
|
+
"next_action": {
|
|
1607
|
+
"status": "done",
|
|
1608
|
+
"summary": "Delivered."
|
|
1609
|
+
}
|
|
1610
|
+
}
|
|
1611
|
+
JSON
|
|
1612
|
+
|
|
1613
|
+
write_json_file "$CRIT1_BUNDLE_REPO/.kontourai/flow-agents/crit1-bundle-task/acceptance.json" <<JSON
|
|
1614
|
+
{
|
|
1615
|
+
"schema_version": "1.0",
|
|
1616
|
+
"task_slug": "crit1-bundle-task",
|
|
1617
|
+
"criteria": [
|
|
1618
|
+
{
|
|
1619
|
+
"id": "pattern-removed",
|
|
1620
|
+
"description": "The removed-pattern string no longer appears in haystack.txt.",
|
|
1621
|
+
"status": "pass",
|
|
1622
|
+
"evidence_refs": [
|
|
1623
|
+
{
|
|
1624
|
+
"kind": "command",
|
|
1625
|
+
"excerpt": "grep -E 'removed-pattern' $CRIT1_BUNDLE_REPO/haystack.txt",
|
|
1626
|
+
"summary": "Bare grep absence check (not self-asserting)."
|
|
1627
|
+
}
|
|
1628
|
+
]
|
|
1629
|
+
}
|
|
1630
|
+
],
|
|
1631
|
+
"goal_fit": {"status": "pass", "summary": "Pattern removal verified."}
|
|
1632
|
+
}
|
|
1633
|
+
JSON
|
|
1634
|
+
|
|
1635
|
+
# trust.bundle claims pass via a `workflow.check.command` claim, WITHOUT any evidence[] item
|
|
1636
|
+
# carrying execution.label -- exactly the "claimed pass but never captured" bundle shape
|
|
1637
|
+
# bundleClaimedPassCommandChecks' part (B) recognizes.
|
|
1638
|
+
python3 - "$CRIT1_BUNDLE_REPO/.kontourai/flow-agents/crit1-bundle-task/trust.bundle" << 'PY'
|
|
1639
|
+
import json, sys
|
|
1640
|
+
bundle_path = sys.argv[1]
|
|
1641
|
+
bundle = {
|
|
1642
|
+
"schemaVersion": 5, "source": "flow-agents/workflow-sidecar",
|
|
1643
|
+
"claims": [{
|
|
1644
|
+
"id": "c1", "subjectId": "crit1-bundle-task/pattern-removed", "subjectType": "flow-step",
|
|
1645
|
+
"claimType": "workflow.check.command",
|
|
1646
|
+
"fieldOrBehavior": "grep -E 'removed-pattern' haystack.txt",
|
|
1647
|
+
"value": "pass", "impactLevel": "high", "status": "verified",
|
|
1648
|
+
"createdAt": "2026-07-05T00:00:00Z", "updatedAt": "2026-07-05T00:00:00Z"
|
|
1649
|
+
}],
|
|
1650
|
+
"evidence": [], "policies": [], "events": []
|
|
1651
|
+
}
|
|
1652
|
+
json.dump(bundle, open(bundle_path, 'w'))
|
|
1653
|
+
PY
|
|
1654
|
+
# No command-log.jsonl in this artifact dir at all -- this claimed-pass check was never captured.
|
|
1655
|
+
|
|
1656
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
1657
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/crit1-bundle.out" 2>"$TMPDIR_EVAL/crit1-bundle.err" <<JSON
|
|
1658
|
+
{"hook_event_name":"Stop","cwd":"$CRIT1_BUNDLE_REPO"}
|
|
1659
|
+
JSON
|
|
1660
|
+
then
|
|
1661
|
+
crit1_bundle_status=0
|
|
1662
|
+
else
|
|
1663
|
+
crit1_bundle_status=$?
|
|
1664
|
+
fi
|
|
1665
|
+
|
|
1666
|
+
if [[ "$crit1_bundle_status" -eq 2 ]]; then
|
|
1667
|
+
_pass "iteration-2 item 1 (CRITICAL, case ii -- trust.bundle present, never captured, live-backstop-only): a terminal (delivered) stop with an ambiguous bundle-sourced claimed-pass command now HARD_BLOCKs (exit 2)"
|
|
1668
|
+
else
|
|
1669
|
+
_fail "iteration-2 item 1 (CRITICAL) REGRESSION: case ii (bundle present, never captured) did NOT block a terminal stop in explicit block mode: exit=$crit1_bundle_status $(cat "$TMPDIR_EVAL/crit1-bundle.out" "$TMPDIR_EVAL/crit1-bundle.err")"
|
|
1670
|
+
fi
|
|
1671
|
+
if grep -q 'NOT_VERIFIED (ambiguous)' "$TMPDIR_EVAL/crit1-bundle.err"; then
|
|
1672
|
+
_pass "iteration-2 item 1 (case ii): the block uses the distinct 'NOT_VERIFIED (ambiguous)' re-record-self-asserting message"
|
|
1673
|
+
else
|
|
1674
|
+
_fail "iteration-2 item 1 (case ii): the distinct ambiguous message was not found: $(cat "$TMPDIR_EVAL/crit1-bundle.out" "$TMPDIR_EVAL/crit1-bundle.err")"
|
|
1675
|
+
fi
|
|
1676
|
+
|
|
1677
|
+
# Site-1 coverage: captureCrossReference's OWN capture-log cross-reference shortcut (distinct
|
|
1678
|
+
# from capturedFailReconciliation's third bucket) -- the claimed-pass command WAS captured, and
|
|
1679
|
+
# its LATEST log entry is ambiguous, at a TERMINAL stop, with NO trust.bundle (so
|
|
1680
|
+
# capturedFailReconciliation's bucket -- which requires a bundle claim -- cannot be what fires
|
|
1681
|
+
# here; only captureCrossReference's own capture-log branch can).
|
|
1682
|
+
CRIT1_CAPTURED_REPO="$TMPDIR_EVAL/crit1-captured-ambiguous/repo"
|
|
1683
|
+
mkdir -p "$CRIT1_CAPTURED_REPO/.kontourai/flow-agents/crit1-captured-task"
|
|
1684
|
+
printf '# Test Repo\n' > "$CRIT1_CAPTURED_REPO/AGENTS.md"
|
|
1685
|
+
|
|
1686
|
+
cat > "$CRIT1_CAPTURED_REPO/.kontourai/flow-agents/crit1-captured-task/crit1-captured-task--deliver.md" <<'MARKDOWN'
|
|
1687
|
+
# Ambiguous captured-log-only terminal task (CRITICAL repro, capture-log shortcut site)
|
|
1688
|
+
|
|
1689
|
+
branch: main
|
|
1690
|
+
worktree: main
|
|
1691
|
+
created: 2026-07-05
|
|
1692
|
+
status: delivered
|
|
1693
|
+
type: deliver
|
|
1694
|
+
|
|
1695
|
+
## Definition Of Done
|
|
1696
|
+
- [x] pattern removed
|
|
1697
|
+
|
|
1698
|
+
## Goal Fit Gate
|
|
1699
|
+
- [x] acceptance verified
|
|
1700
|
+
|
|
1701
|
+
## Verification Report
|
|
1702
|
+
|
|
1703
|
+
Build: PASS
|
|
1704
|
+
|
|
1705
|
+
### Verdict: PASS
|
|
1706
|
+
MARKDOWN
|
|
1707
|
+
|
|
1708
|
+
write_json_file "$CRIT1_CAPTURED_REPO/.kontourai/flow-agents/crit1-captured-task/state.json" <<'JSON'
|
|
1709
|
+
{
|
|
1710
|
+
"schema_version": "1.0",
|
|
1711
|
+
"task_slug": "crit1-captured-task",
|
|
1712
|
+
"status": "delivered",
|
|
1713
|
+
"phase": "done",
|
|
1714
|
+
"updated_at": "2026-07-05T00:00:00Z",
|
|
1715
|
+
"next_action": {
|
|
1716
|
+
"status": "done",
|
|
1717
|
+
"summary": "Delivered."
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1720
|
+
JSON
|
|
1721
|
+
|
|
1722
|
+
write_json_file "$CRIT1_CAPTURED_REPO/.kontourai/flow-agents/crit1-captured-task/evidence.json" <<'JSON'
|
|
1723
|
+
{
|
|
1724
|
+
"schema_version": "1.0",
|
|
1725
|
+
"task_slug": "crit1-captured-task",
|
|
1726
|
+
"verdict": "pass",
|
|
1727
|
+
"checks": [
|
|
1728
|
+
{
|
|
1729
|
+
"id": "pattern-removed",
|
|
1730
|
+
"kind": "command",
|
|
1731
|
+
"status": "pass",
|
|
1732
|
+
"command": "grep -E 'removed-pattern' haystack.txt",
|
|
1733
|
+
"summary": "Confirmed removed-pattern is gone."
|
|
1734
|
+
}
|
|
1735
|
+
],
|
|
1736
|
+
"not_verified_gaps": []
|
|
1737
|
+
}
|
|
1738
|
+
JSON
|
|
1739
|
+
|
|
1740
|
+
# command-log.jsonl HAS an entry for this EXACT command -- its LATEST (only) capture is
|
|
1741
|
+
# ambiguous (bare grep exit 1). No trust.bundle in this artifact dir at all.
|
|
1742
|
+
printf '%s\n' '{"command":"grep -E '"'"'removed-pattern'"'"' haystack.txt","observedResult":"ambiguous","exitCode":1,"capturedAt":"2026-07-05T00:00:00Z","source":"postToolUse-capture"}' \
|
|
1743
|
+
> "$CRIT1_CAPTURED_REPO/.kontourai/flow-agents/crit1-captured-task/command-log.jsonl"
|
|
1744
|
+
|
|
1745
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
1746
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/crit1-captured.out" 2>"$TMPDIR_EVAL/crit1-captured.err" <<JSON
|
|
1747
|
+
{"hook_event_name":"Stop","cwd":"$CRIT1_CAPTURED_REPO"}
|
|
1748
|
+
JSON
|
|
1749
|
+
then
|
|
1750
|
+
crit1_captured_status=0
|
|
1751
|
+
else
|
|
1752
|
+
crit1_captured_status=$?
|
|
1753
|
+
fi
|
|
1754
|
+
|
|
1755
|
+
if [[ "$crit1_captured_status" -eq 2 ]]; then
|
|
1756
|
+
_pass "iteration-2 item 1 (CRITICAL, capture-log cross-reference shortcut site, no bundle): a terminal stop with a CAPTURED ambiguous command claimed pass now HARD_BLOCKs (exit 2)"
|
|
1757
|
+
else
|
|
1758
|
+
_fail "iteration-2 item 1 (CRITICAL) REGRESSION: the capture-log cross-reference shortcut site (captured, ambiguous, no bundle) did NOT block a terminal stop: exit=$crit1_captured_status $(cat "$TMPDIR_EVAL/crit1-captured.out" "$TMPDIR_EVAL/crit1-captured.err")"
|
|
1759
|
+
fi
|
|
1760
|
+
if grep -q 'NOT_VERIFIED (ambiguous)' "$TMPDIR_EVAL/crit1-captured.err"; then
|
|
1761
|
+
_pass "iteration-2 item 1 (capture-log shortcut site): the block uses the distinct 'NOT_VERIFIED (ambiguous)' re-record-self-asserting message"
|
|
1762
|
+
else
|
|
1763
|
+
_fail "iteration-2 item 1 (capture-log shortcut site): the distinct ambiguous message was not found: $(cat "$TMPDIR_EVAL/crit1-captured.out" "$TMPDIR_EVAL/crit1-captured.err")"
|
|
1764
|
+
fi
|
|
1765
|
+
|
|
1766
|
+
# ─── Iteration-2 (re-plan) finding #3 (HIGH), Decision A: no-signal-ambiguous split ───────────
|
|
1767
|
+
# Site-1 coverage (captureCrossReference's own capture-log branch, same site as crit1-captured
|
|
1768
|
+
# above): a GENERIC command ("npm test") whose LATEST capture is `ambiguous` with NO exit code
|
|
1769
|
+
# recoverable at all (`exitCode:null` — e.g. an unreadable/missing codex host banner) is a
|
|
1770
|
+
# DIFFERENT origin from the #362 absence-ambiguous carve-out (bare grep/diff exit EXACTLY 1)
|
|
1771
|
+
# exercised by crit1-captured-task directly above. Decision A: this no-signal case gets
|
|
1772
|
+
# grep/diff-FREE, accurate wording and is warn-only at a TERMINAL stop (never HARD_BLOCKs),
|
|
1773
|
+
# while still blocking a NON-terminal stop via FULL_BLOCK's existing `NOT_VERIFIED —` pattern.
|
|
1774
|
+
# The crit1-captured-task assertions directly above are the regression guard proving the #362
|
|
1775
|
+
# absence-ambiguous case still HARD_BLOCKs a terminal stop, unchanged.
|
|
1776
|
+
echo ""
|
|
1777
|
+
echo "--- iteration-2 (re-plan) finding #3 Decision A: no-signal-ambiguous is warn-only at a terminal stop (site 1) ---"
|
|
1778
|
+
|
|
1779
|
+
NOSIGNAL_TERMINAL_REPO="$TMPDIR_EVAL/nosignal-terminal/repo"
|
|
1780
|
+
mkdir -p "$NOSIGNAL_TERMINAL_REPO/.kontourai/flow-agents/nosignal-terminal-task"
|
|
1781
|
+
printf '# Test Repo\n' > "$NOSIGNAL_TERMINAL_REPO/AGENTS.md"
|
|
1782
|
+
|
|
1783
|
+
cat > "$NOSIGNAL_TERMINAL_REPO/.kontourai/flow-agents/nosignal-terminal-task/nosignal-terminal-task--deliver.md" <<'MARKDOWN'
|
|
1784
|
+
# No-signal ambiguous terminal task (finding #3, generic command, no exit code)
|
|
1785
|
+
|
|
1786
|
+
branch: main
|
|
1787
|
+
worktree: main
|
|
1788
|
+
created: 2026-07-06
|
|
1789
|
+
status: delivered
|
|
1790
|
+
type: deliver
|
|
1791
|
+
|
|
1792
|
+
## Definition Of Done
|
|
1793
|
+
- [x] tests pass
|
|
1794
|
+
|
|
1795
|
+
## Goal Fit Gate
|
|
1796
|
+
- [x] acceptance verified
|
|
1797
|
+
|
|
1798
|
+
## Verification Report
|
|
1799
|
+
|
|
1800
|
+
Build: PASS
|
|
1801
|
+
|
|
1802
|
+
### Verdict: PASS
|
|
1803
|
+
MARKDOWN
|
|
1804
|
+
|
|
1805
|
+
write_json_file "$NOSIGNAL_TERMINAL_REPO/.kontourai/flow-agents/nosignal-terminal-task/state.json" <<'JSON'
|
|
1806
|
+
{
|
|
1807
|
+
"schema_version": "1.0",
|
|
1808
|
+
"task_slug": "nosignal-terminal-task",
|
|
1809
|
+
"status": "delivered",
|
|
1810
|
+
"phase": "done",
|
|
1811
|
+
"updated_at": "2026-07-06T00:00:00Z",
|
|
1812
|
+
"next_action": {
|
|
1813
|
+
"status": "done",
|
|
1814
|
+
"summary": "Delivered."
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
JSON
|
|
1818
|
+
|
|
1819
|
+
write_json_file "$NOSIGNAL_TERMINAL_REPO/.kontourai/flow-agents/nosignal-terminal-task/evidence.json" <<'JSON'
|
|
1820
|
+
{
|
|
1821
|
+
"schema_version": "1.0",
|
|
1822
|
+
"task_slug": "nosignal-terminal-task",
|
|
1823
|
+
"verdict": "pass",
|
|
1824
|
+
"checks": [
|
|
1825
|
+
{
|
|
1826
|
+
"id": "tests-pass",
|
|
1827
|
+
"kind": "command",
|
|
1828
|
+
"status": "pass",
|
|
1829
|
+
"command": "npm test",
|
|
1830
|
+
"summary": "Claimed tests pass."
|
|
1831
|
+
}
|
|
1832
|
+
],
|
|
1833
|
+
"not_verified_gaps": []
|
|
1834
|
+
}
|
|
1835
|
+
JSON
|
|
1836
|
+
|
|
1837
|
+
# command-log.jsonl HAS an entry for this EXACT command -- its LATEST (only) capture is a
|
|
1838
|
+
# no-signal ambiguous (observedResult:"ambiguous", exitCode:null), NOT the #362
|
|
1839
|
+
# absence-ambiguous carve-out (which requires exitCode===1 on a bare grep/diff). No trust.bundle
|
|
1840
|
+
# in this artifact dir at all.
|
|
1841
|
+
printf '%s\n' '{"command":"npm test","observedResult":"ambiguous","exitCode":null,"capturedAt":"2026-07-06T00:00:00Z","source":"postToolUse-capture"}' \
|
|
1842
|
+
> "$NOSIGNAL_TERMINAL_REPO/.kontourai/flow-agents/nosignal-terminal-task/command-log.jsonl"
|
|
1843
|
+
|
|
1844
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
1845
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/nosignal-terminal.out" 2>"$TMPDIR_EVAL/nosignal-terminal.err" <<JSON
|
|
1846
|
+
{"hook_event_name":"Stop","cwd":"$NOSIGNAL_TERMINAL_REPO"}
|
|
1847
|
+
JSON
|
|
1848
|
+
then
|
|
1849
|
+
nosignal_terminal_status=0
|
|
1850
|
+
else
|
|
1851
|
+
nosignal_terminal_status=$?
|
|
1852
|
+
fi
|
|
1853
|
+
|
|
1854
|
+
if [[ "$nosignal_terminal_status" -eq 0 ]]; then
|
|
1855
|
+
_pass "finding #3 Decision A: a terminal (delivered) stop with a no-signal-ambiguous claimed-pass command ('npm test', exitCode:null) does NOT hard-block (exit 0) -- warn-only, no exit code was ever observed on this host"
|
|
1856
|
+
else
|
|
1857
|
+
_fail "finding #3 Decision A REGRESSION: a terminal stop with a no-signal-ambiguous claimed-pass command WRONGLY blocked (exit=$nosignal_terminal_status): $(cat "$TMPDIR_EVAL/nosignal-terminal.out" "$TMPDIR_EVAL/nosignal-terminal.err")"
|
|
1858
|
+
fi
|
|
1859
|
+
|
|
1860
|
+
if ! grep -qi 'grep/diff\|NOT_VERIFIED (ambiguous)' "$TMPDIR_EVAL/nosignal-terminal.err"; then
|
|
1861
|
+
_pass "finding #3 Decision A: the no-signal-ambiguous message contains NO grep/diff wording and NOT the absence-ambiguous 'NOT_VERIFIED (ambiguous)' marker -- accurate wording for a generic command"
|
|
1862
|
+
else
|
|
1863
|
+
_fail "finding #3 Decision A REGRESSION: the no-signal-ambiguous message wrongly carries grep/diff wording or the absence-ambiguous marker: $(cat "$TMPDIR_EVAL/nosignal-terminal.out" "$TMPDIR_EVAL/nosignal-terminal.err")"
|
|
1864
|
+
fi
|
|
1865
|
+
|
|
1866
|
+
if grep -q 'NOT_VERIFIED —' "$TMPDIR_EVAL/nosignal-terminal.err"; then
|
|
1867
|
+
_pass "finding #3 Decision A: the no-signal-ambiguous message still surfaces via the generic 'NOT_VERIFIED —' pattern (warn, not silently dropped)"
|
|
1868
|
+
else
|
|
1869
|
+
_fail "finding #3 Decision A REGRESSION: the no-signal-ambiguous message was dropped entirely: $(cat "$TMPDIR_EVAL/nosignal-terminal.out" "$TMPDIR_EVAL/nosignal-terminal.err")"
|
|
1870
|
+
fi
|
|
1871
|
+
|
|
1872
|
+
# Non-terminal variant: SAME fixture, but status:in_progress/phase:build (not terminal, not
|
|
1873
|
+
# pre-execution) -- FULL_BLOCK's broader `evidence check` / `NOT_VERIFIED —` patterns still
|
|
1874
|
+
# BLOCK a non-terminal stop, proving the split is warn-terminal / block-non-terminal, not a
|
|
1875
|
+
# blanket downgrade to non-blocking.
|
|
1876
|
+
NOSIGNAL_NONTERMINAL_REPO="$TMPDIR_EVAL/nosignal-nonterminal/repo"
|
|
1877
|
+
mkdir -p "$NOSIGNAL_NONTERMINAL_REPO/.kontourai/flow-agents/nosignal-nonterminal-task"
|
|
1878
|
+
printf '# Test Repo\n' > "$NOSIGNAL_NONTERMINAL_REPO/AGENTS.md"
|
|
1879
|
+
|
|
1880
|
+
cat > "$NOSIGNAL_NONTERMINAL_REPO/.kontourai/flow-agents/nosignal-nonterminal-task/nosignal-nonterminal-task--deliver.md" <<'MARKDOWN'
|
|
1881
|
+
# No-signal ambiguous non-terminal task (finding #3)
|
|
1882
|
+
|
|
1883
|
+
branch: main
|
|
1884
|
+
worktree: main
|
|
1885
|
+
created: 2026-07-06
|
|
1886
|
+
status: executing
|
|
1887
|
+
type: deliver
|
|
1888
|
+
|
|
1889
|
+
## Plan
|
|
1890
|
+
|
|
1891
|
+
Same no-signal-ambiguous shape, but NON-terminal (status:in_progress) -- must still BLOCK.
|
|
1892
|
+
MARKDOWN
|
|
1893
|
+
|
|
1894
|
+
write_json_file "$NOSIGNAL_NONTERMINAL_REPO/.kontourai/flow-agents/nosignal-nonterminal-task/state.json" <<'JSON'
|
|
1895
|
+
{
|
|
1896
|
+
"schema_version": "1.0",
|
|
1897
|
+
"task_slug": "nosignal-nonterminal-task",
|
|
1898
|
+
"status": "in_progress",
|
|
1899
|
+
"phase": "build",
|
|
1900
|
+
"updated_at": "2026-07-06T00:00:00Z",
|
|
1901
|
+
"next_action": {
|
|
1902
|
+
"status": "in_progress",
|
|
1903
|
+
"summary": "Mid-build."
|
|
1904
|
+
}
|
|
1905
|
+
}
|
|
1906
|
+
JSON
|
|
1907
|
+
|
|
1908
|
+
write_json_file "$NOSIGNAL_NONTERMINAL_REPO/.kontourai/flow-agents/nosignal-nonterminal-task/evidence.json" <<'JSON'
|
|
1909
|
+
{
|
|
1910
|
+
"schema_version": "1.0",
|
|
1911
|
+
"task_slug": "nosignal-nonterminal-task",
|
|
1912
|
+
"verdict": "pass",
|
|
1913
|
+
"checks": [
|
|
1914
|
+
{
|
|
1915
|
+
"id": "tests-pass",
|
|
1916
|
+
"kind": "command",
|
|
1917
|
+
"status": "pass",
|
|
1918
|
+
"command": "npm test",
|
|
1919
|
+
"summary": "Claimed tests pass."
|
|
1920
|
+
}
|
|
1921
|
+
],
|
|
1922
|
+
"not_verified_gaps": []
|
|
1923
|
+
}
|
|
1924
|
+
JSON
|
|
1925
|
+
|
|
1926
|
+
printf '%s\n' '{"command":"npm test","observedResult":"ambiguous","exitCode":null,"capturedAt":"2026-07-06T00:00:00Z","source":"postToolUse-capture"}' \
|
|
1927
|
+
> "$NOSIGNAL_NONTERMINAL_REPO/.kontourai/flow-agents/nosignal-nonterminal-task/command-log.jsonl"
|
|
1928
|
+
|
|
1929
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
1930
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/nosignal-nonterminal.out" 2>"$TMPDIR_EVAL/nosignal-nonterminal.err" <<JSON
|
|
1931
|
+
{"hook_event_name":"Stop","cwd":"$NOSIGNAL_NONTERMINAL_REPO"}
|
|
1932
|
+
JSON
|
|
1933
|
+
then
|
|
1934
|
+
nosignal_nonterminal_status=0
|
|
1935
|
+
else
|
|
1936
|
+
nosignal_nonterminal_status=$?
|
|
1937
|
+
fi
|
|
1938
|
+
|
|
1939
|
+
if [[ "$nosignal_nonterminal_status" -eq 2 ]]; then
|
|
1940
|
+
_pass "finding #3 Decision A: the SAME no-signal-ambiguous fixture BLOCKS (exit 2) at a non-terminal (executing) stop via FULL_BLOCK -- terminal-warn/non-terminal-block split confirmed"
|
|
1941
|
+
else
|
|
1942
|
+
_fail "finding #3 Decision A REGRESSION: a non-terminal stop with a no-signal-ambiguous claimed-pass command did NOT block: exit=$nosignal_nonterminal_status $(cat "$TMPDIR_EVAL/nosignal-nonterminal.out" "$TMPDIR_EVAL/nosignal-nonterminal.err")"
|
|
1943
|
+
fi
|
|
1944
|
+
|
|
1945
|
+
# Negative control: a LEGITIMATE self-asserting absence check ('! grep -q ...', which exits 0
|
|
1946
|
+
# when the pattern is genuinely absent) claimed pass at a TERMINAL stop must still PASS -- no
|
|
1947
|
+
# new over-block introduced by this fix. Same never-captured/no-bundle shape as case (i), but
|
|
1948
|
+
# the recorded command is self-asserting, not bare.
|
|
1949
|
+
CRIT1_NEGCONTROL_REPO="$TMPDIR_EVAL/crit1-negcontrol/repo"
|
|
1950
|
+
mkdir -p "$CRIT1_NEGCONTROL_REPO/.kontourai/flow-agents/crit1-negcontrol-task"
|
|
1951
|
+
printf '# Test Repo\n' > "$CRIT1_NEGCONTROL_REPO/AGENTS.md"
|
|
1952
|
+
printf 'nothing interesting here\njust some other content\n' > "$CRIT1_NEGCONTROL_REPO/haystack.txt"
|
|
1953
|
+
|
|
1954
|
+
cat > "$CRIT1_NEGCONTROL_REPO/.kontourai/flow-agents/crit1-negcontrol-task/crit1-negcontrol-task--deliver.md" <<'MARKDOWN'
|
|
1955
|
+
# Self-asserting absence check terminal task (negative control -- must NOT block)
|
|
1956
|
+
|
|
1957
|
+
branch: main
|
|
1958
|
+
worktree: main
|
|
1959
|
+
created: 2026-07-05
|
|
1960
|
+
status: delivered
|
|
1961
|
+
type: deliver
|
|
1962
|
+
|
|
1963
|
+
## Definition Of Done
|
|
1964
|
+
- [x] pattern removed
|
|
1965
|
+
|
|
1966
|
+
## Goal Fit Gate
|
|
1967
|
+
- [x] acceptance verified
|
|
1968
|
+
|
|
1969
|
+
## Verification Report
|
|
1970
|
+
|
|
1971
|
+
Build: PASS
|
|
1972
|
+
|
|
1973
|
+
### Verdict: PASS
|
|
1974
|
+
MARKDOWN
|
|
1975
|
+
|
|
1976
|
+
write_json_file "$CRIT1_NEGCONTROL_REPO/.kontourai/flow-agents/crit1-negcontrol-task/state.json" <<'JSON'
|
|
1977
|
+
{
|
|
1978
|
+
"schema_version": "1.0",
|
|
1979
|
+
"task_slug": "crit1-negcontrol-task",
|
|
1980
|
+
"status": "delivered",
|
|
1981
|
+
"phase": "done",
|
|
1982
|
+
"updated_at": "2026-07-05T00:00:00Z",
|
|
1983
|
+
"next_action": {
|
|
1984
|
+
"status": "done",
|
|
1985
|
+
"summary": "Delivered."
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
JSON
|
|
1989
|
+
|
|
1990
|
+
write_json_file "$CRIT1_NEGCONTROL_REPO/.kontourai/flow-agents/crit1-negcontrol-task/acceptance.json" <<JSON
|
|
1991
|
+
{
|
|
1992
|
+
"schema_version": "1.0",
|
|
1993
|
+
"task_slug": "crit1-negcontrol-task",
|
|
1994
|
+
"criteria": [
|
|
1995
|
+
{
|
|
1996
|
+
"id": "pattern-absent",
|
|
1997
|
+
"description": "The removed-pattern string is confirmed absent (self-asserting form).",
|
|
1998
|
+
"status": "pass",
|
|
1999
|
+
"evidence_refs": [
|
|
2000
|
+
{
|
|
2001
|
+
"kind": "command",
|
|
2002
|
+
"excerpt": "! grep -q 'removed-pattern' $CRIT1_NEGCONTROL_REPO/haystack.txt",
|
|
2003
|
+
"summary": "Self-asserting absence check (negated grep -q)."
|
|
2004
|
+
}
|
|
2005
|
+
]
|
|
2006
|
+
}
|
|
2007
|
+
],
|
|
2008
|
+
"goal_fit": {"status": "pass", "summary": "Pattern absence verified (self-asserting)."}
|
|
2009
|
+
}
|
|
2010
|
+
JSON
|
|
2011
|
+
|
|
2012
|
+
write_json_file "$CRIT1_NEGCONTROL_REPO/.kontourai/flow-agents/crit1-negcontrol-task/evidence.json" <<JSON
|
|
2013
|
+
{
|
|
2014
|
+
"schema_version": "1.0",
|
|
2015
|
+
"task_slug": "crit1-negcontrol-task",
|
|
2016
|
+
"verdict": "pass",
|
|
2017
|
+
"checks": [
|
|
2018
|
+
{
|
|
2019
|
+
"id": "pattern-absent",
|
|
2020
|
+
"kind": "command",
|
|
2021
|
+
"status": "pass",
|
|
2022
|
+
"command": "! grep -q 'removed-pattern' $CRIT1_NEGCONTROL_REPO/haystack.txt",
|
|
2023
|
+
"summary": "Confirmed removed-pattern is absent (self-asserting)."
|
|
2024
|
+
}
|
|
2025
|
+
],
|
|
2026
|
+
"not_verified_gaps": []
|
|
2027
|
+
}
|
|
2028
|
+
JSON
|
|
2029
|
+
# No trust.bundle, no command-log.jsonl -- forces the live backstop re-run, which genuinely
|
|
2030
|
+
# executes "! grep -q 'removed-pattern' haystack.txt" and gets a REAL exit 0 (self-asserting).
|
|
2031
|
+
|
|
2032
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
2033
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/crit1-negcontrol.out" 2>"$TMPDIR_EVAL/crit1-negcontrol.err" <<JSON
|
|
2034
|
+
{"hook_event_name":"Stop","cwd":"$CRIT1_NEGCONTROL_REPO"}
|
|
2035
|
+
JSON
|
|
2036
|
+
then
|
|
2037
|
+
crit1_negcontrol_status=0
|
|
2038
|
+
else
|
|
2039
|
+
crit1_negcontrol_status=$?
|
|
2040
|
+
fi
|
|
2041
|
+
|
|
2042
|
+
if [[ "$crit1_negcontrol_status" -eq 0 ]]; then
|
|
2043
|
+
_pass "iteration-2 item 1 negative control: a legitimate self-asserting absence check ('! grep -q ...') claimed pass at a terminal stop still PASSES (exit 0) -- no new over-block introduced by the fix"
|
|
2044
|
+
else
|
|
2045
|
+
_fail "iteration-2 item 1 negative control REGRESSION: a legitimate self-asserting absence check claimed pass at a terminal stop was WRONGLY blocked: exit=$crit1_negcontrol_status $(cat "$TMPDIR_EVAL/crit1-negcontrol.out" "$TMPDIR_EVAL/crit1-negcontrol.err")"
|
|
2046
|
+
fi
|
|
2047
|
+
if ! grep -qi 'NOT_VERIFIED (ambiguous)\|caught false-completion' "$TMPDIR_EVAL/crit1-negcontrol.err"; then
|
|
2048
|
+
_pass "iteration-2 item 1 negative control: no ambiguous or false-completion warning was emitted for the self-asserting form"
|
|
2049
|
+
else
|
|
2050
|
+
_fail "iteration-2 item 1 negative control REGRESSION: an ambiguous or false-completion warning was wrongly emitted for a self-asserting absence check: $(cat "$TMPDIR_EVAL/crit1-negcontrol.out" "$TMPDIR_EVAL/crit1-negcontrol.err")"
|
|
2051
|
+
fi
|
|
2052
|
+
|
|
2053
|
+
# Regression guard: exit codes >= 2 (a REAL tool error, e.g. a missing file) for grep/diff must
|
|
2054
|
+
# STILL hard-fail a terminal stop, unchanged by this fix.
|
|
2055
|
+
CRIT1_EXIT2_REPO="$TMPDIR_EVAL/crit1-exit2-terminal/repo"
|
|
2056
|
+
mkdir -p "$CRIT1_EXIT2_REPO/.kontourai/flow-agents/crit1-exit2-task"
|
|
2057
|
+
printf '# Test Repo\n' > "$CRIT1_EXIT2_REPO/AGENTS.md"
|
|
2058
|
+
|
|
2059
|
+
cat > "$CRIT1_EXIT2_REPO/.kontourai/flow-agents/crit1-exit2-task/crit1-exit2-task--deliver.md" <<'MARKDOWN'
|
|
2060
|
+
# Exit>=2 terminal regression guard (real tool error must still hard-fail)
|
|
2061
|
+
|
|
2062
|
+
branch: main
|
|
2063
|
+
worktree: main
|
|
2064
|
+
created: 2026-07-05
|
|
2065
|
+
status: delivered
|
|
2066
|
+
type: deliver
|
|
2067
|
+
|
|
2068
|
+
## Definition Of Done
|
|
2069
|
+
- [x] pattern removed
|
|
2070
|
+
|
|
2071
|
+
## Goal Fit Gate
|
|
2072
|
+
- [x] acceptance verified
|
|
2073
|
+
|
|
2074
|
+
## Verification Report
|
|
2075
|
+
|
|
2076
|
+
Build: PASS
|
|
2077
|
+
|
|
2078
|
+
### Verdict: PASS
|
|
2079
|
+
MARKDOWN
|
|
2080
|
+
|
|
2081
|
+
write_json_file "$CRIT1_EXIT2_REPO/.kontourai/flow-agents/crit1-exit2-task/state.json" <<'JSON'
|
|
2082
|
+
{
|
|
2083
|
+
"schema_version": "1.0",
|
|
2084
|
+
"task_slug": "crit1-exit2-task",
|
|
2085
|
+
"status": "delivered",
|
|
2086
|
+
"phase": "done",
|
|
2087
|
+
"updated_at": "2026-07-05T00:00:00Z",
|
|
2088
|
+
"next_action": {
|
|
2089
|
+
"status": "done",
|
|
2090
|
+
"summary": "Delivered."
|
|
2091
|
+
}
|
|
2092
|
+
}
|
|
2093
|
+
JSON
|
|
2094
|
+
|
|
2095
|
+
write_json_file "$CRIT1_EXIT2_REPO/.kontourai/flow-agents/crit1-exit2-task/acceptance.json" <<'JSON'
|
|
2096
|
+
{
|
|
2097
|
+
"schema_version": "1.0",
|
|
2098
|
+
"task_slug": "crit1-exit2-task",
|
|
2099
|
+
"criteria": [
|
|
2100
|
+
{
|
|
2101
|
+
"id": "pattern-removed-2",
|
|
2102
|
+
"description": "The removed-pattern string no longer appears in a file that does not exist.",
|
|
2103
|
+
"status": "pass",
|
|
2104
|
+
"evidence_refs": [
|
|
2105
|
+
{
|
|
2106
|
+
"kind": "command",
|
|
2107
|
+
"excerpt": "grep -E 'removed-pattern' does-not-exist-anywhere.txt",
|
|
2108
|
+
"summary": "Bare grep against a nonexistent file (real tool error, not zero-matches)."
|
|
2109
|
+
}
|
|
2110
|
+
]
|
|
2111
|
+
}
|
|
2112
|
+
],
|
|
2113
|
+
"goal_fit": {"status": "pass", "summary": "Pattern removal verified."}
|
|
2114
|
+
}
|
|
2115
|
+
JSON
|
|
2116
|
+
|
|
2117
|
+
write_json_file "$CRIT1_EXIT2_REPO/.kontourai/flow-agents/crit1-exit2-task/evidence.json" <<'JSON'
|
|
2118
|
+
{
|
|
2119
|
+
"schema_version": "1.0",
|
|
2120
|
+
"task_slug": "crit1-exit2-task",
|
|
2121
|
+
"verdict": "pass",
|
|
2122
|
+
"checks": [
|
|
2123
|
+
{
|
|
2124
|
+
"id": "pattern-removed-2",
|
|
2125
|
+
"kind": "command",
|
|
2126
|
+
"status": "pass",
|
|
2127
|
+
"command": "grep -E 'removed-pattern' does-not-exist-anywhere.txt",
|
|
2128
|
+
"summary": "Confirmed removed-pattern is gone."
|
|
2129
|
+
}
|
|
2130
|
+
],
|
|
2131
|
+
"not_verified_gaps": []
|
|
2132
|
+
}
|
|
2133
|
+
JSON
|
|
2134
|
+
|
|
2135
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
2136
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/crit1-exit2.out" 2>"$TMPDIR_EVAL/crit1-exit2.err" <<JSON
|
|
2137
|
+
{"hook_event_name":"Stop","cwd":"$CRIT1_EXIT2_REPO"}
|
|
2138
|
+
JSON
|
|
2139
|
+
then
|
|
2140
|
+
crit1_exit2_status=0
|
|
2141
|
+
else
|
|
2142
|
+
crit1_exit2_status=$?
|
|
2143
|
+
fi
|
|
2144
|
+
|
|
2145
|
+
if [[ "$crit1_exit2_status" -eq 2 ]]; then
|
|
2146
|
+
_pass "iteration-2 item 1 regression guard: exit>=2 (real tool error) for grep/diff still HARD_BLOCKs a terminal stop, unchanged"
|
|
2147
|
+
else
|
|
2148
|
+
_fail "iteration-2 item 1 regression guard REGRESSION: exit>=2 (real tool error) did NOT block a terminal stop: exit=$crit1_exit2_status $(cat "$TMPDIR_EVAL/crit1-exit2.out" "$TMPDIR_EVAL/crit1-exit2.err")"
|
|
2149
|
+
fi
|
|
2150
|
+
|
|
2151
|
+
# ─── Mutation test: neuter the marker-unification fix at ONE common-case emission site ───────
|
|
2152
|
+
# (the live-backstop-re-run branch inside captureCrossReference/runBackstop, the site case (i)
|
|
2153
|
+
# above exercises), reverting it to the pre-fix plain "NOT_VERIFIED —" text -- confirm the
|
|
2154
|
+
# evidence.json-only terminal case (i) goes GREEN-passes again (exit 0, the CRITICAL bug
|
|
2155
|
+
# reproduced), then restore byte-identical and re-confirm it blocks (exit 2) again. This proves
|
|
2156
|
+
# the eval actually exercises the marker-unification fix, not just the pattern's mere presence
|
|
2157
|
+
# in the source.
|
|
2158
|
+
echo ""
|
|
2159
|
+
echo "--- mutation-test: iteration-2 item 1 marker-unification fix (neuter ONE common emission site, confirm case (i) goes green/bug-returns, restore) ---"
|
|
2160
|
+
|
|
2161
|
+
MARKER_MUTATION_SCRATCH="$TMPDIR_EVAL/marker-unification-mutation-scratch"
|
|
2162
|
+
mkdir -p "$MARKER_MUTATION_SCRATCH"
|
|
2163
|
+
cp "$ROOT/scripts/hooks/stop-goal-fit.js" "$MARKER_MUTATION_SCRATCH/stop-goal-fit.orig.js"
|
|
2164
|
+
|
|
2165
|
+
node - "$ROOT/scripts/hooks/stop-goal-fit.js" <<'NODEEOF' 2>"$TMPDIR_EVAL/marker-mutation-patch.err"
|
|
2166
|
+
const fs = require('fs');
|
|
2167
|
+
const file = process.argv[2];
|
|
2168
|
+
let src = fs.readFileSync(file, 'utf8');
|
|
2169
|
+
// Target ONLY the live-backstop-re-run branch's `note` (runBackstop's ambiguous classification,
|
|
2170
|
+
// inside captureCrossReference) -- NOT the capture-log shortcut branch or the
|
|
2171
|
+
// capturedFailReconciliation third bucket -- reverting it to the pre-fix plain "NOT_VERIFIED —"
|
|
2172
|
+
// marker, to prove case (i) above genuinely depends on THIS site's unification.
|
|
2173
|
+
const needle = "const note = `${base} evidence check ${id}: trusted backstop (${trusted.source}) re-run of \"${trusted.argv.join(' ')}\" exited 1 — for grep/diff this may mean zero matches/no differences (PASS for an absence check) or an unintended miss (FAIL for a presence check); NOT_VERIFIED (ambiguous): ${AMBIGUOUS_REMEDIATION} to remove the ambiguity.`;";
|
|
2174
|
+
if (!src.includes(needle)) {
|
|
2175
|
+
process.stderr.write('mutation: runBackstop live-re-run ambiguous note text not found — source pattern drifted, cannot mutation-test\n');
|
|
2176
|
+
process.exit(1);
|
|
2177
|
+
}
|
|
2178
|
+
const mutated = needle.replace('NOT_VERIFIED (ambiguous): ${AMBIGUOUS_REMEDIATION} to remove the ambiguity.', 'NOT_VERIFIED — ${AMBIGUOUS_REMEDIATION} to remove the ambiguity.');
|
|
2179
|
+
src = src.split(needle).join(mutated);
|
|
2180
|
+
fs.writeFileSync(file, src);
|
|
2181
|
+
NODEEOF
|
|
2182
|
+
|
|
2183
|
+
if [[ -s "$TMPDIR_EVAL/marker-mutation-patch.err" ]]; then
|
|
2184
|
+
_fail "mutation-test setup failed (runBackstop live-re-run ambiguous note source pattern did not match scripts/hooks/stop-goal-fit.js), restoring original unmodified: $(cat "$TMPDIR_EVAL/marker-mutation-patch.err")"
|
|
2185
|
+
cp "$MARKER_MUTATION_SCRATCH/stop-goal-fit.orig.js" "$ROOT/scripts/hooks/stop-goal-fit.js"
|
|
2186
|
+
elif ! node --check "$ROOT/scripts/hooks/stop-goal-fit.js" 2>"$TMPDIR_EVAL/marker-mutation-syntax.err"; then
|
|
2187
|
+
_fail "mutation-test setup: mutated stop-goal-fit.js (marker unification neutered) failed a syntax check, restoring original immediately: $(cat "$TMPDIR_EVAL/marker-mutation-syntax.err")"
|
|
2188
|
+
cp "$MARKER_MUTATION_SCRATCH/stop-goal-fit.orig.js" "$ROOT/scripts/hooks/stop-goal-fit.js"
|
|
2189
|
+
else
|
|
2190
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
2191
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/marker-mutation-case1.out" 2>"$TMPDIR_EVAL/marker-mutation-case1.err" <<JSON
|
|
2192
|
+
{"hook_event_name":"Stop","cwd":"$CRIT1_NOBUNDLE_REPO"}
|
|
2193
|
+
JSON
|
|
2194
|
+
then
|
|
2195
|
+
marker_mutation_case1_status=0
|
|
2196
|
+
else
|
|
2197
|
+
marker_mutation_case1_status=$?
|
|
2198
|
+
fi
|
|
2199
|
+
|
|
2200
|
+
cp "$MARKER_MUTATION_SCRATCH/stop-goal-fit.orig.js" "$ROOT/scripts/hooks/stop-goal-fit.js"
|
|
2201
|
+
|
|
2202
|
+
if [[ "$marker_mutation_case1_status" -eq 0 ]]; then
|
|
2203
|
+
_pass "mutation-test: with the marker unification neutered at this one site, case (i) WRONGLY exits 0 again (the CRITICAL bug reproduces -- eval correctly goes red without the fix, proving case (i) exercises this exact site)"
|
|
2204
|
+
else
|
|
2205
|
+
_fail "mutation-test: case (i) still exited nonzero ($marker_mutation_case1_status) even with the marker unification neutered at this site -- case (i) may not be exercising the intended fix"
|
|
2206
|
+
fi
|
|
2207
|
+
fi
|
|
2208
|
+
|
|
2209
|
+
if diff -q "$ROOT/scripts/hooks/stop-goal-fit.js" "$MARKER_MUTATION_SCRATCH/stop-goal-fit.orig.js" >/dev/null 2>&1; then
|
|
2210
|
+
_pass "mutation-test cleanup: scripts/hooks/stop-goal-fit.js is restored byte-identical to its pre-mutation-test content"
|
|
2211
|
+
else
|
|
2212
|
+
_fail "mutation-test cleanup REGRESSION: scripts/hooks/stop-goal-fit.js differs from its own pre-mutation-test content"
|
|
2213
|
+
fi
|
|
2214
|
+
|
|
2215
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
2216
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/marker-restore-recheck.out" 2>"$TMPDIR_EVAL/marker-restore-recheck.err" <<JSON
|
|
2217
|
+
{"hook_event_name":"Stop","cwd":"$CRIT1_NOBUNDLE_REPO"}
|
|
2218
|
+
JSON
|
|
2219
|
+
then
|
|
2220
|
+
marker_restore_status=0
|
|
2221
|
+
else
|
|
2222
|
+
marker_restore_status=$?
|
|
2223
|
+
fi
|
|
2224
|
+
if [[ "$marker_restore_status" -eq 2 ]]; then
|
|
2225
|
+
_pass "mutation-test cleanup re-check: the restored real stop-goal-fit.js blocks case (i) again (fix genuinely back in effect, not just byte-restored)"
|
|
2226
|
+
else
|
|
2227
|
+
_fail "mutation-test cleanup re-check REGRESSION: restored stop-goal-fit.js no longer blocks case (i): exit=$marker_restore_status $(cat "$TMPDIR_EVAL/marker-restore-recheck.out" "$TMPDIR_EVAL/marker-restore-recheck.err")"
|
|
2228
|
+
fi
|
|
2229
|
+
|
|
2230
|
+
# ─── Mutation test: neuter the HARD_BLOCK inclusion of NOT_VERIFIED (ambiguous), confirm the ──
|
|
2231
|
+
# terminal-block fixture above goes red (exit 0 instead of 2), then restore. Same in-place
|
|
2232
|
+
# mutation idiom (and the SAME serial-runner-only constraint, see the doc comment above the
|
|
2233
|
+
# #362 runBackstop mutation test) as the existing mutation test in this file.
|
|
2234
|
+
echo ""
|
|
2235
|
+
echo "--- mutation-test: iteration-2 item 4 HARD_BLOCK NOT_VERIFIED (ambiguous) inclusion (neuter guard in place, confirm terminal-block fixture goes red, restore) ---"
|
|
2236
|
+
|
|
2237
|
+
HARD_BLOCK_MUTATION_SCRATCH="$TMPDIR_EVAL/hardblock-mutation-scratch"
|
|
2238
|
+
mkdir -p "$HARD_BLOCK_MUTATION_SCRATCH"
|
|
2239
|
+
cp "$ROOT/scripts/hooks/stop-goal-fit.js" "$HARD_BLOCK_MUTATION_SCRATCH/stop-goal-fit.orig.js"
|
|
2240
|
+
|
|
2241
|
+
node - "$ROOT/scripts/hooks/stop-goal-fit.js" <<'NODEEOF' 2>"$TMPDIR_EVAL/hardblock-mutation-patch.err"
|
|
2242
|
+
const fs = require('fs');
|
|
2243
|
+
const file = process.argv[2];
|
|
2244
|
+
let src = fs.readFileSync(file, 'utf8');
|
|
2245
|
+
const needle = "const HARD_BLOCK = /contradicts evidence\\.json|caught false-completion|evidence verdict:|evidence check .+ status:|critique status|critique open|required sidecar is missing|command-log integrity check FAILED|gate misconfiguration:|exit-code-laundered|NOT_VERIFIED \\(ambiguous\\)/;";
|
|
2246
|
+
if (!src.includes(needle)) {
|
|
2247
|
+
process.stderr.write('mutation: HARD_BLOCK NOT_VERIFIED (ambiguous) pattern not found — source pattern drifted, cannot mutation-test\n');
|
|
2248
|
+
process.exit(1);
|
|
2249
|
+
}
|
|
2250
|
+
const mutated = "const HARD_BLOCK = /contradicts evidence\\.json|caught false-completion|evidence verdict:|evidence check .+ status:|critique status|critique open|required sidecar is missing|command-log integrity check FAILED|gate misconfiguration:|exit-code-laundered/;";
|
|
2251
|
+
src = src.split(needle).join(mutated);
|
|
2252
|
+
fs.writeFileSync(file, src);
|
|
2253
|
+
NODEEOF
|
|
2254
|
+
|
|
2255
|
+
if [[ -s "$TMPDIR_EVAL/hardblock-mutation-patch.err" ]]; then
|
|
2256
|
+
_fail "mutation-test setup failed (HARD_BLOCK NOT_VERIFIED (ambiguous) source pattern did not match scripts/hooks/stop-goal-fit.js), restoring original unmodified: $(cat "$TMPDIR_EVAL/hardblock-mutation-patch.err")"
|
|
2257
|
+
cp "$HARD_BLOCK_MUTATION_SCRATCH/stop-goal-fit.orig.js" "$ROOT/scripts/hooks/stop-goal-fit.js"
|
|
2258
|
+
elif ! node --check "$ROOT/scripts/hooks/stop-goal-fit.js" 2>"$TMPDIR_EVAL/hardblock-mutation-syntax.err"; then
|
|
2259
|
+
_fail "mutation-test setup: mutated stop-goal-fit.js (HARD_BLOCK ambiguous pattern removed) failed a syntax check, restoring original immediately: $(cat "$TMPDIR_EVAL/hardblock-mutation-syntax.err")"
|
|
2260
|
+
cp "$HARD_BLOCK_MUTATION_SCRATCH/stop-goal-fit.orig.js" "$ROOT/scripts/hooks/stop-goal-fit.js"
|
|
2261
|
+
else
|
|
2262
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
2263
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/hardblock-mutation-terminal.out" 2>"$TMPDIR_EVAL/hardblock-mutation-terminal.err" <<JSON
|
|
2264
|
+
{"hook_event_name":"Stop","cwd":"$AMBIG_TERMINAL_REPO"}
|
|
2265
|
+
JSON
|
|
2266
|
+
then
|
|
2267
|
+
hardblock_mutation_status=0
|
|
2268
|
+
else
|
|
2269
|
+
hardblock_mutation_status=$?
|
|
2270
|
+
fi
|
|
2271
|
+
|
|
2272
|
+
cp "$HARD_BLOCK_MUTATION_SCRATCH/stop-goal-fit.orig.js" "$ROOT/scripts/hooks/stop-goal-fit.js"
|
|
2273
|
+
|
|
2274
|
+
if [[ "$hardblock_mutation_status" -eq 0 ]]; then
|
|
2275
|
+
_pass "mutation-test: with the HARD_BLOCK NOT_VERIFIED (ambiguous) pattern removed, the terminal-block fixture WRONGLY exits 0 again (eval correctly goes red without the fix, proving the fixture exercises it)"
|
|
2276
|
+
else
|
|
2277
|
+
_fail "mutation-test: terminal-block fixture still exited nonzero ($hardblock_mutation_status) even with the HARD_BLOCK ambiguous pattern removed -- fixture may not be exercising the intended guard"
|
|
2278
|
+
fi
|
|
2279
|
+
fi
|
|
2280
|
+
|
|
2281
|
+
if diff -q "$ROOT/scripts/hooks/stop-goal-fit.js" "$HARD_BLOCK_MUTATION_SCRATCH/stop-goal-fit.orig.js" >/dev/null 2>&1; then
|
|
2282
|
+
_pass "mutation-test cleanup: scripts/hooks/stop-goal-fit.js is restored byte-identical to its pre-mutation-test content"
|
|
2283
|
+
else
|
|
2284
|
+
_fail "mutation-test cleanup REGRESSION: scripts/hooks/stop-goal-fit.js differs from its own pre-mutation-test content"
|
|
2285
|
+
fi
|
|
2286
|
+
|
|
2287
|
+
if FLOW_AGENTS_GOAL_FIT_MODE=block FLOW_AGENTS_GOAL_FIT_MAX_BLOCKS=100000 \
|
|
2288
|
+
node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/hardblock-restore-recheck.out" 2>"$TMPDIR_EVAL/hardblock-restore-recheck.err" <<JSON
|
|
2289
|
+
{"hook_event_name":"Stop","cwd":"$AMBIG_TERMINAL_REPO"}
|
|
2290
|
+
JSON
|
|
2291
|
+
then
|
|
2292
|
+
hardblock_restore_status=0
|
|
2293
|
+
else
|
|
2294
|
+
hardblock_restore_status=$?
|
|
2295
|
+
fi
|
|
2296
|
+
if [[ "$hardblock_restore_status" -eq 2 ]]; then
|
|
2297
|
+
_pass "mutation-test cleanup re-check: the restored real stop-goal-fit.js blocks the terminal ambiguous fixture again (guard genuinely back in effect, not just byte-restored)"
|
|
2298
|
+
else
|
|
2299
|
+
_fail "mutation-test cleanup re-check REGRESSION: restored stop-goal-fit.js no longer blocks the terminal ambiguous fixture: exit=$hardblock_restore_status $(cat "$TMPDIR_EVAL/hardblock-restore-recheck.out" "$TMPDIR_EVAL/hardblock-restore-recheck.err")"
|
|
2300
|
+
fi
|
|
2301
|
+
|
|
2302
|
+
|
|
2303
|
+
# ─── Mutation test (#362 runBackstop ambiguous carve-out): temporarily disable the ──────────
|
|
2304
|
+
# `ambiguous = exitCode === 1 && isAmbiguousAbsenceCommand(commandText)` guard IN PLACE on
|
|
2305
|
+
# `scripts/hooks/stop-goal-fit.js` itself (backed up first, restored immediately after) —
|
|
2306
|
+
# `stop-goal-fit.js` requires its sibling `./lib/*.js` helpers via RELATIVE paths, so (unlike
|
|
2307
|
+
# workflow-sidecar.ts's standalone compiled bundle) a scratch copy in an unrelated tmpdir cannot
|
|
2308
|
+
# resolve those requires; the same constraint the reserved-prefix mutation test in
|
|
2309
|
+
# test_workflow_sidecar_writer.sh works around by mutating `build/src/cli/workflow-sidecar.js`
|
|
2310
|
+
# IN PLACE (backup + restore) rather than copying it elsewhere. Confirms the AC4 fixture above
|
|
2311
|
+
# now WRONGLY reports "caught false-completion" again (the eval "goes red" without the guard,
|
|
2312
|
+
# proving AC4 actually exercises it), then restores the original file byte-for-byte and
|
|
2313
|
+
# re-confirms AC4 passes again against the restored file — the mutated state exists on disk for
|
|
2314
|
+
# the shortest possible window (one synchronous node invocation) and is restored even on error
|
|
2315
|
+
# via the wrapping subshell's own control flow.
|
|
2316
|
+
#
|
|
2317
|
+
# ITERATION-2 fix item 5 (MEDIUM, code review): this in-place mutation of the LIVE, real
|
|
2318
|
+
# `scripts/hooks/stop-goal-fit.js` file (not a scratch copy) is SERIAL-RUNNER-SAFE ONLY. It
|
|
2319
|
+
# assumes no other process — a concurrently-running eval suite invocation, a Stop hook firing
|
|
2320
|
+
# from a real agent session in this same worktree, or a second `evals/run.sh` invocation — reads
|
|
2321
|
+
# or executes `scripts/hooks/stop-goal-fit.js` during the narrow window between the mutation
|
|
2322
|
+
# write below and its restore. Running this eval file concurrently with itself, or alongside any
|
|
2323
|
+
# other process that invokes the real Stop hook against this worktree, is UNSUPPORTED and can
|
|
2324
|
+
# produce a spurious pass/fail or (in the worst case, if the process is killed mid-window) leave
|
|
2325
|
+
# the real hook file mutated on disk. This eval MUST be run serially (the default for
|
|
2326
|
+
# `evals/run.sh` and the sweep commands documented in this repo's worker instructions), never
|
|
2327
|
+
# via a parallel/sharded test runner. `test_record_check.sh`'s build-artifact mutation pattern
|
|
2328
|
+
# (mutating a BUILD OUTPUT under `build/`, which is regenerated by `npm run build` and is not the
|
|
2329
|
+
# single live hook path every real session's Stop invokes) is the preferred idiom for a NEW
|
|
2330
|
+
# mutation test where the target is a build artifact rather than a directly-invoked hook script
|
|
2331
|
+
# with sibling-relative requires — prefer that pattern over this one when it is applicable.
|
|
2332
|
+
echo ""
|
|
2333
|
+
echo "--- mutation-test: #362 runBackstop ambiguous carve-out (neuter guard in place, confirm AC4 goes red, restore) ---"
|
|
2334
|
+
|
|
2335
|
+
REAL_STOP_GOAL_FIT="$ROOT/scripts/hooks/stop-goal-fit.js"
|
|
2336
|
+
MUTATION_SCRATCH="$TMPDIR_EVAL/runbackstop-mutation-scratch"
|
|
2337
|
+
mkdir -p "$MUTATION_SCRATCH"
|
|
2338
|
+
cp "$REAL_STOP_GOAL_FIT" "$MUTATION_SCRATCH/stop-goal-fit.orig.js"
|
|
2339
|
+
|
|
2340
|
+
node - "$REAL_STOP_GOAL_FIT" <<'NODEEOF' 2>"$TMPDIR_EVAL/mutation-patch.err"
|
|
2341
|
+
const fs = require('fs');
|
|
2342
|
+
const [, , file] = process.argv;
|
|
2343
|
+
let src = fs.readFileSync(file, 'utf8');
|
|
2344
|
+
const needle = "const ambiguous = exitCode === 1 && isAmbiguousAbsenceCommand(commandText);";
|
|
2345
|
+
if (!src.includes(needle)) {
|
|
2346
|
+
process.stderr.write('mutation: runBackstop ambiguous-guard text not found — source pattern drifted, cannot mutation-test\n');
|
|
2347
|
+
process.exit(1);
|
|
2348
|
+
}
|
|
2349
|
+
src = src.split(needle).join('const ambiguous = false; /* mutation-test: guard neutered */');
|
|
2350
|
+
fs.writeFileSync(file, src);
|
|
2351
|
+
NODEEOF
|
|
2352
|
+
|
|
2353
|
+
if [[ -s "$TMPDIR_EVAL/mutation-patch.err" ]]; then
|
|
2354
|
+
_fail "mutation-test setup failed (runBackstop ambiguous-guard source pattern did not match scripts/hooks/stop-goal-fit.js), restoring original unmodified: $(cat "$TMPDIR_EVAL/mutation-patch.err")"
|
|
2355
|
+
cp "$MUTATION_SCRATCH/stop-goal-fit.orig.js" "$REAL_STOP_GOAL_FIT"
|
|
2356
|
+
elif ! node --check "$REAL_STOP_GOAL_FIT" 2>"$TMPDIR_EVAL/mutation-syntax.err"; then
|
|
2357
|
+
_fail "mutation-test setup: mutated stop-goal-fit.js (ambiguous guard neutered) failed a syntax check, restoring original immediately: $(cat "$TMPDIR_EVAL/mutation-syntax.err")"
|
|
2358
|
+
cp "$MUTATION_SCRATCH/stop-goal-fit.orig.js" "$REAL_STOP_GOAL_FIT"
|
|
2359
|
+
else
|
|
2360
|
+
if node "$REAL_STOP_GOAL_FIT" >"$TMPDIR_EVAL/mutation-ac4.out" 2>"$TMPDIR_EVAL/mutation-ac4.err" <<JSON
|
|
2361
|
+
{"hook_event_name":"Stop","cwd":"$AC362_REPO"}
|
|
2362
|
+
JSON
|
|
2363
|
+
then
|
|
2364
|
+
mutation_ac4_status=0
|
|
2365
|
+
else
|
|
2366
|
+
mutation_ac4_status=$?
|
|
2367
|
+
fi
|
|
2368
|
+
|
|
2369
|
+
# Restore the real file immediately after the single mutated invocation above, before
|
|
2370
|
+
# evaluating the assertion, so the mutated state never persists past this one node call
|
|
2371
|
+
# regardless of the assertion's outcome.
|
|
2372
|
+
cp "$MUTATION_SCRATCH/stop-goal-fit.orig.js" "$REAL_STOP_GOAL_FIT"
|
|
2373
|
+
|
|
2374
|
+
if grep -q 'caught false-completion' "$TMPDIR_EVAL/mutation-ac4.err"; then
|
|
2375
|
+
_pass "mutation-test: with the ambiguous guard neutered, the exact AC4 fixture WRONGLY reports 'caught false-completion' again (eval correctly goes red without the guard, proving AC4 exercises it)"
|
|
2376
|
+
else
|
|
2377
|
+
_fail "mutation-test: AC4 fixture still did NOT report 'caught false-completion' even with the ambiguous guard neutered — AC4 may not be exercising the intended guard: exit=$mutation_ac4_status $(cat "$TMPDIR_EVAL/mutation-ac4.out" "$TMPDIR_EVAL/mutation-ac4.err")"
|
|
2378
|
+
fi
|
|
2379
|
+
fi
|
|
2380
|
+
|
|
2381
|
+
# Restore-verification: confirm the real file is back to its pre-mutation-test byte-identical
|
|
2382
|
+
# state, and re-run the AC4 fixture once more against the restored file to prove the restored
|
|
2383
|
+
# guard is back in effect (classification: 'ambiguous', not the mutated 'fail').
|
|
2384
|
+
if diff -q "$REAL_STOP_GOAL_FIT" "$MUTATION_SCRATCH/stop-goal-fit.orig.js" >/dev/null 2>&1; then
|
|
2385
|
+
_pass "mutation-test cleanup: scripts/hooks/stop-goal-fit.js is restored byte-identical to its pre-mutation-test content"
|
|
2386
|
+
else
|
|
2387
|
+
_fail "mutation-test cleanup REGRESSION: scripts/hooks/stop-goal-fit.js differs from its own pre-mutation-test copy — the real file may have been left altered by this eval"
|
|
2388
|
+
fi
|
|
2389
|
+
|
|
2390
|
+
if node "$REAL_STOP_GOAL_FIT" >"$TMPDIR_EVAL/mutation-restore-recheck.out" 2>"$TMPDIR_EVAL/mutation-restore-recheck.err" <<JSON
|
|
2391
|
+
{"hook_event_name":"Stop","cwd":"$AC362_REPO"}
|
|
2392
|
+
JSON
|
|
2393
|
+
then :; fi
|
|
2394
|
+
if ! grep -q 'caught false-completion' "$TMPDIR_EVAL/mutation-restore-recheck.err" && grep -qi 'ambiguous\|NOT_VERIFIED' "$TMPDIR_EVAL/mutation-restore-recheck.err"; then
|
|
2395
|
+
_pass "mutation-test cleanup re-check: the restored real stop-goal-fit.js classifies the AC4 fixture as ambiguous again (guard is genuinely back in effect, not just byte-restored)"
|
|
2396
|
+
else
|
|
2397
|
+
_fail "mutation-test cleanup re-check REGRESSION: restored stop-goal-fit.js no longer classifies the AC4 fixture as ambiguous: $(cat "$TMPDIR_EVAL/mutation-restore-recheck.out" "$TMPDIR_EVAL/mutation-restore-recheck.err")"
|
|
2398
|
+
fi
|
|
2399
|
+
|
|
2400
|
+
|
|
2401
|
+
# ─── #362: dialect preservation (BRE/ERE) — recorded command's regex dialect/flags survive ────
|
|
2402
|
+
# an unmodified backstop re-run verbatim ────────────────────────────────────────────────────────
|
|
2403
|
+
#
|
|
2404
|
+
# ac-dialect-preservation-required-not-yet-implemented: the recorded command's exact dialect/
|
|
2405
|
+
# flags (e.g. `grep -E`) must be preserved verbatim on every backstop re-run path. A `grep -E`
|
|
2406
|
+
# invocation recorded with an ERE-only construct (unescaped alternation `(foo|bar)`, which is a
|
|
2407
|
+
# literal-character grouping/pipe under BASIC regex, not a metacharacter) must still match under
|
|
2408
|
+
# ERE semantics when the backstop replays it -- proving the recorded string was NOT silently
|
|
2409
|
+
# re-run under BRE (which would treat `(foo|bar)` as the literal substring "(foo|bar)" and fail
|
|
2410
|
+
# to match a fixture that only contains "bar" on its own).
|
|
2411
|
+
echo ""
|
|
2412
|
+
echo "--- #362: dialect preservation (grep -E ERE alternation survives backstop replay verbatim) ---"
|
|
2413
|
+
|
|
2414
|
+
# Direct unit-level pin: resolveTrustedCommand's `acceptance` source must hand back the
|
|
2415
|
+
# recorded command text byte-for-byte in argv[2] (never re-derived/re-quoted), and spawning
|
|
2416
|
+
# that argv directly (mirroring runBackstop's own spawnSync call) must produce a REAL ERE match
|
|
2417
|
+
# (exit 0) against a fixture that only matches under ERE semantics -- if the command were ever
|
|
2418
|
+
# silently re-run under BRE (e.g. via a re-joined/re-quoted argv.join(' ') instead of argv[2]
|
|
2419
|
+
# verbatim), `(foo|bar)` would be treated as a literal string and the match would wrongly fail.
|
|
2420
|
+
DIALECT_FIXTURE_DIR="$TMPDIR_EVAL/ac362-dialect-unit"
|
|
2421
|
+
mkdir -p "$DIALECT_FIXTURE_DIR"
|
|
2422
|
+
printf 'this line only contains bar, never the literal parenthesized alternation\n' > "$DIALECT_FIXTURE_DIR/haystack.txt"
|
|
2423
|
+
|
|
2424
|
+
if node - "$ROOT/scripts/hooks/stop-goal-fit.js" "$DIALECT_FIXTURE_DIR/haystack.txt" <<'NODEEOF' 2>"$TMPDIR_EVAL/ac362-dialect-unit.err"
|
|
2425
|
+
const { spawnSync } = require('child_process');
|
|
2426
|
+
const { resolveTrustedCommand } = require(process.argv[2]);
|
|
2427
|
+
const haystack = process.argv[3];
|
|
2428
|
+
|
|
2429
|
+
const check = { id: 'ere-alternation-check' };
|
|
2430
|
+
const acceptance = {
|
|
2431
|
+
criteria: [
|
|
2432
|
+
{
|
|
2433
|
+
id: 'ere-alternation-check',
|
|
2434
|
+
evidence_refs: [
|
|
2435
|
+
{ kind: 'command', excerpt: `grep -E '(foo|bar)' ${haystack}` },
|
|
2436
|
+
],
|
|
2437
|
+
},
|
|
2438
|
+
],
|
|
2439
|
+
};
|
|
2440
|
+
|
|
2441
|
+
const trusted = resolveTrustedCommand(process.cwd(), process.cwd(), check, acceptance);
|
|
2442
|
+
if (!trusted || trusted.malformed) {
|
|
2443
|
+
console.error(`resolveTrustedCommand did not resolve a trusted command: ${JSON.stringify(trusted)}`);
|
|
2444
|
+
process.exit(1);
|
|
2445
|
+
}
|
|
2446
|
+
if (trusted.argv.length !== 3 || trusted.argv[0] !== 'bash' || trusted.argv[1] !== '-lc') {
|
|
2447
|
+
console.error(`unexpected argv shape: ${JSON.stringify(trusted.argv)}`);
|
|
2448
|
+
process.exit(1);
|
|
2449
|
+
}
|
|
2450
|
+
// DIALECT-PRESERVATION INVARIANT check: argv[2] must be the recorded command text VERBATIM,
|
|
2451
|
+
// byte-for-byte, including its `-E` flag and unescaped `(foo|bar)` alternation -- never
|
|
2452
|
+
// re-derived, re-joined, or re-quoted.
|
|
2453
|
+
const expected = `grep -E '(foo|bar)' ${haystack}`;
|
|
2454
|
+
if (trusted.argv[2] !== expected) {
|
|
2455
|
+
console.error(`argv[2] does not match the recorded command verbatim: got ${JSON.stringify(trusted.argv[2])}, expected ${JSON.stringify(expected)}`);
|
|
2456
|
+
process.exit(1);
|
|
2457
|
+
}
|
|
2458
|
+
|
|
2459
|
+
// Replay argv exactly as runBackstop does (spawnSync(argv[0], argv.slice(1))) -- this is the
|
|
2460
|
+
// REAL dialect-sensitive execution: bash -lc runs grep -E, which interprets `(foo|bar)` as an
|
|
2461
|
+
// ERE alternation and finds the match.
|
|
2462
|
+
const result = spawnSync(trusted.argv[0], trusted.argv.slice(1), { encoding: 'utf8' });
|
|
2463
|
+
if (result.status !== 0) {
|
|
2464
|
+
console.error(`ERE alternation did NOT match under verbatim replay (exit ${result.status}) -- dialect was NOT preserved: stdout=${result.stdout} stderr=${result.stderr}`);
|
|
2465
|
+
process.exit(1);
|
|
2466
|
+
}
|
|
2467
|
+
|
|
2468
|
+
// Negative control: prove the fixture genuinely depends on ERE semantics -- running the SAME
|
|
2469
|
+
// pattern text under BASIC regex (grep without -E) must NOT match, since `(foo|bar)` is then a
|
|
2470
|
+
// literal substring the fixture does not contain. This demonstrates the invariant matters: a
|
|
2471
|
+
// silent BRE downgrade of this exact recorded command would flip the match from found to missed.
|
|
2472
|
+
const bre = spawnSync('bash', ['-lc', `grep '(foo|bar)' ${haystack}`], { encoding: 'utf8' });
|
|
2473
|
+
if (bre.status === 0) {
|
|
2474
|
+
console.error('fixture unexpectedly matched under BASIC regex too -- negative control is not valid, fixture must be ERE-only');
|
|
2475
|
+
process.exit(1);
|
|
2476
|
+
}
|
|
2477
|
+
NODEEOF
|
|
2478
|
+
then
|
|
2479
|
+
_pass "ac-dialect-preservation: resolveTrustedCommand hands back the recorded 'grep -E' command verbatim (byte-for-byte), and replaying it produces a real ERE alternation match -- not silently reinterpreted as BRE"
|
|
2480
|
+
else
|
|
2481
|
+
_fail "ac-dialect-preservation: dialect was NOT preserved verbatim through resolveTrustedCommand/replay: $(cat "$TMPDIR_EVAL/ac362-dialect-unit.err")"
|
|
2482
|
+
fi
|
|
2483
|
+
|
|
2484
|
+
# End-to-end pin: the SAME ERE-alternation shape, recorded as an acceptance criterion's evidence
|
|
2485
|
+
# ref with a claimed pass and NO command-log entry (forcing the trusted backstop re-run path),
|
|
2486
|
+
# must be confirmed as a clean, deterministic pass -- no "caught false-completion" warning and no
|
|
2487
|
+
# "ambiguous"/NOT_VERIFIED warning for this check -- proving the end-to-end Stop-hook backstop
|
|
2488
|
+
# path (not just the unit-level resolveTrustedCommand/runBackstop pairing above) preserves the
|
|
2489
|
+
# recorded command's ERE dialect on replay.
|
|
2490
|
+
AC362_DIALECT_REPO="$TMPDIR_EVAL/ac362-dialect-e2e/repo"
|
|
2491
|
+
mkdir -p "$AC362_DIALECT_REPO/.kontourai/flow-agents/dialect-check-task"
|
|
2492
|
+
printf '# Test Repo\n' > "$AC362_DIALECT_REPO/AGENTS.md"
|
|
2493
|
+
|
|
2494
|
+
# Fixture only matches under ERE semantics (see unit-level negative control above for why).
|
|
2495
|
+
printf 'this line only contains bar, never the literal parenthesized alternation\n' > "$AC362_DIALECT_REPO/haystack.txt"
|
|
2496
|
+
|
|
2497
|
+
cat > "$AC362_DIALECT_REPO/.kontourai/flow-agents/dialect-check-task/dialect-check-task--deliver.md" <<'MARKDOWN'
|
|
2498
|
+
# Dialect check task
|
|
2499
|
+
|
|
2500
|
+
branch: main
|
|
2501
|
+
worktree: main
|
|
2502
|
+
created: 2026-07-01
|
|
2503
|
+
status: executing
|
|
2504
|
+
type: deliver
|
|
2505
|
+
|
|
2506
|
+
## Plan
|
|
2507
|
+
|
|
2508
|
+
Task whose acceptance criterion is recorded as a `grep -E` command with an ERE-only
|
|
2509
|
+
alternation construct, to pin dialect preservation on backstop replay.
|
|
2510
|
+
MARKDOWN
|
|
2511
|
+
|
|
2512
|
+
write_json_file "$AC362_DIALECT_REPO/.kontourai/flow-agents/dialect-check-task/state.json" <<'JSON'
|
|
2513
|
+
{
|
|
2514
|
+
"schema_version": "1.0",
|
|
2515
|
+
"task_slug": "dialect-check-task",
|
|
2516
|
+
"status": "in_progress",
|
|
2517
|
+
"phase": "verification",
|
|
2518
|
+
"updated_at": "2026-07-01T00:00:00Z",
|
|
2519
|
+
"next_action": {
|
|
2520
|
+
"status": "in_progress",
|
|
2521
|
+
"summary": "Verifying the ERE alternation pattern matches."
|
|
2522
|
+
}
|
|
2523
|
+
}
|
|
2524
|
+
JSON
|
|
2525
|
+
|
|
2526
|
+
write_json_file "$AC362_DIALECT_REPO/.kontourai/flow-agents/dialect-check-task/acceptance.json" <<JSON
|
|
2527
|
+
{
|
|
2528
|
+
"schema_version": "1.0",
|
|
2529
|
+
"task_slug": "dialect-check-task",
|
|
2530
|
+
"criteria": [
|
|
2531
|
+
{
|
|
2532
|
+
"id": "ere-alternation-present",
|
|
2533
|
+
"description": "The haystack contains foo or bar (ERE alternation).",
|
|
2534
|
+
"status": "pass",
|
|
2535
|
+
"evidence_refs": [
|
|
2536
|
+
{
|
|
2537
|
+
"kind": "command",
|
|
2538
|
+
"excerpt": "grep -E '(foo|bar)' $AC362_DIALECT_REPO/haystack.txt",
|
|
2539
|
+
"summary": "ERE alternation match (self-asserting presence check, recorded pass)."
|
|
2540
|
+
}
|
|
2541
|
+
]
|
|
2542
|
+
}
|
|
2543
|
+
],
|
|
2544
|
+
"goal_fit": {"status": "pass", "summary": "ERE alternation match verified."}
|
|
2545
|
+
}
|
|
2546
|
+
JSON
|
|
2547
|
+
|
|
2548
|
+
write_json_file "$AC362_DIALECT_REPO/.kontourai/flow-agents/dialect-check-task/evidence.json" <<JSON
|
|
2549
|
+
{
|
|
2550
|
+
"schema_version": "1.0",
|
|
2551
|
+
"task_slug": "dialect-check-task",
|
|
2552
|
+
"verdict": "pass",
|
|
2553
|
+
"checks": [
|
|
2554
|
+
{
|
|
2555
|
+
"id": "ere-alternation-present",
|
|
2556
|
+
"kind": "command",
|
|
2557
|
+
"status": "pass",
|
|
2558
|
+
"command": "grep -E '(foo|bar)' $AC362_DIALECT_REPO/haystack.txt",
|
|
2559
|
+
"summary": "Confirmed ERE alternation match."
|
|
2560
|
+
}
|
|
2561
|
+
],
|
|
2562
|
+
"not_verified_gaps": []
|
|
2563
|
+
}
|
|
2564
|
+
JSON
|
|
2565
|
+
|
|
2566
|
+
# No command-log.jsonl -- forces captureCrossReference into the trusted backstop re-run path.
|
|
2567
|
+
|
|
2568
|
+
if node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/ac362-dialect-e2e.out" 2>"$TMPDIR_EVAL/ac362-dialect-e2e.err" <<JSON
|
|
2569
|
+
{"hook_event_name":"Stop","cwd":"$AC362_DIALECT_REPO"}
|
|
2570
|
+
JSON
|
|
2571
|
+
then
|
|
2572
|
+
ac362_dialect_e2e_status=0
|
|
2573
|
+
else
|
|
2574
|
+
ac362_dialect_e2e_status=$?
|
|
2575
|
+
fi
|
|
2576
|
+
|
|
2577
|
+
if ! grep -qi 'caught false-completion\|ambiguous\|NOT_VERIFIED' "$TMPDIR_EVAL/ac362-dialect-e2e.err"; then
|
|
2578
|
+
_pass "ac-dialect-preservation (end-to-end): the recorded 'grep -E' ERE-alternation evidence ref is confirmed a clean deterministic pass on backstop replay -- no caught-false-completion, no ambiguous/NOT_VERIFIED warning, proving the dialect was preserved through the full Stop-hook path"
|
|
2579
|
+
else
|
|
2580
|
+
_fail "ac-dialect-preservation (end-to-end): backstop replay of the recorded 'grep -E' ERE-alternation command did NOT confirm a clean pass -- possible dialect drift (BRE reinterpretation would miss the match): exit=$ac362_dialect_e2e_status $(cat "$TMPDIR_EVAL/ac362-dialect-e2e.out" "$TMPDIR_EVAL/ac362-dialect-e2e.err")"
|
|
2581
|
+
fi
|
|
2582
|
+
|
|
2583
|
+
# ─── Mutation test (iteration-2 fix item 3/LOW): dialect-preservation evals lacked the ────────
|
|
2584
|
+
# mutation-test wrapper every sibling #362 AC in this file has. Neuter `resolveTrustedCommand`'s
|
|
2585
|
+
# acceptance-branch replay (drop the recorded `-E` flag, forcing a silent BRE reinterpretation of
|
|
2586
|
+
# the ERE-only `(foo|bar)` alternation), confirm BOTH the unit-level and end-to-end dialect evals
|
|
2587
|
+
# above go RED, then restore byte-identical and re-confirm GREEN. Same in-place mutation idiom
|
|
2588
|
+
# (and the SAME serial-runner-only constraint documented above the #362 runBackstop mutation
|
|
2589
|
+
# test) as this file's other mutation tests: this is the live, real `scripts/hooks/
|
|
2590
|
+
# stop-goal-fit.js` (backed up first, restored immediately after), not a scratch copy elsewhere,
|
|
2591
|
+
# because the file requires its sibling `./lib/*.js` helpers via relative paths.
|
|
2592
|
+
echo ""
|
|
2593
|
+
echo "--- mutation-test: ac-dialect-preservation (drop the recorded -E flag in place, confirm both dialect evals go red, restore) ---"
|
|
2594
|
+
|
|
2595
|
+
DIALECT_MUTATION_SCRATCH="$TMPDIR_EVAL/dialect-mutation-scratch"
|
|
2596
|
+
mkdir -p "$DIALECT_MUTATION_SCRATCH"
|
|
2597
|
+
cp "$ROOT/scripts/hooks/stop-goal-fit.js" "$DIALECT_MUTATION_SCRATCH/stop-goal-fit.orig.js"
|
|
2598
|
+
|
|
2599
|
+
node - "$ROOT/scripts/hooks/stop-goal-fit.js" <<'NODEEOF' 2>"$TMPDIR_EVAL/dialect-mutation-patch.err"
|
|
2600
|
+
const fs = require('fs');
|
|
2601
|
+
const file = process.argv[2];
|
|
2602
|
+
let src = fs.readFileSync(file, 'utf8');
|
|
2603
|
+
const needle = "return { argv: ['bash', '-lc', fromAcceptance], cwd: root, source: 'acceptance' };";
|
|
2604
|
+
if (!src.includes(needle)) {
|
|
2605
|
+
process.stderr.write('mutation: resolveTrustedCommand acceptance-branch argv construction not found — source pattern drifted, cannot mutation-test\n');
|
|
2606
|
+
process.exit(1);
|
|
2607
|
+
}
|
|
2608
|
+
// Silently drop the recorded '-E ' flag before replay -- exactly the "silent BRE downgrade"
|
|
2609
|
+
// class of regression the DIALECT-PRESERVATION INVARIANT exists to prevent (e.g. a future
|
|
2610
|
+
// argv.join(' ')/re-derivation refactor that loses a flag along the way).
|
|
2611
|
+
const mutated = "return { argv: ['bash', '-lc', fromAcceptance.replace(/-E /, '')], cwd: root, source: 'acceptance' };";
|
|
2612
|
+
src = src.split(needle).join(mutated);
|
|
2613
|
+
fs.writeFileSync(file, src);
|
|
2614
|
+
NODEEOF
|
|
2615
|
+
|
|
2616
|
+
if [[ -s "$TMPDIR_EVAL/dialect-mutation-patch.err" ]]; then
|
|
2617
|
+
_fail "mutation-test setup failed (resolveTrustedCommand acceptance-branch source pattern did not match scripts/hooks/stop-goal-fit.js), restoring original unmodified: $(cat "$TMPDIR_EVAL/dialect-mutation-patch.err")"
|
|
2618
|
+
cp "$DIALECT_MUTATION_SCRATCH/stop-goal-fit.orig.js" "$ROOT/scripts/hooks/stop-goal-fit.js"
|
|
2619
|
+
elif ! node --check "$ROOT/scripts/hooks/stop-goal-fit.js" 2>"$TMPDIR_EVAL/dialect-mutation-syntax.err"; then
|
|
2620
|
+
_fail "mutation-test setup: mutated stop-goal-fit.js (recorded -E flag dropped) failed a syntax check, restoring original immediately: $(cat "$TMPDIR_EVAL/dialect-mutation-syntax.err")"
|
|
2621
|
+
cp "$DIALECT_MUTATION_SCRATCH/stop-goal-fit.orig.js" "$ROOT/scripts/hooks/stop-goal-fit.js"
|
|
2622
|
+
else
|
|
2623
|
+
# Unit-level: argv[2] must now DIFFER from the recorded command (the -E flag was silently
|
|
2624
|
+
# dropped), and replaying the mutated argv must FAIL the ERE alternation match.
|
|
2625
|
+
if node - "$ROOT/scripts/hooks/stop-goal-fit.js" "$DIALECT_FIXTURE_DIR/haystack.txt" >"$TMPDIR_EVAL/dialect-mutation-unit.out" 2>"$TMPDIR_EVAL/dialect-mutation-unit.err" <<'NODEEOF2'
|
|
2626
|
+
const { spawnSync } = require('child_process');
|
|
2627
|
+
const { resolveTrustedCommand } = require(process.argv[2]);
|
|
2628
|
+
const haystack = process.argv[3];
|
|
2629
|
+
const check = { id: 'ere-alternation-check' };
|
|
2630
|
+
const acceptance = {
|
|
2631
|
+
criteria: [
|
|
2632
|
+
{ id: 'ere-alternation-check', evidence_refs: [{ kind: 'command', excerpt: `grep -E '(foo|bar)' ${haystack}` }] },
|
|
2633
|
+
],
|
|
2634
|
+
};
|
|
2635
|
+
const trusted = resolveTrustedCommand(process.cwd(), process.cwd(), check, acceptance);
|
|
2636
|
+
const expected = `grep -E '(foo|bar)' ${haystack}`;
|
|
2637
|
+
if (trusted.argv[2] === expected) {
|
|
2638
|
+
console.error('mutation had no effect: argv[2] still matches the recorded command verbatim');
|
|
2639
|
+
process.exit(1);
|
|
2640
|
+
}
|
|
2641
|
+
const result = spawnSync(trusted.argv[0], trusted.argv.slice(1), { encoding: 'utf8' });
|
|
2642
|
+
if (result.status === 0) {
|
|
2643
|
+
console.error('mutation had no effect: the mutated replay still matched (expected a miss under BRE)');
|
|
2644
|
+
process.exit(1);
|
|
2645
|
+
}
|
|
2646
|
+
NODEEOF2
|
|
2647
|
+
then
|
|
2648
|
+
dialect_mutation_unit_status=0
|
|
2649
|
+
else
|
|
2650
|
+
dialect_mutation_unit_status=$?
|
|
2651
|
+
fi
|
|
2652
|
+
|
|
2653
|
+
# End-to-end: the SAME dialect e2e fixture (AC362_DIALECT_REPO) must now surface an
|
|
2654
|
+
# ambiguous/NOT_VERIFIED warning (the -E flag drop makes the replay genuinely miss the
|
|
2655
|
+
# ERE-only fixture, which isAmbiguousAbsenceCommand still classifies as a bare grep -- exit 1
|
|
2656
|
+
# on a presence check is ambiguous, not silently a clean pass).
|
|
2657
|
+
if node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/dialect-mutation-e2e.out" 2>"$TMPDIR_EVAL/dialect-mutation-e2e.err" <<JSON
|
|
2658
|
+
{"hook_event_name":"Stop","cwd":"$AC362_DIALECT_REPO"}
|
|
2659
|
+
JSON
|
|
2660
|
+
then :; fi
|
|
2661
|
+
|
|
2662
|
+
cp "$DIALECT_MUTATION_SCRATCH/stop-goal-fit.orig.js" "$ROOT/scripts/hooks/stop-goal-fit.js"
|
|
2663
|
+
|
|
2664
|
+
if [[ "$dialect_mutation_unit_status" -eq 0 ]]; then
|
|
2665
|
+
_pass "mutation-test: unit-level dialect eval goes RED as expected with the recorded -E flag silently dropped (argv no longer verbatim, and the mutated replay misses the ERE-only fixture)"
|
|
2666
|
+
else
|
|
2667
|
+
_fail "mutation-test: unit-level dialect eval did NOT go red with the -E flag dropped -- it may not be exercising the DIALECT-PRESERVATION INVARIANT: $(cat "$TMPDIR_EVAL/dialect-mutation-unit.out" "$TMPDIR_EVAL/dialect-mutation-unit.err")"
|
|
2668
|
+
fi
|
|
2669
|
+
if grep -qi 'ambiguous\|NOT_VERIFIED\|caught false-completion' "$TMPDIR_EVAL/dialect-mutation-e2e.err"; then
|
|
2670
|
+
_pass "mutation-test: end-to-end dialect eval goes RED as expected (an ambiguous/NOT_VERIFIED warning appears) with the recorded -E flag silently dropped"
|
|
2671
|
+
else
|
|
2672
|
+
_fail "mutation-test: end-to-end dialect eval did NOT go red with the -E flag dropped -- it may not be exercising the DIALECT-PRESERVATION INVARIANT: $(cat "$TMPDIR_EVAL/dialect-mutation-e2e.out" "$TMPDIR_EVAL/dialect-mutation-e2e.err")"
|
|
2673
|
+
fi
|
|
2674
|
+
fi
|
|
2675
|
+
|
|
2676
|
+
if diff -q "$ROOT/scripts/hooks/stop-goal-fit.js" "$DIALECT_MUTATION_SCRATCH/stop-goal-fit.orig.js" >/dev/null 2>&1; then
|
|
2677
|
+
_pass "mutation-test cleanup: scripts/hooks/stop-goal-fit.js is restored byte-identical to its pre-mutation-test content"
|
|
2678
|
+
else
|
|
2679
|
+
_fail "mutation-test cleanup REGRESSION: scripts/hooks/stop-goal-fit.js differs from its own pre-mutation-test content"
|
|
2680
|
+
fi
|
|
2681
|
+
|
|
2682
|
+
# Re-confirm GREEN: both dialect evals pass again against the restored file.
|
|
2683
|
+
if node - "$ROOT/scripts/hooks/stop-goal-fit.js" "$DIALECT_FIXTURE_DIR/haystack.txt" >"$TMPDIR_EVAL/dialect-restore-unit.out" 2>"$TMPDIR_EVAL/dialect-restore-unit.err" <<'NODEEOF3'
|
|
2684
|
+
const { spawnSync } = require('child_process');
|
|
2685
|
+
const { resolveTrustedCommand } = require(process.argv[2]);
|
|
2686
|
+
const haystack = process.argv[3];
|
|
2687
|
+
const check = { id: 'ere-alternation-check' };
|
|
2688
|
+
const acceptance = {
|
|
2689
|
+
criteria: [
|
|
2690
|
+
{ id: 'ere-alternation-check', evidence_refs: [{ kind: 'command', excerpt: `grep -E '(foo|bar)' ${haystack}` }] },
|
|
2691
|
+
],
|
|
2692
|
+
};
|
|
2693
|
+
const trusted = resolveTrustedCommand(process.cwd(), process.cwd(), check, acceptance);
|
|
2694
|
+
const expected = `grep -E '(foo|bar)' ${haystack}`;
|
|
2695
|
+
if (trusted.argv[2] !== expected) { console.error('argv[2] not verbatim after restore'); process.exit(1); }
|
|
2696
|
+
const result = spawnSync(trusted.argv[0], trusted.argv.slice(1), { encoding: 'utf8' });
|
|
2697
|
+
if (result.status !== 0) { console.error('ERE alternation did not match after restore'); process.exit(1); }
|
|
2698
|
+
NODEEOF3
|
|
2699
|
+
then
|
|
2700
|
+
_pass "mutation-test cleanup re-check: the restored real stop-goal-fit.js preserves the dialect again (unit-level, not just byte-restored)"
|
|
2701
|
+
else
|
|
2702
|
+
_fail "mutation-test cleanup re-check REGRESSION: restored stop-goal-fit.js no longer preserves the recorded dialect: $(cat "$TMPDIR_EVAL/dialect-restore-unit.out" "$TMPDIR_EVAL/dialect-restore-unit.err")"
|
|
2703
|
+
fi
|
|
2704
|
+
|
|
2705
|
+
if node "$ROOT/scripts/hooks/stop-goal-fit.js" >"$TMPDIR_EVAL/dialect-restore-e2e.out" 2>"$TMPDIR_EVAL/dialect-restore-e2e.err" <<JSON
|
|
2706
|
+
{"hook_event_name":"Stop","cwd":"$AC362_DIALECT_REPO"}
|
|
2707
|
+
JSON
|
|
2708
|
+
then :; fi
|
|
2709
|
+
if ! grep -qi 'ambiguous\|NOT_VERIFIED\|caught false-completion' "$TMPDIR_EVAL/dialect-restore-e2e.err"; then
|
|
2710
|
+
_pass "mutation-test cleanup re-check: the restored real stop-goal-fit.js confirms a clean end-to-end pass again for the dialect fixture"
|
|
2711
|
+
else
|
|
2712
|
+
_fail "mutation-test cleanup re-check REGRESSION: restored stop-goal-fit.js no longer confirms a clean end-to-end pass for the dialect fixture: $(cat "$TMPDIR_EVAL/dialect-restore-e2e.out" "$TMPDIR_EVAL/dialect-restore-e2e.err")"
|
|
2713
|
+
fi
|
|
2714
|
+
|
|
2715
|
+
|
|
685
2716
|
if [[ "$errors" -eq 0 ]]; then
|
|
686
2717
|
echo "Goal Fit hook integration passed."
|
|
687
2718
|
exit 0
|