@kontourai/flow-agents 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/CODEOWNERS +8 -0
- package/.github/workflows/ci.yml +32 -0
- package/AGENTS.md +4 -4
- package/CHANGELOG.md +39 -0
- package/CONTEXT.md +14 -0
- package/README.md +9 -3
- package/agents/dev.json +1 -1
- package/agents/tool-code-reviewer.json +1 -1
- package/agents/tool-planner.json +3 -3
- package/agents/tool-verifier.json +3 -3
- package/build/src/cli/console-learning-projection.js +3 -2
- package/build/src/cli/init.js +104 -21
- package/build/src/cli/kit.js +37 -6
- package/build/src/cli/pull-work-provider.js +1 -1
- package/build/src/cli/usage-feedback.js +3 -3
- package/build/src/cli/validate-hook-influence.js +1 -0
- package/build/src/cli/validate-workflow-artifacts.js +22 -6
- package/build/src/cli/workflow-artifact-cleanup-audit.js +36 -2
- package/build/src/cli/workflow-sidecar.d.ts +14 -7
- package/build/src/cli/workflow-sidecar.js +792 -69
- package/build/src/flow-kit/validate.d.ts +23 -0
- package/build/src/flow-kit/validate.js +67 -1
- package/build/src/index.d.ts +1 -1
- package/build/src/index.js +1 -1
- package/build/src/lib/flow-resolver.d.ts +6 -4
- package/build/src/lib/flow-resolver.js +125 -58
- package/build/src/lib/local-artifact-root.d.ts +14 -2
- package/build/src/lib/local-artifact-root.js +22 -5
- package/build/src/lib/workflow-learning-projection.js +2 -2
- package/build/src/runtime-adapters.d.ts +12 -0
- package/build/src/runtime-adapters.js +56 -16
- package/build/src/tools/build-universal-bundles.js +49 -16
- package/build/src/tools/generate-context-map.js +10 -8
- package/build/src/tools/validate-source-tree.js +23 -1
- package/console.telemetry.json +22 -0
- package/context/contracts/artifact-contract.md +38 -7
- package/context/contracts/builder-kit-workflow-state-contract.md +3 -2
- package/context/contracts/decision-registry-contract.md +263 -0
- package/context/contracts/delivery-contract.md +1 -1
- package/context/contracts/planning-contract.md +2 -1
- package/context/contracts/review-contract.md +5 -3
- package/context/contracts/verification-contract.md +9 -1
- package/context/contracts/work-item-contract.md +1 -1
- package/context/deferred/parallelization.md +1 -1
- package/context/gate-awareness.md +4 -4
- package/context/scripts/git-status.sh +2 -2
- package/context/scripts/hooks/config-protection.js +502 -2
- package/context/scripts/hooks/lib/config-protection-remedies.js +69 -0
- package/context/scripts/hooks/run-hook.js +14 -1
- package/context/scripts/hooks/stop-goal-fit.js +1140 -99
- package/context/scripts/hooks/workflow-steering.js +212 -6
- package/docs/adr/0007-flow-skill-kit-tool-boundary.md +2 -0
- package/docs/adr/0015-flow-flow-agents-boundary-reconciliation.md +35 -0
- package/docs/adr/0019-kit-dependency-ownership.md +62 -0
- package/docs/adr/0020-trust-reconcile-manifest-and-claim-classification.md +290 -0
- package/docs/adr/0021-assignment-leases-and-stale-claim-takeover.md +221 -0
- package/docs/adr/0022-fail-closed-delivery-reconciliation-with-governed-exemptions.md +283 -0
- package/docs/agent-system-guidebook.md +3 -3
- package/docs/agent-usage-feedback-loop.md +6 -6
- package/docs/context-map.md +9 -7
- package/docs/decisions/decision-records.md +57 -0
- package/docs/decisions/index.md +14 -0
- package/docs/decisions/promotion-gate.md +52 -0
- package/docs/developer-architecture.md +1 -1
- package/docs/fixture-ownership.md +3 -0
- package/docs/flow-kit-repository-contract.md +5 -5
- package/docs/getting-started.md +12 -2
- package/docs/integrations/harness-install.md +2 -1
- package/docs/kit-authoring-guide.md +64 -5
- package/docs/knowledge-kit.md +5 -1
- package/docs/learnings/2026-07-improvement-program.md +203 -0
- package/docs/migrations.md +6 -5
- package/docs/north-star.md +2 -2
- package/docs/operating-layers.md +4 -3
- package/docs/repository-structure.md +5 -5
- package/docs/skills-map.md +8 -8
- package/docs/spec/runtime-hook-surface.md +19 -15
- package/docs/spikes/graph-provider-2026-07.md +155 -0
- package/docs/standards-register.md +7 -7
- package/docs/trust-anchor-adoption.md +105 -4
- package/docs/work-item-adapters.md +1 -1
- package/docs/workflow-artifact-lifecycle.md +72 -9
- package/docs/workflow-eval-strategy.md +3 -3
- package/docs/workflow-shared-contracts.md +1 -1
- package/docs/workflow-usage-guide.md +254 -21
- package/evals/acceptance/DEMO-false-completion.md +20 -11
- package/evals/acceptance/prove-capture-teeth-declared.sh +15 -15
- package/evals/acceptance/prove-capture-teeth.sh +11 -11
- package/evals/acceptance/test_kiro_harness.sh +1 -1
- package/evals/ci/antigaming-suite.sh +5 -0
- package/evals/ci/run-baseline.sh +65 -1
- package/evals/fixtures/flow-kit-repository/invalid-bad-dependency/flows/review.flow.json +26 -0
- package/evals/fixtures/flow-kit-repository/invalid-bad-dependency/kit.json +13 -0
- package/evals/fixtures/flow-kit-repository/valid-with-dependency/flows/review.flow.json +26 -0
- package/evals/fixtures/flow-kit-repository/valid-with-dependency/kit.json +13 -0
- package/evals/fixtures/hook-influence/cases.json +32 -0
- package/evals/fixtures/pull-work-provider/github-issues.json +6 -6
- package/evals/fixtures/trust-reconcile-exploits/fabricated-attestation.json +59 -0
- package/evals/fixtures/trust-reconcile-exploits/no-label-bypass.json +36 -0
- package/evals/fixtures/trust-reconcile-exploits/skip-assumed-bypass.json +59 -0
- package/evals/fixtures/trust-reconcile-exploits/status-misassertion.json +48 -0
- package/evals/fixtures/trust-reconcile-exploits/waived-command-check.json +44 -0
- package/evals/fixtures/trust-reconcile-mixed-bundle/mixed-bundle.json +159 -0
- package/evals/fixtures/trust-reconcile-ws3/ws3-bundle.json +920 -0
- package/evals/integration/test_actor_identity.sh +254 -0
- package/evals/integration/test_builder_step_producers.sh +33 -23
- package/evals/integration/test_bundle_install.sh +22 -16
- package/evals/integration/test_bundle_lifecycle.sh +11 -11
- package/evals/integration/test_captured_fail_reconciliation.sh +50 -50
- package/evals/integration/test_checkpoint_signing.sh +43 -0
- package/evals/integration/test_claim_lookup.sh +4 -4
- package/evals/integration/test_codex_hook_resolution.sh +114 -0
- package/evals/integration/test_command_log_concurrency.sh +8 -8
- package/evals/integration/test_command_log_fork_classification.sh +3 -3
- package/evals/integration/test_command_log_integrity.sh +16 -16
- package/evals/integration/test_console_learning_projection.sh +1 -1
- package/evals/integration/test_dual_emit_flow_step.sh +145 -0
- package/evals/integration/test_enforcer_expects_driven.sh +12 -12
- package/evals/integration/test_evidence_capture_hook.sh +14 -13
- package/evals/integration/test_fixture_retirement_audit.sh +2 -2
- package/evals/integration/test_flow_agents_statusline.sh +2 -2
- package/evals/integration/test_flow_kit_install_git.sh +52 -0
- package/evals/integration/test_flow_kit_repository.sh +2 -0
- package/evals/integration/test_flowdef_session_activation.sh +6 -6
- package/evals/integration/test_flowdef_union_floor_regression.sh +429 -0
- package/evals/integration/test_gate_bypass_chain.sh +52 -40
- package/evals/integration/test_gate_lockdown.sh +97 -73
- package/evals/integration/test_gate_review_inquiry_records.sh +3 -3
- package/evals/integration/test_goal_fit_escape_hatch.sh +10 -10
- package/evals/integration/test_goal_fit_ghost_session.sh +104 -0
- package/evals/integration/test_goal_fit_hook.sh +32 -32
- package/evals/integration/test_goal_fit_rederive.sh +9 -9
- package/evals/integration/test_hook_category_behaviors.sh +103 -0
- package/evals/integration/test_hook_influence_cases.sh +1 -0
- package/evals/integration/test_install_merge.sh +262 -1
- package/evals/integration/test_kit_identity_trust.sh +6 -6
- package/evals/integration/test_liveness_conflict_injection.sh +587 -0
- package/evals/integration/test_liveness_heartbeat.sh +677 -0
- package/evals/integration/test_liveness_verdict.sh +394 -0
- package/evals/integration/test_local_flow_kit_install.sh +15 -0
- package/evals/integration/test_phase_map_and_gate_claim.sh +63 -11
- package/evals/integration/test_promote_gate.sh +178 -0
- package/evals/integration/test_publish_delivery.sh +2 -2
- package/evals/integration/test_pull_work_liveness_preflight.sh +386 -0
- package/evals/integration/test_pull_work_provider.sh +2 -2
- package/evals/integration/test_reconcile_soundness.sh +4 -4
- package/evals/integration/test_resolvefirststep_security.sh +22 -0
- package/evals/integration/test_runtime_adapter_activation.sh +87 -18
- package/evals/integration/test_session_resume_roundtrip.sh +147 -0
- package/evals/integration/test_sidecar_field_preservation.sh +249 -0
- package/evals/integration/test_trust_reconcile.sh +2 -2
- package/evals/integration/test_trust_reconcile_manifest.sh +79 -0
- package/evals/integration/test_trust_reconcile_mixed_bundle.sh +96 -0
- package/evals/integration/test_trust_reconcile_negatives.sh +151 -0
- package/evals/integration/test_usage_feedback_global.sh +5 -5
- package/evals/integration/test_validate_artifacts_portability.sh +60 -0
- package/evals/integration/test_verify_cli.sh +2 -2
- package/evals/integration/test_veritas_governance_kit.sh +117 -0
- package/evals/integration/test_workflow_artifact_cleanup_audit.sh +26 -1
- package/evals/integration/test_workflow_artifacts.sh +33 -0
- package/evals/integration/test_workflow_sidecar_writer.sh +723 -74
- package/evals/integration/test_workflow_steering_hook.sh +175 -5
- package/evals/lib/codex-provider.sh +1 -1
- package/evals/lib/node.sh +19 -1
- package/evals/run.sh +22 -0
- package/evals/static/test_decisions.sh +150 -0
- package/evals/static/test_flowdef_codeowners_coverage.sh +56 -0
- package/evals/static/test_package.sh +19 -10
- package/evals/static/test_universal_bundles.sh +48 -3
- package/evals/static/test_workflow_skills.sh +41 -8
- package/kits/builder/flows/build.flow.json +3 -74
- package/kits/builder/flows/publish-learn.flow.json +90 -0
- package/kits/builder/kit.json +11 -0
- package/kits/builder/skills/builder-shape/SKILL.md +1 -1
- package/kits/builder/skills/deliver/SKILL.md +13 -11
- package/kits/builder/skills/design-probe/SKILL.md +4 -4
- package/kits/builder/skills/evidence-gate/SKILL.md +26 -1
- package/kits/builder/skills/execute-plan/SKILL.md +2 -2
- package/kits/builder/skills/fix-bug/SKILL.md +2 -0
- package/kits/builder/skills/gate-review/SKILL.md +6 -6
- package/kits/builder/skills/idea-to-backlog/SKILL.md +1 -1
- package/kits/builder/skills/learning-review/SKILL.md +7 -7
- package/kits/builder/skills/pickup-probe/SKILL.md +5 -4
- package/kits/builder/skills/plan-work/SKILL.md +27 -5
- package/kits/builder/skills/pull-work/SKILL.md +80 -5
- package/kits/builder/skills/release-readiness/SKILL.md +4 -4
- package/kits/builder/skills/review-work/SKILL.md +24 -1
- package/kits/builder/skills/tdd-workflow/SKILL.md +2 -0
- package/kits/builder/skills/verify-work/SKILL.md +8 -1
- package/kits/catalog.json +6 -0
- package/kits/veritas-governance/adapter/readiness-to-trust-bundle.mjs +178 -0
- package/kits/veritas-governance/docs/README.md +75 -0
- package/kits/veritas-governance/fixtures/readiness/not-ready.readiness-report.json +4645 -0
- package/kits/veritas-governance/fixtures/readiness/ready.readiness-report.json +4403 -0
- package/kits/veritas-governance/flows/readiness-check.flow.json +35 -0
- package/kits/veritas-governance/kit.json +16 -0
- package/package.json +8 -4
- package/packaging/README.md +2 -2
- package/packaging/conformance/fixtures/config-protection--allow-read-state-json.json +20 -0
- package/packaging/conformance/fixtures/config-protection--block-write-state-json.json +20 -0
- package/packaging/conformance/fixtures/stop-goal-fit--block-bundle-disputed-claim.json +1 -1
- package/packaging/conformance/fixtures/stop-goal-fit--block-capture-contradicts-claimed-pass.json +1 -1
- package/packaging/manifest.json +9 -10
- package/schemas/decision-record.schema.json +109 -0
- package/schemas/workflow-evidence.schema.json +1 -1
- package/schemas/workflow-state.schema.json +5 -0
- package/scripts/README.md +8 -3
- package/scripts/check-content-boundary.cjs +1 -1
- package/scripts/check-decisions.cjs +356 -0
- package/scripts/ci/derive-claim-status.mjs +74 -0
- package/scripts/ci/trust-reconcile.js +436 -79
- package/scripts/git-status.sh +2 -2
- package/scripts/hooks/claude-telemetry-hook.js +37 -3
- package/scripts/hooks/codex-telemetry-hook.js +36 -2
- package/scripts/hooks/config-protection.js +102 -30
- package/scripts/hooks/evidence-capture.js +3 -3
- package/scripts/hooks/lib/actor-identity.js +368 -0
- package/scripts/hooks/lib/config-protection-remedies.js +69 -0
- package/scripts/hooks/lib/liveness-heartbeat.js +361 -0
- package/scripts/hooks/lib/liveness-policy.js +127 -0
- package/scripts/hooks/lib/liveness-read.js +105 -12
- package/scripts/hooks/lib/liveness-write.js +46 -0
- package/scripts/hooks/lib/local-artifact-paths.js +12 -6
- package/scripts/hooks/opencode-telemetry-hook.js +27 -1
- package/scripts/hooks/pi-telemetry-hook.js +27 -1
- package/scripts/hooks/stop-goal-fit.js +157 -26
- package/scripts/hooks/workflow-steering.js +77 -3
- package/scripts/install-codex-home.sh +138 -38
- package/scripts/statusline/flow-agents-statusline.js +2 -2
- package/src/cli/console-learning-projection.ts +3 -2
- package/src/cli/init.ts +101 -21
- package/src/cli/kit.ts +37 -6
- package/src/cli/public-api.test.mjs +58 -6
- package/src/cli/pull-work-provider.ts +1 -1
- package/src/cli/trust-bundle-policy-order.test.mjs +87 -0
- package/src/cli/usage-feedback.ts +3 -3
- package/src/cli/validate-hook-influence.ts +1 -0
- package/src/cli/validate-workflow-artifacts.ts +22 -6
- package/src/cli/veritas-readiness-adapter.test.mjs +267 -0
- package/src/cli/workflow-artifact-cleanup-audit.ts +35 -2
- package/src/cli/workflow-sidecar.ts +795 -71
- package/src/flow-kit/validate.ts +74 -1
- package/src/index.ts +6 -2
- package/src/lib/flow-resolver.ts +123 -55
- package/src/lib/local-artifact-root.ts +24 -5
- package/src/lib/workflow-learning-projection.ts +2 -2
- package/src/runtime-adapters.ts +61 -15
- package/src/tools/build-universal-bundles.ts +49 -16
- package/src/tools/generate-context-map.ts +10 -8
- package/src/tools/validate-source-tree.ts +23 -1
|
@@ -16,12 +16,12 @@ _pass() { echo " ✓ $1"; }
|
|
|
16
16
|
_fail() { echo " ✗ $1"; errors=$((errors + 1)); }
|
|
17
17
|
|
|
18
18
|
REPO="$TMPDIR_EVAL/repo"
|
|
19
|
-
mkdir -p "$REPO/.flow-agents/steering-demo"
|
|
19
|
+
mkdir -p "$REPO/.kontourai/flow-agents/steering-demo"
|
|
20
20
|
mkdir -p "$REPO/docs"
|
|
21
21
|
printf '# Test Repo\n' > "$REPO/AGENTS.md"
|
|
22
22
|
printf '# Context Map\n' > "$REPO/docs/context-map.md"
|
|
23
23
|
|
|
24
|
-
cat > "$REPO/.flow-agents/steering-demo/state.json" <<'JSON'
|
|
24
|
+
cat > "$REPO/.kontourai/flow-agents/steering-demo/state.json" <<'JSON'
|
|
25
25
|
{
|
|
26
26
|
"schema_version": "1.0",
|
|
27
27
|
"task_slug": "steering-demo",
|
|
@@ -36,7 +36,7 @@ cat > "$REPO/.flow-agents/steering-demo/state.json" <<'JSON'
|
|
|
36
36
|
}
|
|
37
37
|
JSON
|
|
38
38
|
|
|
39
|
-
cat > "$REPO/.flow-agents/steering-demo/critique.json" <<'JSON'
|
|
39
|
+
cat > "$REPO/.kontourai/flow-agents/steering-demo/critique.json" <<'JSON'
|
|
40
40
|
{
|
|
41
41
|
"schema_version": "1.0",
|
|
42
42
|
"task_slug": "steering-demo",
|
|
@@ -217,6 +217,176 @@ else
|
|
|
217
217
|
_fail "Claude hook adapter should not fail for prompt-submit workflow steering"
|
|
218
218
|
fi
|
|
219
219
|
|
|
220
|
+
FRESH_REPO="$TMPDIR_EVAL/fresh-repo"
|
|
221
|
+
mkdir -p "$FRESH_REPO/docs"
|
|
222
|
+
printf '# Fresh Repo\n' > "$FRESH_REPO/AGENTS.md"
|
|
223
|
+
printf '# Context Map\n' > "$FRESH_REPO/docs/context-map.md"
|
|
224
|
+
|
|
225
|
+
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/builder-route.out" 2>"$TMPDIR_EVAL/builder-route.err" <<JSON
|
|
226
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please implement the new settings API and update its tests."}
|
|
227
|
+
JSON
|
|
228
|
+
then
|
|
229
|
+
if rg -q 'BUILDER WORKFLOW ROUTE' "$TMPDIR_EVAL/builder-route.out" && \
|
|
230
|
+
rg -q 'activate `deliver`' "$TMPDIR_EVAL/builder-route.out" && \
|
|
231
|
+
rg -q -- '--flow-id builder.build' "$TMPDIR_EVAL/builder-route.out" && \
|
|
232
|
+
rg -q 'plan-work -> execute-plan -> review-work -> verify-work' "$TMPDIR_EVAL/builder-route.out" && \
|
|
233
|
+
rg -q 'publish/release-readiness and learning-review' "$TMPDIR_EVAL/builder-route.out"; then
|
|
234
|
+
_pass "workflow steering hook routes fresh coding prompts into Builder workflow"
|
|
235
|
+
else
|
|
236
|
+
_fail "workflow steering missed Builder workflow route for coding prompt: $(cat "$TMPDIR_EVAL/builder-route.out")"
|
|
237
|
+
fi
|
|
238
|
+
else
|
|
239
|
+
_fail "workflow steering hook should not fail for fresh coding prompt"
|
|
240
|
+
fi
|
|
241
|
+
|
|
242
|
+
if node "$ROOT/scripts/hooks/claude-hook-adapter.js" UserPromptSubmit prompt:workflow-steering workflow-steering.js standard,strict >"$TMPDIR_EVAL/claude-builder-route.out" 2>"$TMPDIR_EVAL/claude-builder-route.err" <<JSON
|
|
243
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please implement the new settings API and update its tests."}
|
|
244
|
+
JSON
|
|
245
|
+
then
|
|
246
|
+
if node - "$TMPDIR_EVAL/claude-builder-route.out" <<'NODE'
|
|
247
|
+
const fs = require("node:fs");
|
|
248
|
+
const payload = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
249
|
+
const ctx = payload.hookSpecificOutput?.additionalContext || "";
|
|
250
|
+
if (payload.continue !== true) throw new Error("continue not true");
|
|
251
|
+
if (payload.suppressOutput !== false) throw new Error("suppressOutput should be false when guidance exists");
|
|
252
|
+
for (const needle of ["BUILDER WORKFLOW ROUTE", "activate `deliver`", "--flow-id builder.build", "plan-work -> execute-plan -> review-work -> verify-work", "publish/release-readiness and learning-review"]) {
|
|
253
|
+
if (!ctx.includes(needle)) throw new Error(`missing ${needle}`);
|
|
254
|
+
}
|
|
255
|
+
NODE
|
|
256
|
+
then
|
|
257
|
+
_pass "Claude hook adapter surfaces Builder workflow route for coding prompts"
|
|
258
|
+
else
|
|
259
|
+
_fail "Claude hook adapter missed Builder workflow route: $(cat "$TMPDIR_EVAL/claude-builder-route.out") $(cat "$TMPDIR_EVAL/claude-builder-route.err")"
|
|
260
|
+
fi
|
|
261
|
+
else
|
|
262
|
+
_fail "Claude hook adapter should not fail for Builder workflow route"
|
|
263
|
+
fi
|
|
264
|
+
|
|
265
|
+
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/builder-route-review-only.out" 2>"$TMPDIR_EVAL/builder-route-review-only.err" <<JSON
|
|
266
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please review the test coverage and validate whether it is enough. Do not modify files."}
|
|
267
|
+
JSON
|
|
268
|
+
then
|
|
269
|
+
if ! rg -q 'BUILDER WORKFLOW ROUTE' "$TMPDIR_EVAL/builder-route-review-only.out"; then
|
|
270
|
+
_pass "workflow steering hook does not route explicit review-only prompts into Builder workflow"
|
|
271
|
+
else
|
|
272
|
+
_fail "workflow steering incorrectly routed review-only prompt: $(cat "$TMPDIR_EVAL/builder-route-review-only.out")"
|
|
273
|
+
fi
|
|
274
|
+
else
|
|
275
|
+
_fail "workflow steering hook should not fail for review-only prompt"
|
|
276
|
+
fi
|
|
277
|
+
|
|
278
|
+
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/builder-route-validate-only.out" 2>"$TMPDIR_EVAL/builder-route-validate-only.err" <<JSON
|
|
279
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please validate whether the tests are enough. Do not modify files."}
|
|
280
|
+
JSON
|
|
281
|
+
then
|
|
282
|
+
if ! rg -q 'BUILDER WORKFLOW ROUTE' "$TMPDIR_EVAL/builder-route-validate-only.out"; then
|
|
283
|
+
_pass "workflow steering hook does not route explicit validation-only prompts into Builder workflow"
|
|
284
|
+
else
|
|
285
|
+
_fail "workflow steering incorrectly routed validation-only prompt: $(cat "$TMPDIR_EVAL/builder-route-validate-only.out")"
|
|
286
|
+
fi
|
|
287
|
+
else
|
|
288
|
+
_fail "workflow steering hook should not fail for validation-only prompt"
|
|
289
|
+
fi
|
|
290
|
+
|
|
291
|
+
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/builder-route-bare-validate.out" 2>"$TMPDIR_EVAL/builder-route-bare-validate.err" <<JSON
|
|
292
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please validate whether the settings API tests are enough."}
|
|
293
|
+
JSON
|
|
294
|
+
then
|
|
295
|
+
if ! rg -q 'BUILDER WORKFLOW ROUTE' "$TMPDIR_EVAL/builder-route-bare-validate.out"; then
|
|
296
|
+
_pass "workflow steering hook does not route bare validation prompts into Builder workflow"
|
|
297
|
+
else
|
|
298
|
+
_fail "workflow steering incorrectly routed bare validation prompt: $(cat "$TMPDIR_EVAL/builder-route-bare-validate.out")"
|
|
299
|
+
fi
|
|
300
|
+
else
|
|
301
|
+
_fail "workflow steering hook should not fail for bare validation prompt"
|
|
302
|
+
fi
|
|
303
|
+
|
|
304
|
+
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/builder-route-bare-test.out" 2>"$TMPDIR_EVAL/builder-route-bare-test.err" <<JSON
|
|
305
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please test whether this still reproduces."}
|
|
306
|
+
JSON
|
|
307
|
+
then
|
|
308
|
+
if ! rg -q 'BUILDER WORKFLOW ROUTE' "$TMPDIR_EVAL/builder-route-bare-test.out"; then
|
|
309
|
+
_pass "workflow steering hook does not route bare test prompts into Builder workflow"
|
|
310
|
+
else
|
|
311
|
+
_fail "workflow steering incorrectly routed bare test prompt: $(cat "$TMPDIR_EVAL/builder-route-bare-test.out")"
|
|
312
|
+
fi
|
|
313
|
+
else
|
|
314
|
+
_fail "workflow steering hook should not fail for bare test prompt"
|
|
315
|
+
fi
|
|
316
|
+
|
|
317
|
+
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/builder-route-test-question.out" 2>"$TMPDIR_EVAL/builder-route-test-question.err" <<JSON
|
|
318
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"What tests should I run for the settings API?"}
|
|
319
|
+
JSON
|
|
320
|
+
then
|
|
321
|
+
if ! rg -q 'BUILDER WORKFLOW ROUTE' "$TMPDIR_EVAL/builder-route-test-question.out"; then
|
|
322
|
+
_pass "workflow steering hook does not route question-only test prompts into Builder workflow"
|
|
323
|
+
else
|
|
324
|
+
_fail "workflow steering incorrectly routed question-only test prompt: $(cat "$TMPDIR_EVAL/builder-route-test-question.out")"
|
|
325
|
+
fi
|
|
326
|
+
else
|
|
327
|
+
_fail "workflow steering hook should not fail for question-only test prompt"
|
|
328
|
+
fi
|
|
329
|
+
|
|
330
|
+
if node "$ROOT/scripts/hooks/claude-hook-adapter.js" UserPromptSubmit prompt:workflow-steering workflow-steering.js standard,strict >"$TMPDIR_EVAL/claude-builder-route-review-only.out" 2>"$TMPDIR_EVAL/claude-builder-route-review-only.err" <<JSON
|
|
331
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please review the test coverage and validate whether it is enough. Do not modify files."}
|
|
332
|
+
JSON
|
|
333
|
+
then
|
|
334
|
+
if node - "$TMPDIR_EVAL/claude-builder-route-review-only.out" <<'NODE'
|
|
335
|
+
const fs = require("node:fs");
|
|
336
|
+
const payload = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
337
|
+
const ctx = payload.hookSpecificOutput?.additionalContext || "";
|
|
338
|
+
if (payload.continue !== true) throw new Error("continue not true");
|
|
339
|
+
if (ctx.includes("BUILDER WORKFLOW ROUTE")) throw new Error("review-only prompt should not route to Builder workflow");
|
|
340
|
+
NODE
|
|
341
|
+
then
|
|
342
|
+
_pass "Claude hook adapter does not route explicit review-only prompts into Builder workflow"
|
|
343
|
+
else
|
|
344
|
+
_fail "Claude hook adapter incorrectly routed review-only prompt: $(cat "$TMPDIR_EVAL/claude-builder-route-review-only.out") $(cat "$TMPDIR_EVAL/claude-builder-route-review-only.err")"
|
|
345
|
+
fi
|
|
346
|
+
else
|
|
347
|
+
_fail "Claude hook adapter should not fail for review-only prompt"
|
|
348
|
+
fi
|
|
349
|
+
|
|
350
|
+
if node "$ROOT/scripts/hooks/claude-hook-adapter.js" UserPromptSubmit prompt:workflow-steering workflow-steering.js standard,strict >"$TMPDIR_EVAL/claude-builder-route-validate-only.out" 2>"$TMPDIR_EVAL/claude-builder-route-validate-only.err" <<JSON
|
|
351
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please validate whether the tests are enough. Do not modify files."}
|
|
352
|
+
JSON
|
|
353
|
+
then
|
|
354
|
+
if node - "$TMPDIR_EVAL/claude-builder-route-validate-only.out" <<'NODE'
|
|
355
|
+
const fs = require("node:fs");
|
|
356
|
+
const payload = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
357
|
+
const ctx = payload.hookSpecificOutput?.additionalContext || "";
|
|
358
|
+
if (payload.continue !== true) throw new Error("continue not true");
|
|
359
|
+
if (ctx.includes("BUILDER WORKFLOW ROUTE")) throw new Error("validation-only prompt should not route to Builder workflow");
|
|
360
|
+
NODE
|
|
361
|
+
then
|
|
362
|
+
_pass "Claude hook adapter does not route explicit validation-only prompts into Builder workflow"
|
|
363
|
+
else
|
|
364
|
+
_fail "Claude hook adapter incorrectly routed validation-only prompt: $(cat "$TMPDIR_EVAL/claude-builder-route-validate-only.out") $(cat "$TMPDIR_EVAL/claude-builder-route-validate-only.err")"
|
|
365
|
+
fi
|
|
366
|
+
else
|
|
367
|
+
_fail "Claude hook adapter should not fail for validation-only prompt"
|
|
368
|
+
fi
|
|
369
|
+
|
|
370
|
+
if node "$ROOT/scripts/hooks/claude-hook-adapter.js" UserPromptSubmit prompt:workflow-steering workflow-steering.js standard,strict >"$TMPDIR_EVAL/claude-builder-route-test-question.out" 2>"$TMPDIR_EVAL/claude-builder-route-test-question.err" <<JSON
|
|
371
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"What tests should I run for the settings API?"}
|
|
372
|
+
JSON
|
|
373
|
+
then
|
|
374
|
+
if node - "$TMPDIR_EVAL/claude-builder-route-test-question.out" <<'NODE'
|
|
375
|
+
const fs = require("node:fs");
|
|
376
|
+
const payload = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
377
|
+
const ctx = payload.hookSpecificOutput?.additionalContext || "";
|
|
378
|
+
if (payload.continue !== true) throw new Error("continue not true");
|
|
379
|
+
if (ctx.includes("BUILDER WORKFLOW ROUTE")) throw new Error("question-only test prompt should not route to Builder workflow");
|
|
380
|
+
NODE
|
|
381
|
+
then
|
|
382
|
+
_pass "Claude hook adapter does not route question-only test prompts into Builder workflow"
|
|
383
|
+
else
|
|
384
|
+
_fail "Claude hook adapter incorrectly routed question-only test prompt: $(cat "$TMPDIR_EVAL/claude-builder-route-test-question.out") $(cat "$TMPDIR_EVAL/claude-builder-route-test-question.err")"
|
|
385
|
+
fi
|
|
386
|
+
else
|
|
387
|
+
_fail "Claude hook adapter should not fail for question-only test prompt"
|
|
388
|
+
fi
|
|
389
|
+
|
|
220
390
|
if node "$ROOT/scripts/hooks/codex-hook-adapter.js" post:workflow-steering workflow-steering.js standard,strict >"$TMPDIR_EVAL/codex-adapter.out" 2>"$TMPDIR_EVAL/codex-adapter.err" <<JSON
|
|
221
391
|
{"hook_event_name":"PostToolUse","cwd":"$REPO","tool_input":{"command":"Bash","content":{"command":"bash evals/run.sh integration"}},"tool_response":"integration finished"}
|
|
222
392
|
JSON
|
|
@@ -285,7 +455,7 @@ else
|
|
|
285
455
|
_fail "Codex hook adapter should not fail for prompt-submit workflow steering"
|
|
286
456
|
fi
|
|
287
457
|
|
|
288
|
-
cat > "$REPO/.flow-agents/steering-demo/state.json" <<'JSON'
|
|
458
|
+
cat > "$REPO/.kontourai/flow-agents/steering-demo/state.json" <<'JSON'
|
|
289
459
|
{
|
|
290
460
|
"schema_version": "1.0",
|
|
291
461
|
"task_slug": "steering-demo",
|
|
@@ -298,7 +468,7 @@ cat > "$REPO/.flow-agents/steering-demo/state.json" <<'JSON'
|
|
|
298
468
|
}
|
|
299
469
|
}
|
|
300
470
|
JSON
|
|
301
|
-
rm -f "$REPO/.flow-agents/steering-demo/critique.json"
|
|
471
|
+
rm -f "$REPO/.kontourai/flow-agents/steering-demo/critique.json"
|
|
302
472
|
|
|
303
473
|
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/done.out" 2>"$TMPDIR_EVAL/done.err" <<JSON
|
|
304
474
|
{"cwd":"$REPO","tool_input":{"command":"InvokeSubagents","content":{"subagents":[{"agent_name":"tool-verifier"}]}},"tool_response":"verification finished"}
|
package/evals/lib/node.sh
CHANGED
|
@@ -3,8 +3,26 @@
|
|
|
3
3
|
|
|
4
4
|
FLOW_AGENTS_EVAL_ROOT="${ROOT:-${ROOT_DIR:-}}"
|
|
5
5
|
|
|
6
|
+
# Memoized per-process: flow_agents_node is called dozens of times per eval script
|
|
7
|
+
# (often inside a `run_bounded N ...` hang-guard around a single fail-fast assertion).
|
|
8
|
+
# Re-running `npm run build --silent` (npm startup + tsc incremental stat-check) on EVERY
|
|
9
|
+
# call repeatedly re-pays that overhead inside those bounds. On a slow/loaded CI runner a
|
|
10
|
+
# single one of those ~100+ redundant build invocations can occasionally take long enough
|
|
11
|
+
# to blow a tight hang-guard bound before the actual (near-instant) validation ever runs,
|
|
12
|
+
# producing a spurious assertion failure that has nothing to do with the behavior under
|
|
13
|
+
# test (see PR #306 CI trust-reconcile divergence investigation). Source under src/ never
|
|
14
|
+
# changes mid-script for any eval, so building once per process is safe and correct.
|
|
15
|
+
_FLOW_AGENTS_BUILD_DONE=""
|
|
16
|
+
|
|
6
17
|
flow_agents_build_ts() {
|
|
7
|
-
|
|
18
|
+
if [[ "$_FLOW_AGENTS_BUILD_DONE" == "1" ]]; then
|
|
19
|
+
return 0
|
|
20
|
+
fi
|
|
21
|
+
if (cd "$FLOW_AGENTS_EVAL_ROOT" && npm run build --silent >/dev/null); then
|
|
22
|
+
_FLOW_AGENTS_BUILD_DONE="1"
|
|
23
|
+
return 0
|
|
24
|
+
fi
|
|
25
|
+
return 1
|
|
8
26
|
}
|
|
9
27
|
|
|
10
28
|
flow_agents_node() {
|
package/evals/run.sh
CHANGED
|
@@ -135,12 +135,16 @@ run_static() {
|
|
|
135
135
|
echo ""
|
|
136
136
|
bash "$EVAL_DIR/static/test_evidence_refs.sh" || result=1
|
|
137
137
|
echo ""
|
|
138
|
+
bash "$EVAL_DIR/static/test_decisions.sh" || result=1
|
|
139
|
+
echo ""
|
|
138
140
|
bash "$EVAL_DIR/static/test_library_exports.sh" || result=1
|
|
139
141
|
echo ""
|
|
140
142
|
bash "$EVAL_DIR/static/test_console_presets.sh" || result=1
|
|
141
143
|
echo ""
|
|
142
144
|
bash "$EVAL_DIR/static/test_repo_hooks.sh" || result=1
|
|
143
145
|
echo ""
|
|
146
|
+
bash "$EVAL_DIR/static/test_flowdef_codeowners_coverage.sh" || result=1
|
|
147
|
+
echo ""
|
|
144
148
|
bash "$EVAL_DIR/static/test_unit_helpers.sh" || result=1
|
|
145
149
|
return $result
|
|
146
150
|
}
|
|
@@ -179,12 +183,26 @@ run_integration() {
|
|
|
179
183
|
echo ""
|
|
180
184
|
bash "$EVAL_DIR/integration/test_workflow_artifact_cleanup_audit.sh" || result=1
|
|
181
185
|
echo ""
|
|
186
|
+
bash "$EVAL_DIR/integration/test_promote_gate.sh" || result=1
|
|
187
|
+
echo ""
|
|
182
188
|
bash "$EVAL_DIR/integration/test_fixture_retirement_audit.sh" || result=1
|
|
183
189
|
echo ""
|
|
184
190
|
bash "$EVAL_DIR/integration/test_publish_change_helper.sh" || result=1
|
|
185
191
|
echo ""
|
|
186
192
|
bash "$EVAL_DIR/integration/test_workflow_sidecar_writer.sh" || result=1
|
|
187
193
|
echo ""
|
|
194
|
+
bash "$EVAL_DIR/integration/test_sidecar_field_preservation.sh" || result=1
|
|
195
|
+
echo ""
|
|
196
|
+
bash "$EVAL_DIR/integration/test_actor_identity.sh" || result=1
|
|
197
|
+
echo ""
|
|
198
|
+
bash "$EVAL_DIR/integration/test_liveness_heartbeat.sh" || result=1
|
|
199
|
+
echo ""
|
|
200
|
+
bash "$EVAL_DIR/integration/test_pull_work_liveness_preflight.sh" || result=1
|
|
201
|
+
echo ""
|
|
202
|
+
bash "$EVAL_DIR/integration/test_liveness_verdict.sh" || result=1
|
|
203
|
+
echo ""
|
|
204
|
+
bash "$EVAL_DIR/integration/test_liveness_conflict_injection.sh" || result=1
|
|
205
|
+
echo ""
|
|
188
206
|
bash "$EVAL_DIR/integration/test_veritas_governance_adapter.sh" || result=1
|
|
189
207
|
echo ""
|
|
190
208
|
bash "$EVAL_DIR/integration/test_workflow_steering_hook.sh" || result=1
|
|
@@ -203,6 +221,10 @@ run_integration() {
|
|
|
203
221
|
echo ""
|
|
204
222
|
bash "$EVAL_DIR/integration/test_runtime_adapter_activation.sh" || result=1
|
|
205
223
|
echo ""
|
|
224
|
+
bash "$EVAL_DIR/integration/test_validate_artifacts_portability.sh" || result=1
|
|
225
|
+
echo ""
|
|
226
|
+
bash "$EVAL_DIR/integration/test_codex_hook_resolution.sh" || result=1
|
|
227
|
+
echo ""
|
|
206
228
|
bash "$EVAL_DIR/integration/test_bundle_install.sh" || result=1
|
|
207
229
|
echo ""
|
|
208
230
|
bash "$EVAL_DIR/integration/test_bundle_lifecycle.sh" || result=1
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# test_decisions.sh — Decision registry validator + index generator checks.
|
|
3
|
+
# Covers: valid pass, each violation class (bad status, missing subject,
|
|
4
|
+
# secret-shaped evidence, tombstone -> missing slug), and index idempotency.
|
|
5
|
+
set -uo pipefail
|
|
6
|
+
|
|
7
|
+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
8
|
+
SCRIPT="$ROOT/scripts/check-decisions.cjs"
|
|
9
|
+
|
|
10
|
+
errors=0
|
|
11
|
+
pass() { echo " PASS $1"; }
|
|
12
|
+
fail() { echo " FAIL $1"; errors=$((errors + 1)); }
|
|
13
|
+
|
|
14
|
+
echo "=== Decision Registry Checks ==="
|
|
15
|
+
|
|
16
|
+
WORK="$(mktemp -d)"
|
|
17
|
+
trap 'rm -rf "$WORK"' EXIT
|
|
18
|
+
|
|
19
|
+
run_check() { FLOW_AGENTS_DECISIONS_DIR="$1" node "$SCRIPT" check >"$2" 2>&1; }
|
|
20
|
+
gen_index() { FLOW_AGENTS_DECISIONS_DIR="$1" node "$SCRIPT" gen-index >/dev/null 2>&1; }
|
|
21
|
+
|
|
22
|
+
valid_frontmatter() {
|
|
23
|
+
cat <<'EOF'
|
|
24
|
+
---
|
|
25
|
+
status: current
|
|
26
|
+
subject: Example subject
|
|
27
|
+
decided: 2026-07-03
|
|
28
|
+
evidence:
|
|
29
|
+
- kind: issue
|
|
30
|
+
ref: https://github.com/kontourai/flow-agents/issues/310
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
# Example subject
|
|
34
|
+
|
|
35
|
+
The current answer.
|
|
36
|
+
EOF
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
# --- Case 1: valid file passes ---
|
|
40
|
+
D1="$WORK/valid"; mkdir -p "$D1"
|
|
41
|
+
valid_frontmatter >"$D1/example-subject.md"
|
|
42
|
+
gen_index "$D1"
|
|
43
|
+
if run_check "$D1" "$WORK/c1.log"; then
|
|
44
|
+
pass "valid topic file + fresh index passes"
|
|
45
|
+
else
|
|
46
|
+
fail "valid topic file should pass"; cat "$WORK/c1.log"
|
|
47
|
+
fi
|
|
48
|
+
|
|
49
|
+
# --- Case 2: unknown status fails ---
|
|
50
|
+
D2="$WORK/badstatus"; mkdir -p "$D2"
|
|
51
|
+
valid_frontmatter | sed 's/^status: current/status: accepted/' >"$D2/example-subject.md"
|
|
52
|
+
gen_index "$D2"
|
|
53
|
+
if run_check "$D2" "$WORK/c2.log"; then
|
|
54
|
+
fail "unknown status should fail validation"
|
|
55
|
+
else
|
|
56
|
+
grep -q "unknown status" "$WORK/c2.log" && pass "unknown status fails" || { fail "unknown status wrong error"; cat "$WORK/c2.log"; }
|
|
57
|
+
fi
|
|
58
|
+
|
|
59
|
+
# --- Case 3: missing subject fails ---
|
|
60
|
+
D3="$WORK/nosubject"; mkdir -p "$D3"
|
|
61
|
+
valid_frontmatter | grep -v '^subject:' >"$D3/example-subject.md"
|
|
62
|
+
gen_index "$D3"
|
|
63
|
+
if run_check "$D3" "$WORK/c3.log"; then
|
|
64
|
+
fail "missing subject should fail validation"
|
|
65
|
+
else
|
|
66
|
+
grep -q "subject" "$WORK/c3.log" && pass "missing subject fails" || { fail "missing subject wrong error"; cat "$WORK/c3.log"; }
|
|
67
|
+
fi
|
|
68
|
+
|
|
69
|
+
# --- Case 4: secret-shaped evidence ref fails ---
|
|
70
|
+
# The secret-shaped literal is assembled at runtime so no contiguous
|
|
71
|
+
# access-key-shaped token is committed to a tracked file (suite secret-scan gate).
|
|
72
|
+
D4="$WORK/secret"; mkdir -p "$D4"
|
|
73
|
+
FAKE_KEY="AKIA$(printf 'IOSFODNN7EXAMPLE')"
|
|
74
|
+
{
|
|
75
|
+
printf -- '---\n'
|
|
76
|
+
printf 'status: current\n'
|
|
77
|
+
printf 'subject: Example subject\n'
|
|
78
|
+
printf 'decided: 2026-07-03\n'
|
|
79
|
+
printf 'evidence:\n'
|
|
80
|
+
printf ' - kind: url\n'
|
|
81
|
+
printf ' ref: "https://example.com/?token=%s"\n' "$FAKE_KEY"
|
|
82
|
+
printf -- '---\n\n'
|
|
83
|
+
printf '# Example subject\n'
|
|
84
|
+
} >"$D4/example-subject.md"
|
|
85
|
+
gen_index "$D4"
|
|
86
|
+
if run_check "$D4" "$WORK/c4.log"; then
|
|
87
|
+
fail "secret-shaped evidence ref should fail validation"
|
|
88
|
+
else
|
|
89
|
+
grep -q "secret-shaped" "$WORK/c4.log" && pass "secret-shaped evidence fails" || { fail "secret-shaped wrong error"; cat "$WORK/c4.log"; }
|
|
90
|
+
fi
|
|
91
|
+
|
|
92
|
+
# --- Case 5: tombstone pointing at missing slug fails ---
|
|
93
|
+
D5="$WORK/tombstone"; mkdir -p "$D5"
|
|
94
|
+
cat >"$D5/old-topic.md" <<'EOF'
|
|
95
|
+
---
|
|
96
|
+
status: superseded
|
|
97
|
+
subject: Old topic
|
|
98
|
+
decided: 2026-07-03
|
|
99
|
+
superseded_by: does-not-exist
|
|
100
|
+
evidence:
|
|
101
|
+
- kind: issue
|
|
102
|
+
ref: "#310"
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
# Old topic
|
|
106
|
+
EOF
|
|
107
|
+
gen_index "$D5"
|
|
108
|
+
if run_check "$D5" "$WORK/c5.log"; then
|
|
109
|
+
fail "tombstone -> missing slug should fail validation"
|
|
110
|
+
else
|
|
111
|
+
grep -q "missing topic slug" "$WORK/c5.log" && pass "tombstone -> missing slug fails" || { fail "tombstone wrong error"; cat "$WORK/c5.log"; }
|
|
112
|
+
fi
|
|
113
|
+
|
|
114
|
+
# --- Case 6: index idempotency (second gen is diff-clean) ---
|
|
115
|
+
D6="$WORK/idem"; mkdir -p "$D6"
|
|
116
|
+
valid_frontmatter >"$D6/example-subject.md"
|
|
117
|
+
gen_index "$D6"
|
|
118
|
+
cp "$D6/index.md" "$WORK/index-first.md"
|
|
119
|
+
gen_index "$D6"
|
|
120
|
+
if diff -q "$WORK/index-first.md" "$D6/index.md" >/dev/null; then
|
|
121
|
+
pass "index regeneration is idempotent (diff-clean)"
|
|
122
|
+
else
|
|
123
|
+
fail "index regeneration is not idempotent"; diff "$WORK/index-first.md" "$D6/index.md" || true
|
|
124
|
+
fi
|
|
125
|
+
|
|
126
|
+
# --- Case 7: check flags a stale index ---
|
|
127
|
+
D7="$WORK/stale"; mkdir -p "$D7"
|
|
128
|
+
valid_frontmatter >"$D7/example-subject.md"
|
|
129
|
+
gen_index "$D7"
|
|
130
|
+
printf '\nstale line\n' >>"$D7/index.md"
|
|
131
|
+
if run_check "$D7" "$WORK/c7.log"; then
|
|
132
|
+
fail "stale index should fail validation"
|
|
133
|
+
else
|
|
134
|
+
grep -q "stale" "$WORK/c7.log" && pass "stale index detected" || { fail "stale index wrong error"; cat "$WORK/c7.log"; }
|
|
135
|
+
fi
|
|
136
|
+
|
|
137
|
+
# --- Case 8: repo's own registry passes end-to-end ---
|
|
138
|
+
if node "$SCRIPT" check >"$WORK/c8.log" 2>&1; then
|
|
139
|
+
pass "repo docs/decisions registry passes"
|
|
140
|
+
else
|
|
141
|
+
fail "repo docs/decisions registry should pass"; cat "$WORK/c8.log"
|
|
142
|
+
fi
|
|
143
|
+
|
|
144
|
+
echo ""
|
|
145
|
+
if [[ "$errors" -eq 0 ]]; then
|
|
146
|
+
echo "Decision registry checks: all passed."
|
|
147
|
+
exit 0
|
|
148
|
+
fi
|
|
149
|
+
echo "Decision registry checks: $errors failed."
|
|
150
|
+
exit 1
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
5
|
+
cd "$ROOT_DIR"
|
|
6
|
+
|
|
7
|
+
# Regression lock: CODEOWNERS must keep review-gated coverage over the
|
|
8
|
+
# anti-gaming gate's enforcement hooks, the FlowDefinition resolution/trust
|
|
9
|
+
# path, and the kit FlowDefinitions the gate's expects[] taxonomy is derived
|
|
10
|
+
# from (ADR 0016 Abstraction A, ADR 0018 Decision #2). If any of these paths
|
|
11
|
+
# silently drop out of CODEOWNERS, an agent could weaken the gate without
|
|
12
|
+
# owner review. This check must fail loudly, naming the missing path.
|
|
13
|
+
|
|
14
|
+
CODEOWNERS_FILE=".github/CODEOWNERS"
|
|
15
|
+
|
|
16
|
+
pass() {
|
|
17
|
+
echo "PASS: $1"
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
fail() {
|
|
21
|
+
echo "FAIL: $1" >&2
|
|
22
|
+
exit 1
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
require_file() {
|
|
26
|
+
[[ -f "$1" ]] || fail "$1 is missing"
|
|
27
|
+
pass "$1 exists"
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
# Asserts that a non-comment line in CODEOWNERS contains the given
|
|
31
|
+
# path/glob pattern and has an owner assigned to it. Tolerant of leading
|
|
32
|
+
# "/", trailing globs, and surrounding whitespace — it checks coverage,
|
|
33
|
+
# not an exact line match.
|
|
34
|
+
require_codeowner() {
|
|
35
|
+
local pattern="$1"
|
|
36
|
+
local label="$2"
|
|
37
|
+
local match
|
|
38
|
+
match="$(grep -v '^[[:space:]]*#' "$CODEOWNERS_FILE" | grep -F -- "$pattern" || true)"
|
|
39
|
+
[[ -n "$match" ]] || fail "CODEOWNERS coverage regression: no entry found for '$label' (expected pattern containing '$pattern')"
|
|
40
|
+
echo "$match" | grep -Eq '@[A-Za-z0-9_-]+' || fail "CODEOWNERS coverage regression: entry for '$label' has no owner assigned"
|
|
41
|
+
pass "CODEOWNERS covers $label"
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
require_file "$CODEOWNERS_FILE"
|
|
45
|
+
|
|
46
|
+
# NOTE: the flow-resolver.ts pattern below is string-split, matching how other
|
|
47
|
+
# evals/ files reference the same module — otherwise validate-source-tree's
|
|
48
|
+
# legacy-ref scanner mistakes the "lib/" segment for a (missing) top-level path.
|
|
49
|
+
require_codeowner "scripts/hooks/stop-goal-fit.js" "scripts/hooks/stop-goal-fit.js"
|
|
50
|
+
require_codeowner "scripts/hooks/config-protection.js" "scripts/hooks/config-protection.js"
|
|
51
|
+
require_codeowner "scripts/hooks/evidence-capture.js" "scripts/hooks/evidence-capture.js"
|
|
52
|
+
require_codeowner "src/li""b/flow-resolver.ts" "src/li""b/flow-resolver.ts"
|
|
53
|
+
require_codeowner "src/cli/workflow-sidecar.ts" "src/cli/workflow-sidecar.ts"
|
|
54
|
+
require_codeowner "kits/*/flows/*.flow.json" "kits/*/flows/*.flow.json (kit FlowDefinitions)"
|
|
55
|
+
|
|
56
|
+
echo "FlowDefinition-related CODEOWNERS coverage checks passed."
|
|
@@ -87,6 +87,7 @@ for (const entry of required) {
|
|
|
87
87
|
const requiredExcludes = [
|
|
88
88
|
"!evals/cases/dev/node_modules/",
|
|
89
89
|
"!**/.flow-agents/",
|
|
90
|
+
"!**/.kontourai/",
|
|
90
91
|
"!**/.surface/",
|
|
91
92
|
"!**/.telemetry/",
|
|
92
93
|
"!**/.veritas/",
|
|
@@ -99,7 +100,7 @@ const forbidden = [
|
|
|
99
100
|
".agents/",
|
|
100
101
|
".codex/",
|
|
101
102
|
".claude/",
|
|
102
|
-
".
|
|
103
|
+
".kontourai/",
|
|
103
104
|
".surface/",
|
|
104
105
|
".telemetry/",
|
|
105
106
|
".veritas/",
|
|
@@ -120,7 +121,7 @@ NODE
|
|
|
120
121
|
if node - "$ROOT_DIR/console.telemetry.json" <<'NODE'
|
|
121
122
|
const fs = require("node:fs");
|
|
122
123
|
const descriptor = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
123
|
-
for (const id of ["builder.shape", "builder.build"]) {
|
|
124
|
+
for (const id of ["builder.shape", "builder.build", "builder.publish-learn"]) {
|
|
124
125
|
const flow = (descriptor.flows || []).find((candidate) => candidate.id === id);
|
|
125
126
|
if (!flow) throw new Error(`missing ${id} flow descriptor`);
|
|
126
127
|
if (!flow.detailAttributes || Array.isArray(flow.detailAttributes) || typeof flow.detailAttributes !== "object") {
|
|
@@ -142,14 +143,14 @@ NODE
|
|
|
142
143
|
else
|
|
143
144
|
_pass "tracked source has no legacy Flow Agents rename references"
|
|
144
145
|
fi
|
|
145
|
-
if (cd "$ROOT_DIR" && FLOW_AGENTS_CONTENT_BOUNDARY_FILES='.flow-agents/example/state.json' node scripts/check-content-boundary.cjs >/tmp/content-boundary-runtime.out 2>&1); then
|
|
146
|
+
if (cd "$ROOT_DIR" && FLOW_AGENTS_CONTENT_BOUNDARY_FILES='.kontourai/flow-agents/example/state.json' node scripts/check-content-boundary.cjs >/tmp/content-boundary-runtime.out 2>&1); then
|
|
146
147
|
_fail "content boundary allows ordinary workflow runtime artifacts"
|
|
147
148
|
elif rg -q 'Flow Agents runtime artifact must not be tracked' /tmp/content-boundary-runtime.out; then
|
|
148
149
|
_pass "content boundary blocks ordinary workflow runtime artifacts"
|
|
149
150
|
else
|
|
150
151
|
_fail "content boundary runtime rejection was not actionable"
|
|
151
152
|
fi
|
|
152
|
-
if (cd "$ROOT_DIR" && FLOW_AGENTS_CONTENT_BOUNDARY_FILES='.flow-agents/nested/example/closeout.md' node scripts/check-content-boundary.cjs >/tmp/content-boundary-nested.out 2>&1); then
|
|
153
|
+
if (cd "$ROOT_DIR" && FLOW_AGENTS_CONTENT_BOUNDARY_FILES='.kontourai/flow-agents/nested/example/closeout.md' node scripts/check-content-boundary.cjs >/tmp/content-boundary-nested.out 2>&1); then
|
|
153
154
|
_fail "content boundary allows nested workflow runtime artifacts"
|
|
154
155
|
elif rg -q 'Flow Agents runtime artifact must not be tracked' /tmp/content-boundary-nested.out; then
|
|
155
156
|
_pass "content boundary blocks nested workflow runtime artifacts"
|
|
@@ -157,7 +158,7 @@ NODE
|
|
|
157
158
|
_fail "content boundary nested runtime rejection was not actionable"
|
|
158
159
|
fi
|
|
159
160
|
current_branch="$(cd "$ROOT_DIR" && git branch --show-current 2>/dev/null || true)"
|
|
160
|
-
tracked_runtime_artifacts="$(cd "$ROOT_DIR" && git ls-files -- '.flow-agents' 2>/dev/null || true)"
|
|
161
|
+
tracked_runtime_artifacts="$(cd "$ROOT_DIR" && git ls-files -- '.kontourai/flow-agents' 2>/dev/null || true)"
|
|
161
162
|
if [[ "$current_branch" == "main" && -n "$tracked_runtime_artifacts" ]]; then
|
|
162
163
|
printf '%s\n' "$tracked_runtime_artifacts" >/tmp/tracked-flow-agent-runtime-artifacts.txt
|
|
163
164
|
_fail "main contains tracked workflow runtime artifacts (see /tmp/tracked-flow-agent-runtime-artifacts.txt)"
|
|
@@ -174,12 +175,12 @@ NODE
|
|
|
174
175
|
else
|
|
175
176
|
_fail "installed bundle is missing Kit Catalog or Builder Kit manifest"
|
|
176
177
|
fi
|
|
177
|
-
if [[ -f "$PACKAGE_DIR/kits/builder/flows/shape.flow.json" && -f "$PACKAGE_DIR/kits/builder/flows/build.flow.json" ]]; then
|
|
178
|
+
if [[ -f "$PACKAGE_DIR/kits/builder/flows/shape.flow.json" && -f "$PACKAGE_DIR/kits/builder/flows/build.flow.json" && -f "$PACKAGE_DIR/kits/builder/flows/publish-learn.flow.json" ]]; then
|
|
178
179
|
_pass "installed bundle includes Builder Kit Flow Definitions"
|
|
179
180
|
else
|
|
180
181
|
_fail "installed bundle is missing Builder Kit Flow Definitions"
|
|
181
182
|
fi
|
|
182
|
-
if node - "$PACKAGE_DIR/kits/catalog.json" "$PACKAGE_DIR/kits/builder/kit.json" "$PACKAGE_DIR/kits/builder/flows/shape.flow.json" "$PACKAGE_DIR/kits/builder/flows/build.flow.json" <<'NODE'
|
|
183
|
+
if node - "$PACKAGE_DIR/kits/catalog.json" "$PACKAGE_DIR/kits/builder/kit.json" "$PACKAGE_DIR/kits/builder/flows/shape.flow.json" "$PACKAGE_DIR/kits/builder/flows/build.flow.json" "$PACKAGE_DIR/kits/builder/flows/publish-learn.flow.json" <<'NODE'
|
|
183
184
|
const fs = require("node:fs");
|
|
184
185
|
for (const file of process.argv.slice(2)) JSON.parse(fs.readFileSync(file, "utf8"));
|
|
185
186
|
console.log("ok");
|
|
@@ -189,9 +190,10 @@ NODE
|
|
|
189
190
|
else
|
|
190
191
|
_fail "installed kit JSON parse failed"
|
|
191
192
|
fi
|
|
192
|
-
if node - "$PACKAGE_DIR/kits/builder/flows/build.flow.json" <<'NODE'
|
|
193
|
+
if node - "$PACKAGE_DIR/kits/builder/flows/build.flow.json" "$PACKAGE_DIR/kits/builder/flows/publish-learn.flow.json" <<'NODE'
|
|
193
194
|
const fs = require("node:fs");
|
|
194
195
|
const flow = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
196
|
+
const publishLearn = JSON.parse(fs.readFileSync(process.argv[3], "utf8"));
|
|
195
197
|
const steps = Object.fromEntries((flow.steps || []).map((step) => [step.id, step.next]));
|
|
196
198
|
if (steps["pull-work"] !== "design-probe") throw new Error("pull-work should route to design-probe");
|
|
197
199
|
if (steps["design-probe"] !== "plan") throw new Error("design-probe should route to plan");
|
|
@@ -208,13 +210,20 @@ for (const gateId of ["verify-gate", "merge-ready-gate"]) {
|
|
|
208
210
|
for (const [reason, target] of Object.entries(expected)) if (gate.on_route_back?.[reason] !== target) throw new Error(`${gateId} ${reason} should route to ${target}`);
|
|
209
211
|
if (gate.route_back_policy?.on_exceeded !== "block") throw new Error(`${gateId} route_back_policy should block on exceeded attempts`);
|
|
210
212
|
}
|
|
211
|
-
|
|
213
|
+
for (const stepId of ["pr-open", "merge-ready-ci", "learn"]) {
|
|
214
|
+
const step = (flow.steps || []).find((item) => item.id === stepId);
|
|
215
|
+
if (step?.uses_flow !== "builder.publish-learn") throw new Error(`${stepId} should compose builder.publish-learn`);
|
|
216
|
+
}
|
|
217
|
+
const expectations = [
|
|
218
|
+
...Object.values(flow.gates || {}).flatMap((gate) => gate.expects || []),
|
|
219
|
+
...Object.values(publishLearn.gates || {}).flatMap((gate) => gate.expects || []),
|
|
220
|
+
];
|
|
212
221
|
if (!expectations.length) throw new Error("Builder build flow should declare gate expectations");
|
|
213
222
|
for (const expectation of expectations) {
|
|
214
223
|
if (expectation.kind !== "trust.bundle") throw new Error(`${expectation.id || "<unknown>"} should remain a trust.bundle expectation`);
|
|
215
224
|
if (!expectation.bundle_claim?.claimType || !expectation.bundle_claim?.accepted_statuses) throw new Error(`${expectation.id || "<unknown>"} should declare bundle_claim claimType and accepted statuses`);
|
|
216
225
|
}
|
|
217
|
-
const flowText = JSON.stringify(flow).toLowerCase();
|
|
226
|
+
const flowText = JSON.stringify({ flow, publishLearn }).toLowerCase();
|
|
218
227
|
for (const term of ["veritas", "trust_provider", "trust-provider", "provider_name", "provider_ref", "veritas_policy", "veritas_readiness"]) {
|
|
219
228
|
if (flowText.includes(term)) throw new Error(`Builder build flow should not name provider-specific trust field: ${term}`);
|
|
220
229
|
}
|