@kontourai/flow-agents 2.3.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/CODEOWNERS +16 -0
- package/.github/workflows/ci.yml +44 -0
- package/.github/workflows/trust-reconcile.yml +62 -4
- package/AGENTS.md +4 -4
- package/CHANGELOG.md +62 -0
- package/CONTEXT.md +35 -0
- package/README.md +9 -3
- package/agents/dev.json +1 -1
- package/agents/tool-code-reviewer.json +1 -1
- package/agents/tool-planner.json +3 -3
- package/agents/tool-verifier.json +3 -3
- package/build/src/cli/assignment-provider.d.ts +1 -0
- package/build/src/cli/assignment-provider.js +748 -0
- package/build/src/cli/console-learning-projection.js +3 -2
- package/build/src/cli/effective-assignment-provider-settings.d.ts +1 -0
- package/build/src/cli/effective-assignment-provider-settings.js +125 -0
- package/build/src/cli/init.js +104 -21
- package/build/src/cli/kit.js +37 -6
- package/build/src/cli/pull-work-provider.js +1 -1
- package/build/src/cli/usage-feedback.js +3 -3
- package/build/src/cli/validate-hook-influence.js +1 -0
- package/build/src/cli/validate-workflow-artifacts.js +27 -7
- package/build/src/cli/workflow-artifact-cleanup-audit.js +36 -2
- package/build/src/cli/workflow-sidecar.d.ts +14 -7
- package/build/src/cli/workflow-sidecar.js +938 -168
- package/build/src/cli.js +6 -0
- package/build/src/flow-kit/validate.d.ts +23 -0
- package/build/src/flow-kit/validate.js +67 -1
- package/build/src/index.d.ts +1 -1
- package/build/src/index.js +1 -1
- package/build/src/lib/flow-resolver.d.ts +6 -4
- package/build/src/lib/flow-resolver.js +125 -58
- package/build/src/lib/local-artifact-root.d.ts +14 -2
- package/build/src/lib/local-artifact-root.js +22 -5
- package/build/src/lib/workflow-learning-projection.js +2 -2
- package/build/src/runtime-adapters.d.ts +12 -0
- package/build/src/runtime-adapters.js +56 -16
- package/build/src/tools/build-universal-bundles.js +49 -16
- package/build/src/tools/generate-context-map.js +10 -8
- package/build/src/tools/validate-source-tree.js +24 -1
- package/console.telemetry.json +22 -0
- package/context/contracts/artifact-contract.md +40 -7
- package/context/contracts/assignment-provider-contract.md +239 -0
- package/context/contracts/builder-kit-workflow-state-contract.md +5 -2
- package/context/contracts/decision-registry-contract.md +265 -0
- package/context/contracts/delivery-contract.md +3 -1
- package/context/contracts/execution-contract.md +25 -0
- package/context/contracts/governance-adapter-contract.md +2 -0
- package/context/contracts/knowledge-store-contract.md +197 -0
- package/context/contracts/planning-contract.md +4 -1
- package/context/contracts/review-contract.md +7 -3
- package/context/contracts/sandbox-policy.md +2 -0
- package/context/contracts/standing-directives.md +13 -0
- package/context/contracts/verification-contract.md +11 -1
- package/context/contracts/work-item-contract.md +3 -1
- package/context/deferred/parallelization.md +1 -1
- package/context/gate-awareness.md +4 -4
- package/context/scripts/git-status.sh +2 -2
- package/context/scripts/hooks/config-protection.js +502 -2
- package/context/scripts/hooks/lib/config-protection-remedies.js +69 -0
- package/context/scripts/hooks/run-hook.js +14 -1
- package/context/scripts/hooks/stop-goal-fit.js +1140 -99
- package/context/scripts/hooks/workflow-steering.js +212 -6
- package/context/settings/assignment-provider-settings.json +33 -0
- package/docs/adr/0007-flow-skill-kit-tool-boundary.md +2 -0
- package/docs/adr/0015-flow-flow-agents-boundary-reconciliation.md +35 -0
- package/docs/adr/0019-kit-dependency-ownership.md +62 -0
- package/docs/adr/0020-trust-reconcile-manifest-and-claim-classification.md +290 -0
- package/docs/adr/0021-assignment-leases-and-stale-claim-takeover.md +221 -0
- package/docs/adr/0022-fail-closed-delivery-reconciliation-with-governed-exemptions.md +463 -0
- package/docs/agent-system-guidebook.md +3 -3
- package/docs/agent-usage-feedback-loop.md +6 -6
- package/docs/context-map.md +10 -7
- package/docs/decisions/decision-records.md +57 -0
- package/docs/decisions/index.md +17 -0
- package/docs/decisions/knowledge-store-provider.md +51 -0
- package/docs/decisions/model-routing.md +63 -0
- package/docs/decisions/promotion-gate.md +52 -0
- package/docs/decisions/standing-directives.md +66 -0
- package/docs/developer-architecture.md +1 -1
- package/docs/fixture-ownership.md +4 -0
- package/docs/flow-kit-repository-contract.md +5 -5
- package/docs/getting-started.md +12 -2
- package/docs/integrations/harness-install.md +2 -1
- package/docs/kit-authoring-guide.md +64 -5
- package/docs/knowledge-kit.md +5 -1
- package/docs/learnings/2026-07-improvement-program.md +203 -0
- package/docs/migrations.md +6 -5
- package/docs/north-star.md +2 -2
- package/docs/operating-layers.md +4 -3
- package/docs/repository-structure.md +5 -5
- package/docs/skills-map.md +8 -8
- package/docs/spec/runtime-hook-surface.md +19 -15
- package/docs/spikes/graph-provider-2026-07.md +155 -0
- package/docs/standards-register.md +7 -7
- package/docs/trust-anchor-adoption.md +105 -4
- package/docs/work-item-adapters.md +1 -1
- package/docs/workflow-artifact-lifecycle.md +72 -9
- package/docs/workflow-eval-strategy.md +3 -3
- package/docs/workflow-shared-contracts.md +3 -2
- package/docs/workflow-usage-guide.md +260 -21
- package/evals/acceptance/DEMO-false-completion.md +20 -11
- package/evals/acceptance/prove-capture-teeth-declared.sh +15 -15
- package/evals/acceptance/prove-capture-teeth.sh +11 -11
- package/evals/acceptance/test_kiro_harness.sh +1 -1
- package/evals/ci/antigaming-suite.sh +5 -0
- package/evals/ci/run-baseline.sh +71 -1
- package/evals/fixtures/assignment-provider/actor-a.json +6 -0
- package/evals/fixtures/assignment-provider/actor-b.json +6 -0
- package/evals/fixtures/assignment-provider/github-issue-claimed.json +27 -0
- package/evals/fixtures/assignment-provider/github-issue-unassigned.json +7 -0
- package/evals/fixtures/assignment-provider/liveness-fresh.json +9 -0
- package/evals/fixtures/assignment-provider/liveness-stale.json +9 -0
- package/evals/fixtures/flow-kit-repository/invalid-bad-dependency/flows/review.flow.json +26 -0
- package/evals/fixtures/flow-kit-repository/invalid-bad-dependency/kit.json +13 -0
- package/evals/fixtures/flow-kit-repository/valid-with-dependency/flows/review.flow.json +26 -0
- package/evals/fixtures/flow-kit-repository/valid-with-dependency/kit.json +13 -0
- package/evals/fixtures/hook-influence/cases.json +32 -0
- package/evals/fixtures/pull-work-provider/github-issues.json +6 -6
- package/evals/fixtures/trust-reconcile-exploits/fabricated-attestation.json +59 -0
- package/evals/fixtures/trust-reconcile-exploits/no-label-bypass.json +36 -0
- package/evals/fixtures/trust-reconcile-exploits/skip-assumed-bypass.json +59 -0
- package/evals/fixtures/trust-reconcile-exploits/status-misassertion.json +48 -0
- package/evals/fixtures/trust-reconcile-exploits/waived-command-check.json +44 -0
- package/evals/fixtures/trust-reconcile-mixed-bundle/mixed-bundle.json +159 -0
- package/evals/fixtures/trust-reconcile-ws3/ws3-bundle.json +920 -0
- package/evals/integration/test_actor_identity.sh +254 -0
- package/evals/integration/test_assignment_provider_github.sh +318 -0
- package/evals/integration/test_assignment_provider_local_file.sh +222 -0
- package/evals/integration/test_builder_step_producers.sh +33 -23
- package/evals/integration/test_bundle_install.sh +22 -16
- package/evals/integration/test_bundle_lifecycle.sh +11 -11
- package/evals/integration/test_captured_fail_reconciliation.sh +50 -50
- package/evals/integration/test_checkpoint_signing.sh +43 -0
- package/evals/integration/test_claim_lookup.sh +4 -4
- package/evals/integration/test_codex_hook_resolution.sh +114 -0
- package/evals/integration/test_command_log_concurrency.sh +8 -8
- package/evals/integration/test_command_log_fork_classification.sh +3 -3
- package/evals/integration/test_command_log_integrity.sh +16 -16
- package/evals/integration/test_console_learning_projection.sh +1 -1
- package/evals/integration/test_critique_supersession_roundtrip.sh +182 -0
- package/evals/integration/test_dual_emit_flow_step.sh +145 -0
- package/evals/integration/test_enforcer_expects_driven.sh +12 -12
- package/evals/integration/test_evidence_capture_hook.sh +14 -13
- package/evals/integration/test_fixture_retirement_audit.sh +2 -2
- package/evals/integration/test_flow_agents_statusline.sh +2 -2
- package/evals/integration/test_flow_kit_install_git.sh +52 -0
- package/evals/integration/test_flow_kit_repository.sh +2 -0
- package/evals/integration/test_flowdef_session_activation.sh +6 -6
- package/evals/integration/test_flowdef_union_floor_regression.sh +429 -0
- package/evals/integration/test_gate_bypass_chain.sh +52 -40
- package/evals/integration/test_gate_lockdown.sh +97 -73
- package/evals/integration/test_gate_review_inquiry_records.sh +3 -3
- package/evals/integration/test_goal_fit_escape_hatch.sh +10 -10
- package/evals/integration/test_goal_fit_ghost_session.sh +104 -0
- package/evals/integration/test_goal_fit_hook.sh +32 -32
- package/evals/integration/test_goal_fit_rederive.sh +9 -9
- package/evals/integration/test_hook_category_behaviors.sh +103 -0
- package/evals/integration/test_hook_influence_cases.sh +1 -0
- package/evals/integration/test_install_merge.sh +262 -1
- package/evals/integration/test_kit_identity_trust.sh +6 -6
- package/evals/integration/test_liveness_conflict_injection.sh +587 -0
- package/evals/integration/test_liveness_heartbeat.sh +677 -0
- package/evals/integration/test_liveness_verdict.sh +394 -0
- package/evals/integration/test_local_flow_kit_install.sh +15 -0
- package/evals/integration/test_phase_map_and_gate_claim.sh +63 -11
- package/evals/integration/test_promote_gate.sh +178 -0
- package/evals/integration/test_publish_delivery.sh +23 -6
- package/evals/integration/test_pull_work_assignment_join.sh +132 -0
- package/evals/integration/test_pull_work_liveness_preflight.sh +394 -0
- package/evals/integration/test_pull_work_provider.sh +2 -2
- package/evals/integration/test_reconcile_soundness.sh +37 -13
- package/evals/integration/test_resolvefirststep_security.sh +22 -0
- package/evals/integration/test_runtime_adapter_activation.sh +87 -18
- package/evals/integration/test_session_resume_roundtrip.sh +147 -0
- package/evals/integration/test_sidecar_field_preservation.sh +249 -0
- package/evals/integration/test_trust_reconcile.sh +11 -10
- package/evals/integration/test_trust_reconcile_manifest.sh +79 -0
- package/evals/integration/test_trust_reconcile_mixed_bundle.sh +96 -0
- package/evals/integration/test_trust_reconcile_negatives.sh +759 -0
- package/evals/integration/test_usage_feedback_global.sh +5 -5
- package/evals/integration/test_validate_artifacts_portability.sh +60 -0
- package/evals/integration/test_verify_cli.sh +2 -2
- package/evals/integration/test_veritas_governance_kit.sh +117 -0
- package/evals/integration/test_workflow_artifact_cleanup_audit.sh +26 -1
- package/evals/integration/test_workflow_artifacts.sh +33 -0
- package/evals/integration/test_workflow_sidecar_writer.sh +802 -74
- package/evals/integration/test_workflow_steering_hook.sh +175 -5
- package/evals/lib/codex-provider.sh +1 -1
- package/evals/lib/node.sh +19 -1
- package/evals/run.sh +32 -0
- package/evals/static/test_decisions.sh +150 -0
- package/evals/static/test_flowdef_codeowners_coverage.sh +62 -0
- package/evals/static/test_knowledge_providers.sh +23 -0
- package/evals/static/test_package.sh +19 -10
- package/evals/static/test_universal_bundles.sh +48 -3
- package/evals/static/test_workflow_skills.sh +41 -8
- package/kits/builder/flows/build.flow.json +3 -74
- package/kits/builder/flows/publish-learn.flow.json +90 -0
- package/kits/builder/kit.json +11 -0
- package/kits/builder/skills/builder-shape/SKILL.md +1 -1
- package/kits/builder/skills/deliver/SKILL.md +32 -11
- package/kits/builder/skills/design-probe/SKILL.md +4 -4
- package/kits/builder/skills/evidence-gate/SKILL.md +26 -1
- package/kits/builder/skills/execute-plan/SKILL.md +2 -2
- package/kits/builder/skills/fix-bug/SKILL.md +2 -0
- package/kits/builder/skills/gate-review/SKILL.md +6 -6
- package/kits/builder/skills/idea-to-backlog/SKILL.md +1 -1
- package/kits/builder/skills/learning-review/SKILL.md +7 -7
- package/kits/builder/skills/pickup-probe/SKILL.md +5 -4
- package/kits/builder/skills/plan-work/SKILL.md +27 -5
- package/kits/builder/skills/pull-work/SKILL.md +148 -5
- package/kits/builder/skills/release-readiness/SKILL.md +4 -4
- package/kits/builder/skills/review-work/SKILL.md +24 -1
- package/kits/builder/skills/tdd-workflow/SKILL.md +2 -0
- package/kits/builder/skills/verify-work/SKILL.md +8 -1
- package/kits/catalog.json +6 -0
- package/kits/knowledge/kit.json +35 -0
- package/kits/knowledge/providers/conformance/fixtures/git-repo/CONTEXT.md +12 -0
- package/kits/knowledge/providers/conformance/fixtures/git-repo/docs/decisions/old-sprocket-shape.md +13 -0
- package/kits/knowledge/providers/conformance/fixtures/git-repo/docs/decisions/sprocket-shape.md +14 -0
- package/kits/knowledge/providers/conformance/fixtures/git-repo/docs/decisions/widget-format.md +14 -0
- package/kits/knowledge/providers/conformance/fixtures/git-repo/docs/learnings/fixture-learning.md +7 -0
- package/kits/knowledge/providers/conformance/fixtures/work-item/issues.json +30 -0
- package/kits/knowledge/providers/conformance/suite.test.js +125 -0
- package/kits/knowledge/providers/git-repo/index.js +236 -0
- package/kits/knowledge/providers/health/health-pass.test.js +99 -0
- package/kits/knowledge/providers/health/index.js +153 -0
- package/kits/knowledge/providers/index.js +24 -0
- package/kits/knowledge/providers/lib/model.js +91 -0
- package/kits/knowledge/providers/lib/schema-validate.js +119 -0
- package/kits/knowledge/providers/markdown-vault/index.js +169 -0
- package/kits/knowledge/providers/work-item/index.js +204 -0
- package/kits/veritas-governance/adapter/readiness-to-trust-bundle.mjs +178 -0
- package/kits/veritas-governance/docs/README.md +75 -0
- package/kits/veritas-governance/fixtures/readiness/not-ready.readiness-report.json +4645 -0
- package/kits/veritas-governance/fixtures/readiness/ready.readiness-report.json +4403 -0
- package/kits/veritas-governance/flows/readiness-check.flow.json +35 -0
- package/kits/veritas-governance/kit.json +16 -0
- package/package.json +11 -5
- package/packaging/README.md +2 -2
- package/packaging/conformance/fixtures/config-protection--allow-read-state-json.json +20 -0
- package/packaging/conformance/fixtures/config-protection--block-write-state-json.json +20 -0
- package/packaging/conformance/fixtures/stop-goal-fit--block-bundle-disputed-claim.json +1 -1
- package/packaging/conformance/fixtures/stop-goal-fit--block-capture-contradicts-claimed-pass.json +1 -1
- package/packaging/manifest.json +9 -10
- package/schemas/assignment-provider-settings.schema.json +125 -0
- package/schemas/decision-record.schema.json +109 -0
- package/schemas/knowledge/edge.schema.json +54 -0
- package/schemas/knowledge/health-report.schema.json +45 -0
- package/schemas/knowledge/node.schema.json +49 -0
- package/schemas/knowledge/proposal.schema.json +53 -0
- package/schemas/workflow-evidence.schema.json +1 -1
- package/schemas/workflow-state.schema.json +5 -0
- package/scripts/README.md +8 -3
- package/scripts/check-content-boundary.cjs +1 -1
- package/scripts/check-decisions.cjs +356 -0
- package/scripts/ci/derive-claim-status.mjs +74 -0
- package/scripts/ci/trust-reconcile.js +957 -103
- package/scripts/git-status.sh +2 -2
- package/scripts/hooks/claude-telemetry-hook.js +37 -3
- package/scripts/hooks/codex-telemetry-hook.js +36 -2
- package/scripts/hooks/config-protection.js +102 -30
- package/scripts/hooks/evidence-capture.js +3 -3
- package/scripts/hooks/lib/actor-identity.js +368 -0
- package/scripts/hooks/lib/config-protection-remedies.js +69 -0
- package/scripts/hooks/lib/liveness-heartbeat.js +361 -0
- package/scripts/hooks/lib/liveness-policy.js +127 -0
- package/scripts/hooks/lib/liveness-read.js +105 -12
- package/scripts/hooks/lib/liveness-write.js +46 -0
- package/scripts/hooks/lib/local-artifact-paths.js +12 -6
- package/scripts/hooks/opencode-telemetry-hook.js +27 -1
- package/scripts/hooks/pi-telemetry-hook.js +27 -1
- package/scripts/hooks/stop-goal-fit.js +157 -26
- package/scripts/hooks/workflow-steering.js +77 -3
- package/scripts/install-codex-home.sh +138 -38
- package/scripts/statusline/flow-agents-statusline.js +2 -2
- package/src/cli/assignment-provider.ts +845 -0
- package/src/cli/console-learning-projection.ts +3 -2
- package/src/cli/effective-assignment-provider-settings.ts +112 -0
- package/src/cli/init.ts +101 -21
- package/src/cli/kit.ts +37 -6
- package/src/cli/public-api.test.mjs +58 -6
- package/src/cli/pull-work-provider.ts +1 -1
- package/src/cli/trust-bundle-policy-order.test.mjs +87 -0
- package/src/cli/usage-feedback.ts +3 -3
- package/src/cli/validate-hook-influence.ts +1 -0
- package/src/cli/validate-workflow-artifacts.ts +27 -7
- package/src/cli/veritas-readiness-adapter.test.mjs +267 -0
- package/src/cli/workflow-artifact-cleanup-audit.ts +35 -2
- package/src/cli/workflow-sidecar.ts +933 -168
- package/src/cli.ts +6 -0
- package/src/flow-kit/validate.ts +74 -1
- package/src/index.ts +6 -2
- package/src/lib/flow-resolver.ts +123 -55
- package/src/lib/local-artifact-root.ts +24 -5
- package/src/lib/workflow-learning-projection.ts +2 -2
- package/src/runtime-adapters.ts +61 -15
- package/src/tools/build-universal-bundles.ts +49 -16
- package/src/tools/generate-context-map.ts +10 -8
- package/src/tools/validate-source-tree.ts +24 -1
|
@@ -16,12 +16,12 @@ _pass() { echo " ✓ $1"; }
|
|
|
16
16
|
_fail() { echo " ✗ $1"; errors=$((errors + 1)); }
|
|
17
17
|
|
|
18
18
|
REPO="$TMPDIR_EVAL/repo"
|
|
19
|
-
mkdir -p "$REPO/.flow-agents/steering-demo"
|
|
19
|
+
mkdir -p "$REPO/.kontourai/flow-agents/steering-demo"
|
|
20
20
|
mkdir -p "$REPO/docs"
|
|
21
21
|
printf '# Test Repo\n' > "$REPO/AGENTS.md"
|
|
22
22
|
printf '# Context Map\n' > "$REPO/docs/context-map.md"
|
|
23
23
|
|
|
24
|
-
cat > "$REPO/.flow-agents/steering-demo/state.json" <<'JSON'
|
|
24
|
+
cat > "$REPO/.kontourai/flow-agents/steering-demo/state.json" <<'JSON'
|
|
25
25
|
{
|
|
26
26
|
"schema_version": "1.0",
|
|
27
27
|
"task_slug": "steering-demo",
|
|
@@ -36,7 +36,7 @@ cat > "$REPO/.flow-agents/steering-demo/state.json" <<'JSON'
|
|
|
36
36
|
}
|
|
37
37
|
JSON
|
|
38
38
|
|
|
39
|
-
cat > "$REPO/.flow-agents/steering-demo/critique.json" <<'JSON'
|
|
39
|
+
cat > "$REPO/.kontourai/flow-agents/steering-demo/critique.json" <<'JSON'
|
|
40
40
|
{
|
|
41
41
|
"schema_version": "1.0",
|
|
42
42
|
"task_slug": "steering-demo",
|
|
@@ -217,6 +217,176 @@ else
|
|
|
217
217
|
_fail "Claude hook adapter should not fail for prompt-submit workflow steering"
|
|
218
218
|
fi
|
|
219
219
|
|
|
220
|
+
FRESH_REPO="$TMPDIR_EVAL/fresh-repo"
|
|
221
|
+
mkdir -p "$FRESH_REPO/docs"
|
|
222
|
+
printf '# Fresh Repo\n' > "$FRESH_REPO/AGENTS.md"
|
|
223
|
+
printf '# Context Map\n' > "$FRESH_REPO/docs/context-map.md"
|
|
224
|
+
|
|
225
|
+
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/builder-route.out" 2>"$TMPDIR_EVAL/builder-route.err" <<JSON
|
|
226
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please implement the new settings API and update its tests."}
|
|
227
|
+
JSON
|
|
228
|
+
then
|
|
229
|
+
if rg -q 'BUILDER WORKFLOW ROUTE' "$TMPDIR_EVAL/builder-route.out" && \
|
|
230
|
+
rg -q 'activate `deliver`' "$TMPDIR_EVAL/builder-route.out" && \
|
|
231
|
+
rg -q -- '--flow-id builder.build' "$TMPDIR_EVAL/builder-route.out" && \
|
|
232
|
+
rg -q 'plan-work -> execute-plan -> review-work -> verify-work' "$TMPDIR_EVAL/builder-route.out" && \
|
|
233
|
+
rg -q 'publish/release-readiness and learning-review' "$TMPDIR_EVAL/builder-route.out"; then
|
|
234
|
+
_pass "workflow steering hook routes fresh coding prompts into Builder workflow"
|
|
235
|
+
else
|
|
236
|
+
_fail "workflow steering missed Builder workflow route for coding prompt: $(cat "$TMPDIR_EVAL/builder-route.out")"
|
|
237
|
+
fi
|
|
238
|
+
else
|
|
239
|
+
_fail "workflow steering hook should not fail for fresh coding prompt"
|
|
240
|
+
fi
|
|
241
|
+
|
|
242
|
+
if node "$ROOT/scripts/hooks/claude-hook-adapter.js" UserPromptSubmit prompt:workflow-steering workflow-steering.js standard,strict >"$TMPDIR_EVAL/claude-builder-route.out" 2>"$TMPDIR_EVAL/claude-builder-route.err" <<JSON
|
|
243
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please implement the new settings API and update its tests."}
|
|
244
|
+
JSON
|
|
245
|
+
then
|
|
246
|
+
if node - "$TMPDIR_EVAL/claude-builder-route.out" <<'NODE'
|
|
247
|
+
const fs = require("node:fs");
|
|
248
|
+
const payload = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
249
|
+
const ctx = payload.hookSpecificOutput?.additionalContext || "";
|
|
250
|
+
if (payload.continue !== true) throw new Error("continue not true");
|
|
251
|
+
if (payload.suppressOutput !== false) throw new Error("suppressOutput should be false when guidance exists");
|
|
252
|
+
for (const needle of ["BUILDER WORKFLOW ROUTE", "activate `deliver`", "--flow-id builder.build", "plan-work -> execute-plan -> review-work -> verify-work", "publish/release-readiness and learning-review"]) {
|
|
253
|
+
if (!ctx.includes(needle)) throw new Error(`missing ${needle}`);
|
|
254
|
+
}
|
|
255
|
+
NODE
|
|
256
|
+
then
|
|
257
|
+
_pass "Claude hook adapter surfaces Builder workflow route for coding prompts"
|
|
258
|
+
else
|
|
259
|
+
_fail "Claude hook adapter missed Builder workflow route: $(cat "$TMPDIR_EVAL/claude-builder-route.out") $(cat "$TMPDIR_EVAL/claude-builder-route.err")"
|
|
260
|
+
fi
|
|
261
|
+
else
|
|
262
|
+
_fail "Claude hook adapter should not fail for Builder workflow route"
|
|
263
|
+
fi
|
|
264
|
+
|
|
265
|
+
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/builder-route-review-only.out" 2>"$TMPDIR_EVAL/builder-route-review-only.err" <<JSON
|
|
266
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please review the test coverage and validate whether it is enough. Do not modify files."}
|
|
267
|
+
JSON
|
|
268
|
+
then
|
|
269
|
+
if ! rg -q 'BUILDER WORKFLOW ROUTE' "$TMPDIR_EVAL/builder-route-review-only.out"; then
|
|
270
|
+
_pass "workflow steering hook does not route explicit review-only prompts into Builder workflow"
|
|
271
|
+
else
|
|
272
|
+
_fail "workflow steering incorrectly routed review-only prompt: $(cat "$TMPDIR_EVAL/builder-route-review-only.out")"
|
|
273
|
+
fi
|
|
274
|
+
else
|
|
275
|
+
_fail "workflow steering hook should not fail for review-only prompt"
|
|
276
|
+
fi
|
|
277
|
+
|
|
278
|
+
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/builder-route-validate-only.out" 2>"$TMPDIR_EVAL/builder-route-validate-only.err" <<JSON
|
|
279
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please validate whether the tests are enough. Do not modify files."}
|
|
280
|
+
JSON
|
|
281
|
+
then
|
|
282
|
+
if ! rg -q 'BUILDER WORKFLOW ROUTE' "$TMPDIR_EVAL/builder-route-validate-only.out"; then
|
|
283
|
+
_pass "workflow steering hook does not route explicit validation-only prompts into Builder workflow"
|
|
284
|
+
else
|
|
285
|
+
_fail "workflow steering incorrectly routed validation-only prompt: $(cat "$TMPDIR_EVAL/builder-route-validate-only.out")"
|
|
286
|
+
fi
|
|
287
|
+
else
|
|
288
|
+
_fail "workflow steering hook should not fail for validation-only prompt"
|
|
289
|
+
fi
|
|
290
|
+
|
|
291
|
+
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/builder-route-bare-validate.out" 2>"$TMPDIR_EVAL/builder-route-bare-validate.err" <<JSON
|
|
292
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please validate whether the settings API tests are enough."}
|
|
293
|
+
JSON
|
|
294
|
+
then
|
|
295
|
+
if ! rg -q 'BUILDER WORKFLOW ROUTE' "$TMPDIR_EVAL/builder-route-bare-validate.out"; then
|
|
296
|
+
_pass "workflow steering hook does not route bare validation prompts into Builder workflow"
|
|
297
|
+
else
|
|
298
|
+
_fail "workflow steering incorrectly routed bare validation prompt: $(cat "$TMPDIR_EVAL/builder-route-bare-validate.out")"
|
|
299
|
+
fi
|
|
300
|
+
else
|
|
301
|
+
_fail "workflow steering hook should not fail for bare validation prompt"
|
|
302
|
+
fi
|
|
303
|
+
|
|
304
|
+
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/builder-route-bare-test.out" 2>"$TMPDIR_EVAL/builder-route-bare-test.err" <<JSON
|
|
305
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please test whether this still reproduces."}
|
|
306
|
+
JSON
|
|
307
|
+
then
|
|
308
|
+
if ! rg -q 'BUILDER WORKFLOW ROUTE' "$TMPDIR_EVAL/builder-route-bare-test.out"; then
|
|
309
|
+
_pass "workflow steering hook does not route bare test prompts into Builder workflow"
|
|
310
|
+
else
|
|
311
|
+
_fail "workflow steering incorrectly routed bare test prompt: $(cat "$TMPDIR_EVAL/builder-route-bare-test.out")"
|
|
312
|
+
fi
|
|
313
|
+
else
|
|
314
|
+
_fail "workflow steering hook should not fail for bare test prompt"
|
|
315
|
+
fi
|
|
316
|
+
|
|
317
|
+
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/builder-route-test-question.out" 2>"$TMPDIR_EVAL/builder-route-test-question.err" <<JSON
|
|
318
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"What tests should I run for the settings API?"}
|
|
319
|
+
JSON
|
|
320
|
+
then
|
|
321
|
+
if ! rg -q 'BUILDER WORKFLOW ROUTE' "$TMPDIR_EVAL/builder-route-test-question.out"; then
|
|
322
|
+
_pass "workflow steering hook does not route question-only test prompts into Builder workflow"
|
|
323
|
+
else
|
|
324
|
+
_fail "workflow steering incorrectly routed question-only test prompt: $(cat "$TMPDIR_EVAL/builder-route-test-question.out")"
|
|
325
|
+
fi
|
|
326
|
+
else
|
|
327
|
+
_fail "workflow steering hook should not fail for question-only test prompt"
|
|
328
|
+
fi
|
|
329
|
+
|
|
330
|
+
if node "$ROOT/scripts/hooks/claude-hook-adapter.js" UserPromptSubmit prompt:workflow-steering workflow-steering.js standard,strict >"$TMPDIR_EVAL/claude-builder-route-review-only.out" 2>"$TMPDIR_EVAL/claude-builder-route-review-only.err" <<JSON
|
|
331
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please review the test coverage and validate whether it is enough. Do not modify files."}
|
|
332
|
+
JSON
|
|
333
|
+
then
|
|
334
|
+
if node - "$TMPDIR_EVAL/claude-builder-route-review-only.out" <<'NODE'
|
|
335
|
+
const fs = require("node:fs");
|
|
336
|
+
const payload = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
337
|
+
const ctx = payload.hookSpecificOutput?.additionalContext || "";
|
|
338
|
+
if (payload.continue !== true) throw new Error("continue not true");
|
|
339
|
+
if (ctx.includes("BUILDER WORKFLOW ROUTE")) throw new Error("review-only prompt should not route to Builder workflow");
|
|
340
|
+
NODE
|
|
341
|
+
then
|
|
342
|
+
_pass "Claude hook adapter does not route explicit review-only prompts into Builder workflow"
|
|
343
|
+
else
|
|
344
|
+
_fail "Claude hook adapter incorrectly routed review-only prompt: $(cat "$TMPDIR_EVAL/claude-builder-route-review-only.out") $(cat "$TMPDIR_EVAL/claude-builder-route-review-only.err")"
|
|
345
|
+
fi
|
|
346
|
+
else
|
|
347
|
+
_fail "Claude hook adapter should not fail for review-only prompt"
|
|
348
|
+
fi
|
|
349
|
+
|
|
350
|
+
if node "$ROOT/scripts/hooks/claude-hook-adapter.js" UserPromptSubmit prompt:workflow-steering workflow-steering.js standard,strict >"$TMPDIR_EVAL/claude-builder-route-validate-only.out" 2>"$TMPDIR_EVAL/claude-builder-route-validate-only.err" <<JSON
|
|
351
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"Please validate whether the tests are enough. Do not modify files."}
|
|
352
|
+
JSON
|
|
353
|
+
then
|
|
354
|
+
if node - "$TMPDIR_EVAL/claude-builder-route-validate-only.out" <<'NODE'
|
|
355
|
+
const fs = require("node:fs");
|
|
356
|
+
const payload = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
357
|
+
const ctx = payload.hookSpecificOutput?.additionalContext || "";
|
|
358
|
+
if (payload.continue !== true) throw new Error("continue not true");
|
|
359
|
+
if (ctx.includes("BUILDER WORKFLOW ROUTE")) throw new Error("validation-only prompt should not route to Builder workflow");
|
|
360
|
+
NODE
|
|
361
|
+
then
|
|
362
|
+
_pass "Claude hook adapter does not route explicit validation-only prompts into Builder workflow"
|
|
363
|
+
else
|
|
364
|
+
_fail "Claude hook adapter incorrectly routed validation-only prompt: $(cat "$TMPDIR_EVAL/claude-builder-route-validate-only.out") $(cat "$TMPDIR_EVAL/claude-builder-route-validate-only.err")"
|
|
365
|
+
fi
|
|
366
|
+
else
|
|
367
|
+
_fail "Claude hook adapter should not fail for validation-only prompt"
|
|
368
|
+
fi
|
|
369
|
+
|
|
370
|
+
if node "$ROOT/scripts/hooks/claude-hook-adapter.js" UserPromptSubmit prompt:workflow-steering workflow-steering.js standard,strict >"$TMPDIR_EVAL/claude-builder-route-test-question.out" 2>"$TMPDIR_EVAL/claude-builder-route-test-question.err" <<JSON
|
|
371
|
+
{"hook_event_name":"UserPromptSubmit","cwd":"$FRESH_REPO","prompt":"What tests should I run for the settings API?"}
|
|
372
|
+
JSON
|
|
373
|
+
then
|
|
374
|
+
if node - "$TMPDIR_EVAL/claude-builder-route-test-question.out" <<'NODE'
|
|
375
|
+
const fs = require("node:fs");
|
|
376
|
+
const payload = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
|
|
377
|
+
const ctx = payload.hookSpecificOutput?.additionalContext || "";
|
|
378
|
+
if (payload.continue !== true) throw new Error("continue not true");
|
|
379
|
+
if (ctx.includes("BUILDER WORKFLOW ROUTE")) throw new Error("question-only test prompt should not route to Builder workflow");
|
|
380
|
+
NODE
|
|
381
|
+
then
|
|
382
|
+
_pass "Claude hook adapter does not route question-only test prompts into Builder workflow"
|
|
383
|
+
else
|
|
384
|
+
_fail "Claude hook adapter incorrectly routed question-only test prompt: $(cat "$TMPDIR_EVAL/claude-builder-route-test-question.out") $(cat "$TMPDIR_EVAL/claude-builder-route-test-question.err")"
|
|
385
|
+
fi
|
|
386
|
+
else
|
|
387
|
+
_fail "Claude hook adapter should not fail for question-only test prompt"
|
|
388
|
+
fi
|
|
389
|
+
|
|
220
390
|
if node "$ROOT/scripts/hooks/codex-hook-adapter.js" post:workflow-steering workflow-steering.js standard,strict >"$TMPDIR_EVAL/codex-adapter.out" 2>"$TMPDIR_EVAL/codex-adapter.err" <<JSON
|
|
221
391
|
{"hook_event_name":"PostToolUse","cwd":"$REPO","tool_input":{"command":"Bash","content":{"command":"bash evals/run.sh integration"}},"tool_response":"integration finished"}
|
|
222
392
|
JSON
|
|
@@ -285,7 +455,7 @@ else
|
|
|
285
455
|
_fail "Codex hook adapter should not fail for prompt-submit workflow steering"
|
|
286
456
|
fi
|
|
287
457
|
|
|
288
|
-
cat > "$REPO/.flow-agents/steering-demo/state.json" <<'JSON'
|
|
458
|
+
cat > "$REPO/.kontourai/flow-agents/steering-demo/state.json" <<'JSON'
|
|
289
459
|
{
|
|
290
460
|
"schema_version": "1.0",
|
|
291
461
|
"task_slug": "steering-demo",
|
|
@@ -298,7 +468,7 @@ cat > "$REPO/.flow-agents/steering-demo/state.json" <<'JSON'
|
|
|
298
468
|
}
|
|
299
469
|
}
|
|
300
470
|
JSON
|
|
301
|
-
rm -f "$REPO/.flow-agents/steering-demo/critique.json"
|
|
471
|
+
rm -f "$REPO/.kontourai/flow-agents/steering-demo/critique.json"
|
|
302
472
|
|
|
303
473
|
if node "$ROOT/scripts/hooks/workflow-steering.js" >"$TMPDIR_EVAL/done.out" 2>"$TMPDIR_EVAL/done.err" <<JSON
|
|
304
474
|
{"cwd":"$REPO","tool_input":{"command":"InvokeSubagents","content":{"subagents":[{"agent_name":"tool-verifier"}]}},"tool_response":"verification finished"}
|
package/evals/lib/node.sh
CHANGED
|
@@ -3,8 +3,26 @@
|
|
|
3
3
|
|
|
4
4
|
FLOW_AGENTS_EVAL_ROOT="${ROOT:-${ROOT_DIR:-}}"
|
|
5
5
|
|
|
6
|
+
# Memoized per-process: flow_agents_node is called dozens of times per eval script
|
|
7
|
+
# (often inside a `run_bounded N ...` hang-guard around a single fail-fast assertion).
|
|
8
|
+
# Re-running `npm run build --silent` (npm startup + tsc incremental stat-check) on EVERY
|
|
9
|
+
# call repeatedly re-pays that overhead inside those bounds. On a slow/loaded CI runner a
|
|
10
|
+
# single one of those ~100+ redundant build invocations can occasionally take long enough
|
|
11
|
+
# to blow a tight hang-guard bound before the actual (near-instant) validation ever runs,
|
|
12
|
+
# producing a spurious assertion failure that has nothing to do with the behavior under
|
|
13
|
+
# test (see PR #306 CI trust-reconcile divergence investigation). Source under src/ never
|
|
14
|
+
# changes mid-script for any eval, so building once per process is safe and correct.
|
|
15
|
+
_FLOW_AGENTS_BUILD_DONE=""
|
|
16
|
+
|
|
6
17
|
flow_agents_build_ts() {
|
|
7
|
-
|
|
18
|
+
if [[ "$_FLOW_AGENTS_BUILD_DONE" == "1" ]]; then
|
|
19
|
+
return 0
|
|
20
|
+
fi
|
|
21
|
+
if (cd "$FLOW_AGENTS_EVAL_ROOT" && npm run build --silent >/dev/null); then
|
|
22
|
+
_FLOW_AGENTS_BUILD_DONE="1"
|
|
23
|
+
return 0
|
|
24
|
+
fi
|
|
25
|
+
return 1
|
|
8
26
|
}
|
|
9
27
|
|
|
10
28
|
flow_agents_node() {
|
package/evals/run.sh
CHANGED
|
@@ -135,13 +135,19 @@ run_static() {
|
|
|
135
135
|
echo ""
|
|
136
136
|
bash "$EVAL_DIR/static/test_evidence_refs.sh" || result=1
|
|
137
137
|
echo ""
|
|
138
|
+
bash "$EVAL_DIR/static/test_decisions.sh" || result=1
|
|
139
|
+
echo ""
|
|
138
140
|
bash "$EVAL_DIR/static/test_library_exports.sh" || result=1
|
|
139
141
|
echo ""
|
|
140
142
|
bash "$EVAL_DIR/static/test_console_presets.sh" || result=1
|
|
141
143
|
echo ""
|
|
142
144
|
bash "$EVAL_DIR/static/test_repo_hooks.sh" || result=1
|
|
143
145
|
echo ""
|
|
146
|
+
bash "$EVAL_DIR/static/test_flowdef_codeowners_coverage.sh" || result=1
|
|
147
|
+
echo ""
|
|
144
148
|
bash "$EVAL_DIR/static/test_unit_helpers.sh" || result=1
|
|
149
|
+
echo ""
|
|
150
|
+
bash "$EVAL_DIR/static/test_knowledge_providers.sh" || result=1
|
|
145
151
|
return $result
|
|
146
152
|
}
|
|
147
153
|
|
|
@@ -179,12 +185,32 @@ run_integration() {
|
|
|
179
185
|
echo ""
|
|
180
186
|
bash "$EVAL_DIR/integration/test_workflow_artifact_cleanup_audit.sh" || result=1
|
|
181
187
|
echo ""
|
|
188
|
+
bash "$EVAL_DIR/integration/test_promote_gate.sh" || result=1
|
|
189
|
+
echo ""
|
|
182
190
|
bash "$EVAL_DIR/integration/test_fixture_retirement_audit.sh" || result=1
|
|
183
191
|
echo ""
|
|
184
192
|
bash "$EVAL_DIR/integration/test_publish_change_helper.sh" || result=1
|
|
185
193
|
echo ""
|
|
186
194
|
bash "$EVAL_DIR/integration/test_workflow_sidecar_writer.sh" || result=1
|
|
187
195
|
echo ""
|
|
196
|
+
bash "$EVAL_DIR/integration/test_sidecar_field_preservation.sh" || result=1
|
|
197
|
+
echo ""
|
|
198
|
+
bash "$EVAL_DIR/integration/test_actor_identity.sh" || result=1
|
|
199
|
+
echo ""
|
|
200
|
+
bash "$EVAL_DIR/integration/test_assignment_provider_local_file.sh" || result=1
|
|
201
|
+
echo ""
|
|
202
|
+
bash "$EVAL_DIR/integration/test_assignment_provider_github.sh" || result=1
|
|
203
|
+
echo ""
|
|
204
|
+
bash "$EVAL_DIR/integration/test_pull_work_assignment_join.sh" || result=1
|
|
205
|
+
echo ""
|
|
206
|
+
bash "$EVAL_DIR/integration/test_liveness_heartbeat.sh" || result=1
|
|
207
|
+
echo ""
|
|
208
|
+
bash "$EVAL_DIR/integration/test_pull_work_liveness_preflight.sh" || result=1
|
|
209
|
+
echo ""
|
|
210
|
+
bash "$EVAL_DIR/integration/test_liveness_verdict.sh" || result=1
|
|
211
|
+
echo ""
|
|
212
|
+
bash "$EVAL_DIR/integration/test_liveness_conflict_injection.sh" || result=1
|
|
213
|
+
echo ""
|
|
188
214
|
bash "$EVAL_DIR/integration/test_veritas_governance_adapter.sh" || result=1
|
|
189
215
|
echo ""
|
|
190
216
|
bash "$EVAL_DIR/integration/test_workflow_steering_hook.sh" || result=1
|
|
@@ -203,6 +229,10 @@ run_integration() {
|
|
|
203
229
|
echo ""
|
|
204
230
|
bash "$EVAL_DIR/integration/test_runtime_adapter_activation.sh" || result=1
|
|
205
231
|
echo ""
|
|
232
|
+
bash "$EVAL_DIR/integration/test_validate_artifacts_portability.sh" || result=1
|
|
233
|
+
echo ""
|
|
234
|
+
bash "$EVAL_DIR/integration/test_codex_hook_resolution.sh" || result=1
|
|
235
|
+
echo ""
|
|
206
236
|
bash "$EVAL_DIR/integration/test_bundle_install.sh" || result=1
|
|
207
237
|
echo ""
|
|
208
238
|
bash "$EVAL_DIR/integration/test_bundle_lifecycle.sh" || result=1
|
|
@@ -219,6 +249,8 @@ run_integration() {
|
|
|
219
249
|
echo ""
|
|
220
250
|
bash "$EVAL_DIR/integration/test_flowdef_session_history_preservation.sh" || result=1
|
|
221
251
|
echo ""
|
|
252
|
+
bash "$EVAL_DIR/integration/test_critique_supersession_roundtrip.sh" || result=1
|
|
253
|
+
echo ""
|
|
222
254
|
bash "$EVAL_DIR/integration/test_flowdef_session_activation.sh" || result=1
|
|
223
255
|
echo ""
|
|
224
256
|
bash "$EVAL_DIR/integration/test_trust_checkpoint.sh" || result=1
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# test_decisions.sh — Decision registry validator + index generator checks.
|
|
3
|
+
# Covers: valid pass, each violation class (bad status, missing subject,
|
|
4
|
+
# secret-shaped evidence, tombstone -> missing slug), and index idempotency.
|
|
5
|
+
set -uo pipefail
|
|
6
|
+
|
|
7
|
+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
8
|
+
SCRIPT="$ROOT/scripts/check-decisions.cjs"
|
|
9
|
+
|
|
10
|
+
errors=0
|
|
11
|
+
pass() { echo " PASS $1"; }
|
|
12
|
+
fail() { echo " FAIL $1"; errors=$((errors + 1)); }
|
|
13
|
+
|
|
14
|
+
echo "=== Decision Registry Checks ==="
|
|
15
|
+
|
|
16
|
+
WORK="$(mktemp -d)"
|
|
17
|
+
trap 'rm -rf "$WORK"' EXIT
|
|
18
|
+
|
|
19
|
+
run_check() { FLOW_AGENTS_DECISIONS_DIR="$1" node "$SCRIPT" check >"$2" 2>&1; }
|
|
20
|
+
gen_index() { FLOW_AGENTS_DECISIONS_DIR="$1" node "$SCRIPT" gen-index >/dev/null 2>&1; }
|
|
21
|
+
|
|
22
|
+
valid_frontmatter() {
|
|
23
|
+
cat <<'EOF'
|
|
24
|
+
---
|
|
25
|
+
status: current
|
|
26
|
+
subject: Example subject
|
|
27
|
+
decided: 2026-07-03
|
|
28
|
+
evidence:
|
|
29
|
+
- kind: issue
|
|
30
|
+
ref: https://github.com/kontourai/flow-agents/issues/310
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
# Example subject
|
|
34
|
+
|
|
35
|
+
The current answer.
|
|
36
|
+
EOF
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
# --- Case 1: valid file passes ---
|
|
40
|
+
D1="$WORK/valid"; mkdir -p "$D1"
|
|
41
|
+
valid_frontmatter >"$D1/example-subject.md"
|
|
42
|
+
gen_index "$D1"
|
|
43
|
+
if run_check "$D1" "$WORK/c1.log"; then
|
|
44
|
+
pass "valid topic file + fresh index passes"
|
|
45
|
+
else
|
|
46
|
+
fail "valid topic file should pass"; cat "$WORK/c1.log"
|
|
47
|
+
fi
|
|
48
|
+
|
|
49
|
+
# --- Case 2: unknown status fails ---
|
|
50
|
+
D2="$WORK/badstatus"; mkdir -p "$D2"
|
|
51
|
+
valid_frontmatter | sed 's/^status: current/status: accepted/' >"$D2/example-subject.md"
|
|
52
|
+
gen_index "$D2"
|
|
53
|
+
if run_check "$D2" "$WORK/c2.log"; then
|
|
54
|
+
fail "unknown status should fail validation"
|
|
55
|
+
else
|
|
56
|
+
grep -q "unknown status" "$WORK/c2.log" && pass "unknown status fails" || { fail "unknown status wrong error"; cat "$WORK/c2.log"; }
|
|
57
|
+
fi
|
|
58
|
+
|
|
59
|
+
# --- Case 3: missing subject fails ---
|
|
60
|
+
D3="$WORK/nosubject"; mkdir -p "$D3"
|
|
61
|
+
valid_frontmatter | grep -v '^subject:' >"$D3/example-subject.md"
|
|
62
|
+
gen_index "$D3"
|
|
63
|
+
if run_check "$D3" "$WORK/c3.log"; then
|
|
64
|
+
fail "missing subject should fail validation"
|
|
65
|
+
else
|
|
66
|
+
grep -q "subject" "$WORK/c3.log" && pass "missing subject fails" || { fail "missing subject wrong error"; cat "$WORK/c3.log"; }
|
|
67
|
+
fi
|
|
68
|
+
|
|
69
|
+
# --- Case 4: secret-shaped evidence ref fails ---
|
|
70
|
+
# The secret-shaped literal is assembled at runtime so no contiguous
|
|
71
|
+
# access-key-shaped token is committed to a tracked file (suite secret-scan gate).
|
|
72
|
+
D4="$WORK/secret"; mkdir -p "$D4"
|
|
73
|
+
FAKE_KEY="AKIA$(printf 'IOSFODNN7EXAMPLE')"
|
|
74
|
+
{
|
|
75
|
+
printf -- '---\n'
|
|
76
|
+
printf 'status: current\n'
|
|
77
|
+
printf 'subject: Example subject\n'
|
|
78
|
+
printf 'decided: 2026-07-03\n'
|
|
79
|
+
printf 'evidence:\n'
|
|
80
|
+
printf ' - kind: url\n'
|
|
81
|
+
printf ' ref: "https://example.com/?token=%s"\n' "$FAKE_KEY"
|
|
82
|
+
printf -- '---\n\n'
|
|
83
|
+
printf '# Example subject\n'
|
|
84
|
+
} >"$D4/example-subject.md"
|
|
85
|
+
gen_index "$D4"
|
|
86
|
+
if run_check "$D4" "$WORK/c4.log"; then
|
|
87
|
+
fail "secret-shaped evidence ref should fail validation"
|
|
88
|
+
else
|
|
89
|
+
grep -q "secret-shaped" "$WORK/c4.log" && pass "secret-shaped evidence fails" || { fail "secret-shaped wrong error"; cat "$WORK/c4.log"; }
|
|
90
|
+
fi
|
|
91
|
+
|
|
92
|
+
# --- Case 5: tombstone pointing at missing slug fails ---
|
|
93
|
+
D5="$WORK/tombstone"; mkdir -p "$D5"
|
|
94
|
+
cat >"$D5/old-topic.md" <<'EOF'
|
|
95
|
+
---
|
|
96
|
+
status: superseded
|
|
97
|
+
subject: Old topic
|
|
98
|
+
decided: 2026-07-03
|
|
99
|
+
superseded_by: does-not-exist
|
|
100
|
+
evidence:
|
|
101
|
+
- kind: issue
|
|
102
|
+
ref: "#310"
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
# Old topic
|
|
106
|
+
EOF
|
|
107
|
+
gen_index "$D5"
|
|
108
|
+
if run_check "$D5" "$WORK/c5.log"; then
|
|
109
|
+
fail "tombstone -> missing slug should fail validation"
|
|
110
|
+
else
|
|
111
|
+
grep -q "missing topic slug" "$WORK/c5.log" && pass "tombstone -> missing slug fails" || { fail "tombstone wrong error"; cat "$WORK/c5.log"; }
|
|
112
|
+
fi
|
|
113
|
+
|
|
114
|
+
# --- Case 6: index idempotency (second gen is diff-clean) ---
|
|
115
|
+
D6="$WORK/idem"; mkdir -p "$D6"
|
|
116
|
+
valid_frontmatter >"$D6/example-subject.md"
|
|
117
|
+
gen_index "$D6"
|
|
118
|
+
cp "$D6/index.md" "$WORK/index-first.md"
|
|
119
|
+
gen_index "$D6"
|
|
120
|
+
if diff -q "$WORK/index-first.md" "$D6/index.md" >/dev/null; then
|
|
121
|
+
pass "index regeneration is idempotent (diff-clean)"
|
|
122
|
+
else
|
|
123
|
+
fail "index regeneration is not idempotent"; diff "$WORK/index-first.md" "$D6/index.md" || true
|
|
124
|
+
fi
|
|
125
|
+
|
|
126
|
+
# --- Case 7: check flags a stale index ---
|
|
127
|
+
D7="$WORK/stale"; mkdir -p "$D7"
|
|
128
|
+
valid_frontmatter >"$D7/example-subject.md"
|
|
129
|
+
gen_index "$D7"
|
|
130
|
+
printf '\nstale line\n' >>"$D7/index.md"
|
|
131
|
+
if run_check "$D7" "$WORK/c7.log"; then
|
|
132
|
+
fail "stale index should fail validation"
|
|
133
|
+
else
|
|
134
|
+
grep -q "stale" "$WORK/c7.log" && pass "stale index detected" || { fail "stale index wrong error"; cat "$WORK/c7.log"; }
|
|
135
|
+
fi
|
|
136
|
+
|
|
137
|
+
# --- Case 8: repo's own registry passes end-to-end ---
|
|
138
|
+
if node "$SCRIPT" check >"$WORK/c8.log" 2>&1; then
|
|
139
|
+
pass "repo docs/decisions registry passes"
|
|
140
|
+
else
|
|
141
|
+
fail "repo docs/decisions registry should pass"; cat "$WORK/c8.log"
|
|
142
|
+
fi
|
|
143
|
+
|
|
144
|
+
echo ""
|
|
145
|
+
if [[ "$errors" -eq 0 ]]; then
|
|
146
|
+
echo "Decision registry checks: all passed."
|
|
147
|
+
exit 0
|
|
148
|
+
fi
|
|
149
|
+
echo "Decision registry checks: $errors failed."
|
|
150
|
+
exit 1
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
5
|
+
cd "$ROOT_DIR"
|
|
6
|
+
|
|
7
|
+
# Regression lock: CODEOWNERS must keep review-gated coverage over the
|
|
8
|
+
# anti-gaming gate's enforcement hooks, the FlowDefinition resolution/trust
|
|
9
|
+
# path, and the kit FlowDefinitions the gate's expects[] taxonomy is derived
|
|
10
|
+
# from (ADR 0016 Abstraction A, ADR 0018 Decision #2). If any of these paths
|
|
11
|
+
# silently drop out of CODEOWNERS, an agent could weaken the gate without
|
|
12
|
+
# owner review. This check must fail loudly, naming the missing path.
|
|
13
|
+
|
|
14
|
+
CODEOWNERS_FILE=".github/CODEOWNERS"
|
|
15
|
+
|
|
16
|
+
pass() {
|
|
17
|
+
echo "PASS: $1"
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
fail() {
|
|
21
|
+
echo "FAIL: $1" >&2
|
|
22
|
+
exit 1
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
require_file() {
|
|
26
|
+
[[ -f "$1" ]] || fail "$1 is missing"
|
|
27
|
+
pass "$1 exists"
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
# Asserts that a non-comment line in CODEOWNERS contains the given
|
|
31
|
+
# path/glob pattern and has an owner assigned to it. Tolerant of leading
|
|
32
|
+
# "/", trailing globs, and surrounding whitespace — it checks coverage,
|
|
33
|
+
# not an exact line match.
|
|
34
|
+
require_codeowner() {
|
|
35
|
+
local pattern="$1"
|
|
36
|
+
local label="$2"
|
|
37
|
+
local match
|
|
38
|
+
match="$(grep -v '^[[:space:]]*#' "$CODEOWNERS_FILE" | grep -F -- "$pattern" || true)"
|
|
39
|
+
[[ -n "$match" ]] || fail "CODEOWNERS coverage regression: no entry found for '$label' (expected pattern containing '$pattern')"
|
|
40
|
+
echo "$match" | grep -Eq '@[A-Za-z0-9_-]+' || fail "CODEOWNERS coverage regression: entry for '$label' has no owner assigned"
|
|
41
|
+
pass "CODEOWNERS covers $label"
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
require_file "$CODEOWNERS_FILE"
|
|
45
|
+
|
|
46
|
+
# NOTE: the flow-resolver.ts pattern below is string-split, matching how other
|
|
47
|
+
# evals/ files reference the same module — otherwise validate-source-tree's
|
|
48
|
+
# legacy-ref scanner mistakes the "lib/" segment for a (missing) top-level path.
|
|
49
|
+
require_codeowner "scripts/hooks/stop-goal-fit.js" "scripts/hooks/stop-goal-fit.js"
|
|
50
|
+
require_codeowner "scripts/hooks/config-protection.js" "scripts/hooks/config-protection.js"
|
|
51
|
+
require_codeowner "scripts/hooks/evidence-capture.js" "scripts/hooks/evidence-capture.js"
|
|
52
|
+
require_codeowner "src/li""b/flow-resolver.ts" "src/li""b/flow-resolver.ts"
|
|
53
|
+
require_codeowner "src/cli/workflow-sidecar.ts" "src/cli/workflow-sidecar.ts"
|
|
54
|
+
require_codeowner "kits/*/flows/*.flow.json" "kits/*/flows/*.flow.json (kit FlowDefinitions)"
|
|
55
|
+
|
|
56
|
+
# ADR 0022 §2: delivery/DECLARED is the no-agent-delivery exemption marker
|
|
57
|
+
# scripts/ci/trust-reconcile.js reads — a self-asserted governance artifact (names
|
|
58
|
+
# approved_by) that must stay owner-review-gated the same as the other verify-config/
|
|
59
|
+
# anchor paths above (#301's file-level half; server-side review enforcement is #225).
|
|
60
|
+
require_codeowner "delivery/DECLARED" "delivery/DECLARED (ADR 0022 §2 no-agent-delivery marker)"
|
|
61
|
+
|
|
62
|
+
echo "FlowDefinition-related CODEOWNERS coverage checks passed."
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# test_knowledge_providers.sh — Layer 1: Knowledge Store Provider contract
|
|
3
|
+
# conformance + provider-agnostic health verbs (issue #317).
|
|
4
|
+
#
|
|
5
|
+
# Runs the node:test suites that back AC1–AC3:
|
|
6
|
+
# - providers/conformance/suite.test.js (AC3: all three providers conform)
|
|
7
|
+
# - providers/health/health-pass.test.js (AC1: vault+git-repo reports;
|
|
8
|
+
# AC2: work-item seeded dupe + broken blocker)
|
|
9
|
+
# Deterministic, dependency-free, fixture-driven — never touches a real board.
|
|
10
|
+
set -uo pipefail
|
|
11
|
+
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
12
|
+
cd "$ROOT_DIR"
|
|
13
|
+
|
|
14
|
+
echo "── Knowledge store provider conformance + health (node --test) ──"
|
|
15
|
+
|
|
16
|
+
if node --test \
|
|
17
|
+
kits/knowledge/providers/conformance/suite.test.js \
|
|
18
|
+
kits/knowledge/providers/health/health-pass.test.js; then
|
|
19
|
+
echo " PASS: knowledge store provider conformance + health verbs"
|
|
20
|
+
else
|
|
21
|
+
echo " FAIL: knowledge store provider conformance + health verbs"
|
|
22
|
+
exit 1
|
|
23
|
+
fi
|