@kontourai/flow-agents 3.2.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/add-to-project.yml +15 -0
- package/.github/workflows/ci.yml +173 -0
- package/CHANGELOG.md +54 -0
- package/CONTEXT.md +5 -1
- package/README.md +19 -8
- package/build/src/builder-flow-run-adapter.d.ts +80 -0
- package/build/src/builder-flow-run-adapter.js +241 -0
- package/build/src/builder-flow-runtime.d.ts +16 -0
- package/build/src/builder-flow-runtime.js +290 -0
- package/build/src/cli/assignment-provider.js +10 -1
- package/build/src/cli/builder-run.d.ts +1 -0
- package/build/src/cli/builder-run.js +27 -0
- package/build/src/cli/effective-backlog-settings.js +70 -2
- package/build/src/cli/init.d.ts +34 -0
- package/build/src/cli/init.js +341 -61
- package/build/src/cli/kit.js +55 -12
- package/build/src/cli/pull-work-provider.js +346 -5
- package/build/src/cli/skill-drift-check.d.ts +1 -0
- package/build/src/cli/skill-drift-check.js +165 -0
- package/build/src/cli/telemetry-doctor.d.ts +37 -0
- package/build/src/cli/telemetry-doctor.js +53 -6
- package/build/src/cli/validate-hook-influence.js +37 -7
- package/build/src/cli/workflow-artifact-cleanup-audit.js +418 -11
- package/build/src/cli/workflow-sidecar.d.ts +310 -1
- package/build/src/cli/workflow-sidecar.js +1914 -126
- package/build/src/cli.js +5 -0
- package/build/src/flow-kit/validate.d.ts +54 -34
- package/build/src/flow-kit/validate.js +237 -26
- package/build/src/index.d.ts +2 -0
- package/build/src/index.js +1 -0
- package/build/src/lib/console-connect-options.d.ts +97 -0
- package/build/src/lib/console-connect-options.js +199 -0
- package/build/src/lib/console-telemetry-validate.d.ts +49 -0
- package/build/src/lib/console-telemetry-validate.js +91 -0
- package/build/src/lib/flow-resolver.d.ts +54 -1
- package/build/src/lib/flow-resolver.js +112 -5
- package/build/src/lib/fs.d.ts +17 -0
- package/build/src/lib/fs.js +172 -0
- package/build/src/lib/local-artifact-root.d.ts +44 -1
- package/build/src/lib/local-artifact-root.js +131 -3
- package/build/src/runtime-adapters.d.ts +39 -3
- package/build/src/runtime-adapters.js +77 -31
- package/build/src/tools/build-universal-bundles.js +40 -2
- package/build/src/tools/codex-agent-routing.d.ts +2 -0
- package/build/src/tools/codex-agent-routing.js +49 -0
- package/build/src/tools/generate-context-map.js +1 -0
- package/build/src/tools/validate-source-tree.js +30 -3
- package/context/contracts/artifact-contract.md +16 -2
- package/context/scripts/hooks/lib/kit-catalog.js +235 -0
- package/context/scripts/hooks/lib/runnable-command.js +177 -0
- package/context/scripts/hooks/stop-goal-fit.js +278 -48
- package/context/scripts/hooks/workflow-steering.js +194 -22
- package/context/scripts/package.json +3 -0
- package/context/scripts/telemetry/install-console-config.sh +25 -4
- package/context/scripts/telemetry/lib/config.sh +102 -12
- package/context/scripts/telemetry/lib/pricing.sh +50 -0
- package/context/scripts/telemetry/lib/session.sh +3 -0
- package/context/scripts/telemetry/lib/transport.sh +87 -0
- package/context/scripts/telemetry/lib/usage.sh +205 -4
- package/context/scripts/telemetry/telemetry.conf +6 -0
- package/context/scripts/telemetry/telemetry.sh +48 -0
- package/context/settings/workspace-backlog-provider-settings.example.json +48 -0
- package/docs/agent-usage-feedback-loop.md +35 -0
- package/docs/architecture-engine-and-kits.md +110 -0
- package/docs/context-map.md +2 -0
- package/docs/coordination-guide.md +370 -0
- package/docs/decisions/agent-coordination.md +26 -9
- package/docs/decisions/embeddable-engine.md +152 -0
- package/docs/decisions/index.md +5 -3
- package/docs/decisions/trust-ledger-retention.md +88 -0
- package/docs/decisions/trust-reconcile.md +42 -9
- package/docs/decisions/workflow-enforcement.md +31 -9
- package/docs/fixture-ownership.md +6 -2
- package/docs/implementing-trust-reconciliation.md +129 -0
- package/docs/index.md +23 -9
- package/docs/integrations/flow-agents-console.md +275 -0
- package/docs/integrations/index.md +4 -0
- package/docs/kit-authoring-guide.md +52 -21
- package/docs/spec/builder-flow-runtime.md +80 -0
- package/docs/spec/runtime-hook-surface.md +45 -1
- package/docs/specs/economics-record-contract.md +270 -0
- package/docs/specs/harness-capability-matrix.md +74 -0
- package/docs/specs/learning-review-proposals-contract.md +340 -0
- package/docs/specs/routing-efficiency-review.md +59 -0
- package/docs/verifiable-trust.md +74 -25
- package/docs/workflow-artifact-lifecycle.md +38 -1
- package/docs/workflow-usage-guide.md +10 -0
- package/evals/acceptance/prove-capture-teeth.sh +132 -0
- package/evals/ci/antigaming-suite.sh +2 -0
- package/evals/ci/run-baseline.sh +78 -4
- package/evals/fixtures/economics/acceptance.json +12 -0
- package/evals/fixtures/economics/agents/tool-worker-1/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-2/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-3/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-4/events.jsonl +1 -0
- package/evals/fixtures/economics/agents/tool-worker-5/events.jsonl +2 -0
- package/evals/fixtures/economics/critique.json +22 -0
- package/evals/fixtures/economics/expected-record.json +71 -0
- package/evals/fixtures/economics/session-usage-event.json +1 -0
- package/evals/fixtures/economics/state.json +11 -0
- package/evals/fixtures/economics/transcript.jsonl +3 -0
- package/evals/fixtures/hook-influence/cases.json +7 -7
- package/evals/fixtures/learning-review-proposals/balanced/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/economics.jsonl +5 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-1/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-2/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-3/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-4/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-5/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/expected-aggregates.json +30 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/expected-aggregates.md +66 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-1/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-1/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-2/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-2/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-3/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-3/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-4/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-4/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-5/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-6/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/repeat-window/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/under-threshold/economics.jsonl +3 -0
- package/evals/fixtures/reconcile-preflight/disputed-critique-unsuperseded.json +48 -0
- package/evals/fixtures/reconcile-preflight/standalone-disputed-session-local.json +59 -0
- package/evals/fixtures/telemetry/usage-transcript-sample.jsonl +4 -0
- package/evals/fixtures/trust-reconcile-exploits/mcp-degrade.json +42 -0
- package/evals/integration/test_builder_entry_enforcement.sh +241 -0
- package/evals/integration/test_builder_step_producers.sh +18 -10
- package/evals/integration/test_bundle_install.sh +172 -0
- package/evals/integration/test_checkpoint_signing.sh +10 -2
- package/evals/integration/test_ci_actor_identity.sh +221 -0
- package/evals/integration/test_console_tenant_isolation.sh +167 -0
- package/evals/integration/test_critique_supersession_roundtrip.sh +4 -1
- package/evals/integration/test_dual_emit_flow_step.sh +10 -4
- package/evals/integration/test_economics_record.sh +674 -0
- package/evals/integration/test_effective_backlog_settings.sh +1 -1
- package/evals/integration/test_evidence_capture_hook.sh +17 -2
- package/evals/integration/test_exemption_usage_review.sh +198 -0
- package/evals/integration/test_fixture_retirement_audit.sh +2 -2
- package/evals/integration/test_flow_kit_install_git.sh +83 -0
- package/evals/integration/test_flowdef_session_activation.sh +0 -1
- package/evals/integration/test_flowdef_session_history_preservation.sh +13 -3
- package/evals/integration/test_gate_lockdown.sh +7 -0
- package/evals/integration/test_gate_review_inquiry_records.sh +9 -1
- package/evals/integration/test_goal_fit_hook.sh +2031 -0
- package/evals/integration/test_hook_category_behaviors.sh +8 -1
- package/evals/integration/test_hook_influence_cases.sh +25 -1
- package/evals/integration/test_install_merge.sh +227 -2
- package/evals/integration/test_kit_conformance_levels.sh +6 -6
- package/evals/integration/test_learning_review_proposals.sh +329 -0
- package/evals/integration/test_liveness_conflict_injection.sh +26 -22
- package/evals/integration/test_liveness_console_relay.sh +166 -0
- package/evals/integration/test_liveness_heartbeat.sh +17 -17
- package/evals/integration/test_liveness_worktree_root.sh +575 -0
- package/evals/integration/test_phase_map_and_gate_claim.sh +6 -1
- package/evals/integration/test_publish_delivery.sh +389 -2
- package/evals/integration/test_pull_work_board.sh +200 -0
- package/evals/integration/test_pull_work_provider.sh +1 -1
- package/evals/integration/test_reconcile_preflight.sh +304 -0
- package/evals/integration/test_record_check.sh +378 -0
- package/evals/integration/test_routing_efficiency.sh +71 -0
- package/evals/integration/test_runtime_adapter_activation.sh +28 -0
- package/evals/integration/test_session_resume_roundtrip.sh +16 -19
- package/evals/integration/test_skill_drift_check.sh +870 -0
- package/evals/integration/test_takeover_protocol.sh +340 -0
- package/evals/integration/test_telemetry.sh +445 -0
- package/evals/integration/test_telemetry_doctor.sh +66 -0
- package/evals/integration/test_telemetry_usage_pipeline.sh +228 -0
- package/evals/integration/test_trust_reconcile_negatives.sh +121 -13
- package/evals/integration/test_trust_reconcile_trailer_diagnostic.sh +247 -0
- package/evals/integration/test_usage_cost.sh +61 -0
- package/evals/integration/test_verify_hold.sh +910 -0
- package/evals/integration/test_veritas_governance_kit.sh +257 -0
- package/evals/integration/test_workflow_artifact_cleanup_audit.sh +575 -3
- package/evals/integration/test_workflow_sidecar_writer.sh +1395 -0
- package/evals/integration/test_workflow_steering_hook.sh +157 -16
- package/evals/integration/test_workspace_settings.sh +176 -0
- package/evals/lib/env.sh +26 -0
- package/evals/lib/node.sh +8 -0
- package/evals/run.sh +37 -0
- package/evals/static/test_ci_integration_coverage.sh +115 -0
- package/evals/static/test_declared_scope_forms_documented.sh +114 -0
- package/evals/static/test_universal_bundles.sh +34 -0
- package/evals/static/test_validate_source_kit_asset_scope.sh +259 -0
- package/evals/static/test_workflow_skills.sh +1 -1
- package/kits/builder/flows/build.flow.json +9 -18
- package/kits/builder/flows/publish-learn.flow.json +5 -1
- package/kits/builder/kit.json +120 -0
- package/kits/builder/skills/continue-work/SKILL.md +2 -0
- package/kits/builder/skills/deliver/SKILL.md +115 -0
- package/kits/builder/skills/evidence-gate/SKILL.md +12 -0
- package/kits/builder/skills/execute-plan/SKILL.md +9 -0
- package/kits/builder/skills/learning-review/SKILL.md +51 -0
- package/kits/builder/skills/plan-work/SKILL.md +17 -20
- package/kits/builder/skills/pull-work/SKILL.md +33 -2
- package/kits/builder/skills/release-readiness/SKILL.md +12 -0
- package/kits/knowledge/kit.json +9 -0
- package/kits/veritas-governance/docs/README.md +113 -7
- package/kits/veritas-governance/fixtures/exemption/approved.trust-bundle.json +74 -0
- package/kits/veritas-governance/fixtures/exemption/not-approved.trust-bundle.json +74 -0
- package/kits/veritas-governance/fixtures/exemption-review/mixed-fresh-stale.DECLARED.json +14 -0
- package/kits/veritas-governance/flows/exemption-issuance.flow.json +35 -0
- package/kits/veritas-governance/kit.json +19 -0
- package/kits/veritas-governance/skills/exemption-usage-review/SKILL.md +128 -0
- package/kits/veritas-governance/skills/exemption-usage-review/review-exemptions.mjs +231 -0
- package/package.json +2 -2
- package/packaging/manifest.json +29 -0
- package/schemas/backlog-provider-settings.schema.json +13 -0
- package/schemas/workflow-state.schema.json +44 -0
- package/scripts/README.md +4 -0
- package/scripts/check-content-boundary.cjs +8 -1
- package/scripts/ci/trust-reconcile.js +214 -253
- package/scripts/hooks/codex-hook-adapter.js +77 -2
- package/scripts/hooks/evidence-capture.js +38 -5
- package/scripts/hooks/lib/actor-identity.js +82 -0
- package/scripts/hooks/lib/codex-exit-code.js +316 -0
- package/scripts/hooks/lib/kit-catalog.js +235 -0
- package/scripts/hooks/lib/liveness-write.js +28 -1
- package/scripts/hooks/lib/local-artifact-paths.js +97 -1
- package/scripts/hooks/lib/runnable-command.js +177 -0
- package/scripts/hooks/lib/skill-drift.js +350 -0
- package/scripts/hooks/stop-goal-fit.js +278 -48
- package/scripts/hooks/workflow-steering.js +194 -22
- package/scripts/install-codex-home.sh +97 -47
- package/scripts/install-merge.js +72 -14
- package/scripts/install-owned-files.js +178 -0
- package/scripts/lib/reconcile-shape.js +381 -0
- package/scripts/liveness/relay.sh +84 -0
- package/scripts/telemetry/economics-record.schema.json +145 -0
- package/scripts/telemetry/economics-record.sh +331 -0
- package/scripts/telemetry/install-console-config.sh +25 -4
- package/scripts/telemetry/learning-review-decide.sh +124 -0
- package/scripts/telemetry/learning-review-proposals.schema.json +161 -0
- package/scripts/telemetry/learning-review-proposals.sh +484 -0
- package/scripts/telemetry/lib/config.sh +102 -12
- package/scripts/telemetry/lib/pricing.sh +14 -6
- package/scripts/telemetry/lib/session.sh +3 -0
- package/scripts/telemetry/lib/transport.sh +133 -15
- package/scripts/telemetry/lib/usage.sh +121 -28
- package/scripts/telemetry/routing-efficiency.sh +0 -0
- package/scripts/telemetry/telemetry.conf +6 -0
- package/scripts/telemetry/telemetry.sh +48 -0
- package/src/builder-flow-run-adapter.ts +357 -0
- package/src/builder-flow-runtime.ts +348 -0
- package/src/cli/assignment-provider.ts +12 -1
- package/src/cli/builder-flow-run-adapter.test.mjs +495 -0
- package/src/cli/builder-flow-runtime.test.mjs +213 -0
- package/src/cli/builder-run.ts +28 -0
- package/src/cli/codex-agent-routing.test.mjs +44 -0
- package/src/cli/codex-exit-code.test.mjs +207 -0
- package/src/cli/console-connect-options.test.mjs +329 -0
- package/src/cli/console-telemetry-validate.test.mjs +157 -0
- package/src/cli/effective-backlog-settings.ts +68 -2
- package/src/cli/flow-resolver-composition.test.mjs +72 -0
- package/src/cli/init.test.mjs +161 -0
- package/src/cli/init.ts +407 -62
- package/src/cli/kit-metadata-security.test.mjs +443 -0
- package/src/cli/kit.ts +50 -12
- package/src/cli/pull-work-provider.ts +377 -3
- package/src/cli/sidecar-pure-helpers.test.mjs +64 -0
- package/src/cli/skill-drift-check.ts +196 -0
- package/src/cli/telemetry-doctor.test.mjs +53 -0
- package/src/cli/telemetry-doctor.ts +50 -7
- package/src/cli/validate-hook-influence.ts +37 -6
- package/src/cli/workflow-artifact-cleanup-audit.ts +483 -10
- package/src/cli/workflow-sidecar.ts +1980 -119
- package/src/cli.ts +5 -0
- package/src/flow-kit/validate.ts +277 -38
- package/src/index.ts +19 -0
- package/src/lib/console-connect-options.ts +261 -0
- package/src/lib/console-telemetry-validate.ts +88 -0
- package/src/lib/flow-resolver.ts +117 -4
- package/src/lib/fs.ts +160 -0
- package/src/lib/local-artifact-root.ts +129 -3
- package/src/runtime-adapters.ts +113 -33
- package/src/tools/build-universal-bundles.ts +36 -2
- package/src/tools/codex-agent-routing.ts +48 -0
- package/src/tools/generate-context-map.ts +1 -0
- package/src/tools/validate-source-tree.ts +29 -3
package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-4/trust.bundle
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 3,
|
|
3
|
+
"source": "flow-agents/workflow-sidecar;statusFunctionVersion=1",
|
|
4
|
+
"claims": [
|
|
5
|
+
{
|
|
6
|
+
"id": "task-lr-ef-4-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass",
|
|
7
|
+
"subjectType": "workflow-check",
|
|
8
|
+
"subjectId": "task-lr-ef-4/unit-tests",
|
|
9
|
+
"facet": "flow-agents.workflow",
|
|
10
|
+
"claimType": "builder.verify.tests",
|
|
11
|
+
"fieldOrBehavior": "unit tests pass",
|
|
12
|
+
"value": "pass",
|
|
13
|
+
"createdAt": "2026-07-04T00:00:00Z",
|
|
14
|
+
"updatedAt": "2026-07-04T00:00:00Z",
|
|
15
|
+
"status": "verified"
|
|
16
|
+
}
|
|
17
|
+
],
|
|
18
|
+
"evidence": [],
|
|
19
|
+
"events": [],
|
|
20
|
+
"policies": []
|
|
21
|
+
}
|
package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-5/trust.bundle
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 3,
|
|
3
|
+
"source": "flow-agents/workflow-sidecar;statusFunctionVersion=1",
|
|
4
|
+
"claims": [
|
|
5
|
+
{
|
|
6
|
+
"id": "task-lr-ef-5-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass",
|
|
7
|
+
"subjectType": "workflow-check",
|
|
8
|
+
"subjectId": "task-lr-ef-5/unit-tests",
|
|
9
|
+
"facet": "flow-agents.workflow",
|
|
10
|
+
"claimType": "builder.verify.tests",
|
|
11
|
+
"fieldOrBehavior": "unit tests pass",
|
|
12
|
+
"value": "pass",
|
|
13
|
+
"createdAt": "2026-07-04T00:00:00Z",
|
|
14
|
+
"updatedAt": "2026-07-04T00:00:00Z",
|
|
15
|
+
"status": "verified"
|
|
16
|
+
}
|
|
17
|
+
],
|
|
18
|
+
"evidence": [],
|
|
19
|
+
"events": [],
|
|
20
|
+
"policies": []
|
|
21
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"pp-r1","at":1751500000000,"task_slug":"task-lr-pp-1","model":"claude-opus-4-8","cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.1,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":1,"verification_verdict":"PASS"}}
|
|
2
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"pp-r2","at":1751500001000,"task_slug":"task-lr-pp-2","model":"claude-opus-4-8","cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.1,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":1,"verification_verdict":"PASS"}}
|
|
3
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"pp-r3","at":1751500002000,"task_slug":"task-lr-pp-3","model":"claude-opus-4-8","cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.1,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":1,"verification_verdict":"PASS"}}
|
|
4
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"pp-r4","at":1751500003000,"task_slug":"task-lr-pp-4","model":"claude-opus-4-8","cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.2,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":1,"verification_verdict":"PASS"}}
|
|
5
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"pp-r5","at":1751500004000,"task_slug":"task-lr-pp-5","model":"claude-opus-4-8","cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.2,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":1,"verification_verdict":"PASS"}}
|
|
6
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"pp-r6","at":1751500005000,"task_slug":"task-lr-pp-6","model":"claude-opus-4-8","cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.2,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":1,"verification_verdict":"PASS"}}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
{
|
|
2
|
+
"by_kit": [
|
|
3
|
+
{
|
|
4
|
+
"kit_id": "builder",
|
|
5
|
+
"runs": 6,
|
|
6
|
+
"first_half_avg_cost_usd": 0.1,
|
|
7
|
+
"second_half_avg_cost_usd": 0.2,
|
|
8
|
+
"cost_trend_pct": 100,
|
|
9
|
+
"first_half_findings_total": 6,
|
|
10
|
+
"second_half_findings_total": 6,
|
|
11
|
+
"findings_delta_pct": 0,
|
|
12
|
+
"avg_wall_clock_s": 100,
|
|
13
|
+
"avg_human_wait_s": 10,
|
|
14
|
+
"route_back_rate": 0.5,
|
|
15
|
+
"caught_false_completions_total": 6
|
|
16
|
+
}
|
|
17
|
+
],
|
|
18
|
+
"by_gate": [
|
|
19
|
+
{
|
|
20
|
+
"gate_id": "unit tests pass",
|
|
21
|
+
"fire_count": 4,
|
|
22
|
+
"correct_count": 1,
|
|
23
|
+
"false_block_count": 3,
|
|
24
|
+
"missed_block_count": 0,
|
|
25
|
+
"false_block_rate": 0.75,
|
|
26
|
+
"avg_wall_clock_s_when_fired": 100,
|
|
27
|
+
"avg_human_wait_s_when_fired": 10
|
|
28
|
+
}
|
|
29
|
+
]
|
|
30
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# `pattern-present` — hand-computed aggregate arithmetic (AC1)
|
|
2
|
+
|
|
3
|
+
Six `kontour.console.economics` records in `economics.jsonl`, ordered ascending by `.at`
|
|
4
|
+
(`pp-r1` .. `pp-r6`), all joined (via `sessions/<task_slug>/trust.bundle`, one `builder.verify.tests`
|
|
5
|
+
claim each) to `kit_id = "builder"`. Four of the six sessions (`task-lr-pp-1..4`) also carry a
|
|
6
|
+
`gate-review.inquiries.json` with one `InquiryRecord` each for gate `"unit tests pass"`; the
|
|
7
|
+
remaining two (`task-lr-pp-5`, `task-lr-pp-6`) have no gate-review join (counted in `notes`).
|
|
8
|
+
|
|
9
|
+
Raw per-record inputs (identical across records except `cost` and — for the first four — the
|
|
10
|
+
gate calibration):
|
|
11
|
+
|
|
12
|
+
| run_id | at | cost.estimated_cost_usd | findings (high+low) | time.wall_clock_s | time.human_wait_s | iterations.count | iterations.route_backs | defects.caught_false_completions | gate calibration |
|
|
13
|
+
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
|
14
|
+
| pp-r1 | 1751500000000 | 0.10 | 1+1=2 | 100 | 10 | 2 | 1 | 1 | correct |
|
|
15
|
+
| pp-r2 | 1751500001000 | 0.10 | 1+1=2 | 100 | 10 | 2 | 1 | 1 | false_block |
|
|
16
|
+
| pp-r3 | 1751500002000 | 0.10 | 1+1=2 | 100 | 10 | 2 | 1 | 1 | false_block |
|
|
17
|
+
| pp-r4 | 1751500003000 | 0.20 | 1+1=2 | 100 | 10 | 2 | 1 | 1 | false_block |
|
|
18
|
+
| pp-r5 | 1751500004000 | 0.20 | 1+1=2 | 100 | 10 | 2 | 1 | 1 | (no gate join) |
|
|
19
|
+
| pp-r6 | 1751500005000 | 0.20 | 1+1=2 | 100 | 10 | 2 | 1 | 1 | (no gate join) |
|
|
20
|
+
|
|
21
|
+
## `by_kit[]` — `kit_id: "builder"` (runs = 6)
|
|
22
|
+
|
|
23
|
+
`half_point = floor(6 / 2) = 3` → first half = {pp-r1, pp-r2, pp-r3} (oldest 3 by `.at`), second
|
|
24
|
+
half = {pp-r4, pp-r5, pp-r6} (newest 3).
|
|
25
|
+
|
|
26
|
+
- `first_half_avg_cost_usd` = (0.10 + 0.10 + 0.10) / 3 = **0.10**
|
|
27
|
+
- `second_half_avg_cost_usd` = (0.20 + 0.20 + 0.20) / 3 = **0.20**
|
|
28
|
+
- `cost_trend_pct` = ((0.20 − 0.10) / 0.10) × 100 = **100** ( ≥ `LR_COST_RISE_PCT` default 25 )
|
|
29
|
+
- `first_half_findings_total` = 2 + 2 + 2 = **6**
|
|
30
|
+
- `second_half_findings_total` = 2 + 2 + 2 = **6**
|
|
31
|
+
- `findings_delta_pct` = first half is non-zero (6), so `((6 − 6) / 6) × 100` = **0** ( ≤
|
|
32
|
+
`LR_FLAT_FINDINGS_PCT` default 10 — flat, not rising )
|
|
33
|
+
- `avg_wall_clock_s` = (100 × 6) / 6 = **100** (all six records, not half-split)
|
|
34
|
+
- `avg_human_wait_s` = (10 × 6) / 6 = **10**
|
|
35
|
+
- `route_back_rate` = sum(route_backs) / sum(count) = (1×6) / (2×6) = 6 / 12 = **0.5**
|
|
36
|
+
- `caught_false_completions_total` = 1 × 6 = **6**
|
|
37
|
+
|
|
38
|
+
`runs (6) >= LR_MIN_KIT_SAMPLE (6)` AND `cost_trend_pct (100) >= LR_COST_RISE_PCT (25)` AND
|
|
39
|
+
`findings_delta_pct (0) <= LR_FLAT_FINDINGS_PCT (10)` → **`kit-review-cost-inflation` fires** for
|
|
40
|
+
`{kind:"kit", id:"builder"}` (AC2).
|
|
41
|
+
|
|
42
|
+
## `by_gate[]` — `gate_id: "unit tests pass"` (fire_count = 4)
|
|
43
|
+
|
|
44
|
+
Gate rows come only from `task-lr-pp-1..4`'s `gate-review.inquiries.json` (one row each);
|
|
45
|
+
`task-lr-pp-5`/`-6` contribute none (2 sessions without a gate-review join, per `notes`).
|
|
46
|
+
|
|
47
|
+
- `fire_count` = 4 (pp-r1 correct, pp-r2/pp-r3/pp-r4 false_block)
|
|
48
|
+
- `correct_count` = 1 (pp-r1)
|
|
49
|
+
- `false_block_count` = 3 (pp-r2, pp-r3, pp-r4)
|
|
50
|
+
- `missed_block_count` = 0
|
|
51
|
+
- `false_block_rate` = 3 / 4 = **0.75** ( ≥ `LR_GATE_FALSE_BLOCK_RATE` default 0.5 )
|
|
52
|
+
- `avg_wall_clock_s_when_fired` = mean of the parent record's `time.wall_clock_s` over the 4
|
|
53
|
+
distinct records with a `fired:true` row for this gate (pp-r1..pp-r4, all 100) = **100**
|
|
54
|
+
- `avg_human_wait_s_when_fired` = same, `time.human_wait_s` (all 10) = **10**
|
|
55
|
+
|
|
56
|
+
`fire_count (4) >= LR_MIN_GATE_SAMPLE (3)` AND `false_block_rate (0.75) >= LR_GATE_FALSE_BLOCK_RATE
|
|
57
|
+
(0.5)` → **`gate-false-block-review` fires** for `{kind:"gate", id:"unit tests pass"}` (AC2).
|
|
58
|
+
`gate-well-calibrated` does NOT fire (`false_block_count` is 3, not 0).
|
|
59
|
+
|
|
60
|
+
These are the exact numbers asserted (4-decimal-rounded per the contract's `round4`, though every
|
|
61
|
+
value here is already exact at 0–2 decimal places) in
|
|
62
|
+
`evals/integration/test_learning_review_proposals.sh` and recorded verbatim in
|
|
63
|
+
`expected-aggregates.json` — computed from the formulas in
|
|
64
|
+
`docs/specs/learning-review-proposals-contract.md`, independently of running
|
|
65
|
+
`scripts/telemetry/learning-review-proposals.sh` (then cross-checked against the script's actual
|
|
66
|
+
output as a confirmation, not a derivation).
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"id": "task-lr-pp-1-gr-1",
|
|
4
|
+
"inquiry": {
|
|
5
|
+
"id": "task-lr-pp-1-gr-1",
|
|
6
|
+
"question": "Was gate action on claim task-lr-pp-1/unit-tests justified?",
|
|
7
|
+
"askedBy": "gate-review",
|
|
8
|
+
"askedAt": "2026-07-03T00:05:00Z",
|
|
9
|
+
"target": { "subjectType": "workflow-check", "subjectId": "task-lr-pp-1/unit-tests", "fieldOrBehavior": "unit tests pass" }
|
|
10
|
+
},
|
|
11
|
+
"outcome": "matched",
|
|
12
|
+
"resolutionPath": { "claimIds": ["task-lr-pp-1-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass"] },
|
|
13
|
+
"answer": {
|
|
14
|
+
"status": "disputed",
|
|
15
|
+
"value": {
|
|
16
|
+
"calibration": "correct",
|
|
17
|
+
"advisoryFix": "No action needed; gate correctly blocked on a disputed claim.",
|
|
18
|
+
"gateFired": true,
|
|
19
|
+
"sessionSlug": "task-lr-pp-1"
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"inputSnapshot": [{ "claimId": "task-lr-pp-1-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass", "status": "disputed" }],
|
|
23
|
+
"statusFunctionVersion": "1",
|
|
24
|
+
"resolvedAt": "2026-07-03T00:05:00Z"
|
|
25
|
+
}
|
|
26
|
+
]
|
package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-1/trust.bundle
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 3,
|
|
3
|
+
"source": "flow-agents/workflow-sidecar;statusFunctionVersion=1",
|
|
4
|
+
"claims": [
|
|
5
|
+
{
|
|
6
|
+
"id": "task-lr-pp-1-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass",
|
|
7
|
+
"subjectType": "workflow-check",
|
|
8
|
+
"subjectId": "task-lr-pp-1/unit-tests",
|
|
9
|
+
"facet": "flow-agents.workflow",
|
|
10
|
+
"claimType": "builder.verify.tests",
|
|
11
|
+
"fieldOrBehavior": "unit tests pass",
|
|
12
|
+
"value": "pass",
|
|
13
|
+
"createdAt": "2026-07-03T00:00:00Z",
|
|
14
|
+
"updatedAt": "2026-07-03T00:00:00Z",
|
|
15
|
+
"status": "disputed"
|
|
16
|
+
}
|
|
17
|
+
],
|
|
18
|
+
"evidence": [],
|
|
19
|
+
"events": [],
|
|
20
|
+
"policies": []
|
|
21
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"id": "task-lr-pp-2-gr-1",
|
|
4
|
+
"inquiry": {
|
|
5
|
+
"id": "task-lr-pp-2-gr-1",
|
|
6
|
+
"question": "Was gate action on claim task-lr-pp-2/unit-tests justified?",
|
|
7
|
+
"askedBy": "gate-review",
|
|
8
|
+
"askedAt": "2026-07-03T00:05:00Z",
|
|
9
|
+
"target": { "subjectType": "workflow-check", "subjectId": "task-lr-pp-2/unit-tests", "fieldOrBehavior": "unit tests pass" }
|
|
10
|
+
},
|
|
11
|
+
"outcome": "matched",
|
|
12
|
+
"resolutionPath": { "claimIds": ["task-lr-pp-2-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass"] },
|
|
13
|
+
"answer": {
|
|
14
|
+
"status": "verified",
|
|
15
|
+
"value": {
|
|
16
|
+
"calibration": "false_block",
|
|
17
|
+
"advisoryFix": "Investigate why the gate blocked when the claim was already verified passing.",
|
|
18
|
+
"gateFired": true,
|
|
19
|
+
"sessionSlug": "task-lr-pp-2"
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"inputSnapshot": [{ "claimId": "task-lr-pp-2-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass", "status": "verified" }],
|
|
23
|
+
"statusFunctionVersion": "1",
|
|
24
|
+
"resolvedAt": "2026-07-03T00:05:00Z"
|
|
25
|
+
}
|
|
26
|
+
]
|
package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-2/trust.bundle
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 3,
|
|
3
|
+
"source": "flow-agents/workflow-sidecar;statusFunctionVersion=1",
|
|
4
|
+
"claims": [
|
|
5
|
+
{
|
|
6
|
+
"id": "task-lr-pp-2-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass",
|
|
7
|
+
"subjectType": "workflow-check",
|
|
8
|
+
"subjectId": "task-lr-pp-2/unit-tests",
|
|
9
|
+
"facet": "flow-agents.workflow",
|
|
10
|
+
"claimType": "builder.verify.tests",
|
|
11
|
+
"fieldOrBehavior": "unit tests pass",
|
|
12
|
+
"value": "pass",
|
|
13
|
+
"createdAt": "2026-07-03T00:00:00Z",
|
|
14
|
+
"updatedAt": "2026-07-03T00:00:00Z",
|
|
15
|
+
"status": "verified"
|
|
16
|
+
}
|
|
17
|
+
],
|
|
18
|
+
"evidence": [],
|
|
19
|
+
"events": [],
|
|
20
|
+
"policies": []
|
|
21
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"id": "task-lr-pp-3-gr-1",
|
|
4
|
+
"inquiry": {
|
|
5
|
+
"id": "task-lr-pp-3-gr-1",
|
|
6
|
+
"question": "Was gate action on claim task-lr-pp-3/unit-tests justified?",
|
|
7
|
+
"askedBy": "gate-review",
|
|
8
|
+
"askedAt": "2026-07-03T00:05:00Z",
|
|
9
|
+
"target": { "subjectType": "workflow-check", "subjectId": "task-lr-pp-3/unit-tests", "fieldOrBehavior": "unit tests pass" }
|
|
10
|
+
},
|
|
11
|
+
"outcome": "matched",
|
|
12
|
+
"resolutionPath": { "claimIds": ["task-lr-pp-3-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass"] },
|
|
13
|
+
"answer": {
|
|
14
|
+
"status": "verified",
|
|
15
|
+
"value": {
|
|
16
|
+
"calibration": "false_block",
|
|
17
|
+
"advisoryFix": "Investigate why the gate blocked when the claim was already verified passing.",
|
|
18
|
+
"gateFired": true,
|
|
19
|
+
"sessionSlug": "task-lr-pp-3"
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"inputSnapshot": [{ "claimId": "task-lr-pp-3-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass", "status": "verified" }],
|
|
23
|
+
"statusFunctionVersion": "1",
|
|
24
|
+
"resolvedAt": "2026-07-03T00:05:00Z"
|
|
25
|
+
}
|
|
26
|
+
]
|
package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-3/trust.bundle
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 3,
|
|
3
|
+
"source": "flow-agents/workflow-sidecar;statusFunctionVersion=1",
|
|
4
|
+
"claims": [
|
|
5
|
+
{
|
|
6
|
+
"id": "task-lr-pp-3-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass",
|
|
7
|
+
"subjectType": "workflow-check",
|
|
8
|
+
"subjectId": "task-lr-pp-3/unit-tests",
|
|
9
|
+
"facet": "flow-agents.workflow",
|
|
10
|
+
"claimType": "builder.verify.tests",
|
|
11
|
+
"fieldOrBehavior": "unit tests pass",
|
|
12
|
+
"value": "pass",
|
|
13
|
+
"createdAt": "2026-07-03T00:00:00Z",
|
|
14
|
+
"updatedAt": "2026-07-03T00:00:00Z",
|
|
15
|
+
"status": "verified"
|
|
16
|
+
}
|
|
17
|
+
],
|
|
18
|
+
"evidence": [],
|
|
19
|
+
"events": [],
|
|
20
|
+
"policies": []
|
|
21
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"id": "task-lr-pp-4-gr-1",
|
|
4
|
+
"inquiry": {
|
|
5
|
+
"id": "task-lr-pp-4-gr-1",
|
|
6
|
+
"question": "Was gate action on claim task-lr-pp-4/unit-tests justified?",
|
|
7
|
+
"askedBy": "gate-review",
|
|
8
|
+
"askedAt": "2026-07-03T00:05:00Z",
|
|
9
|
+
"target": { "subjectType": "workflow-check", "subjectId": "task-lr-pp-4/unit-tests", "fieldOrBehavior": "unit tests pass" }
|
|
10
|
+
},
|
|
11
|
+
"outcome": "matched",
|
|
12
|
+
"resolutionPath": { "claimIds": ["task-lr-pp-4-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass"] },
|
|
13
|
+
"answer": {
|
|
14
|
+
"status": "verified",
|
|
15
|
+
"value": {
|
|
16
|
+
"calibration": "false_block",
|
|
17
|
+
"advisoryFix": "Investigate why the gate blocked when the claim was already verified passing.",
|
|
18
|
+
"gateFired": true,
|
|
19
|
+
"sessionSlug": "task-lr-pp-4"
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"inputSnapshot": [{ "claimId": "task-lr-pp-4-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass", "status": "verified" }],
|
|
23
|
+
"statusFunctionVersion": "1",
|
|
24
|
+
"resolvedAt": "2026-07-03T00:05:00Z"
|
|
25
|
+
}
|
|
26
|
+
]
|
package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-4/trust.bundle
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 3,
|
|
3
|
+
"source": "flow-agents/workflow-sidecar;statusFunctionVersion=1",
|
|
4
|
+
"claims": [
|
|
5
|
+
{
|
|
6
|
+
"id": "task-lr-pp-4-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass",
|
|
7
|
+
"subjectType": "workflow-check",
|
|
8
|
+
"subjectId": "task-lr-pp-4/unit-tests",
|
|
9
|
+
"facet": "flow-agents.workflow",
|
|
10
|
+
"claimType": "builder.verify.tests",
|
|
11
|
+
"fieldOrBehavior": "unit tests pass",
|
|
12
|
+
"value": "pass",
|
|
13
|
+
"createdAt": "2026-07-03T00:00:00Z",
|
|
14
|
+
"updatedAt": "2026-07-03T00:00:00Z",
|
|
15
|
+
"status": "verified"
|
|
16
|
+
}
|
|
17
|
+
],
|
|
18
|
+
"evidence": [],
|
|
19
|
+
"events": [],
|
|
20
|
+
"policies": []
|
|
21
|
+
}
|
package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-5/trust.bundle
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 3,
|
|
3
|
+
"source": "flow-agents/workflow-sidecar;statusFunctionVersion=1",
|
|
4
|
+
"claims": [
|
|
5
|
+
{
|
|
6
|
+
"id": "task-lr-pp-5-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass",
|
|
7
|
+
"subjectType": "workflow-check",
|
|
8
|
+
"subjectId": "task-lr-pp-5/unit-tests",
|
|
9
|
+
"facet": "flow-agents.workflow",
|
|
10
|
+
"claimType": "builder.verify.tests",
|
|
11
|
+
"fieldOrBehavior": "unit tests pass",
|
|
12
|
+
"value": "pass",
|
|
13
|
+
"createdAt": "2026-07-03T00:00:00Z",
|
|
14
|
+
"updatedAt": "2026-07-03T00:00:00Z",
|
|
15
|
+
"status": "verified"
|
|
16
|
+
}
|
|
17
|
+
],
|
|
18
|
+
"evidence": [],
|
|
19
|
+
"events": [],
|
|
20
|
+
"policies": []
|
|
21
|
+
}
|
package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-6/trust.bundle
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 3,
|
|
3
|
+
"source": "flow-agents/workflow-sidecar;statusFunctionVersion=1",
|
|
4
|
+
"claims": [
|
|
5
|
+
{
|
|
6
|
+
"id": "task-lr-pp-6-gate-claim-unit-tests.flow-agents-workflow.unit-tests-pass",
|
|
7
|
+
"subjectType": "workflow-check",
|
|
8
|
+
"subjectId": "task-lr-pp-6/unit-tests",
|
|
9
|
+
"facet": "flow-agents.workflow",
|
|
10
|
+
"claimType": "builder.verify.tests",
|
|
11
|
+
"fieldOrBehavior": "unit tests pass",
|
|
12
|
+
"value": "pass",
|
|
13
|
+
"createdAt": "2026-07-03T00:00:00Z",
|
|
14
|
+
"updatedAt": "2026-07-03T00:00:00Z",
|
|
15
|
+
"status": "verified"
|
|
16
|
+
}
|
|
17
|
+
],
|
|
18
|
+
"evidence": [],
|
|
19
|
+
"events": [],
|
|
20
|
+
"policies": []
|
|
21
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"rw-r1","at":1751800000000,"task_slug":null,"cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.10,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":0,"verification_verdict":"PASS"}}
|
|
2
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"rw-r2","at":1751800001000,"task_slug":null,"cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.10,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":0,"verification_verdict":"PASS"}}
|
|
3
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"rw-r3","at":1751800002000,"task_slug":null,"cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.10,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":0,"verification_verdict":"PASS"}}
|
|
4
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"rw-r4","at":1751800003000,"task_slug":null,"cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.30,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":0,"verification_verdict":"PASS"}}
|
|
5
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"rw-r5","at":1751800004000,"task_slug":null,"cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.30,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":0,"verification_verdict":"PASS"}}
|
|
6
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"rw-r6","at":1751800005000,"task_slug":null,"cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.30,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":0,"verification_verdict":"PASS"}}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"ut-r1","at":1751700000000,"task_slug":null,"cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.10,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":0,"verification_verdict":"PASS"}}
|
|
2
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"ut-r2","at":1751700001000,"task_slug":null,"cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.20,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":0,"verification_verdict":"PASS"}}
|
|
3
|
+
{"schema":"kontour.console.economics","version":"0.1","run_id":"ut-r3","at":1751700002000,"task_slug":null,"cost":{"input_tokens":1000,"output_tokens":500,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"estimated_cost_usd":0.30,"by_model":[]},"time":{"wall_clock_s":100,"human_wait_s":10},"iterations":{"count":2,"route_backs":1},"defects":{"gate_fires":1,"findings_by_severity":{"critical":0,"high":1,"medium":0,"low":1},"caught_false_completions":0,"verification_verdict":"PASS"}}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 5,
|
|
3
|
+
"source": "reconcile-preflight-fixture:disputed-critique-unsuperseded",
|
|
4
|
+
"note": "#356 shape 3: an un-superseded workflow.critique.review claim with a 'fail' verdict derives status 'disputed' — session-local-failed divergence (a disputed critique always blocks reconcile unless a later same-reviewer pass superseded it via metadata.superseded_by).",
|
|
5
|
+
"claims": [
|
|
6
|
+
{
|
|
7
|
+
"id": "c-disputed-critique",
|
|
8
|
+
"claimType": "workflow.critique.review",
|
|
9
|
+
"value": "fail",
|
|
10
|
+
"status": "disputed",
|
|
11
|
+
"subjectId": "reconcile-preflight-fixture/review-1",
|
|
12
|
+
"facet": "flow-agents.workflow",
|
|
13
|
+
"subjectType": "workflow-critique",
|
|
14
|
+
"fieldOrBehavior": "Found a critical security issue in the auth handler.",
|
|
15
|
+
"createdAt": "2026-07-04T00:00:00Z",
|
|
16
|
+
"updatedAt": "2026-07-04T00:00:00Z",
|
|
17
|
+
"impactLevel": "medium",
|
|
18
|
+
"verificationPolicyId": "policy:workflow.critique.review",
|
|
19
|
+
"metadata": { "origin": "critique", "reviewer": "tool-code-reviewer", "reviewed_at": "2026-07-04T00:00:00Z" }
|
|
20
|
+
}
|
|
21
|
+
],
|
|
22
|
+
"evidence": [],
|
|
23
|
+
"policies": [
|
|
24
|
+
{
|
|
25
|
+
"id": "policy:workflow.critique.review",
|
|
26
|
+
"claimType": "workflow.critique.review",
|
|
27
|
+
"requiredEvidence": [],
|
|
28
|
+
"acceptanceCriteria": ["A verified verification event must support a workflow.critique.review claim."],
|
|
29
|
+
"reviewAuthority": "system",
|
|
30
|
+
"validityRule": { "kind": "manual" },
|
|
31
|
+
"stalenessTriggers": [],
|
|
32
|
+
"conflictRules": [],
|
|
33
|
+
"impactLevel": "medium"
|
|
34
|
+
}
|
|
35
|
+
],
|
|
36
|
+
"events": [
|
|
37
|
+
{
|
|
38
|
+
"id": "evt-disputed-critique",
|
|
39
|
+
"claimId": "c-disputed-critique",
|
|
40
|
+
"status": "disputed",
|
|
41
|
+
"actor": "flow-agents/workflow-sidecar",
|
|
42
|
+
"method": "validation",
|
|
43
|
+
"evidenceIds": [],
|
|
44
|
+
"createdAt": "2026-07-04T00:00:00Z",
|
|
45
|
+
"verifiedAt": "2026-07-04T00:00:00Z"
|
|
46
|
+
}
|
|
47
|
+
]
|
|
48
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 5,
|
|
3
|
+
"source": "reconcile-preflight-fixture:standalone-disputed-session-local",
|
|
4
|
+
"note": "#356 shape 4 (the #292/#384 variant): a NON-critique session-local claim (workflow.check.external, human_attestation-backed) with a re-derived status of 'disputed' and no waiver. Distinct from shape 3 (an un-superseded disputed CRITIQUE) only in claimType/subject — both land on the same session-local-failed divergence, proving a disjoint pre-existing failure can never be smuggled in as a standalone claim; the fix hint directs the agent to document it as prose in a waived non-command summary instead.",
|
|
5
|
+
"claims": [
|
|
6
|
+
{
|
|
7
|
+
"id": "c-standalone-disputed",
|
|
8
|
+
"claimType": "workflow.check.external",
|
|
9
|
+
"value": "fail",
|
|
10
|
+
"status": "disputed",
|
|
11
|
+
"subjectId": "reconcile-preflight-fixture/pre-existing-flaky-integration",
|
|
12
|
+
"facet": "flow-agents.workflow",
|
|
13
|
+
"subjectType": "workflow-check",
|
|
14
|
+
"fieldOrBehavior": "pre-existing integration test failure unrelated to this change",
|
|
15
|
+
"createdAt": "2026-07-04T00:00:00Z",
|
|
16
|
+
"updatedAt": "2026-07-04T00:00:00Z",
|
|
17
|
+
"impactLevel": "high",
|
|
18
|
+
"verificationPolicyId": "policy:workflow.check.external:human_attestation"
|
|
19
|
+
}
|
|
20
|
+
],
|
|
21
|
+
"evidence": [
|
|
22
|
+
{
|
|
23
|
+
"id": "ev-standalone-disputed",
|
|
24
|
+
"claimId": "c-standalone-disputed",
|
|
25
|
+
"evidenceType": "human_attestation",
|
|
26
|
+
"method": "attestation",
|
|
27
|
+
"sourceRef": "reconcile-preflight-fixture/evidence.json",
|
|
28
|
+
"excerptOrSummary": "pre-existing integration test failure, unrelated to this change, observed but not fixed",
|
|
29
|
+
"observedAt": "2026-07-04T00:00:00Z",
|
|
30
|
+
"collectedBy": "flow-agents/workflow-sidecar",
|
|
31
|
+
"passing": false
|
|
32
|
+
}
|
|
33
|
+
],
|
|
34
|
+
"policies": [
|
|
35
|
+
{
|
|
36
|
+
"id": "policy:workflow.check.external:human_attestation",
|
|
37
|
+
"claimType": "workflow.check.external",
|
|
38
|
+
"requiredEvidence": ["human_attestation"],
|
|
39
|
+
"acceptanceCriteria": ["A verified verification event must support a workflow.check.external claim."],
|
|
40
|
+
"reviewAuthority": "system",
|
|
41
|
+
"validityRule": { "kind": "manual" },
|
|
42
|
+
"stalenessTriggers": [],
|
|
43
|
+
"conflictRules": [],
|
|
44
|
+
"impactLevel": "high"
|
|
45
|
+
}
|
|
46
|
+
],
|
|
47
|
+
"events": [
|
|
48
|
+
{
|
|
49
|
+
"id": "evt-standalone-disputed",
|
|
50
|
+
"claimId": "c-standalone-disputed",
|
|
51
|
+
"status": "disputed",
|
|
52
|
+
"actor": "flow-agents/workflow-sidecar",
|
|
53
|
+
"method": "attestation",
|
|
54
|
+
"evidenceIds": ["ev-standalone-disputed"],
|
|
55
|
+
"createdAt": "2026-07-04T00:00:00Z",
|
|
56
|
+
"verifiedAt": "2026-07-04T00:00:00Z"
|
|
57
|
+
}
|
|
58
|
+
]
|
|
59
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
{"type":"user","message":{"role":"user","content":[{"type":"text","text":"hermetic fixture prompt"}]}}
|
|
2
|
+
{"type":"assistant","message":{"model":"claude-opus-4-8","usage":{"input_tokens":1200,"output_tokens":3400,"cache_creation_input_tokens":500,"cache_read_input_tokens":600000}}}
|
|
3
|
+
{"type":"assistant","message":{"model":"claude-opus-4-8","usage":{"input_tokens":300,"output_tokens":900,"cache_creation_input_tokens":0,"cache_read_input_tokens":0}}}
|
|
4
|
+
{"type":"assistant","message":{"model":"claude-fable-5","usage":{"input_tokens":50,"output_tokens":200,"cache_creation_input_tokens":0,"cache_read_input_tokens":0}}}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 5,
|
|
3
|
+
"source": "ws8-exploit-fixture:mcp-degrade",
|
|
4
|
+
"note": "ISSUE 492 DELIVERABLE 2: an MCP-shaped self-reported pass must not launder into a CI-verified kind:'command' check. The evidence deliberately carries a test_output execution.label that names an MCP tool, not a run-baseline.sh manifest command. CI must reject it as not-run / not-in-manifest; at most this kind of non-native pass can remain a session-local external attestation, never RECONCILED.",
|
|
5
|
+
"claims": [
|
|
6
|
+
{
|
|
7
|
+
"id": "c-mcp-degrade-test",
|
|
8
|
+
"claimType": "workflow.check.test",
|
|
9
|
+
"value": "pass",
|
|
10
|
+
"status": "verified",
|
|
11
|
+
"subjectId": "mcp/unit-tests",
|
|
12
|
+
"facet": "flow-agents.workflow",
|
|
13
|
+
"subjectType": "workflow-check",
|
|
14
|
+
"fieldOrBehavior": "unit tests passed via MCP runner",
|
|
15
|
+
"createdAt": "2026-07-07T00:00:00Z",
|
|
16
|
+
"updatedAt": "2026-07-07T00:00:00Z",
|
|
17
|
+
"impactLevel": "high",
|
|
18
|
+
"verificationPolicyId": "policy:workflow.check.test"
|
|
19
|
+
}
|
|
20
|
+
],
|
|
21
|
+
"evidence": [
|
|
22
|
+
{
|
|
23
|
+
"id": "ev-mcp-degrade-test",
|
|
24
|
+
"claimId": "c-mcp-degrade-test",
|
|
25
|
+
"evidenceType": "test_output",
|
|
26
|
+
"method": "validation",
|
|
27
|
+
"sourceRef": "mcp://runner/run_tests",
|
|
28
|
+
"excerptOrSummary": "MCP runner reported tests passed",
|
|
29
|
+
"observedAt": "2026-07-07T00:00:00Z",
|
|
30
|
+
"collectedBy": "mcp__runner__run_tests",
|
|
31
|
+
"passing": true,
|
|
32
|
+
"execution": {
|
|
33
|
+
"runner": "mcp",
|
|
34
|
+
"label": "mcp__runner__run_tests",
|
|
35
|
+
"isError": false,
|
|
36
|
+
"exitCode": 0
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
],
|
|
40
|
+
"policies": [],
|
|
41
|
+
"events": []
|
|
42
|
+
}
|