@kontourai/flow-agents 3.3.0 → 3.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/add-to-project.yml +15 -0
- package/.github/workflows/ci.yml +161 -0
- package/CHANGELOG.md +48 -0
- package/CONTEXT.md +5 -1
- package/README.md +19 -8
- package/build/src/builder-flow-run-adapter.d.ts +80 -0
- package/build/src/builder-flow-run-adapter.js +241 -0
- package/build/src/builder-flow-runtime.d.ts +16 -0
- package/build/src/builder-flow-runtime.js +290 -0
- package/build/src/cli/builder-run.d.ts +1 -0
- package/build/src/cli/builder-run.js +27 -0
- package/build/src/cli/effective-backlog-settings.js +70 -2
- package/build/src/cli/init.d.ts +34 -0
- package/build/src/cli/init.js +341 -61
- package/build/src/cli/kit.js +55 -12
- package/build/src/cli/pull-work-provider.js +346 -5
- package/build/src/cli/skill-drift-check.d.ts +1 -0
- package/build/src/cli/skill-drift-check.js +165 -0
- package/build/src/cli/telemetry-doctor.d.ts +37 -0
- package/build/src/cli/telemetry-doctor.js +53 -6
- package/build/src/cli/validate-hook-influence.js +37 -7
- package/build/src/cli/workflow-sidecar.d.ts +93 -8
- package/build/src/cli/workflow-sidecar.js +1175 -158
- package/build/src/cli.js +5 -0
- package/build/src/flow-kit/validate.d.ts +54 -34
- package/build/src/flow-kit/validate.js +237 -26
- package/build/src/index.d.ts +2 -0
- package/build/src/index.js +1 -0
- package/build/src/lib/console-connect-options.d.ts +97 -0
- package/build/src/lib/console-connect-options.js +199 -0
- package/build/src/lib/console-telemetry-validate.d.ts +49 -0
- package/build/src/lib/console-telemetry-validate.js +91 -0
- package/build/src/lib/flow-resolver.d.ts +56 -3
- package/build/src/lib/flow-resolver.js +151 -11
- package/build/src/lib/fs.d.ts +17 -0
- package/build/src/lib/fs.js +172 -0
- package/build/src/lib/local-artifact-root.d.ts +44 -1
- package/build/src/lib/local-artifact-root.js +131 -3
- package/build/src/runtime-adapters.d.ts +39 -3
- package/build/src/runtime-adapters.js +77 -31
- package/build/src/tools/build-universal-bundles.js +40 -2
- package/build/src/tools/codex-agent-routing.d.ts +2 -0
- package/build/src/tools/codex-agent-routing.js +49 -0
- package/build/src/tools/generate-context-map.js +1 -0
- package/build/src/tools/validate-source-tree.js +27 -1
- package/context/scripts/hooks/lib/kit-catalog.js +235 -0
- package/context/scripts/hooks/lib/runnable-command.js +177 -0
- package/context/scripts/hooks/stop-goal-fit.js +278 -48
- package/context/scripts/hooks/workflow-steering.js +121 -21
- package/context/scripts/package.json +3 -0
- package/context/scripts/telemetry/install-console-config.sh +25 -4
- package/context/scripts/telemetry/lib/config.sh +102 -12
- package/context/scripts/telemetry/lib/pricing.sh +50 -0
- package/context/scripts/telemetry/lib/session.sh +3 -0
- package/context/scripts/telemetry/lib/transport.sh +87 -0
- package/context/scripts/telemetry/lib/usage.sh +205 -4
- package/context/scripts/telemetry/telemetry.conf +6 -0
- package/context/scripts/telemetry/telemetry.sh +48 -0
- package/context/settings/workspace-backlog-provider-settings.example.json +48 -0
- package/docs/agent-usage-feedback-loop.md +35 -0
- package/docs/architecture-engine-and-kits.md +110 -0
- package/docs/context-map.md +2 -0
- package/docs/decisions/embeddable-engine.md +152 -0
- package/docs/decisions/index.md +3 -1
- package/docs/decisions/trust-ledger-retention.md +88 -0
- package/docs/decisions/workflow-enforcement.md +31 -9
- package/docs/fixture-ownership.md +3 -0
- package/docs/implementing-trust-reconciliation.md +129 -0
- package/docs/index.md +19 -9
- package/docs/integrations/flow-agents-console.md +167 -0
- package/docs/kit-authoring-guide.md +52 -21
- package/docs/spec/builder-flow-runtime.md +80 -0
- package/docs/spec/runtime-hook-surface.md +45 -1
- package/docs/specs/economics-record-contract.md +270 -0
- package/docs/specs/harness-capability-matrix.md +74 -0
- package/docs/specs/learning-review-proposals-contract.md +340 -0
- package/docs/specs/routing-efficiency-review.md +59 -0
- package/docs/verifiable-trust.md +74 -25
- package/docs/workflow-usage-guide.md +10 -0
- package/evals/acceptance/prove-capture-teeth.sh +132 -0
- package/evals/ci/antigaming-suite.sh +1 -0
- package/evals/ci/run-baseline.sh +72 -4
- package/evals/fixtures/economics/acceptance.json +12 -0
- package/evals/fixtures/economics/agents/tool-worker-1/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-2/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-3/events.jsonl +2 -0
- package/evals/fixtures/economics/agents/tool-worker-4/events.jsonl +1 -0
- package/evals/fixtures/economics/agents/tool-worker-5/events.jsonl +2 -0
- package/evals/fixtures/economics/critique.json +22 -0
- package/evals/fixtures/economics/expected-record.json +71 -0
- package/evals/fixtures/economics/session-usage-event.json +1 -0
- package/evals/fixtures/economics/state.json +11 -0
- package/evals/fixtures/economics/transcript.jsonl +3 -0
- package/evals/fixtures/hook-influence/cases.json +7 -7
- package/evals/fixtures/learning-review-proposals/balanced/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/economics.jsonl +5 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-1/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-2/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-3/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-4/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/effect-follow-up/sessions/task-lr-ef-5/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/expected-aggregates.json +30 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/expected-aggregates.md +66 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-1/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-1/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-2/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-2/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-3/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-3/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-4/gate-review.inquiries.json +26 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-4/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-5/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/pattern-present/sessions/task-lr-pp-6/trust.bundle +21 -0
- package/evals/fixtures/learning-review-proposals/repeat-window/economics.jsonl +6 -0
- package/evals/fixtures/learning-review-proposals/under-threshold/economics.jsonl +3 -0
- package/evals/fixtures/telemetry/usage-transcript-sample.jsonl +4 -0
- package/evals/fixtures/trust-reconcile-exploits/mcp-degrade.json +42 -0
- package/evals/integration/test_builder_entry_enforcement.sh +241 -0
- package/evals/integration/test_builder_step_producers.sh +18 -10
- package/evals/integration/test_bundle_install.sh +172 -0
- package/evals/integration/test_console_tenant_isolation.sh +167 -0
- package/evals/integration/test_critique_supersession_roundtrip.sh +4 -1
- package/evals/integration/test_dual_emit_flow_step.sh +10 -4
- package/evals/integration/test_economics_record.sh +674 -0
- package/evals/integration/test_effective_backlog_settings.sh +1 -1
- package/evals/integration/test_evidence_capture_hook.sh +17 -2
- package/evals/integration/test_exemption_usage_review.sh +198 -0
- package/evals/integration/test_fixture_retirement_audit.sh +2 -2
- package/evals/integration/test_flow_kit_install_git.sh +83 -0
- package/evals/integration/test_flowdef_session_activation.sh +0 -1
- package/evals/integration/test_flowdef_session_history_preservation.sh +13 -3
- package/evals/integration/test_gate_lockdown.sh +7 -0
- package/evals/integration/test_gate_review_inquiry_records.sh +9 -1
- package/evals/integration/test_goal_fit_hook.sh +2031 -0
- package/evals/integration/test_hook_category_behaviors.sh +8 -1
- package/evals/integration/test_hook_influence_cases.sh +25 -1
- package/evals/integration/test_install_merge.sh +227 -2
- package/evals/integration/test_kit_conformance_levels.sh +6 -6
- package/evals/integration/test_learning_review_proposals.sh +329 -0
- package/evals/integration/test_liveness_conflict_injection.sh +26 -22
- package/evals/integration/test_liveness_console_relay.sh +166 -0
- package/evals/integration/test_liveness_heartbeat.sh +17 -17
- package/evals/integration/test_liveness_worktree_root.sh +575 -0
- package/evals/integration/test_phase_map_and_gate_claim.sh +6 -1
- package/evals/integration/test_publish_delivery.sh +331 -1
- package/evals/integration/test_pull_work_board.sh +200 -0
- package/evals/integration/test_pull_work_provider.sh +1 -1
- package/evals/integration/test_record_check.sh +378 -0
- package/evals/integration/test_routing_efficiency.sh +71 -0
- package/evals/integration/test_runtime_adapter_activation.sh +28 -0
- package/evals/integration/test_session_resume_roundtrip.sh +16 -19
- package/evals/integration/test_skill_drift_check.sh +870 -0
- package/evals/integration/test_telemetry.sh +445 -0
- package/evals/integration/test_telemetry_doctor.sh +66 -0
- package/evals/integration/test_telemetry_usage_pipeline.sh +228 -0
- package/evals/integration/test_trust_reconcile_negatives.sh +30 -13
- package/evals/integration/test_trust_reconcile_trailer_diagnostic.sh +247 -0
- package/evals/integration/test_usage_cost.sh +61 -0
- package/evals/integration/test_workflow_sidecar_writer.sh +1395 -0
- package/evals/integration/test_workflow_steering_hook.sh +157 -16
- package/evals/integration/test_workspace_settings.sh +176 -0
- package/evals/lib/env.sh +26 -0
- package/evals/lib/node.sh +8 -0
- package/evals/run.sh +29 -0
- package/evals/static/test_ci_integration_coverage.sh +115 -0
- package/evals/static/test_declared_scope_forms_documented.sh +114 -0
- package/evals/static/test_universal_bundles.sh +34 -0
- package/evals/static/test_validate_source_kit_asset_scope.sh +259 -0
- package/evals/static/test_workflow_skills.sh +1 -1
- package/kits/builder/flows/build.flow.json +9 -18
- package/kits/builder/flows/publish-learn.flow.json +5 -1
- package/kits/builder/kit.json +120 -0
- package/kits/builder/skills/deliver/SKILL.md +42 -0
- package/kits/builder/skills/evidence-gate/SKILL.md +12 -0
- package/kits/builder/skills/execute-plan/SKILL.md +9 -0
- package/kits/builder/skills/learning-review/SKILL.md +51 -0
- package/kits/builder/skills/plan-work/SKILL.md +17 -20
- package/kits/builder/skills/pull-work/SKILL.md +21 -0
- package/kits/builder/skills/release-readiness/SKILL.md +12 -0
- package/kits/knowledge/kit.json +9 -0
- package/kits/veritas-governance/docs/README.md +35 -7
- package/kits/veritas-governance/fixtures/exemption-review/mixed-fresh-stale.DECLARED.json +14 -0
- package/kits/veritas-governance/kit.json +14 -0
- package/kits/veritas-governance/skills/exemption-usage-review/SKILL.md +128 -0
- package/kits/veritas-governance/skills/exemption-usage-review/review-exemptions.mjs +231 -0
- package/package.json +2 -2
- package/packaging/manifest.json +29 -0
- package/schemas/backlog-provider-settings.schema.json +13 -0
- package/schemas/workflow-state.schema.json +44 -0
- package/scripts/README.md +4 -0
- package/scripts/check-content-boundary.cjs +8 -1
- package/scripts/ci/trust-reconcile.js +136 -0
- package/scripts/hooks/codex-hook-adapter.js +77 -2
- package/scripts/hooks/evidence-capture.js +38 -5
- package/scripts/hooks/lib/codex-exit-code.js +316 -0
- package/scripts/hooks/lib/kit-catalog.js +235 -0
- package/scripts/hooks/lib/liveness-write.js +28 -1
- package/scripts/hooks/lib/local-artifact-paths.js +97 -1
- package/scripts/hooks/lib/runnable-command.js +177 -0
- package/scripts/hooks/lib/skill-drift.js +350 -0
- package/scripts/hooks/stop-goal-fit.js +278 -48
- package/scripts/hooks/workflow-steering.js +121 -21
- package/scripts/install-codex-home.sh +97 -47
- package/scripts/install-merge.js +72 -14
- package/scripts/install-owned-files.js +178 -0
- package/scripts/liveness/relay.sh +84 -0
- package/scripts/telemetry/economics-record.schema.json +145 -0
- package/scripts/telemetry/economics-record.sh +331 -0
- package/scripts/telemetry/install-console-config.sh +25 -4
- package/scripts/telemetry/learning-review-decide.sh +124 -0
- package/scripts/telemetry/learning-review-proposals.schema.json +161 -0
- package/scripts/telemetry/learning-review-proposals.sh +484 -0
- package/scripts/telemetry/lib/config.sh +102 -12
- package/scripts/telemetry/lib/pricing.sh +14 -6
- package/scripts/telemetry/lib/session.sh +3 -0
- package/scripts/telemetry/lib/transport.sh +133 -15
- package/scripts/telemetry/lib/usage.sh +121 -28
- package/scripts/telemetry/routing-efficiency.sh +0 -0
- package/scripts/telemetry/telemetry.conf +6 -0
- package/scripts/telemetry/telemetry.sh +48 -0
- package/src/builder-flow-run-adapter.ts +357 -0
- package/src/builder-flow-runtime.ts +348 -0
- package/src/cli/builder-flow-run-adapter.test.mjs +495 -0
- package/src/cli/builder-flow-runtime.test.mjs +213 -0
- package/src/cli/builder-run.ts +28 -0
- package/src/cli/codex-agent-routing.test.mjs +44 -0
- package/src/cli/codex-exit-code.test.mjs +207 -0
- package/src/cli/console-connect-options.test.mjs +329 -0
- package/src/cli/console-telemetry-validate.test.mjs +157 -0
- package/src/cli/effective-backlog-settings.ts +68 -2
- package/src/cli/flow-resolver-composition.test.mjs +101 -0
- package/src/cli/init.test.mjs +161 -0
- package/src/cli/init.ts +407 -62
- package/src/cli/kit-metadata-security.test.mjs +443 -0
- package/src/cli/kit.ts +50 -12
- package/src/cli/pull-work-provider.ts +377 -3
- package/src/cli/sidecar-pure-helpers.test.mjs +64 -0
- package/src/cli/skill-drift-check.ts +196 -0
- package/src/cli/telemetry-doctor.test.mjs +53 -0
- package/src/cli/telemetry-doctor.ts +50 -7
- package/src/cli/validate-hook-influence.ts +37 -6
- package/src/cli/workflow-sidecar.ts +1150 -151
- package/src/cli.ts +5 -0
- package/src/flow-kit/validate.ts +277 -38
- package/src/index.ts +19 -0
- package/src/lib/console-connect-options.ts +261 -0
- package/src/lib/console-telemetry-validate.ts +88 -0
- package/src/lib/flow-resolver.ts +153 -10
- package/src/lib/fs.ts +160 -0
- package/src/lib/local-artifact-root.ts +129 -3
- package/src/runtime-adapters.ts +113 -33
- package/src/tools/build-universal-bundles.ts +36 -2
- package/src/tools/codex-agent-routing.ts +48 -0
- package/src/tools/generate-context-map.ts +1 -0
- package/src/tools/validate-source-tree.ts +26 -1
|
@@ -3,6 +3,7 @@ import fs from "node:fs";
|
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { loadJson, readText, root, walkFiles, writeText } from "./common.js";
|
|
6
|
+
import { CODEX_REASONING_EFFORTS, codexAgentRoutingErrors } from "./codex-agent-routing.js";
|
|
6
7
|
import { DURABLE_FLOW_AGENTS_DIR, FLOW_AGENTS_RUNTIME_DIR } from "../lib/local-artifact-root.js";
|
|
7
8
|
|
|
8
9
|
type Agent = Record<string, unknown> & { name: string; prompt: string };
|
|
@@ -188,8 +189,36 @@ function mapped(mapName: string, value: unknown): string {
|
|
|
188
189
|
const map = manifest[mapName];
|
|
189
190
|
return map[modelPrefix(String(value ?? ""))] ?? map.default;
|
|
190
191
|
}
|
|
192
|
+
function codexAgentOverride(spec: Agent): Record<string, unknown> | undefined {
|
|
193
|
+
const override = manifest.codex_agent_map?.[spec.name];
|
|
194
|
+
if (override === undefined) return undefined;
|
|
195
|
+
if (!override || typeof override !== "object" || Array.isArray(override)) {
|
|
196
|
+
throw new Error(`packaging/manifest.json: codex_agent_map.${spec.name} must be an object`);
|
|
197
|
+
}
|
|
198
|
+
const keys = Object.keys(override).sort();
|
|
199
|
+
if (keys.join(",") !== "model,reasoning_effort") {
|
|
200
|
+
throw new Error(`packaging/manifest.json: codex_agent_map.${spec.name} must contain exactly model and reasoning_effort`);
|
|
201
|
+
}
|
|
202
|
+
return override;
|
|
203
|
+
}
|
|
204
|
+
function validateCodexModel(model: string, context: string): string {
|
|
205
|
+
const allowed = new Set(manifest.codex?.allowed_agent_models ?? []);
|
|
206
|
+
if (!allowed.has(model)) throw new Error(`packaging/manifest.json: ${context} uses unsupported Codex agent model '${model}'`);
|
|
207
|
+
return model;
|
|
208
|
+
}
|
|
191
209
|
function resolveCodexModel(spec: Agent): string {
|
|
192
|
-
|
|
210
|
+
const override = codexAgentOverride(spec)?.model;
|
|
211
|
+
if (typeof override === "string" && override) return validateCodexModel(override, `codex_agent_map.${spec.name}.model`);
|
|
212
|
+
const fallback = ["tool-agent-delegate", "tool-agent-handoff"].includes(spec.name) ? "gpt-5.4-mini" : mapped("codex_model_map", spec.model);
|
|
213
|
+
return validateCodexModel(fallback, `resolved model for ${spec.name}`);
|
|
214
|
+
}
|
|
215
|
+
function resolveCodexReasoningEffort(spec: Agent): string {
|
|
216
|
+
const override = codexAgentOverride(spec)?.reasoning_effort;
|
|
217
|
+
const effort = typeof override === "string" && override ? override : mapped("codex_reasoning_map", spec.model);
|
|
218
|
+
if (!CODEX_REASONING_EFFORTS.has(effort)) {
|
|
219
|
+
throw new Error(`packaging/manifest.json: resolved reasoning effort for ${spec.name} is invalid: '${effort}'`);
|
|
220
|
+
}
|
|
221
|
+
return effort;
|
|
193
222
|
}
|
|
194
223
|
function normalizeCodexText(text: string): string {
|
|
195
224
|
const replacements: Record<string, string> = { "—": "-", "–": "-", "→": "->", "←": "<-", "’": "'", "“": '"', "”": '"', "⚡": "[quick]", "🖥️": "[interactive]" };
|
|
@@ -230,7 +259,7 @@ function codexNicknames(agentName: string): string[] {
|
|
|
230
259
|
function exportCodexAgent(spec: Agent): string {
|
|
231
260
|
const prompt = appendExportNote(normalizeCodexText(sanitizeText(spec.prompt, "codex", "<bundle-root>")), "Exported from a Kiro agent spec. Hooks, resource auto-loading, and tool allowlists were converted into prompt guidance only. Any explicit Kiro scheduler commands are preserved as optional shell workflows.");
|
|
232
261
|
const description = normalizeCodexText(String(spec.description ?? "")).replace(/"/g, "'");
|
|
233
|
-
return `# exported from agents/${spec.name}.json\nname = "${spec.name}"\nnickname_candidates = ${JSON.stringify(codexNicknames(spec.name))}\ndescription = "${description}"\nmodel = "${resolveCodexModel(spec)}"\nmodel_reasoning_effort = "${
|
|
262
|
+
return `# exported from agents/${spec.name}.json\nname = "${spec.name}"\nnickname_candidates = ${JSON.stringify(codexNicknames(spec.name))}\ndescription = "${description}"\nmodel = "${resolveCodexModel(spec)}"\nmodel_reasoning_effort = "${resolveCodexReasoningEffort(spec)}"\ndeveloper_instructions = ${JSON.stringify(prompt)}\n`;
|
|
234
263
|
}
|
|
235
264
|
function tomlValue(value: unknown): string {
|
|
236
265
|
if (typeof value === "boolean") return value ? "true" : "false";
|
|
@@ -733,6 +762,11 @@ export function main(): number {
|
|
|
733
762
|
return 1;
|
|
734
763
|
}
|
|
735
764
|
const agents = fs.readdirSync(path.join(root, "agents")).filter((name) => name.endsWith(".json")).sort().map((name) => loadJson<Agent>(path.join(root, "agents", name)));
|
|
765
|
+
const routingErrors = codexAgentRoutingErrors(manifest, agents.map((agent) => agent.name));
|
|
766
|
+
if (routingErrors.length) {
|
|
767
|
+
for (const error of routingErrors) console.error(`flow-agents: packaging/manifest.json: ${error}`);
|
|
768
|
+
return 1;
|
|
769
|
+
}
|
|
736
770
|
buildBase(agents);
|
|
737
771
|
buildKiro(agents);
|
|
738
772
|
buildClaudeCode(agents);
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
export const CODEX_REASONING_EFFORTS = new Set(["low", "medium", "high", "xhigh", "max", "ultra"]);
|
|
2
|
+
|
|
3
|
+
export function codexAgentRoutingErrors(manifest: any, agentNames: Iterable<string>): string[] {
|
|
4
|
+
const errors: string[] = [];
|
|
5
|
+
const knownAgents = new Set(agentNames);
|
|
6
|
+
const allowedModels = manifest.codex?.allowed_agent_models;
|
|
7
|
+
if (!Array.isArray(allowedModels) || allowedModels.length === 0) {
|
|
8
|
+
errors.push("codex.allowed_agent_models must be a non-empty list");
|
|
9
|
+
}
|
|
10
|
+
const allowedModelSet = new Set(Array.isArray(allowedModels) ? allowedModels : []);
|
|
11
|
+
for (const model of allowedModelSet) {
|
|
12
|
+
if (typeof model !== "string" || !model) errors.push("codex.allowed_agent_models entries must be non-empty strings");
|
|
13
|
+
}
|
|
14
|
+
if (allowedModelSet.size !== (Array.isArray(allowedModels) ? allowedModels.length : 0)) {
|
|
15
|
+
errors.push("codex.allowed_agent_models must not contain duplicates");
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
for (const [family, model] of Object.entries(manifest.codex_model_map ?? {})) {
|
|
19
|
+
if (!allowedModelSet.has(model)) errors.push(`codex_model_map.${family} uses unsupported model '${String(model)}'`);
|
|
20
|
+
}
|
|
21
|
+
for (const [family, effort] of Object.entries(manifest.codex_reasoning_map ?? {})) {
|
|
22
|
+
if (!CODEX_REASONING_EFFORTS.has(String(effort))) errors.push(`codex_reasoning_map.${family} uses invalid reasoning effort '${String(effort)}'`);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const agentMap = manifest.codex_agent_map;
|
|
26
|
+
if (!agentMap || typeof agentMap !== "object" || Array.isArray(agentMap)) {
|
|
27
|
+
errors.push("codex_agent_map must be an object");
|
|
28
|
+
return errors;
|
|
29
|
+
}
|
|
30
|
+
for (const [agentName, override] of Object.entries(agentMap)) {
|
|
31
|
+
if (!knownAgents.has(agentName)) errors.push(`codex_agent_map agent '${agentName}' does not exist`);
|
|
32
|
+
if (!override || typeof override !== "object" || Array.isArray(override)) {
|
|
33
|
+
errors.push(`codex_agent_map.${agentName} must be an object`);
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
const entry = override as Record<string, unknown>;
|
|
37
|
+
if (Object.keys(entry).sort().join(",") !== "model,reasoning_effort") {
|
|
38
|
+
errors.push(`codex_agent_map.${agentName} must contain exactly model and reasoning_effort`);
|
|
39
|
+
}
|
|
40
|
+
if (!allowedModelSet.has(entry.model)) {
|
|
41
|
+
errors.push(`codex_agent_map.${agentName}.model uses unsupported model '${String(entry.model)}'`);
|
|
42
|
+
}
|
|
43
|
+
if (!CODEX_REASONING_EFFORTS.has(String(entry.reasoning_effort))) {
|
|
44
|
+
errors.push(`codex_agent_map.${agentName}.reasoning_effort is invalid: '${String(entry.reasoning_effort)}'`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return errors;
|
|
48
|
+
}
|
|
@@ -28,6 +28,7 @@ const commands = [
|
|
|
28
28
|
["Claim lookup", "npm run workflow:sidecar -- claim <id> <dir>"],
|
|
29
29
|
["Context map drift", "npm run context-map:check"],
|
|
30
30
|
["Bundle build", "npm run build:bundles"],
|
|
31
|
+
["Skill drift check", "flow-agents skill-drift-check"],
|
|
31
32
|
];
|
|
32
33
|
|
|
33
34
|
function frontmatter(text: string): Record<string, string> {
|
|
@@ -3,6 +3,7 @@ import fs from "node:fs";
|
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { validateKitRepository as validateFlowKitRepository } from "../flow-kit/validate.js";
|
|
6
|
+
import { codexAgentRoutingErrors } from "./codex-agent-routing.js";
|
|
6
7
|
import { loadJson, readText, rel, root, walkFiles } from "./common.js";
|
|
7
8
|
|
|
8
9
|
class Reporter {
|
|
@@ -31,6 +32,8 @@ const mirroredFiles = new Map<string, { mirror: string; allowedDifferences: Arra
|
|
|
31
32
|
// fails `npm run validate:source` instead of silently accumulating.
|
|
32
33
|
["scripts/hooks/config-protection.js", { mirror: "context/scripts/hooks/config-protection.js", allowedDifferences: [] }],
|
|
33
34
|
["scripts/hooks/lib/config-protection-remedies.js", { mirror: "context/scripts/hooks/lib/config-protection-remedies.js", allowedDifferences: [] }],
|
|
35
|
+
["scripts/hooks/lib/kit-catalog.js", { mirror: "context/scripts/hooks/lib/kit-catalog.js", allowedDifferences: [] }],
|
|
36
|
+
["scripts/hooks/lib/runnable-command.js", { mirror: "context/scripts/hooks/lib/runnable-command.js", allowedDifferences: [] }],
|
|
34
37
|
["scripts/hooks/stop-goal-fit.js", { mirror: "context/scripts/hooks/stop-goal-fit.js", allowedDifferences: [] }],
|
|
35
38
|
["scripts/hooks/run-hook.js", { mirror: "context/scripts/hooks/run-hook.js", allowedDifferences: [] }],
|
|
36
39
|
["scripts/hooks/workflow-steering.js", { mirror: "context/scripts/hooks/workflow-steering.js", allowedDifferences: [] }],
|
|
@@ -82,9 +85,11 @@ const hookFilePolicies = new Map<string, { category: string; requiredNeedles: st
|
|
|
82
85
|
["scripts/hooks/desktop-notify.sh", { category: "local notification helper", requiredNeedles: ["desktop-notify.sh", "osascript"] }],
|
|
83
86
|
["scripts/hooks/lib/actor-identity.js", { category: "shared hook library", requiredNeedles: ["resolveActor", "FLOW_AGENTS_ACTOR"] }],
|
|
84
87
|
["scripts/hooks/lib/audit-transport.sh", { category: "shared hook library", requiredNeedles: ["audit_emit"] }],
|
|
88
|
+
["scripts/hooks/lib/codex-exit-code.js", { category: "shared hook library", requiredNeedles: ["extractExitCodeFromBanner", "readExitCodeFromRollout"] }],
|
|
85
89
|
["scripts/hooks/lib/config-protection-remedies.js", { category: "shared hook library", requiredNeedles: ["SANCTIONED_REMEDIES", "REMEDY_COMMAND_CANDIDATES"] }],
|
|
86
90
|
["scripts/hooks/lib/current-pointer.js", { category: "shared hook library", requiredNeedles: ["readCurrentPointer", "perActorCurrentFile"] }],
|
|
87
91
|
["scripts/hooks/lib/hook-flags.js", { category: "shared hook library", requiredNeedles: ["isHookEnabled"] }],
|
|
92
|
+
["scripts/hooks/lib/kit-catalog.js", { category: "shared hook library", requiredNeedles: ["readKitManifests", "workflowTriggersFor"] }],
|
|
88
93
|
["scripts/hooks/lib/liveness-heartbeat.js", { category: "shared hook library", requiredNeedles: ["maybeEmitHeartbeat", "resolveHeartbeatThrottleSeconds"] }],
|
|
89
94
|
["scripts/hooks/lib/liveness-policy.js", { category: "shared hook library", requiredNeedles: ["isLivenessEnabled", "resolveTtlSeconds"] }],
|
|
90
95
|
["scripts/hooks/lib/liveness-read.js", { category: "shared hook library", requiredNeedles: ["freshHolders", "readLivenessEvents"] }],
|
|
@@ -92,18 +97,23 @@ const hookFilePolicies = new Map<string, { category: string; requiredNeedles: st
|
|
|
92
97
|
["scripts/hooks/lib/local-artifact-paths.js", { category: "shared hook library", requiredNeedles: ["flowAgentsArtifactRoot", "defaultArtifactRootForRead"] }],
|
|
93
98
|
["scripts/hooks/lib/patterns.sh", { category: "shared hook library", requiredNeedles: ["_detect_secrets"] }],
|
|
94
99
|
["scripts/hooks/lib/resolve-formatter.js", { category: "shared hook library", requiredNeedles: ["resolveFormatter"] }],
|
|
100
|
+
["scripts/hooks/lib/runnable-command.js", { category: "shared hook library", requiredNeedles: ["isRunnableCommandText"] }],
|
|
101
|
+
["scripts/hooks/lib/skill-drift.js", { category: "shared hook library", requiredNeedles: ["compareSkillDrift", "buildManifest"] }],
|
|
95
102
|
]);
|
|
96
103
|
const fixtureOwnerPolicies = new Map<string, { owners: string[]; classification: string }>([
|
|
97
104
|
["evals/fixtures/assignment-provider", { owners: ["evals/integration/test_assignment_provider_local_file.sh", "evals/integration/test_assignment_provider_github.sh", "evals/integration/test_pull_work_assignment_join.sh", "evals/integration/test_ensure_session_ownership_guard.sh"], classification: "AssignmentProvider local-file and GitHub render/status fixtures (#290); hostile-effective-state.json is the #291 ensure-session ownership guard's AC9 sanitization fixture" }],
|
|
98
105
|
["evals/fixtures/backlog-provider-settings", { owners: ["evals/integration/test_effective_backlog_settings.sh"], classification: "settings precedence fixtures" }],
|
|
99
106
|
["evals/fixtures/builder-kit-workflow-state", { owners: ["evals/static/test_workflow_skills.sh"], classification: "Builder Kit workflow-state fixtures" }],
|
|
100
107
|
["evals/fixtures/console-learning-projection", { owners: ["evals/integration/test_console_learning_projection.sh"], classification: "console learning projection fixtures" }],
|
|
108
|
+
["evals/fixtures/economics", { owners: ["evals/integration/test_economics_record.sh"], classification: "per-run kit-economics record fixtures (#349): a transcript with .message.usage blocks, state.json/acceptance.json/critique.json join sources, a session.usage event, and the golden expected kontour.console.economics record" }],
|
|
101
109
|
["evals/fixtures/flow-kit-repository", { owners: ["evals/integration/test_flow_kit_repository.sh", "evals/integration/test_local_flow_kit_install.sh", "evals/integration/test_runtime_adapter_activation.sh", "evals/integration/test_activate_npx_context.sh", "evals/integration/test_flow_kit_install_git.sh", "evals/static/test_workflow_skills.sh"], classification: "Flow Kit repository contract fixtures" }],
|
|
102
110
|
["evals/fixtures/kit-conformance-levels", { owners: ["evals/integration/test_kit_conformance_levels.sh"], classification: "K-level conformance and consumer-target derivation fixtures" }],
|
|
103
111
|
["evals/fixtures/hook-influence", { owners: ["evals/integration/test_hook_influence_cases.sh", "evals/static/test_workflow_skills.sh", "scripts/validate-hook-influence-cases.js"], classification: "hook influence behavioral cases" }],
|
|
112
|
+
["evals/fixtures/learning-review-proposals", { owners: ["evals/integration/test_learning_review_proposals.sh"], classification: "learning-review kit/gate tuning proposal fixtures (#352): pattern-present (engineered cost-inflation + gate false-block-rate pattern, with sessions/ trust.bundle + gate-review.inquiries.json joins and a hand-computed expected-aggregates.json), balanced (proportional cost/findings movement -> zero proposals), under-threshold (below LR_MIN_WINDOW_SAMPLE), repeat-window (idempotency), and effect-follow-up (later-window effect-fill pass for a ratified proposal)" }],
|
|
104
113
|
["evals/fixtures/pull-work-provider", { owners: ["evals/integration/test_pull_work_provider.sh"], classification: "work item provider normalization fixtures" }],
|
|
105
114
|
["evals/fixtures/pull-work-wip-shepherding", { owners: ["evals/static/test_workflow_skills.sh"], classification: "WIP shepherding state fixtures" }],
|
|
106
115
|
["evals/fixtures/surface-trust", { owners: ["evals/integration/test_workflow_sidecar_writer.sh"], classification: "Surface trust evidence fixtures" }],
|
|
116
|
+
["evals/fixtures/telemetry", { owners: ["evals/integration/test_usage_cost.sh", "evals/integration/test_telemetry_usage_pipeline.sh"], classification: "hermetic Stop-hook usage transcript fixture (#defect-2026-07 telemetry-usage-cost-extraction): a multi-model JSONL transcript with .message.model + .message.usage token blocks proving real token/cost/model extraction end-to-end" }],
|
|
107
117
|
["evals/fixtures/reconcile-preflight", { owners: ["evals/integration/test_reconcile_preflight.sh"], classification: "#356 reconcile-preflight shape fixtures not already covered by trust-reconcile-exploits (un-superseded disputed critique, standalone disputed session-local claim)" }],
|
|
108
118
|
["evals/fixtures/trust-reconcile-exploits", { owners: ["evals/integration/test_trust_reconcile_negatives.sh", "evals/integration/test_reconcile_preflight.sh"], classification: "WS8 trust-reconcile anti-gaming exploit fixtures (frozen negative regressions); also reused by the #356 local reconcile-preflight eval (same shapes, no forked copies)" }],
|
|
109
119
|
["evals/fixtures/trust-reconcile-mixed-bundle", { owners: ["evals/integration/test_trust_reconcile_mixed_bundle.sh", "evals/integration/test_reconcile_preflight.sh"], classification: "WS8 trust-reconcile mixed-evidence end-to-end proof fixture; also reused by the #356 reconcile-preflight eval as its CLEAN-BUNDLE (AC4) case" }],
|
|
@@ -130,6 +140,9 @@ const allowedPythonCommandFiles = [
|
|
|
130
140
|
// as prose). These are literal recognizer tokens, NOT a flow-agents Python dependency — no
|
|
131
141
|
// .py file is ever invoked — so the two byte-identical mirror copies are exempt here.
|
|
132
142
|
/^(context\/)?scripts\/hooks\/stop-goal-fit\.js$/,
|
|
143
|
+
// #412: the recognizer allowlist itself now lives in scripts/hooks/lib/runnable-command.js
|
|
144
|
+
// (single source of truth), so the same literal-token exemption applies there too.
|
|
145
|
+
/^(context\/)?scripts\/hooks\/lib\/runnable-command\.js$/,
|
|
133
146
|
];
|
|
134
147
|
|
|
135
148
|
function tryLoadJson(file: string, reporter: Reporter): any {
|
|
@@ -197,6 +210,7 @@ function validateManifest(reporter: Reporter, manifest: any, agentNames: Set<str
|
|
|
197
210
|
}
|
|
198
211
|
for (const dir of manifest.optional_copy_dirs ?? []) if (!fs.existsSync(path.join(root, dir))) console.log(`warning: ${rel(manifestPath)} optional_copy_dirs entry absent: ${dir}`);
|
|
199
212
|
for (const agent of manifest.codex?.excluded_agents ?? []) reporter.check(agentNames.has(agent), `${rel(manifestPath)}: codex excluded agent '${agent}' does not exist`);
|
|
213
|
+
for (const error of codexAgentRoutingErrors(manifest, agentNames)) reporter.fail(`${rel(manifestPath)}: ${error}`);
|
|
200
214
|
}
|
|
201
215
|
async function validateKitRepository(kitDir: string, reporter: Reporter): Promise<void> {
|
|
202
216
|
if (!fs.existsSync(kitDir) || !fs.statSync(kitDir).isDirectory()) { reporter.fail(`${rel(kitDir)}: kit directory does not exist`); return; }
|
|
@@ -241,6 +255,16 @@ function validateLegacyRefs(reporter: Reporter): void {
|
|
|
241
255
|
// Collect all kit-owned asset relative paths so legacy-ref scanning can skip matches
|
|
242
256
|
// that are subpaths of kit-owned assets. E.g. legacyRefRe matches "skills/plan-work/SKILL.md"
|
|
243
257
|
// within "kits/builder/skills/plan-work/SKILL.md"; the kit declares and validates these.
|
|
258
|
+
//
|
|
259
|
+
// Deliberately NO directory-level exemption: a ref must be an exact registered path, or a
|
|
260
|
+
// parent directory of one, to be skipped. A kit that ships a helper file living alongside a
|
|
261
|
+
// registered asset (e.g. a skill's own script next to its SKILL.md) must register that
|
|
262
|
+
// helper as its own kit.json entry (e.g. under an "assets" section) rather than relying on
|
|
263
|
+
// sibling-directory proximity — widening the exemption to "any file in the same directory
|
|
264
|
+
// as a registered path" would silence real missing-path detection for every OTHER file that
|
|
265
|
+
// happens to sit next to a registered asset, not just the intended one. See
|
|
266
|
+
// kits/veritas-governance/kit.json's "assets" section for the registration this exact case
|
|
267
|
+
// uses.
|
|
244
268
|
const kitOwnedSubPaths = new Set<string>();
|
|
245
269
|
const kitsDir = path.join(root, "kits");
|
|
246
270
|
if (fs.existsSync(kitsDir)) {
|
|
@@ -271,7 +295,8 @@ function validateLegacyRefs(reporter: Reporter): void {
|
|
|
271
295
|
if (ref.split(/[\\/]/).includes("node_modules")) continue;
|
|
272
296
|
// Skip refs that are declared kit-owned asset paths or their parent directories
|
|
273
297
|
// (e.g. "skills/plan-work/SKILL.md" or "skills/plan-work" matched inside
|
|
274
|
-
// "kits/builder/skills/plan-work/SKILL.md" in eval files).
|
|
298
|
+
// "kits/builder/skills/plan-work/SKILL.md" in eval files). No directory-level
|
|
299
|
+
// exemption — see the kitOwnedSubPaths comment above.
|
|
275
300
|
if (kitOwnedSubPaths.has(ref) || [...kitOwnedSubPaths].some((p) => p.startsWith(ref + "/"))) continue;
|
|
276
301
|
const candidates = [path.join(root, ref), ...(ref.startsWith("evals/") ? [] : [path.join(root, "evals", ref)])];
|
|
277
302
|
if (!candidates.some(fs.existsSync)) reporter.fail(`${rel(file)}: references missing source path: ${ref}`);
|