@raishin/vanguard-frontier-agentic 3.1.1 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/.cursor-plugin/plugin.json +1 -1
- package/.github/plugin/marketplace.json +1 -1
- package/catalog/asset-integrity.json +59 -24
- package/catalog/model-assignments.json +19613 -0
- package/catalog/model-policy.json +371 -0
- package/catalog/model-registry.json +134 -0
- package/catalog/skill-manifest.json +18 -18
- package/package.json +7 -2
- package/plugins/vanguard-frontier-agentic/.codex-plugin/plugin.json +1 -1
- package/schemas/model-policy.schema.json +60 -0
- package/schemas/model-registry.schema.json +127 -0
- package/scripts/model-policy.mjs +1353 -0
- package/skills/gcp/gcp-firebase-developer/SKILL.md +1 -1
- package/skills/gcp/gcp-gke-platform-operator/SKILL.md +1 -1
- package/skills/salesforce/salesforce-agentforce-stdm-observer-skill/SKILL.md +1 -1
- package/skills/salesforce/salesforce-apex-log-analyzer-skill/SKILL.md +1 -1
- package/skills/salesforce/salesforce-apex-test-runner-skill/SKILL.md +1 -1
- package/skills/salesforce/salesforce-soql-explorer-skill/SKILL.md +1 -1
- package/tests/validate-skill-coherence.py +364 -0
|
@@ -0,0 +1,1353 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Per-harness model + reasoning-effort policy engine for agent harness variants.
|
|
4
|
+
*
|
|
5
|
+
* Canonical intent lives in catalog/model-policy.json. This script projects
|
|
6
|
+
* that policy into the executable harness files (codex.toml `model` /
|
|
7
|
+
* `model_reasoning_effort` / `model_provider`, claude-code `.agent.md`
|
|
8
|
+
* frontmatter `model:` / `effort:`, cursor `.agent.md` frontmatter `model:`)
|
|
9
|
+
* and regenerates the resolved index catalog/model-assignments.json that
|
|
10
|
+
* read-side tooling (vfa-tui) displays.
|
|
11
|
+
*
|
|
12
|
+
* Every model name and reasoning-effort value a rule projects is checked
|
|
13
|
+
* against catalog/model-registry.json (schema: schemas/model-registry.schema.json),
|
|
14
|
+
* a verified per-harness capability matrix. The engine fails closed: a model
|
|
15
|
+
* that isn't registered, or a reasoning effort a model/namespace doesn't
|
|
16
|
+
* support, is a validation error rather than a value silently projected that
|
|
17
|
+
* would 404 or be dropped at request time. See docs/model-policy-matrix.md
|
|
18
|
+
* for the human-readable matrix and .claude/skills/model-registry-refresh/
|
|
19
|
+
* SKILL.md for the refresh workflow.
|
|
20
|
+
*
|
|
21
|
+
* codex model_provider: derived from the registry namespace that classifies
|
|
22
|
+
* the assigned model (openai -> no line projected/default provider;
|
|
23
|
+
* ollama/openrouter -> `model_provider = "ollama"`/`"openrouter"` projected).
|
|
24
|
+
* The operator's codex config must define a matching [model_providers.<id>]
|
|
25
|
+
* table for the route to actually work — the registry only flags the intent.
|
|
26
|
+
*
|
|
27
|
+
* Reasoning-effort vocabularies are per harness (catalog/model-registry.json
|
|
28
|
+
* `reasoning_key` / `reasoning_efforts`) and, where the registry narrows
|
|
29
|
+
* further, per model/namespace: codex projects `model_reasoning_effort` in
|
|
30
|
+
* codex.toml; claude-code now also projects reasoning via the subagent
|
|
31
|
+
* frontmatter `effort:` key; cursor has no reasoning passthrough.
|
|
32
|
+
*
|
|
33
|
+
* Model lifecycle (catalog/model-registry.json `status` / `retirement_date` /
|
|
34
|
+
* `successor` on model entries): behavior is driven ONLY by the committed
|
|
35
|
+
* `status` field, never Date.now(). "retiring" projects the pinned model
|
|
36
|
+
* unchanged and emits a warning naming the documented successor. "retired"
|
|
37
|
+
* projects the documented successor instead (chain-followed through any
|
|
38
|
+
* further "retired" links) in place of the pinned model, and emits a louder
|
|
39
|
+
* warning until the policy rule is migrated to pin the successor directly.
|
|
40
|
+
* Warnings never fail `check` or change the exit code; they are aggregated
|
|
41
|
+
* by message text and printed once each, with an affected-assignment count.
|
|
42
|
+
*
|
|
43
|
+
* Policy semantics:
|
|
44
|
+
* - Scopes: "all", "provider:<id>", "role:<id>", "agent:<id>".
|
|
45
|
+
* - Precedence: agent > role > provider > all, per field (model and
|
|
46
|
+
* reasoning_effort resolve independently).
|
|
47
|
+
* - Two rules in the same tier that disagree on the same agent+harness+field
|
|
48
|
+
* are a hard error (roles overlap by design, so equal values are allowed;
|
|
49
|
+
* conflicting values must be settled by an agent-level rule).
|
|
50
|
+
* - "auto" clears the field: the managed line is removed and the harness
|
|
51
|
+
* runtime default applies. Absence of any rule means "auto".
|
|
52
|
+
* - Fields are only projected into harnesses that support them (see
|
|
53
|
+
* HARNESS_CAPABILITIES). A rule targeting an unsupported field fails
|
|
54
|
+
* `check` — intent that cannot be enforced is treated as an error, not
|
|
55
|
+
* silently recorded.
|
|
56
|
+
*
|
|
57
|
+
* Commands:
|
|
58
|
+
* report [--json] print resolved assignments per agent x harness
|
|
59
|
+
* check validate policy + detect drift (exit 1 on violation)
|
|
60
|
+
* apply [--dry-run] project policy into harness files + assignments index
|
|
61
|
+
* set --scope <all|provider=ID|role=ID|agent=ID|agents=a,b> --harness <id>
|
|
62
|
+
* [--model <name|auto>] [--reasoning <effort|auto>] [--dry-run]
|
|
63
|
+
* upsert rule(s), then apply
|
|
64
|
+
* import-current [--force]
|
|
65
|
+
* seed the policy from the values currently present in
|
|
66
|
+
* harness files (bootstrap; refuses to overwrite an
|
|
67
|
+
* existing policy without --force)
|
|
68
|
+
*
|
|
69
|
+
* After a non-dry-run apply/set, refresh the integrity manifest:
|
|
70
|
+
* npm run asset-integrity:write
|
|
71
|
+
*/
|
|
72
|
+
|
|
73
|
+
import { readFileSync, writeFileSync } from "node:fs";
|
|
74
|
+
import { createHash } from "node:crypto";
|
|
75
|
+
import { dirname, isAbsolute, join, relative } from "node:path";
|
|
76
|
+
import { fileURLToPath } from "node:url";
|
|
77
|
+
|
|
78
|
+
const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
|
|
79
|
+
const policyPath = join(repoRoot, "catalog", "model-policy.json");
|
|
80
|
+
const assignmentsPath = join(repoRoot, "catalog", "model-assignments.json");
|
|
81
|
+
const agentsCatalogPath = join(repoRoot, "catalog", "agents.json");
|
|
82
|
+
const rolesCatalogPath = join(repoRoot, "catalog", "install-roles.json");
|
|
83
|
+
const registryPath = join(repoRoot, "catalog", "model-registry.json");
|
|
84
|
+
|
|
85
|
+
/** Which policy fields each harness's executable format can express.
|
|
86
|
+
* Only verified, officially supported keys are projected; inventing metadata
|
|
87
|
+
* fields in executable agent files is forbidden (see CLAUDE.md cross-platform
|
|
88
|
+
* asset rule). Extend this table only with documented harness support.
|
|
89
|
+
* `reasoning_key` is the config key used to project reasoning_effort
|
|
90
|
+
* (codex.toml key / frontmatter key); null = no projectable reasoning field. */
|
|
91
|
+
const HARNESS_CAPABILITIES = {
|
|
92
|
+
codex: {
|
|
93
|
+
model: true,
|
|
94
|
+
reasoning_effort: true,
|
|
95
|
+
reasoning_key: "model_reasoning_effort",
|
|
96
|
+
variant: "codex",
|
|
97
|
+
file: "codex.toml",
|
|
98
|
+
},
|
|
99
|
+
"claude-code": {
|
|
100
|
+
model: true,
|
|
101
|
+
reasoning_effort: true,
|
|
102
|
+
reasoning_key: "effort",
|
|
103
|
+
variant: "claude-code",
|
|
104
|
+
file: "claude-code.agent.md",
|
|
105
|
+
},
|
|
106
|
+
cursor: { model: true, reasoning_effort: false, reasoning_key: null, variant: "cursor", file: "cursor.agent.md" },
|
|
107
|
+
copilot: { model: false, reasoning_effort: false, reasoning_key: null, variant: "copilot", file: "copilot.agent.md" },
|
|
108
|
+
gemini: { model: false, reasoning_effort: false, reasoning_key: null, variant: "gemini", file: "gemini.agent.md" },
|
|
109
|
+
kiro: { model: false, reasoning_effort: false, reasoning_key: null, variant: "kiro-ide", file: "kiro-ide.agent.md" },
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
/** Harness-specific model-name character shape. codex model names may carry
|
|
113
|
+
* ':' (Ollama name:tag) and '/' (OpenRouter author/model); every other
|
|
114
|
+
* harness falls back to SAFE_VALUE. Values are embedded only inside
|
|
115
|
+
* double-quoted TOML strings / YAML frontmatter and passed as argv, never
|
|
116
|
+
* through a shell, so this is a shape check, not a shell-safety check. */
|
|
117
|
+
const MODEL_CHARSETS = { codex: /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/ };
|
|
118
|
+
|
|
119
|
+
const SAFE_VALUE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
120
|
+
const SCOPE_TIERS = ["all", "provider", "role", "agent"];
|
|
121
|
+
|
|
122
|
+
/** Registry-hardening limits. The registry is attacker-controllable catalog
|
|
123
|
+
* data in a PR; every value below is compiled/interpolated by this script, so
|
|
124
|
+
* it is validated fail-closed before use. */
|
|
125
|
+
const MAX_MATCH_PATTERN_LENGTH = 200;
|
|
126
|
+
/** A quantifier ( + * {n,} ) applied to a group that itself ends in an
|
|
127
|
+
* unbounded quantifier — the catastrophic-backtracking shape, e.g. (a+)+. */
|
|
128
|
+
const NESTED_QUANTIFIER_RE = /\([^()]*[+*][^()]*\)[+*{]/;
|
|
129
|
+
/** Provider ids and reasoning-effort tokens embedded into codex.toml /
|
|
130
|
+
* frontmatter. Lowercase-alnum with internal separators; no quotes, spaces,
|
|
131
|
+
* newlines, or shell/markup metacharacters. */
|
|
132
|
+
const SAFE_TOKEN = /^[a-z0-9][a-z0-9._-]*$/;
|
|
133
|
+
|
|
134
|
+
// ── shared loading ───────────────────────────────────────────────────────────
|
|
135
|
+
|
|
136
|
+
/** Resolve a harness-variant path and confirm it resolves to a file under the
|
|
137
|
+
* `agents/` tree. Returns the absolute path, or null if it escapes the
|
|
138
|
+
* repository (`..` / absolute) OR resolves anywhere outside `agents/`. Catalog
|
|
139
|
+
* data (agent.path / harness_variants) is attacker-controllable in a PR, and
|
|
140
|
+
* these paths are both read and written; repo-containment alone is not enough
|
|
141
|
+
* because sensitive files like `.git/config`, `package.json`, or a root file
|
|
142
|
+
* are lexically *inside* the repo. Every one of the 1,100+ real variant paths
|
|
143
|
+
* lives under `agents/`, so scoping the guard there is both correct and tight —
|
|
144
|
+
* it blocks `.git/`, root files, and traversal in one check. Stricter than the
|
|
145
|
+
* `path_is_inside_repo` sibling validators (which this hereby supersedes for
|
|
146
|
+
* the codex/read-write path); lexical only, no symlink resolution. */
|
|
147
|
+
function resolveInsideRepo(relPath) {
|
|
148
|
+
if (typeof relPath !== "string" || relPath.length === 0) return null;
|
|
149
|
+
const abs = join(repoRoot, relPath);
|
|
150
|
+
const rel = relative(repoRoot, abs);
|
|
151
|
+
if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) return null;
|
|
152
|
+
if (rel.split(/[\\/]/)[0] !== "agents") return null;
|
|
153
|
+
return abs;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** Read a file as UTF-8, or return null if it does not exist. Avoids the
|
|
157
|
+
* TOCTOU race of existsSync-then-read: a single syscall either returns the
|
|
158
|
+
* contents or reports ENOENT, leaving no window between check and use. */
|
|
159
|
+
function readFileOrNull(path) {
|
|
160
|
+
try {
|
|
161
|
+
return readFileSync(path, "utf8");
|
|
162
|
+
} catch (e) {
|
|
163
|
+
if (e && e.code === "ENOENT") return null;
|
|
164
|
+
throw e;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function loadJson(path, label) {
|
|
169
|
+
const raw = readFileOrNull(path);
|
|
170
|
+
if (raw === null) {
|
|
171
|
+
fail(2, `ERROR: ${label} not found at ${relative(repoRoot, path)}`);
|
|
172
|
+
}
|
|
173
|
+
try {
|
|
174
|
+
return JSON.parse(raw);
|
|
175
|
+
} catch (e) {
|
|
176
|
+
fail(2, `ERROR: ${label} is not valid JSON: ${e.message}`);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Load JSON, returning `fallback` when the file is absent. */
|
|
181
|
+
function loadJsonOrDefault(path, label, fallback) {
|
|
182
|
+
const raw = readFileOrNull(path);
|
|
183
|
+
if (raw === null) return fallback;
|
|
184
|
+
try {
|
|
185
|
+
return JSON.parse(raw);
|
|
186
|
+
} catch (e) {
|
|
187
|
+
fail(2, `ERROR: ${label} is not valid JSON: ${e.message}`);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function fail(code, ...lines) {
|
|
192
|
+
for (const l of lines) console.error(l);
|
|
193
|
+
process.exit(code);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function loadAgents() {
|
|
197
|
+
const catalog = loadJson(agentsCatalogPath, "agent catalog");
|
|
198
|
+
return catalog.filter((e) => e.type === "agent");
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function loadRoles() {
|
|
202
|
+
return loadJson(rolesCatalogPath, "role catalog").roles;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function sha256Hex(text) {
|
|
206
|
+
return createHash("sha256").update(text).digest("hex");
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// ── model registry (catalog/model-registry.json) ────────────────────────────
|
|
210
|
+
|
|
211
|
+
/** Structural validation of the raw registry document. Returns a list of
|
|
212
|
+
* error strings (empty = valid). Does not build the compiled lookup
|
|
213
|
+
* structure — call compileRegistry() once this returns no errors. */
|
|
214
|
+
function validateRegistry(doc) {
|
|
215
|
+
const errors = [];
|
|
216
|
+
if (!doc || typeof doc !== "object" || Array.isArray(doc)) {
|
|
217
|
+
return ["model registry root must be an object"];
|
|
218
|
+
}
|
|
219
|
+
if (doc.manifest_version !== 1) {
|
|
220
|
+
errors.push("manifest_version must be 1");
|
|
221
|
+
}
|
|
222
|
+
if (!doc.harnesses || typeof doc.harnesses !== "object" || Array.isArray(doc.harnesses)) {
|
|
223
|
+
errors.push("harnesses must be an object");
|
|
224
|
+
return errors;
|
|
225
|
+
}
|
|
226
|
+
const requiredHarnesses = Object.entries(HARNESS_CAPABILITIES)
|
|
227
|
+
.filter(([, caps]) => caps.model)
|
|
228
|
+
.map(([h]) => h);
|
|
229
|
+
for (const h of requiredHarnesses) {
|
|
230
|
+
if (!doc.harnesses[h]) errors.push(`harnesses.${h} is required`);
|
|
231
|
+
}
|
|
232
|
+
for (const [harness, hdef] of Object.entries(doc.harnesses)) {
|
|
233
|
+
const where = `harnesses.${harness}`;
|
|
234
|
+
if (!hdef || typeof hdef !== "object") {
|
|
235
|
+
errors.push(`${where} must be an object`);
|
|
236
|
+
continue;
|
|
237
|
+
}
|
|
238
|
+
if (hdef.reasoning_key !== null && typeof hdef.reasoning_key !== "string") {
|
|
239
|
+
errors.push(`${where}.reasoning_key must be a string or null`);
|
|
240
|
+
}
|
|
241
|
+
if (!Array.isArray(hdef.reasoning_efforts)) {
|
|
242
|
+
errors.push(`${where}.reasoning_efforts must be an array of strings`);
|
|
243
|
+
} else {
|
|
244
|
+
// Each vocab element is interpolated into codex.toml / frontmatter
|
|
245
|
+
// (`model_reasoning_effort = "..."` / `effort: "..."`); charset-check it
|
|
246
|
+
// so a crafted registry can't smuggle a quote/newline through membership.
|
|
247
|
+
for (const eff of hdef.reasoning_efforts) {
|
|
248
|
+
if (typeof eff !== "string" || !SAFE_TOKEN.test(eff)) {
|
|
249
|
+
errors.push(
|
|
250
|
+
`${where}.reasoning_efforts contains an unsafe value ${JSON.stringify(eff)}`,
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
if (!Array.isArray(hdef.namespaces) || hdef.namespaces.length === 0) {
|
|
256
|
+
errors.push(`${where}.namespaces must be a non-empty array`);
|
|
257
|
+
continue;
|
|
258
|
+
}
|
|
259
|
+
const efforts = Array.isArray(hdef.reasoning_efforts) ? hdef.reasoning_efforts : [];
|
|
260
|
+
hdef.namespaces.forEach((ns, i) => {
|
|
261
|
+
const nsWhere = `${where}.namespaces[${i}]`;
|
|
262
|
+
if (!ns || typeof ns.id !== "string") {
|
|
263
|
+
errors.push(`${nsWhere}.id is required`);
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
if (typeof ns.match !== "string") {
|
|
267
|
+
errors.push(`${nsWhere}.match must be a string`);
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
if (!ns.match.startsWith("^") || !ns.match.endsWith("$")) {
|
|
271
|
+
errors.push(`${nsWhere}.match must be anchored (^...$): "${ns.match}"`);
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
// ReDoS guard: registry data is attacker-controllable in a PR, and this
|
|
275
|
+
// pattern is compiled and run in validate:model-policy (CI). Reject
|
|
276
|
+
// over-long patterns and nested unbounded quantifiers (the classic
|
|
277
|
+
// catastrophic-backtracking shape, e.g. (a+)+, (a*)+ ) before compiling.
|
|
278
|
+
// Node's RegExp has no step budget, so this is the only line of defense.
|
|
279
|
+
if (ns.match.length > MAX_MATCH_PATTERN_LENGTH) {
|
|
280
|
+
errors.push(
|
|
281
|
+
`${nsWhere}.match is too long (${ns.match.length} > ${MAX_MATCH_PATTERN_LENGTH}); simplify the namespace pattern`,
|
|
282
|
+
);
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
if (NESTED_QUANTIFIER_RE.test(ns.match)) {
|
|
286
|
+
errors.push(
|
|
287
|
+
`${nsWhere}.match contains a nested unbounded quantifier (ReDoS risk): "${ns.match}"`,
|
|
288
|
+
);
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
let re;
|
|
292
|
+
try {
|
|
293
|
+
re = new RegExp(ns.match);
|
|
294
|
+
} catch (e) {
|
|
295
|
+
errors.push(`${nsWhere}.match is not a valid regular expression: ${e.message}`);
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
if (ns.membership !== "closed" && ns.membership !== "open") {
|
|
299
|
+
errors.push(`${nsWhere}.membership must be "closed" or "open"`);
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
if (ns.membership === "closed" && !Array.isArray(ns.models)) {
|
|
303
|
+
errors.push(`${nsWhere}: membership "closed" requires a models array`);
|
|
304
|
+
}
|
|
305
|
+
// model_provider is interpolated into codex.toml (`model_provider = "..."`).
|
|
306
|
+
if (
|
|
307
|
+
ns.model_provider !== undefined &&
|
|
308
|
+
ns.model_provider !== null &&
|
|
309
|
+
(typeof ns.model_provider !== "string" || !SAFE_TOKEN.test(ns.model_provider))
|
|
310
|
+
) {
|
|
311
|
+
errors.push(
|
|
312
|
+
`${nsWhere}.model_provider must be null or a safe token, got ${JSON.stringify(ns.model_provider)}`,
|
|
313
|
+
);
|
|
314
|
+
}
|
|
315
|
+
if (Array.isArray(ns.models)) {
|
|
316
|
+
const nsModelIds = new Set(
|
|
317
|
+
ns.models.filter((m) => m && typeof m.id === "string").map((m) => m.id),
|
|
318
|
+
);
|
|
319
|
+
const modelsById = new Map(
|
|
320
|
+
ns.models.filter((m) => m && typeof m.id === "string").map((m) => [m.id, m]),
|
|
321
|
+
);
|
|
322
|
+
ns.models.forEach((model, j) => {
|
|
323
|
+
const mWhere = `${nsWhere}.models[${j}]`;
|
|
324
|
+
if (!model || typeof model.id !== "string") {
|
|
325
|
+
errors.push(`${mWhere}.id is required`);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
if (!re.test(model.id)) {
|
|
329
|
+
errors.push(
|
|
330
|
+
`${mWhere}: model id "${model.id}" does not match namespace "${ns.id}" match pattern "${ns.match}"`,
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
// model.id is interpolated into `model = "..."`; a crafted registry
|
|
334
|
+
// could pair a permissive namespace `match` with a quote/newline in
|
|
335
|
+
// the id, so charset-check it against the harness's own shape.
|
|
336
|
+
const idCharset = MODEL_CHARSETS[harness] ?? SAFE_VALUE;
|
|
337
|
+
if (!idCharset.test(model.id)) {
|
|
338
|
+
errors.push(
|
|
339
|
+
`${mWhere}: model id "${model.id}" contains characters unsafe for ${harness} projection`,
|
|
340
|
+
);
|
|
341
|
+
}
|
|
342
|
+
if (model.reasoning_efforts !== undefined) {
|
|
343
|
+
if (!Array.isArray(model.reasoning_efforts)) {
|
|
344
|
+
errors.push(`${mWhere}.reasoning_efforts must be an array`);
|
|
345
|
+
} else {
|
|
346
|
+
for (const eff of model.reasoning_efforts) {
|
|
347
|
+
if (!efforts.includes(eff)) {
|
|
348
|
+
errors.push(
|
|
349
|
+
`${mWhere}.reasoning_efforts value "${eff}" is not in the "${harness}" reasoning_efforts vocabulary`,
|
|
350
|
+
);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
if (
|
|
356
|
+
model.status !== undefined &&
|
|
357
|
+
!["available", "retiring", "retired"].includes(model.status)
|
|
358
|
+
) {
|
|
359
|
+
errors.push(`${mWhere}.status must be one of available|retiring|retired`);
|
|
360
|
+
}
|
|
361
|
+
if (model.retirement_date !== undefined) {
|
|
362
|
+
if (
|
|
363
|
+
typeof model.retirement_date !== "string" ||
|
|
364
|
+
!/^\d{4}-\d{2}-\d{2}$/.test(model.retirement_date)
|
|
365
|
+
) {
|
|
366
|
+
errors.push(`${mWhere}.retirement_date must match YYYY-MM-DD`);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
if (model.successor !== undefined) {
|
|
370
|
+
if (typeof model.successor !== "string" || !nsModelIds.has(model.successor)) {
|
|
371
|
+
errors.push(
|
|
372
|
+
`${mWhere}.successor "${model.successor}" must name an existing model id in namespace "${ns.id}"`,
|
|
373
|
+
);
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
if (model.status === "retired" && !model.successor) {
|
|
377
|
+
errors.push(`${mWhere}: status "retired" requires a successor`);
|
|
378
|
+
}
|
|
379
|
+
});
|
|
380
|
+
// Successor chains must terminate at a model whose status is not
|
|
381
|
+
// "retired" within 5 hops, with no cycles — mirrors resolveLifecycle's
|
|
382
|
+
// rule of following successor links only while the current entry is
|
|
383
|
+
// "retired".
|
|
384
|
+
ns.models.forEach((model, j) => {
|
|
385
|
+
if (!model || typeof model.id !== "string" || model.successor === undefined) return;
|
|
386
|
+
const mWhere = `${nsWhere}.models[${j}]`;
|
|
387
|
+
const visited = new Set([model.id]);
|
|
388
|
+
let current = model;
|
|
389
|
+
let hops = 0;
|
|
390
|
+
while (current.status === "retired") {
|
|
391
|
+
const nextId = current.successor;
|
|
392
|
+
if (nextId === undefined || !modelsById.has(nextId)) break; // reported separately
|
|
393
|
+
if (visited.has(nextId)) {
|
|
394
|
+
errors.push(`${mWhere}: successor chain from "${model.id}" has a cycle at "${nextId}"`);
|
|
395
|
+
break;
|
|
396
|
+
}
|
|
397
|
+
hops++;
|
|
398
|
+
if (hops > 5) {
|
|
399
|
+
errors.push(
|
|
400
|
+
`${mWhere}: successor chain from "${model.id}" does not terminate within 5 hops`,
|
|
401
|
+
);
|
|
402
|
+
break;
|
|
403
|
+
}
|
|
404
|
+
visited.add(nextId);
|
|
405
|
+
current = modelsById.get(nextId);
|
|
406
|
+
}
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
if (ns.reasoning_efforts !== undefined) {
|
|
410
|
+
if (!Array.isArray(ns.reasoning_efforts)) {
|
|
411
|
+
errors.push(`${nsWhere}.reasoning_efforts must be an array`);
|
|
412
|
+
} else {
|
|
413
|
+
for (const eff of ns.reasoning_efforts) {
|
|
414
|
+
if (!efforts.includes(eff)) {
|
|
415
|
+
errors.push(
|
|
416
|
+
`${nsWhere}.reasoning_efforts value "${eff}" is not in the "${harness}" reasoning_efforts vocabulary`,
|
|
417
|
+
);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
return errors;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/** Build the compiled lookup structure from an already-validated registry
|
|
428
|
+
* document: Map harness -> { reasoningKey, efforts, namespaces }, where each
|
|
429
|
+
* namespace carries a compiled RegExp and a Map(id -> model entry). */
|
|
430
|
+
function compileRegistry(doc) {
|
|
431
|
+
const compiled = new Map();
|
|
432
|
+
for (const [harness, hdef] of Object.entries(doc.harnesses)) {
|
|
433
|
+
const namespaces = (hdef.namespaces || []).map((ns) => {
|
|
434
|
+
const models = new Map();
|
|
435
|
+
for (const model of ns.models || []) {
|
|
436
|
+
models.set(model.id, {
|
|
437
|
+
...model,
|
|
438
|
+
status: model.status ?? "available",
|
|
439
|
+
retirement_date: model.retirement_date ?? null,
|
|
440
|
+
successor: model.successor ?? null,
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
return {
|
|
444
|
+
id: ns.id,
|
|
445
|
+
re: new RegExp(ns.match),
|
|
446
|
+
membership: ns.membership,
|
|
447
|
+
modelProvider: ns.model_provider ?? null,
|
|
448
|
+
nsEfforts: ns.reasoning_efforts ?? null,
|
|
449
|
+
models,
|
|
450
|
+
};
|
|
451
|
+
});
|
|
452
|
+
compiled.set(harness, {
|
|
453
|
+
reasoningKey: hdef.reasoning_key ?? null,
|
|
454
|
+
efforts: hdef.reasoning_efforts || [],
|
|
455
|
+
namespaces,
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
return compiled;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
function loadRegistry() {
|
|
462
|
+
const doc = loadJson(registryPath, "model registry");
|
|
463
|
+
const errors = validateRegistry(doc);
|
|
464
|
+
if (errors.length > 0) {
|
|
465
|
+
fail(2, "ERROR: model registry is invalid:", ...errors.map((e) => " " + e));
|
|
466
|
+
}
|
|
467
|
+
return compileRegistry(doc);
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/** First namespace (in registry order) whose match regex accepts `value`,
|
|
471
|
+
* or null if none does. */
|
|
472
|
+
function classifyModel(harnessReg, value) {
|
|
473
|
+
if (!harnessReg) return null;
|
|
474
|
+
for (const ns of harnessReg.namespaces) {
|
|
475
|
+
if (ns.re.test(value)) return ns;
|
|
476
|
+
}
|
|
477
|
+
return null;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/** Verified reasoning-effort vocabulary for a specific model value, or null
|
|
481
|
+
* if the model doesn't classify into any namespace at all (distinct from an
|
|
482
|
+
* empty array, which means "classified, but no reasoning passthrough"). */
|
|
483
|
+
function effortsForModel(harnessReg, value) {
|
|
484
|
+
const ns = classifyModel(harnessReg, value);
|
|
485
|
+
if (!ns) return null;
|
|
486
|
+
if (ns.membership === "closed" && ns.models.has(value)) {
|
|
487
|
+
const entry = ns.models.get(value);
|
|
488
|
+
if (entry.reasoning_efforts !== undefined) return entry.reasoning_efforts;
|
|
489
|
+
}
|
|
490
|
+
return ns.nsEfforts ?? harnessReg.efforts;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/** Resolve provider lifecycle for one classified model value: follow
|
|
494
|
+
* `successor` links while the current entry's `status` is "retired"
|
|
495
|
+
* (registry validation already guarantees these chains terminate at a
|
|
496
|
+
* non-retired model within 5 hops with no cycles), producing the effective
|
|
497
|
+
* model to project plus an operator-facing warning. Open namespaces and
|
|
498
|
+
* values that don't classify into a closed namespace, or aren't listed in
|
|
499
|
+
* one, are returned unchanged with no warning — lifecycle tracking only
|
|
500
|
+
* applies to the verified allowlist. Never consults the wall clock. */
|
|
501
|
+
function resolveLifecycle(harnessReg, ns, value) {
|
|
502
|
+
if (!ns || ns.membership !== "closed" || !ns.models.has(value)) {
|
|
503
|
+
return { model: value, fallbackFrom: null, warning: null };
|
|
504
|
+
}
|
|
505
|
+
const original = ns.models.get(value);
|
|
506
|
+
let current = original;
|
|
507
|
+
while (current.status === "retired" && current.successor && ns.models.has(current.successor)) {
|
|
508
|
+
current = ns.models.get(current.successor);
|
|
509
|
+
}
|
|
510
|
+
if (current.id !== value) {
|
|
511
|
+
let warning = `model "${value}" was retired by the provider — projecting documented successor "${current.id}"; migrate the policy rule`;
|
|
512
|
+
if (current.status === "retiring") {
|
|
513
|
+
const chainedDatePart = current.retirement_date ? ` on ${current.retirement_date}` : "";
|
|
514
|
+
warning += ` (note: "${current.id}" is itself scheduled for retirement${chainedDatePart} — successor: ${current.successor ?? "none announced"})`;
|
|
515
|
+
}
|
|
516
|
+
return {
|
|
517
|
+
model: current.id,
|
|
518
|
+
fallbackFrom: value,
|
|
519
|
+
warning,
|
|
520
|
+
};
|
|
521
|
+
}
|
|
522
|
+
if (original.status === "retiring") {
|
|
523
|
+
const datePart = original.retirement_date ? ` on ${original.retirement_date}` : "";
|
|
524
|
+
return {
|
|
525
|
+
model: value,
|
|
526
|
+
fallbackFrom: null,
|
|
527
|
+
warning: `model "${value}" is scheduled for retirement${datePart} — documented successor: ${original.successor ?? "none announced"}; plan a policy migration`,
|
|
528
|
+
};
|
|
529
|
+
}
|
|
530
|
+
return { model: value, fallbackFrom: null, warning: null };
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
// ── policy parsing + validation ──────────────────────────────────────────────
|
|
534
|
+
|
|
535
|
+
function parseScope(scope) {
|
|
536
|
+
if (scope === "all") return { tier: "all", id: null };
|
|
537
|
+
const m = /^(provider|role|agent):([a-z0-9][a-z0-9-]*)$/.exec(scope);
|
|
538
|
+
if (!m) return null;
|
|
539
|
+
return { tier: m[1], id: m[2] };
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/** Structural + referential validation. Returns a list of error strings. */
|
|
543
|
+
function validatePolicy(policy, agents, roles, registry) {
|
|
544
|
+
const errors = [];
|
|
545
|
+
if (!policy || typeof policy !== "object" || Array.isArray(policy)) {
|
|
546
|
+
return ["policy root must be an object"];
|
|
547
|
+
}
|
|
548
|
+
if (policy.manifest_version !== 1) {
|
|
549
|
+
errors.push("manifest_version must be 1");
|
|
550
|
+
}
|
|
551
|
+
if (policy.defaults) {
|
|
552
|
+
if (policy.defaults.model !== "auto" || policy.defaults.reasoning_effort !== "auto") {
|
|
553
|
+
errors.push('defaults must be {"model": "auto", "reasoning_effort": "auto"}');
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
if (!Array.isArray(policy.rules)) {
|
|
557
|
+
errors.push("rules must be an array");
|
|
558
|
+
return errors;
|
|
559
|
+
}
|
|
560
|
+
const agentIds = new Set(agents.map((a) => a.id));
|
|
561
|
+
const providers = new Set(agents.map((a) => a.provider));
|
|
562
|
+
const seen = new Set();
|
|
563
|
+
policy.rules.forEach((rule, i) => {
|
|
564
|
+
const where = `rules[${i}]`;
|
|
565
|
+
const known = new Set(["scope", "harness", "model", "reasoning_effort"]);
|
|
566
|
+
for (const k of Object.keys(rule)) {
|
|
567
|
+
if (!known.has(k)) errors.push(`${where}: unknown field "${k}"`);
|
|
568
|
+
}
|
|
569
|
+
const scope = typeof rule.scope === "string" ? parseScope(rule.scope) : null;
|
|
570
|
+
if (!scope) {
|
|
571
|
+
errors.push(`${where}: invalid scope "${rule.scope}"`);
|
|
572
|
+
return;
|
|
573
|
+
}
|
|
574
|
+
const caps = HARNESS_CAPABILITIES[rule.harness];
|
|
575
|
+
if (!caps) {
|
|
576
|
+
errors.push(`${where}: unknown harness "${rule.harness}"`);
|
|
577
|
+
return;
|
|
578
|
+
}
|
|
579
|
+
if (rule.model === undefined && rule.reasoning_effort === undefined) {
|
|
580
|
+
errors.push(`${where}: must set model and/or reasoning_effort`);
|
|
581
|
+
}
|
|
582
|
+
if (rule.model !== undefined) {
|
|
583
|
+
if (!caps.model) {
|
|
584
|
+
errors.push(`${where}: harness "${rule.harness}" does not support model pinning`);
|
|
585
|
+
} else if (rule.model !== "auto") {
|
|
586
|
+
const charset = MODEL_CHARSETS[rule.harness] ?? SAFE_VALUE;
|
|
587
|
+
if (typeof rule.model !== "string" || !charset.test(rule.model)) {
|
|
588
|
+
errors.push(`${where}: unsafe model value "${rule.model}"`);
|
|
589
|
+
} else {
|
|
590
|
+
const harnessReg = registry.get(rule.harness);
|
|
591
|
+
if (!harnessReg) {
|
|
592
|
+
errors.push(`${where}: no model registry entry for harness "${rule.harness}"`);
|
|
593
|
+
} else {
|
|
594
|
+
const ns = classifyModel(harnessReg, rule.model);
|
|
595
|
+
if (!ns) {
|
|
596
|
+
const nsList = harnessReg.namespaces.map((n) => `${n.id}: ${n.re.source}`).join("; ");
|
|
597
|
+
errors.push(
|
|
598
|
+
`${where}: model "${rule.model}" does not match any ${rule.harness} model namespace (${nsList}); see docs/model-policy-matrix.md`,
|
|
599
|
+
);
|
|
600
|
+
} else if (ns.membership === "closed" && !ns.models.has(rule.model)) {
|
|
601
|
+
errors.push(
|
|
602
|
+
`${where}: model "${rule.model}" is not in the verified model registry for ${rule.harness} namespace "${ns.id}" (catalog/model-registry.json); verify it against official docs and add it via the model-registry-refresh workflow (.claude/skills/model-registry-refresh/SKILL.md)`,
|
|
603
|
+
);
|
|
604
|
+
} else if (ns.membership === "closed" && ns.models.has(rule.model)) {
|
|
605
|
+
// Defense-in-depth: registry validation already requires a
|
|
606
|
+
// successor whenever status is "retired", but a rule pinning
|
|
607
|
+
// such a model with no successor to fall back to must still be
|
|
608
|
+
// rejected here so it can never be applied.
|
|
609
|
+
const entry = ns.models.get(rule.model);
|
|
610
|
+
if (entry.status === "retired" && !entry.successor) {
|
|
611
|
+
errors.push(
|
|
612
|
+
`${where}: model "${rule.model}" was retired with no documented successor — the rule must be migrated`,
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
if (rule.reasoning_effort !== undefined) {
|
|
621
|
+
if (!caps.reasoning_effort) {
|
|
622
|
+
errors.push(`${where}: harness "${rule.harness}" does not support reasoning_effort`);
|
|
623
|
+
} else if (rule.reasoning_effort !== "auto") {
|
|
624
|
+
const harnessReg = registry.get(rule.harness);
|
|
625
|
+
const efforts = harnessReg ? harnessReg.efforts : [];
|
|
626
|
+
if (!efforts.includes(rule.reasoning_effort)) {
|
|
627
|
+
errors.push(`${where}: reasoning_effort "${rule.reasoning_effort}" must be auto|${efforts.join("|")}`);
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
if (scope.tier === "provider" && !providers.has(scope.id)) {
|
|
632
|
+
errors.push(`${where}: unknown provider "${scope.id}"`);
|
|
633
|
+
}
|
|
634
|
+
if (scope.tier === "role" && !roles[scope.id]) {
|
|
635
|
+
errors.push(`${where}: unknown role "${scope.id}"`);
|
|
636
|
+
}
|
|
637
|
+
if (scope.tier === "agent" && !agentIds.has(scope.id)) {
|
|
638
|
+
errors.push(`${where}: unknown agent "${scope.id}"`);
|
|
639
|
+
}
|
|
640
|
+
const dupKey = `${rule.scope}::${rule.harness}`;
|
|
641
|
+
if (seen.has(dupKey)) {
|
|
642
|
+
errors.push(`${where}: duplicate rule for scope "${rule.scope}" harness "${rule.harness}"`);
|
|
643
|
+
}
|
|
644
|
+
seen.add(dupKey);
|
|
645
|
+
});
|
|
646
|
+
return errors;
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
// ── resolution ───────────────────────────────────────────────────────────────
|
|
650
|
+
|
|
651
|
+
/** Resolve effective {model, reasoning_effort, sources} for one agent+harness.
|
|
652
|
+
*
|
|
653
|
+
* Only the highest-priority tier (agent > role > provider > all) that actually
|
|
654
|
+
* matches a field decides that field. A conflict is reported *only* when the
|
|
655
|
+
* winning tier itself carries two different values — lower-tier disagreements
|
|
656
|
+
* are irrelevant because a higher tier overrides them. This preserves the
|
|
657
|
+
* documented escape hatch: adding a single `agent:<id>` rule resolves a
|
|
658
|
+
* role-overlap conflict instead of being rejected alongside it. */
|
|
659
|
+
function resolveAgentHarness(agent, harness, policy, roleIndex, errors) {
|
|
660
|
+
const result = {
|
|
661
|
+
model: "auto",
|
|
662
|
+
reasoning_effort: "auto",
|
|
663
|
+
model_source: "default",
|
|
664
|
+
reasoning_source: "default",
|
|
665
|
+
};
|
|
666
|
+
// Collect matching rules per field, grouped by tier.
|
|
667
|
+
const hitsByTier = { model: new Map(), reasoning_effort: new Map() };
|
|
668
|
+
for (const rule of policy.rules) {
|
|
669
|
+
if (rule.harness !== harness) continue;
|
|
670
|
+
const scope = parseScope(rule.scope);
|
|
671
|
+
if (!scope) continue;
|
|
672
|
+
const matches =
|
|
673
|
+
scope.tier === "all" ||
|
|
674
|
+
(scope.tier === "provider" && scope.id === agent.provider) ||
|
|
675
|
+
(scope.tier === "role" && (roleIndex.get(scope.id) || new Set()).has(agent.id)) ||
|
|
676
|
+
(scope.tier === "agent" && scope.id === agent.id);
|
|
677
|
+
if (!matches) continue;
|
|
678
|
+
for (const field of ["model", "reasoning_effort"]) {
|
|
679
|
+
if (rule[field] === undefined) continue;
|
|
680
|
+
if (!hitsByTier[field].has(scope.tier)) hitsByTier[field].set(scope.tier, []);
|
|
681
|
+
hitsByTier[field].get(scope.tier).push({ rule, value: rule[field] });
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
for (const field of ["model", "reasoning_effort"]) {
|
|
685
|
+
// Highest-priority tier with any matching rule wins outright.
|
|
686
|
+
let winningTier = null;
|
|
687
|
+
for (const tier of SCOPE_TIERS) {
|
|
688
|
+
if (hitsByTier[field].has(tier)) winningTier = tier;
|
|
689
|
+
}
|
|
690
|
+
if (winningTier === null) continue;
|
|
691
|
+
const hits = hitsByTier[field].get(winningTier);
|
|
692
|
+
const values = new Set(hits.map((h) => h.value));
|
|
693
|
+
if (values.size > 1) {
|
|
694
|
+
errors.push(
|
|
695
|
+
`conflict: agent "${agent.id}" harness "${harness}" gets ${field} values ` +
|
|
696
|
+
`[${[...values].join(", ")}] from ${winningTier}-tier rules ` +
|
|
697
|
+
`(${hits.map((h) => h.rule.scope).join(", ")}); add an agent-level override`,
|
|
698
|
+
);
|
|
699
|
+
continue;
|
|
700
|
+
}
|
|
701
|
+
result[field] = hits[0].value;
|
|
702
|
+
const src = field === "model" ? "model_source" : "reasoning_source";
|
|
703
|
+
result[src] = hits[0].rule.scope;
|
|
704
|
+
}
|
|
705
|
+
return result;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
function buildRoleIndex(roles) {
|
|
709
|
+
const index = new Map();
|
|
710
|
+
for (const [roleId, role] of Object.entries(roles)) {
|
|
711
|
+
index.set(roleId, new Set(role.agents || []));
|
|
712
|
+
}
|
|
713
|
+
return index;
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
/** Resolve the full assignment table. Returns { assignments, errors, warnings }.
|
|
717
|
+
* `warnings` is a flat array of message strings, one per assignment carrying
|
|
718
|
+
* a lifecycle warning (duplicates included; callers aggregate for display). */
|
|
719
|
+
function resolveAll(policy, agents, roles, registry) {
|
|
720
|
+
const errors = [];
|
|
721
|
+
const warnings = [];
|
|
722
|
+
const roleIndex = buildRoleIndex(roles);
|
|
723
|
+
const assignments = [];
|
|
724
|
+
const sorted = [...agents].sort((a, b) => a.id.localeCompare(b.id));
|
|
725
|
+
for (const agent of sorted) {
|
|
726
|
+
for (const [harness, caps] of Object.entries(HARNESS_CAPABILITIES)) {
|
|
727
|
+
if (!caps.model && !caps.reasoning_effort) continue;
|
|
728
|
+
const variantRel =
|
|
729
|
+
agent.harness_variants?.[caps.variant] ?? `${agent.path}/harnesses/${caps.file}`;
|
|
730
|
+
const variantAbs = resolveInsideRepo(variantRel);
|
|
731
|
+
if (variantAbs === null) {
|
|
732
|
+
errors.push(
|
|
733
|
+
`unsafe variant path for agent "${agent.id}" harness "${harness}": "${variantRel}" escapes the repository`,
|
|
734
|
+
);
|
|
735
|
+
continue;
|
|
736
|
+
}
|
|
737
|
+
if (readFileOrNull(variantAbs) === null) continue;
|
|
738
|
+
const r = resolveAgentHarness(agent, harness, policy, roleIndex, errors);
|
|
739
|
+
const harnessReg = registry.get(harness);
|
|
740
|
+
// Resolve provider lifecycle (status: available|retiring|retired) on
|
|
741
|
+
// the winning model BEFORE model_provider/efforts derivation, so a
|
|
742
|
+
// "retired" pin projects its documented successor everywhere else.
|
|
743
|
+
let effectiveModel = r.model;
|
|
744
|
+
let modelFallbackFrom = null;
|
|
745
|
+
let modelWarning = null;
|
|
746
|
+
if (r.model !== "auto") {
|
|
747
|
+
const pinnedNs = classifyModel(harnessReg, r.model);
|
|
748
|
+
const lifecycle = resolveLifecycle(harnessReg, pinnedNs, r.model);
|
|
749
|
+
effectiveModel = lifecycle.model;
|
|
750
|
+
modelFallbackFrom = lifecycle.fallbackFrom;
|
|
751
|
+
modelWarning = lifecycle.warning;
|
|
752
|
+
if (modelWarning) warnings.push(modelWarning);
|
|
753
|
+
}
|
|
754
|
+
const modelProvider =
|
|
755
|
+
effectiveModel === "auto"
|
|
756
|
+
? null
|
|
757
|
+
: classifyModel(harnessReg, effectiveModel)?.modelProvider ?? null;
|
|
758
|
+
if (caps.reasoning_effort && effectiveModel !== "auto" && r.reasoning_effort !== "auto") {
|
|
759
|
+
const efforts = effortsForModel(harnessReg, effectiveModel);
|
|
760
|
+
if (efforts !== null && !efforts.includes(r.reasoning_effort)) {
|
|
761
|
+
errors.push(
|
|
762
|
+
`incompatible: agent "${agent.id}" harness "${harness}" model "${effectiveModel}" (from ${r.model_source}) ` +
|
|
763
|
+
`does not support reasoning_effort "${r.reasoning_effort}" (from ${r.reasoning_source}); ` +
|
|
764
|
+
(efforts.length
|
|
765
|
+
? `supported: ${efforts.join("|")}`
|
|
766
|
+
: "no verified reasoning passthrough for this route — set reasoning to auto"),
|
|
767
|
+
);
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
assignments.push({
|
|
771
|
+
agent_id: agent.id,
|
|
772
|
+
harness,
|
|
773
|
+
model: effectiveModel === "auto" ? null : effectiveModel,
|
|
774
|
+
model_provider: modelProvider,
|
|
775
|
+
model_fallback_from: modelFallbackFrom,
|
|
776
|
+
model_warning: modelWarning,
|
|
777
|
+
reasoning_effort: caps.reasoning_effort
|
|
778
|
+
? r.reasoning_effort === "auto"
|
|
779
|
+
? null
|
|
780
|
+
: r.reasoning_effort
|
|
781
|
+
: null,
|
|
782
|
+
model_source: r.model_source,
|
|
783
|
+
reasoning_source: caps.reasoning_effort ? r.reasoning_source : "default",
|
|
784
|
+
file: variantRel,
|
|
785
|
+
});
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
return { assignments, errors, warnings };
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
// ── surgical file editing ────────────────────────────────────────────────────
|
|
792
|
+
|
|
793
|
+
/** Set/replace/remove a managed top-level key in a codex.toml file.
|
|
794
|
+
* Only lines before the first table header ([...]) and outside triple-quoted
|
|
795
|
+
* strings are considered, so prose inside developer_instructions can never be
|
|
796
|
+
* mistaken for a config key. `value === null` removes the line. */
|
|
797
|
+
function editTomlKey(content, key, value) {
|
|
798
|
+
const lines = content.split("\n");
|
|
799
|
+
let inTriple = false;
|
|
800
|
+
const keyRe = new RegExp(`^${key}\\s*=`);
|
|
801
|
+
let keyLine = -1;
|
|
802
|
+
let nameLine = -1;
|
|
803
|
+
let descLine = -1;
|
|
804
|
+
let modelLine = -1;
|
|
805
|
+
for (let i = 0; i < lines.length; i++) {
|
|
806
|
+
const tripleCount = (lines[i].match(/"""/g) || []).length;
|
|
807
|
+
if (!inTriple) {
|
|
808
|
+
// Stop at the first table header ([...]): only top-level keys are managed.
|
|
809
|
+
if (/^\[/.test(lines[i])) {
|
|
810
|
+
break;
|
|
811
|
+
}
|
|
812
|
+
if (keyRe.test(lines[i]) && keyLine === -1) keyLine = i;
|
|
813
|
+
if (/^name\s*=/.test(lines[i])) nameLine = i;
|
|
814
|
+
if (/^description\s*=/.test(lines[i]) && descLine === -1) descLine = i;
|
|
815
|
+
// Capture the real top-level `model =` line in the SAME triple-quote-aware
|
|
816
|
+
// pass, so a decoy `model = ...` line inside a triple-quoted string can
|
|
817
|
+
// never be chosen as the insertion anchor (which would splice the new key
|
|
818
|
+
// into the middle of that string).
|
|
819
|
+
if (/^model\s*=/.test(lines[i]) && modelLine === -1) modelLine = i;
|
|
820
|
+
}
|
|
821
|
+
if (tripleCount % 2 === 1) inTriple = !inTriple;
|
|
822
|
+
}
|
|
823
|
+
const rendered = value === null ? null : `${key} = "${value}"`;
|
|
824
|
+
if (keyLine >= 0) {
|
|
825
|
+
if (rendered === null) {
|
|
826
|
+
lines.splice(keyLine, 1);
|
|
827
|
+
} else if (lines[keyLine] !== rendered) {
|
|
828
|
+
lines[keyLine] = rendered;
|
|
829
|
+
} else {
|
|
830
|
+
return content;
|
|
831
|
+
}
|
|
832
|
+
return lines.join("\n");
|
|
833
|
+
}
|
|
834
|
+
if (rendered === null) return content;
|
|
835
|
+
// Insert after `model =` for the reasoning key, else after description/name.
|
|
836
|
+
let anchor = descLine >= 0 ? descLine : nameLine;
|
|
837
|
+
if (key === "model_reasoning_effort" && modelLine >= 0) {
|
|
838
|
+
anchor = modelLine;
|
|
839
|
+
}
|
|
840
|
+
lines.splice(anchor + 1, 0, rendered);
|
|
841
|
+
return lines.join("\n");
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
/** Set/replace/remove a managed key in a YAML frontmatter block (the block
|
|
845
|
+
* between the leading `---` fence pair). `value === null` removes the line.
|
|
846
|
+
* Insertion order: `model` is inserted just before the closing fence (its
|
|
847
|
+
* long-standing position); any other key (e.g. `effort`) is inserted directly
|
|
848
|
+
* after an existing `model:` line when present, else also before the closing
|
|
849
|
+
* fence. */
|
|
850
|
+
function editFrontmatterKey(content, key, value) {
|
|
851
|
+
const lines = content.split("\n");
|
|
852
|
+
if (lines[0] !== "---") return content;
|
|
853
|
+
let close = -1;
|
|
854
|
+
for (let i = 1; i < lines.length; i++) {
|
|
855
|
+
if (lines[i] === "---") {
|
|
856
|
+
close = i;
|
|
857
|
+
break;
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
if (close === -1) return content;
|
|
861
|
+
const keyRe = new RegExp(`^${key}:`);
|
|
862
|
+
const rendered = value === null ? null : `${key}: "${value}"`;
|
|
863
|
+
for (let i = 1; i < close; i++) {
|
|
864
|
+
if (keyRe.test(lines[i])) {
|
|
865
|
+
if (rendered === null) {
|
|
866
|
+
lines.splice(i, 1);
|
|
867
|
+
} else if (lines[i] !== rendered) {
|
|
868
|
+
lines[i] = rendered;
|
|
869
|
+
} else {
|
|
870
|
+
return content;
|
|
871
|
+
}
|
|
872
|
+
return lines.join("\n");
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
if (rendered === null) return content;
|
|
876
|
+
let insertAt = close;
|
|
877
|
+
if (key !== "model") {
|
|
878
|
+
for (let i = 1; i < close; i++) {
|
|
879
|
+
if (/^model:/.test(lines[i])) {
|
|
880
|
+
insertAt = i + 1;
|
|
881
|
+
break;
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
lines.splice(insertAt, 0, rendered);
|
|
886
|
+
return lines.join("\n");
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
/** Compute the projected content for one assignment. */
|
|
890
|
+
function projectFile(content, assignment) {
|
|
891
|
+
if (assignment.harness === "codex") {
|
|
892
|
+
let next = editTomlKey(content, "model", assignment.model);
|
|
893
|
+
next = editTomlKey(next, "model_reasoning_effort", assignment.reasoning_effort);
|
|
894
|
+
next = editTomlKey(next, "model_provider", assignment.model_provider);
|
|
895
|
+
return next;
|
|
896
|
+
}
|
|
897
|
+
const caps = HARNESS_CAPABILITIES[assignment.harness];
|
|
898
|
+
let next = editFrontmatterKey(content, "model", assignment.model);
|
|
899
|
+
if (caps.reasoning_effort && caps.reasoning_key) {
|
|
900
|
+
next = editFrontmatterKey(next, caps.reasoning_key, assignment.reasoning_effort);
|
|
901
|
+
}
|
|
902
|
+
return next;
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
// ── assignments index ────────────────────────────────────────────────────────
|
|
906
|
+
|
|
907
|
+
function renderAssignments(policyText, assignments) {
|
|
908
|
+
const capabilities = {};
|
|
909
|
+
for (const [harness, caps] of Object.entries(HARNESS_CAPABILITIES)) {
|
|
910
|
+
capabilities[harness] = { model: caps.model, reasoning_effort: caps.reasoning_effort };
|
|
911
|
+
}
|
|
912
|
+
const doc = {
|
|
913
|
+
manifest_version: 1,
|
|
914
|
+
generated_by: "scripts/model-policy.mjs",
|
|
915
|
+
policy_sha256: sha256Hex(policyText),
|
|
916
|
+
capabilities,
|
|
917
|
+
assignments: assignments.map(({ file, ...rest }) => rest),
|
|
918
|
+
};
|
|
919
|
+
return JSON.stringify(doc, null, 2) + "\n";
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
function canonicalPolicyText(policy) {
|
|
923
|
+
const tierRank = Object.fromEntries(SCOPE_TIERS.map((t, i) => [t, i]));
|
|
924
|
+
const rules = [...policy.rules].sort((a, b) => {
|
|
925
|
+
if (a.harness !== b.harness) return a.harness.localeCompare(b.harness);
|
|
926
|
+
const ta = parseScope(a.scope);
|
|
927
|
+
const tb = parseScope(b.scope);
|
|
928
|
+
if (tierRank[ta.tier] !== tierRank[tb.tier]) return tierRank[ta.tier] - tierRank[tb.tier];
|
|
929
|
+
return a.scope.localeCompare(b.scope);
|
|
930
|
+
});
|
|
931
|
+
const doc = {
|
|
932
|
+
manifest_version: 1,
|
|
933
|
+
description:
|
|
934
|
+
"Per-harness model and reasoning-effort policy for agent harness variants. " +
|
|
935
|
+
"Managed by scripts/model-policy.mjs (vfa-tui Model Policy view or CLI). " +
|
|
936
|
+
"'auto' omits the field so the harness runtime default applies. " +
|
|
937
|
+
"Precedence: agent > role > provider > all.",
|
|
938
|
+
defaults: { model: "auto", reasoning_effort: "auto" },
|
|
939
|
+
rules: rules.map((r) => {
|
|
940
|
+
const out = { scope: r.scope, harness: r.harness };
|
|
941
|
+
if (r.model !== undefined) out.model = r.model;
|
|
942
|
+
if (r.reasoning_effort !== undefined) out.reasoning_effort = r.reasoning_effort;
|
|
943
|
+
return out;
|
|
944
|
+
}),
|
|
945
|
+
};
|
|
946
|
+
return JSON.stringify(doc, null, 2) + "\n";
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
// ── plan / apply ─────────────────────────────────────────────────────────────
|
|
950
|
+
|
|
951
|
+
/** Compute the full projection plan: file changes + new assignments text. */
|
|
952
|
+
function computePlan(policy, agents, roles, registry) {
|
|
953
|
+
const { assignments, errors, warnings } = resolveAll(policy, agents, roles, registry);
|
|
954
|
+
const changes = [];
|
|
955
|
+
for (const a of assignments) {
|
|
956
|
+
const abs = join(repoRoot, a.file);
|
|
957
|
+
const current = readFileSync(abs, "utf8");
|
|
958
|
+
const next = projectFile(current, a);
|
|
959
|
+
if (next !== current) {
|
|
960
|
+
changes.push({ file: a.file, assignment: a, next });
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
return { assignments, errors, warnings, changes };
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
/** Print aggregated lifecycle warnings to stdout: identical warning texts
|
|
967
|
+
* are grouped and printed once each, with the count of assignments they
|
|
968
|
+
* affect. Warnings never change the exit code. */
|
|
969
|
+
function printWarnings(warnings) {
|
|
970
|
+
const counts = new Map();
|
|
971
|
+
for (const w of warnings) counts.set(w, (counts.get(w) || 0) + 1);
|
|
972
|
+
for (const [text, count] of counts) {
|
|
973
|
+
console.log(`WARNING: ${text} [affects ${count} assignment(s)]`);
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
function describeChange(c) {
|
|
978
|
+
const model = c.assignment.model ?? "auto";
|
|
979
|
+
const reasoning = c.assignment.reasoning_effort ?? "auto";
|
|
980
|
+
let detail;
|
|
981
|
+
if (c.assignment.harness === "codex") {
|
|
982
|
+
detail = `model=${model} provider=${c.assignment.model_provider ?? "default"} reasoning=${reasoning}`;
|
|
983
|
+
} else if (c.assignment.harness === "claude-code") {
|
|
984
|
+
detail = `model=${model} effort=${reasoning}`;
|
|
985
|
+
} else {
|
|
986
|
+
detail = `model=${model}`;
|
|
987
|
+
}
|
|
988
|
+
return `file update: ${c.file} (${detail})`;
|
|
989
|
+
}
|
|
990
|
+
|
|
991
|
+
function runApply({ dryRun }) {
|
|
992
|
+
const registry = loadRegistry();
|
|
993
|
+
const agents = loadAgents();
|
|
994
|
+
const roles = loadRoles();
|
|
995
|
+
const policy = loadJson(policyPath, "model policy");
|
|
996
|
+
const structuralErrors = validatePolicy(policy, agents, roles, registry);
|
|
997
|
+
if (structuralErrors.length > 0) {
|
|
998
|
+
fail(1, "ERROR: model policy is invalid:", ...structuralErrors.map((e) => " " + e));
|
|
999
|
+
}
|
|
1000
|
+
const { assignments, errors, warnings, changes } = computePlan(policy, agents, roles, registry);
|
|
1001
|
+
if (errors.length > 0) {
|
|
1002
|
+
fail(1, "ERROR: model policy conflicts:", ...[...new Set(errors)].map((e) => " " + e));
|
|
1003
|
+
}
|
|
1004
|
+
const policyText = canonicalPolicyText(policy);
|
|
1005
|
+
const assignmentsText = renderAssignments(policyText, assignments);
|
|
1006
|
+
const assignmentsStale = readFileOrNull(assignmentsPath) !== assignmentsText;
|
|
1007
|
+
|
|
1008
|
+
for (const c of changes) console.log(describeChange(c));
|
|
1009
|
+
printWarnings(warnings);
|
|
1010
|
+
if (dryRun) {
|
|
1011
|
+
console.log(
|
|
1012
|
+
`dry-run: ${changes.length} file(s) would change; assignments index ${assignmentsStale ? "would be rewritten" : "already in sync"}`,
|
|
1013
|
+
);
|
|
1014
|
+
return;
|
|
1015
|
+
}
|
|
1016
|
+
for (const c of changes) writeFileSync(join(repoRoot, c.file), c.next);
|
|
1017
|
+
const currentPolicyText = readFileSync(policyPath, "utf8");
|
|
1018
|
+
if (currentPolicyText !== policyText) writeFileSync(policyPath, policyText);
|
|
1019
|
+
if (assignmentsStale) writeFileSync(assignmentsPath, assignmentsText);
|
|
1020
|
+
console.log(
|
|
1021
|
+
`OK: applied model policy (${changes.length} file(s) changed, ${assignments.length} assignments)`,
|
|
1022
|
+
);
|
|
1023
|
+
if (changes.length > 0 || assignmentsStale) {
|
|
1024
|
+
console.log("reminder: run `npm run asset-integrity:write` before committing");
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
function runCheck() {
|
|
1029
|
+
const registry = loadRegistry();
|
|
1030
|
+
const agents = loadAgents();
|
|
1031
|
+
const roles = loadRoles();
|
|
1032
|
+
const policy = loadJson(policyPath, "model policy");
|
|
1033
|
+
const structuralErrors = validatePolicy(policy, agents, roles, registry);
|
|
1034
|
+
if (structuralErrors.length > 0) {
|
|
1035
|
+
fail(1, "ERROR: model policy is invalid:", ...structuralErrors.map((e) => " " + e));
|
|
1036
|
+
}
|
|
1037
|
+
const { assignments, errors, warnings, changes } = computePlan(policy, agents, roles, registry);
|
|
1038
|
+
const problems = [...new Set(errors)];
|
|
1039
|
+
for (const c of changes) {
|
|
1040
|
+
problems.push(`drift: ${c.file} does not match the model policy`);
|
|
1041
|
+
}
|
|
1042
|
+
const policyText = canonicalPolicyText(policy);
|
|
1043
|
+
const assignmentsText = renderAssignments(policyText, assignments);
|
|
1044
|
+
const currentAssignments = readFileOrNull(assignmentsPath);
|
|
1045
|
+
if (currentAssignments === null) {
|
|
1046
|
+
problems.push("catalog/model-assignments.json is missing; run npm run model-policy:apply");
|
|
1047
|
+
} else if (currentAssignments !== assignmentsText) {
|
|
1048
|
+
problems.push("catalog/model-assignments.json is stale; run npm run model-policy:apply");
|
|
1049
|
+
}
|
|
1050
|
+
if (problems.length > 0) {
|
|
1051
|
+
fail(
|
|
1052
|
+
1,
|
|
1053
|
+
"ERROR: model policy check failed:",
|
|
1054
|
+
...problems.map((p) => " " + p),
|
|
1055
|
+
"fix: adjust catalog/model-policy.json or run npm run model-policy:apply",
|
|
1056
|
+
);
|
|
1057
|
+
}
|
|
1058
|
+
// Warnings are informational only — they never affect the exit code, even
|
|
1059
|
+
// when problems is otherwise empty (check stays green with warnings).
|
|
1060
|
+
printWarnings(warnings);
|
|
1061
|
+
console.log(
|
|
1062
|
+
`OK: model policy in sync (${policy.rules.length} rules, ${assignments.length} assignments)`,
|
|
1063
|
+
);
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
function runReport({ json }) {
|
|
1067
|
+
const registry = loadRegistry();
|
|
1068
|
+
const agents = loadAgents();
|
|
1069
|
+
const roles = loadRoles();
|
|
1070
|
+
const policy = loadJson(policyPath, "model policy");
|
|
1071
|
+
const structuralErrors = validatePolicy(policy, agents, roles, registry);
|
|
1072
|
+
if (structuralErrors.length > 0) {
|
|
1073
|
+
fail(1, "ERROR: model policy is invalid:", ...structuralErrors.map((e) => " " + e));
|
|
1074
|
+
}
|
|
1075
|
+
const { assignments, errors, warnings } = resolveAll(policy, agents, roles, registry);
|
|
1076
|
+
if (errors.length > 0) {
|
|
1077
|
+
fail(1, "ERROR: model policy conflicts:", ...[...new Set(errors)].map((e) => " " + e));
|
|
1078
|
+
}
|
|
1079
|
+
if (json) {
|
|
1080
|
+
console.log(renderAssignments(canonicalPolicyText(policy), assignments).trimEnd());
|
|
1081
|
+
return;
|
|
1082
|
+
}
|
|
1083
|
+
for (const a of assignments) {
|
|
1084
|
+
const model = a.model ?? "auto";
|
|
1085
|
+
const reasoning = a.reasoning_effort ?? "auto";
|
|
1086
|
+
console.log(`${a.agent_id} ${a.harness} model=${model} reasoning=${reasoning}`);
|
|
1087
|
+
}
|
|
1088
|
+
printWarnings(warnings);
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
// ── set ──────────────────────────────────────────────────────────────────────
|
|
1092
|
+
|
|
1093
|
+
function parseSetArgs(argv) {
|
|
1094
|
+
const args = { scopes: [], harness: null, model: undefined, reasoning: undefined, dryRun: false };
|
|
1095
|
+
for (let i = 0; i < argv.length; i++) {
|
|
1096
|
+
const a = argv[i];
|
|
1097
|
+
if (a === "--dry-run") {
|
|
1098
|
+
args.dryRun = true;
|
|
1099
|
+
} else if (a === "--scope") {
|
|
1100
|
+
const v = argv[++i];
|
|
1101
|
+
if (v === undefined) fail(2, "ERROR: --scope requires a value");
|
|
1102
|
+
if (v === "all") {
|
|
1103
|
+
args.scopes.push("all");
|
|
1104
|
+
} else {
|
|
1105
|
+
const m = /^(provider|role|agent|agents)=(.+)$/.exec(v);
|
|
1106
|
+
if (!m) fail(2, `ERROR: invalid --scope "${v}"`);
|
|
1107
|
+
if (m[1] === "agents") {
|
|
1108
|
+
for (const id of m[2].split(",").filter(Boolean)) args.scopes.push(`agent:${id}`);
|
|
1109
|
+
} else {
|
|
1110
|
+
args.scopes.push(`${m[1]}:${m[2]}`);
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
} else if (a === "--harness") {
|
|
1114
|
+
args.harness = argv[++i];
|
|
1115
|
+
} else if (a === "--model") {
|
|
1116
|
+
args.model = argv[++i];
|
|
1117
|
+
} else if (a === "--reasoning") {
|
|
1118
|
+
args.reasoning = argv[++i];
|
|
1119
|
+
} else {
|
|
1120
|
+
fail(2, `ERROR: unknown argument "${a}"`);
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
if (args.scopes.length === 0) fail(2, "ERROR: --scope is required");
|
|
1124
|
+
if (!args.harness) fail(2, "ERROR: --harness is required");
|
|
1125
|
+
if (args.model === undefined && args.reasoning === undefined) {
|
|
1126
|
+
fail(2, "ERROR: provide --model and/or --reasoning");
|
|
1127
|
+
}
|
|
1128
|
+
return args;
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
function runSet(argv) {
|
|
1132
|
+
const args = parseSetArgs(argv);
|
|
1133
|
+
const registry = loadRegistry();
|
|
1134
|
+
const agents = loadAgents();
|
|
1135
|
+
const roles = loadRoles();
|
|
1136
|
+
const policy = loadJsonOrDefault(policyPath, "model policy", {
|
|
1137
|
+
manifest_version: 1,
|
|
1138
|
+
rules: [],
|
|
1139
|
+
});
|
|
1140
|
+
|
|
1141
|
+
for (const scope of args.scopes) {
|
|
1142
|
+
const existing = policy.rules.find((r) => r.scope === scope && r.harness === args.harness);
|
|
1143
|
+
const rule = existing ?? { scope, harness: args.harness };
|
|
1144
|
+
if (args.model !== undefined) rule.model = args.model;
|
|
1145
|
+
if (args.reasoning !== undefined) rule.reasoning_effort = args.reasoning;
|
|
1146
|
+
if (!existing) policy.rules.push(rule);
|
|
1147
|
+
console.log(
|
|
1148
|
+
`policy rule: ${scope} ${args.harness}` +
|
|
1149
|
+
(args.model !== undefined ? ` model=${args.model}` : "") +
|
|
1150
|
+
(args.reasoning !== undefined ? ` reasoning=${args.reasoning}` : ""),
|
|
1151
|
+
);
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
const structuralErrors = validatePolicy(policy, agents, roles, registry);
|
|
1155
|
+
if (structuralErrors.length > 0) {
|
|
1156
|
+
fail(1, "ERROR: resulting policy would be invalid:", ...structuralErrors.map((e) => " " + e));
|
|
1157
|
+
}
|
|
1158
|
+
const { assignments, errors, warnings, changes } = computePlan(policy, agents, roles, registry);
|
|
1159
|
+
if (errors.length > 0) {
|
|
1160
|
+
fail(1, "ERROR: resulting policy has conflicts:", ...[...new Set(errors)].map((e) => " " + e));
|
|
1161
|
+
}
|
|
1162
|
+
for (const c of changes) console.log(describeChange(c));
|
|
1163
|
+
printWarnings(warnings);
|
|
1164
|
+
if (args.dryRun) {
|
|
1165
|
+
console.log(`dry-run: ${changes.length} file(s) would change; policy not written`);
|
|
1166
|
+
return;
|
|
1167
|
+
}
|
|
1168
|
+
const policyText = canonicalPolicyText(policy);
|
|
1169
|
+
writeFileSync(policyPath, policyText);
|
|
1170
|
+
for (const c of changes) writeFileSync(join(repoRoot, c.file), c.next);
|
|
1171
|
+
writeFileSync(assignmentsPath, renderAssignments(policyText, assignments));
|
|
1172
|
+
console.log(
|
|
1173
|
+
`OK: policy updated (${policy.rules.length} rules), ${changes.length} file(s) changed`,
|
|
1174
|
+
);
|
|
1175
|
+
console.log("reminder: run `npm run asset-integrity:write` before committing");
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
// ── import-current ───────────────────────────────────────────────────────────
|
|
1179
|
+
|
|
1180
|
+
function readCurrentValue(content, harness, key) {
|
|
1181
|
+
if (harness === "codex") {
|
|
1182
|
+
const lines = content.split("\n");
|
|
1183
|
+
let inTriple = false;
|
|
1184
|
+
const keyRe = new RegExp(`^${key}\\s*=\\s*"(.*)"\\s*$`);
|
|
1185
|
+
for (const line of lines) {
|
|
1186
|
+
const tripleCount = (line.match(/"""/g) || []).length;
|
|
1187
|
+
if (!inTriple) {
|
|
1188
|
+
if (/^\[/.test(line)) break;
|
|
1189
|
+
const m = keyRe.exec(line);
|
|
1190
|
+
if (m) return m[1];
|
|
1191
|
+
}
|
|
1192
|
+
if (tripleCount % 2 === 1) inTriple = !inTriple;
|
|
1193
|
+
}
|
|
1194
|
+
return "auto";
|
|
1195
|
+
}
|
|
1196
|
+
const lines = content.split("\n");
|
|
1197
|
+
if (lines[0] !== "---") return "auto";
|
|
1198
|
+
const keyRe = new RegExp(`^${key}:\\s*"?([^"]*)"?\\s*$`);
|
|
1199
|
+
for (let i = 1; i < lines.length; i++) {
|
|
1200
|
+
if (lines[i] === "---") break;
|
|
1201
|
+
const m = keyRe.exec(lines[i]);
|
|
1202
|
+
if (m) return m[1];
|
|
1203
|
+
}
|
|
1204
|
+
return "auto";
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
/** Pick the most common value; ties prefer "auto", then lexicographic. */
|
|
1208
|
+
function majority(counter) {
|
|
1209
|
+
let best = null;
|
|
1210
|
+
for (const [value, count] of [...counter.entries()].sort((a, b) => {
|
|
1211
|
+
if (b[1] !== a[1]) return b[1] - a[1];
|
|
1212
|
+
if (a[0] === "auto") return -1;
|
|
1213
|
+
if (b[0] === "auto") return 1;
|
|
1214
|
+
return a[0].localeCompare(b[0]);
|
|
1215
|
+
})) {
|
|
1216
|
+
best = { value, count };
|
|
1217
|
+
break;
|
|
1218
|
+
}
|
|
1219
|
+
return best;
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
/** Derive a minimal rule set (all -> provider -> agent) reproducing the
|
|
1223
|
+
* observed per-agent values for one harness+field. */
|
|
1224
|
+
function minimizeRules(observed, harness, field) {
|
|
1225
|
+
const rules = [];
|
|
1226
|
+
const global = new Map();
|
|
1227
|
+
for (const { value } of observed) global.set(value, (global.get(value) || 0) + 1);
|
|
1228
|
+
const globalBest = majority(global);
|
|
1229
|
+
const allValue = globalBest ? globalBest.value : "auto";
|
|
1230
|
+
if (allValue !== "auto") {
|
|
1231
|
+
rules.push({ scope: "all", harness, [field]: allValue });
|
|
1232
|
+
}
|
|
1233
|
+
const byProvider = new Map();
|
|
1234
|
+
for (const o of observed) {
|
|
1235
|
+
if (!byProvider.has(o.provider)) byProvider.set(o.provider, []);
|
|
1236
|
+
byProvider.get(o.provider).push(o);
|
|
1237
|
+
}
|
|
1238
|
+
for (const [provider, entries] of [...byProvider.entries()].sort()) {
|
|
1239
|
+
const counter = new Map();
|
|
1240
|
+
for (const { value } of entries) counter.set(value, (counter.get(value) || 0) + 1);
|
|
1241
|
+
const best = majority(counter);
|
|
1242
|
+
let effective = allValue;
|
|
1243
|
+
if (best.value !== allValue && best.count > entries.length / 2) {
|
|
1244
|
+
rules.push({ scope: `provider:${provider}`, harness, [field]: best.value });
|
|
1245
|
+
effective = best.value;
|
|
1246
|
+
}
|
|
1247
|
+
for (const o of entries.sort((a, b) => a.agent_id.localeCompare(b.agent_id))) {
|
|
1248
|
+
if (o.value !== effective) {
|
|
1249
|
+
rules.push({ scope: `agent:${o.agent_id}`, harness, [field]: o.value });
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
return rules;
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
function runImportCurrent({ force, dryRun }) {
|
|
1257
|
+
if (readFileOrNull(policyPath) !== null && !force) {
|
|
1258
|
+
fail(2, "ERROR: catalog/model-policy.json already exists; pass --force to regenerate");
|
|
1259
|
+
}
|
|
1260
|
+
const registry = loadRegistry();
|
|
1261
|
+
const agents = loadAgents();
|
|
1262
|
+
const roles = loadRoles();
|
|
1263
|
+
const rawRules = [];
|
|
1264
|
+
for (const [harness, caps] of Object.entries(HARNESS_CAPABILITIES)) {
|
|
1265
|
+
if (!caps.model && !caps.reasoning_effort) continue;
|
|
1266
|
+
const observedModel = [];
|
|
1267
|
+
const observedReasoning = [];
|
|
1268
|
+
for (const agent of agents) {
|
|
1269
|
+
const variantRel =
|
|
1270
|
+
agent.harness_variants?.[caps.variant] ?? `${agent.path}/harnesses/${caps.file}`;
|
|
1271
|
+
const abs = resolveInsideRepo(variantRel);
|
|
1272
|
+
if (abs === null) continue;
|
|
1273
|
+
const content = readFileOrNull(abs);
|
|
1274
|
+
if (content === null) continue;
|
|
1275
|
+
if (caps.model) {
|
|
1276
|
+
observedModel.push({
|
|
1277
|
+
agent_id: agent.id,
|
|
1278
|
+
provider: agent.provider,
|
|
1279
|
+
value: readCurrentValue(content, harness, "model"),
|
|
1280
|
+
});
|
|
1281
|
+
}
|
|
1282
|
+
if (caps.reasoning_effort) {
|
|
1283
|
+
observedReasoning.push({
|
|
1284
|
+
agent_id: agent.id,
|
|
1285
|
+
provider: agent.provider,
|
|
1286
|
+
value: readCurrentValue(content, harness, caps.reasoning_key),
|
|
1287
|
+
});
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
if (caps.model) rawRules.push(...minimizeRules(observedModel, harness, "model"));
|
|
1291
|
+
if (caps.reasoning_effort) {
|
|
1292
|
+
rawRules.push(...minimizeRules(observedReasoning, harness, "reasoning_effort"));
|
|
1293
|
+
}
|
|
1294
|
+
}
|
|
1295
|
+
// Merge model + reasoning rules that share scope+harness.
|
|
1296
|
+
const merged = new Map();
|
|
1297
|
+
for (const r of rawRules) {
|
|
1298
|
+
const key = `${r.scope}::${r.harness}`;
|
|
1299
|
+
merged.set(key, { ...(merged.get(key) || {}), ...r });
|
|
1300
|
+
}
|
|
1301
|
+
const policy = { manifest_version: 1, rules: [...merged.values()] };
|
|
1302
|
+
const structuralErrors = validatePolicy(policy, agents, roles, registry);
|
|
1303
|
+
if (structuralErrors.length > 0) {
|
|
1304
|
+
fail(1, "ERROR: imported policy is invalid:", ...structuralErrors.map((e) => " " + e));
|
|
1305
|
+
}
|
|
1306
|
+
const { assignments, errors, changes } = computePlan(policy, agents, roles, registry);
|
|
1307
|
+
if (errors.length > 0) {
|
|
1308
|
+
fail(1, "ERROR: imported policy has conflicts:", ...[...new Set(errors)].map((e) => " " + e));
|
|
1309
|
+
}
|
|
1310
|
+
if (changes.length > 0) {
|
|
1311
|
+
fail(
|
|
1312
|
+
1,
|
|
1313
|
+
"ERROR: imported policy does not reproduce the current tree (bug):",
|
|
1314
|
+
...changes.map((c) => " " + describeChange(c)),
|
|
1315
|
+
);
|
|
1316
|
+
}
|
|
1317
|
+
if (dryRun) {
|
|
1318
|
+
console.log(`dry-run: would write policy with ${policy.rules.length} rules (0 file changes)`);
|
|
1319
|
+
return;
|
|
1320
|
+
}
|
|
1321
|
+
const policyText = canonicalPolicyText(policy);
|
|
1322
|
+
writeFileSync(policyPath, policyText);
|
|
1323
|
+
writeFileSync(assignmentsPath, renderAssignments(policyText, assignments));
|
|
1324
|
+
console.log(
|
|
1325
|
+
`OK: imported model policy (${policy.rules.length} rules, ${assignments.length} assignments, 0 file changes)`,
|
|
1326
|
+
);
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
// ── main ─────────────────────────────────────────────────────────────────────
|
|
1330
|
+
|
|
1331
|
+
const [command, ...rest] = process.argv.slice(2);
|
|
1332
|
+
switch (command) {
|
|
1333
|
+
case "report":
|
|
1334
|
+
runReport({ json: rest.includes("--json") });
|
|
1335
|
+
break;
|
|
1336
|
+
case "check":
|
|
1337
|
+
runCheck();
|
|
1338
|
+
break;
|
|
1339
|
+
case "apply":
|
|
1340
|
+
runApply({ dryRun: rest.includes("--dry-run") });
|
|
1341
|
+
break;
|
|
1342
|
+
case "set":
|
|
1343
|
+
runSet(rest);
|
|
1344
|
+
break;
|
|
1345
|
+
case "import-current":
|
|
1346
|
+
runImportCurrent({ force: rest.includes("--force"), dryRun: rest.includes("--dry-run") });
|
|
1347
|
+
break;
|
|
1348
|
+
default:
|
|
1349
|
+
fail(
|
|
1350
|
+
2,
|
|
1351
|
+
"usage: node scripts/model-policy.mjs <report|check|apply|set|import-current> [options]",
|
|
1352
|
+
);
|
|
1353
|
+
}
|