@crewhaus/spec 0.5.7 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +61148 -26327
- package/dist/index.js +1307 -103
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -14,6 +14,28 @@ const safeName = z
|
|
|
14
14
|
.string()
|
|
15
15
|
.min(1)
|
|
16
16
|
.regex(/^[\w .:-]+$/, "name may contain only letters, digits, spaces, and '_ . - :' (no newlines, quotes, slashes, or comment/template delimiters)");
|
|
17
|
+
/**
|
|
18
|
+
* 0.6.0 §4.1 — the `models:` profile-name grammar. Deliberately NARROWER
|
|
19
|
+
* than `safeName`: a profile name becomes an arm id in `arms.jsonl`, a key
|
|
20
|
+
* in generated source and the `$<name>` reference sigil every model slot
|
|
21
|
+
* accepts, so the charset is lowercase-first `[a-z][a-z0-9_-]{0,63}` —
|
|
22
|
+
* first-character-disjoint from the documented `$UPPER_SNAKE` env-ref
|
|
23
|
+
* convention (`ENV_REF_RE` in the compiler), so a profile ref can never read
|
|
24
|
+
* as a secret ref and vice versa.
|
|
25
|
+
*
|
|
26
|
+
* DUPLICATED from `PROFILE_NAME_RE` in `@crewhaus/model-plan` (the runtime
|
|
27
|
+
* resolver's source of truth) because the spec package stays
|
|
28
|
+
* dependency-light — it must not import runtime packages. Keep in sync; the
|
|
29
|
+
* compiler PR that depends on both packages pins the two literals equal.
|
|
30
|
+
*/
|
|
31
|
+
export const SPEC_PROFILE_NAME_RE = /^[a-z][a-z0-9_-]{0,63}$/;
|
|
32
|
+
const profileName = z
|
|
33
|
+
.string()
|
|
34
|
+
.regex(SPEC_PROFILE_NAME_RE, "profile name must be lowercase-first: letters, digits, '_' and '-' only, at most 64 characters (/^[a-z][a-z0-9_-]{0,63}$/)");
|
|
35
|
+
/** The `$<profile>` reference form every model slot accepts (0.6.0 §4.1). */
|
|
36
|
+
function profileRefName(value) {
|
|
37
|
+
return value.startsWith("$") ? value.slice(1) : undefined;
|
|
38
|
+
}
|
|
17
39
|
/**
|
|
18
40
|
* v0 spec schema — a discriminated union over `target`.
|
|
19
41
|
*
|
|
@@ -96,6 +118,35 @@ const permissionsBlock = z
|
|
|
96
118
|
* retries in the background, registering the peer's tools when it lands.
|
|
97
119
|
*/
|
|
98
120
|
const mcpRequiredField = z.boolean().optional();
|
|
121
|
+
/**
|
|
122
|
+
* 0.6.0 §5.5 — MCP tool trust flags, NARROWING-ONLY. Every MCP tool is
|
|
123
|
+
* `readOnly: false` today and therefore asks in default mode; `tool_flags`
|
|
124
|
+
* lets a spec tighten what the runtime knows about a server's tools
|
|
125
|
+
* (`defaults` for every tool on the server, `per_tool` for named ones).
|
|
126
|
+
*
|
|
127
|
+
* SECURITY: the enumerated key set is `{readOnly: true, destructive: true,
|
|
128
|
+
* requireJustification: true}` and each value is the literal `true` — a spec
|
|
129
|
+
* may never clear `requireJustification`, never set `scope: internal` on an
|
|
130
|
+
* `mcp__*` tool and never touch `ioCapability`. Loosening any of those would
|
|
131
|
+
* punch straight through the egress chokepoint, which keys on
|
|
132
|
+
* `scope === "external"`, so the schema rejects the loosening direction at
|
|
133
|
+
* parse time (defense in depth, mirroring `permissions.mode: bypass`).
|
|
134
|
+
*/
|
|
135
|
+
const MCP_TOOL_FLAG_FORBIDDEN_KEYS = ["scope", "ioCapability", "classifyOutput"];
|
|
136
|
+
const mcpToolFlagsEntrySchema = z
|
|
137
|
+
.object({
|
|
138
|
+
readOnly: z.literal(true).optional(),
|
|
139
|
+
destructive: z.literal(true).optional(),
|
|
140
|
+
requireJustification: z.literal(true).optional(),
|
|
141
|
+
})
|
|
142
|
+
.strict(`mcp_servers.<name>.tool_flags may only TIGHTEN a tool's trust flags (readOnly: true, destructive: true, requireJustification: true); ${MCP_TOOL_FLAG_FORBIDDEN_KEYS.join(", ")} and every other RegisteredTool property are tool-author facts a spec cannot override`);
|
|
143
|
+
const mcpToolFlagsBlock = z
|
|
144
|
+
.object({
|
|
145
|
+
defaults: mcpToolFlagsEntrySchema.optional(),
|
|
146
|
+
per_tool: z.record(z.string().min(1), mcpToolFlagsEntrySchema).optional(),
|
|
147
|
+
})
|
|
148
|
+
.strict()
|
|
149
|
+
.optional();
|
|
99
150
|
const stdioMcpConfig = z
|
|
100
151
|
.object({
|
|
101
152
|
transport: z.literal("stdio"),
|
|
@@ -103,6 +154,7 @@ const stdioMcpConfig = z
|
|
|
103
154
|
args: z.array(z.string()).optional(),
|
|
104
155
|
env: z.record(z.string()).optional(),
|
|
105
156
|
required: mcpRequiredField,
|
|
157
|
+
tool_flags: mcpToolFlagsBlock,
|
|
106
158
|
})
|
|
107
159
|
.strict();
|
|
108
160
|
const sseMcpConfig = z
|
|
@@ -111,46 +163,15 @@ const sseMcpConfig = z
|
|
|
111
163
|
url: z.string().url(),
|
|
112
164
|
headers: z.record(z.string()).optional(),
|
|
113
165
|
required: mcpRequiredField,
|
|
166
|
+
tool_flags: mcpToolFlagsBlock,
|
|
114
167
|
})
|
|
115
168
|
.strict();
|
|
116
169
|
const mcpServerConfigSchema = z.discriminatedUnion("transport", [stdioMcpConfig, sseMcpConfig]);
|
|
117
170
|
const mcpServersBlock = z.record(z.string().min(1), mcpServerConfigSchema).optional();
|
|
118
|
-
// Section 13 — sub-agent definitions
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
-
//
|
|
122
|
-
const subAgentDefinitionSchema = z
|
|
123
|
-
.object({
|
|
124
|
-
description: z.string().min(1),
|
|
125
|
-
instructions: z.string().min(1),
|
|
126
|
-
tools: z.array(z.string().min(1)).optional(),
|
|
127
|
-
model: z.string().min(1).optional(),
|
|
128
|
-
permissions: z
|
|
129
|
-
.union([
|
|
130
|
-
z.enum(["inherit", "scoped"]),
|
|
131
|
-
z
|
|
132
|
-
.object({
|
|
133
|
-
allow: z.array(z.string().min(1)),
|
|
134
|
-
deny: z.array(z.string().min(1)),
|
|
135
|
-
})
|
|
136
|
-
.strict(),
|
|
137
|
-
])
|
|
138
|
-
.optional(),
|
|
139
|
-
inherit_bypass: z.boolean().optional(),
|
|
140
|
-
/**
|
|
141
|
-
* Item 2 (G31 — A2A federation) — wire this sub-agent to a REMOTE peer
|
|
142
|
-
* instead of spawning it locally. `url` is the peer deployment's base
|
|
143
|
-
* URL; the spawner routes the Task call through `@crewhaus/federation-
|
|
144
|
-
* router` to the peer's inbound A2A handler (whose Agent Card lives at
|
|
145
|
-
* `<url>/.well-known/agent-card.json`), mapping the federation envelope
|
|
146
|
-
* onto A2A message/task semantics. Present ⇒ the entry is a federated
|
|
147
|
-
* peer reference; `description`/`instructions` still describe it to the
|
|
148
|
-
* parent's Task tool (the remote peer owns its own prompt).
|
|
149
|
-
*/
|
|
150
|
-
federation: z.object({ url: z.string().url() }).strict().optional(),
|
|
151
|
-
})
|
|
152
|
-
.strict();
|
|
153
|
-
const subAgentsBlock = z.record(safeName, subAgentDefinitionSchema).optional();
|
|
171
|
+
// Section 13 — sub-agent definitions (`subAgentDefinitionSchema` /
|
|
172
|
+
// `subAgentsBlock`) are declared below the model-profile section: from
|
|
173
|
+
// 0.6.0 a sub-agent carries the same routing blocks as an agent (§7.7), so
|
|
174
|
+
// its schema depends on `modelPoolBlock` and `thinkingBlock`.
|
|
154
175
|
/**
|
|
155
176
|
* Section 14 — per-tool runtime config map. Tool-specific schemas live
|
|
156
177
|
* inside each tool package; the spec layer treats every value as opaque
|
|
@@ -319,68 +340,29 @@ const modelTiersBlock = z
|
|
|
319
340
|
* floor and the cost/latency reward references.
|
|
320
341
|
*
|
|
321
342
|
* Omitted entirely → single-model behaviour, byte-identical bundles.
|
|
343
|
+
*
|
|
344
|
+
* 0.6.0 §7.1 — the pool is ALSO the hybrid container: `rules`, `directives`,
|
|
345
|
+
* `classifier`, `strategy`, `reward` and `scope` are declared as SIBLINGS of
|
|
346
|
+
* `routing`/`learning` (the optimizer whitelists those two wholesale, and
|
|
347
|
+
* none of the new blocks is optimizer-tunable), and every candidate may
|
|
348
|
+
* carry the per-model profile fields inline (`modelPoolCandidateSchema`).
|
|
349
|
+
* The block itself is declared below the model-profile section it depends
|
|
350
|
+
* on; see `modelPoolBlock` there.
|
|
322
351
|
*/
|
|
323
|
-
const modelPoolBlock = z
|
|
324
|
-
.object({
|
|
325
|
-
candidates: z
|
|
326
|
-
.array(z
|
|
327
|
-
.object({
|
|
328
|
-
model: z.string().min(1),
|
|
329
|
-
tags: z.array(z.string().min(1)).default([]),
|
|
330
|
-
})
|
|
331
|
-
.strict())
|
|
332
|
-
.min(2),
|
|
333
|
-
policy: z.enum(["static", "heuristic", "learned"]).default("heuristic"),
|
|
334
|
-
objective: z
|
|
335
|
-
.object({
|
|
336
|
-
quality: z.number().min(0).optional(),
|
|
337
|
-
cost: z.number().min(0).optional(),
|
|
338
|
-
latency: z.number().min(0).optional(),
|
|
339
|
-
})
|
|
340
|
-
.strict()
|
|
341
|
-
.optional(),
|
|
342
|
-
routing: z
|
|
343
|
-
.object({
|
|
344
|
-
contextTokenThreshold: z.number().int().positive().optional(),
|
|
345
|
-
toolsToDefault: z.boolean().optional(),
|
|
346
|
-
firstTurnToDefault: z.boolean().optional(),
|
|
347
|
-
priorToolDensityThreshold: z.number().int().positive().optional(),
|
|
348
|
-
strongTag: z.string().min(1).optional(),
|
|
349
|
-
cheapTag: z.string().min(1).optional(),
|
|
350
|
-
})
|
|
351
|
-
.strict()
|
|
352
|
-
.optional(),
|
|
353
|
-
learning: z
|
|
354
|
-
.object({
|
|
355
|
-
minSamplesPerArm: z.number().int().positive().optional(),
|
|
356
|
-
costRefUsd: z.number().positive().optional(),
|
|
357
|
-
latencyRefMs: z.number().int().positive().optional(),
|
|
358
|
-
// ε for ε-greedy online exploration once every arm clears the sample
|
|
359
|
-
// floor (fraction of exploit-phase turns that try a non-best model).
|
|
360
|
-
// Default 0 → deterministic explore-then-exploit, no RNG.
|
|
361
|
-
explorationRate: z.number().min(0).max(1).optional(),
|
|
362
|
-
// Fixed exploration seed for reproducible-across-runs behaviour (e.g.
|
|
363
|
-
// tests). Omitted → the runtime seeds from the sessionId, so each run
|
|
364
|
-
// explores differently while still replaying from its own transcript.
|
|
365
|
-
seed: z.string().min(1).optional(),
|
|
366
|
-
// Exploit-phase exploration strategy. "epsilon-greedy" (default) uses
|
|
367
|
-
// explorationRate; "thompson" draws each arm from its reward posterior
|
|
368
|
-
// and self-balances (explorationRate is then ignored).
|
|
369
|
-
bandit: z.enum(["epsilon-greedy", "thompson"]).optional(),
|
|
370
|
-
})
|
|
371
|
-
.strict()
|
|
372
|
-
.optional(),
|
|
373
|
-
})
|
|
374
|
-
.strict()
|
|
375
|
-
.optional();
|
|
376
352
|
/**
|
|
377
353
|
* Mutual-exclusion refine shared by every agent block that carries model
|
|
378
354
|
* routing: `model_pool` is the superset of both the two-tier router and the
|
|
379
355
|
* ordered failover chain, so declaring it alongside either is an error rather
|
|
380
356
|
* than an ambiguous double-route. (Per-candidate failover chains compose in a
|
|
381
357
|
* later release; this release keeps precedence unambiguous.)
|
|
358
|
+
*
|
|
359
|
+
* 0.6.0 §4.1 — the same refine rejects `temperature` beside `thinking` on
|
|
360
|
+
* one block: the Anthropic API returns 400 for an explicit temperature
|
|
361
|
+
* alongside extended thinking (`adapter-anthropic`'s translate documents the
|
|
362
|
+
* silent drop), so the pair is a parse error rather than a surprise.
|
|
382
363
|
*/
|
|
383
364
|
function refineModelSelection(agent, ctx) {
|
|
365
|
+
refineTemperatureThinking(agent, ctx);
|
|
384
366
|
if (agent.model_pool === undefined)
|
|
385
367
|
return;
|
|
386
368
|
if (agent.model_tiers !== undefined) {
|
|
@@ -398,6 +380,21 @@ function refineModelSelection(agent, ctx) {
|
|
|
398
380
|
});
|
|
399
381
|
}
|
|
400
382
|
}
|
|
383
|
+
/**
|
|
384
|
+
* 0.6.0 §4.1 — `temperature` and `thinking` on ONE block (agent / step /
|
|
385
|
+
* node / role / sub-agent / profile / pool candidate) is a parse error: the
|
|
386
|
+
* Anthropic API rejects an explicit temperature alongside extended thinking
|
|
387
|
+
* with a 400, and the adapter would otherwise drop the pin silently.
|
|
388
|
+
*/
|
|
389
|
+
function refineTemperatureThinking(block, ctx) {
|
|
390
|
+
if (block.temperature !== undefined && block.thinking !== undefined) {
|
|
391
|
+
ctx.addIssue({
|
|
392
|
+
code: z.ZodIssueCode.custom,
|
|
393
|
+
message: "temperature and thinking are mutually exclusive on one block — the Anthropic API rejects an explicit temperature alongside extended thinking (400); declare one or the other",
|
|
394
|
+
path: ["temperature"],
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
}
|
|
401
398
|
/**
|
|
402
399
|
* Section 17 — optional override for the model used by
|
|
403
400
|
* `compaction-autocompact` when summarising long conversations. Defaults
|
|
@@ -528,14 +525,40 @@ const failureTaxonomyBlock = z.array(failureTaxonomyEntrySchema).optional();
|
|
|
528
525
|
* Item 27 — run-level spend cap with a degradation ladder. Generalizes the
|
|
529
526
|
* optimizer's `--budget-usd` to normal runs. `usd` is the dollar ceiling;
|
|
530
527
|
* when the run's accrued spend reaches it, `on_exceed` decides:
|
|
531
|
-
* - `{ action: "stop" }` — end the run
|
|
528
|
+
* - `{ action: "stop" }` — end the run. 0.6.0 (§7.12): the cap is
|
|
529
|
+
* checked before EVERY model call, tool iterations included, so a
|
|
530
|
+
* runaway tool loop stops at the cap with a classified
|
|
531
|
+
* `crewhaus_budget` failure (the REPL's pre-turn check still ends an
|
|
532
|
+
* idle run cleanly before the next turn opens).
|
|
532
533
|
* - `{ action: "degrade", model }` — re-resolve the primary model to the
|
|
533
|
-
* cheaper `model` (one rung) and continue
|
|
534
|
-
*
|
|
535
|
-
*
|
|
536
|
-
*
|
|
537
|
-
*
|
|
534
|
+
* cheaper `model` (one rung) and continue: the rung serves every
|
|
535
|
+
* remaining model call of the turn in which the degrade fired (a
|
|
536
|
+
* mid-turn degrade finishes its tool loop; a single-turn host gets one
|
|
537
|
+
* complete degraded reply), and the run ends cleanly at the next turn
|
|
538
|
+
* boundary. Under a `model_pool` the rung does not
|
|
539
|
+
* swap the adapter: it becomes the FORCED pool candidate (`model_route`
|
|
540
|
+
* policy `forced`, reason `budget_degrade`). A `model` outside the pool
|
|
541
|
+
* roster is a compile WARNING (`budget-degrade-outside-pool`) plus an
|
|
542
|
+
* extra always-eligible rung — never a parse error.
|
|
543
|
+
* `scope` (0.6.0, default `run`) decides what the cap bounds: `run` meters
|
|
544
|
+
* this process's spend only; `session` also seeds the meter on `--resume`
|
|
545
|
+
* (and the channel/managed resume-per-message pattern) from the
|
|
546
|
+
* `cost_accrual` lines the session log already persists, so the cap bounds
|
|
547
|
+
* the whole conversation rather than one inbound message. Carried on the
|
|
548
|
+
* same interactive shapes as the failover chain (cli, channel, managed)
|
|
549
|
+
* plus the single-turn shapes. `on_exceed.model` follows the agent.model
|
|
538
550
|
* grammar. Defaults to `{ action: "stop" }` when `on_exceed` is omitted.
|
|
551
|
+
*
|
|
552
|
+
* `judge_share` (0.6.0 §6.2, default 0.3) is the fraction of `usd` the
|
|
553
|
+
* AUXILIARY model calls may spend between them — the in-loop `evaluation:`
|
|
554
|
+
* judge, `kind: judge` gates, compaction summaries, and (0.6.x) guide,
|
|
555
|
+
* classifier, consult, committee and shadow calls. Those calls ride the run
|
|
556
|
+
* bus with a `role` from 0.6.0, so they count toward `usd` like any other
|
|
557
|
+
* call; the share is the sub-cap inside it. Reaching the share records
|
|
558
|
+
* `reason: judge_share_exhausted` on the `eval_graded` event (the judge
|
|
559
|
+
* keeps judging under the total cap — a cascade consumes the signal to
|
|
560
|
+
* serve its strong rung directly). Deliberately EXCLUDED from the optimizer
|
|
561
|
+
* whitelist: a spend split is a human-owned policy, not a quality dial.
|
|
539
562
|
*/
|
|
540
563
|
const budgetBlock = z
|
|
541
564
|
.object({
|
|
@@ -546,6 +569,17 @@ const budgetBlock = z
|
|
|
546
569
|
z.object({ action: z.literal("degrade"), model: z.string().min(1) }).strict(),
|
|
547
570
|
])
|
|
548
571
|
.default({ action: "stop" }),
|
|
572
|
+
// 0.6.0 — optional, NO zod default: an absent key stays absent on the
|
|
573
|
+
// parsed spec and the IR, so pre-0.6.0 specs lower byte-identically.
|
|
574
|
+
scope: z.enum(["run", "session"]).optional(),
|
|
575
|
+
// 0.6.0 §6.2 — same discipline: optional, no zod default (the runtime
|
|
576
|
+
// owns the 0.3 default), so an absent key never reaches the bundle.
|
|
577
|
+
judge_share: z
|
|
578
|
+
.number()
|
|
579
|
+
.min(0)
|
|
580
|
+
.max(1)
|
|
581
|
+
.optional()
|
|
582
|
+
.describe("fraction of usd the auxiliary model calls (judge, compaction, guide, …) may spend; default 0.3"),
|
|
549
583
|
})
|
|
550
584
|
.strict()
|
|
551
585
|
.optional();
|
|
@@ -565,7 +599,9 @@ const budgetBlock = z
|
|
|
565
599
|
* - `{ type: llm_judge, criteria, model? }` — a model scores the reply
|
|
566
600
|
* in [0,1] against `criteria`; `model` defaults to the shape's primary
|
|
567
601
|
* model (the `cheapest` sentinel resolves like `compaction.model`).
|
|
568
|
-
* Judge calls are METERED into the run budget.
|
|
602
|
+
* Judge calls are METERED into the run budget: from 0.6.0 they ride the
|
|
603
|
+
* run bus with `role: "judge"` and count toward `budget.usd` under
|
|
604
|
+
* `budget.judge_share`.
|
|
569
605
|
* - `{ type: contains, value }` / `{ type: regex, value }` —
|
|
570
606
|
* deterministic pass/fail text checks (no threshold; no model spend).
|
|
571
607
|
*
|
|
@@ -573,6 +609,66 @@ const budgetBlock = z
|
|
|
573
609
|
* it with a deterministic grader is a parse error. `.strict()` throughout
|
|
574
610
|
* so a typo'd sub-key fails the build.
|
|
575
611
|
*/
|
|
612
|
+
/**
|
|
613
|
+
* 0.6.0 §6.2 — the judge-panel knobs shared by the in-loop `evaluation.grader`
|
|
614
|
+
* (`llm_judge`) and the `kind: judge` gate. They map one-to-one onto
|
|
615
|
+
* `createJudgeGrader(rubric, {judges, repeats, temperature, target})` in
|
|
616
|
+
* `@crewhaus/eval-judge`:
|
|
617
|
+
* - `judges` — a PANEL of judge models (each a grammar string or a
|
|
618
|
+
* `$profile`); mutually exclusive with the single `model`.
|
|
619
|
+
* - `repeats` — k repeat verdicts per judge, folded by median. It must be
|
|
620
|
+
* ODD: an even count has no median without a tie-break, and
|
|
621
|
+
* `createJudgeGrader` refuses one outright. The bound lives HERE rather
|
|
622
|
+
* than in the emitters so the refusal lands at parse time — before 0.6.0
|
|
623
|
+
* PR 13b these knobs were inert and an even count merely did nothing;
|
|
624
|
+
* now every judge site honours them, so an unrefused `repeats: 2` would
|
|
625
|
+
* throw mid-run (and `optimize` could patch one in, since
|
|
626
|
+
* `evaluation.grader.repeats` / `steps[].judge.repeats` are optimizable
|
|
627
|
+
* and applySpecPatch re-parses through this schema).
|
|
628
|
+
* - `temperature` — the judge's pinned sampling temperature (0 is the
|
|
629
|
+
* judge-bias literature's recommendation).
|
|
630
|
+
* - `target` — `output` (default) grades the final text; `transcript`
|
|
631
|
+
* grades the run trajectory digest.
|
|
632
|
+
* Categorical rubrics still reject panels and repeats (documented in
|
|
633
|
+
* eval-grader, not changed here).
|
|
634
|
+
*/
|
|
635
|
+
const judgePanelFields = {
|
|
636
|
+
judges: z
|
|
637
|
+
.array(z.string().min(1))
|
|
638
|
+
.min(1)
|
|
639
|
+
.optional()
|
|
640
|
+
.describe("judge PANEL: model ids (or $profile refs) whose verdicts are folded; mutually exclusive with model"),
|
|
641
|
+
repeats: z
|
|
642
|
+
.number()
|
|
643
|
+
.int()
|
|
644
|
+
.min(1)
|
|
645
|
+
.max(9)
|
|
646
|
+
.refine((n) => n % 2 === 1, {
|
|
647
|
+
message: "repeats must be odd — verdicts are folded by median, and an even count cannot break a tie",
|
|
648
|
+
})
|
|
649
|
+
.optional()
|
|
650
|
+
.describe("repeat verdicts per judge, folded by median; must be odd (default 1)"),
|
|
651
|
+
temperature: z
|
|
652
|
+
.number()
|
|
653
|
+
.min(0)
|
|
654
|
+
.max(2)
|
|
655
|
+
.optional()
|
|
656
|
+
.describe("pinned judge sampling temperature (0 recommended for stable verdicts)"),
|
|
657
|
+
target: z
|
|
658
|
+
.enum(["output", "transcript"])
|
|
659
|
+
.optional()
|
|
660
|
+
.describe("what the judge grades: the final text (output, default) or the run trajectory"),
|
|
661
|
+
};
|
|
662
|
+
/** `judges` (a panel) and `model` (one judge) name the same slot twice. */
|
|
663
|
+
function refineJudgePanel(block, ctx, label) {
|
|
664
|
+
if (block.judges !== undefined && block.model !== undefined) {
|
|
665
|
+
ctx.addIssue({
|
|
666
|
+
code: z.ZodIssueCode.custom,
|
|
667
|
+
path: ["judges"],
|
|
668
|
+
message: `${label}.judges (a judge panel) and ${label}.model (one judge) are mutually exclusive — declare the panel OR the single judge`,
|
|
669
|
+
});
|
|
670
|
+
}
|
|
671
|
+
}
|
|
576
672
|
const evaluationGraderSchema = z.discriminatedUnion("type", [
|
|
577
673
|
z
|
|
578
674
|
.object({
|
|
@@ -581,11 +677,13 @@ const evaluationGraderSchema = z.discriminatedUnion("type", [
|
|
|
581
677
|
.string()
|
|
582
678
|
.min(1)
|
|
583
679
|
.optional()
|
|
584
|
-
.describe("judge model id; defaults to the shape's primary model (the cheapest sentinel resolves at compile time)"),
|
|
680
|
+
.describe("judge model id or $profile ref; defaults to the shape's primary model (the cheapest sentinel resolves at compile time)"),
|
|
585
681
|
criteria: z
|
|
586
682
|
.string()
|
|
587
683
|
.min(1)
|
|
588
684
|
.describe("what a passing reply must satisfy — the judge scores the final text against this"),
|
|
685
|
+
// 0.6.0 §6.2 — judge panels, repeats, pinned temperature, target.
|
|
686
|
+
...judgePanelFields,
|
|
589
687
|
})
|
|
590
688
|
.strict()
|
|
591
689
|
.describe("model-scored grader: an LLM judges the final text in [0,1] against criteria"),
|
|
@@ -614,9 +712,9 @@ const evaluationBlock = z
|
|
|
614
712
|
.optional()
|
|
615
713
|
.describe("passing score in 0..1 (default 0.7); llm_judge grader only"),
|
|
616
714
|
on_fail: z
|
|
617
|
-
.enum(["retry", "halt", "note"])
|
|
715
|
+
.enum(["retry", "halt", "note", "escalate"])
|
|
618
716
|
.optional()
|
|
619
|
-
.describe("below-threshold behaviour: retry re-prompts with the judge rationale (default), halt aborts the turn classified, note emits a trace event only"),
|
|
717
|
+
.describe("below-threshold behaviour: retry re-prompts with the judge rationale (default), halt aborts the turn classified, note emits a trace event only, escalate (0.6.0) re-runs the turn on the pool's strategy.cascade.escalate_to candidate (else the strongest candidate)"),
|
|
620
718
|
max_retries: z
|
|
621
719
|
.number()
|
|
622
720
|
.int()
|
|
@@ -624,9 +722,22 @@ const evaluationBlock = z
|
|
|
624
722
|
.max(5)
|
|
625
723
|
.optional()
|
|
626
724
|
.describe("hard cap on evaluation-triggered retries per turn (default 1)"),
|
|
725
|
+
/**
|
|
726
|
+
* 0.6.0 §4.3 — silences the judge-independence lint (`crewhaus lint` /
|
|
727
|
+
* `doctor --philosophy-alignment` warn when a pooled or strategy spec's
|
|
728
|
+
* judge is the serving arm itself). Never optimizer-tunable: it is a
|
|
729
|
+
* measurement-integrity waiver, not a quality knob.
|
|
730
|
+
*/
|
|
731
|
+
allow_self_judge: z
|
|
732
|
+
.boolean()
|
|
733
|
+
.optional()
|
|
734
|
+
.describe("0.6.0: accept a judge that is also a serving arm (silences the judge-independence lint)"),
|
|
627
735
|
})
|
|
628
736
|
.strict()
|
|
629
737
|
.superRefine((e, ctx) => {
|
|
738
|
+
if (e.grader.type === "llm_judge") {
|
|
739
|
+
refineJudgePanel(e.grader, ctx, "evaluation.grader");
|
|
740
|
+
}
|
|
630
741
|
if (e.threshold !== undefined && e.grader.type !== "llm_judge") {
|
|
631
742
|
ctx.addIssue({
|
|
632
743
|
code: z.ZodIssueCode.custom,
|
|
@@ -664,7 +775,8 @@ const evaluationBlock = z
|
|
|
664
775
|
*
|
|
665
776
|
* `model` defaults to the shape's top-level `model` (the `cheapest`
|
|
666
777
|
* sentinel resolves like `compaction.model`). Judge calls are METERED into
|
|
667
|
-
* the run budget.
|
|
778
|
+
* the run budget: from 0.6.0 they publish on the run bus with
|
|
779
|
+
* `role: "judge"` (the workflow shape's meter spans the whole run).
|
|
668
780
|
*/
|
|
669
781
|
const judgeGateBlock = z
|
|
670
782
|
.object({
|
|
@@ -676,7 +788,7 @@ const judgeGateBlock = z
|
|
|
676
788
|
.string()
|
|
677
789
|
.min(1)
|
|
678
790
|
.optional()
|
|
679
|
-
.describe("judge model id; defaults to the shape's top-level model"),
|
|
791
|
+
.describe("judge model id or $profile ref; defaults to the shape's top-level model"),
|
|
680
792
|
threshold: z.number().min(0).max(1).optional().describe("passing score in 0..1 (default 0.7)"),
|
|
681
793
|
on_fail: z
|
|
682
794
|
.enum(["retry_previous", "halt", "continue"])
|
|
@@ -689,8 +801,22 @@ const judgeGateBlock = z
|
|
|
689
801
|
.max(5)
|
|
690
802
|
.optional()
|
|
691
803
|
.describe("hard cap on judge-triggered re-runs of the gated step/node (default 1)"),
|
|
804
|
+
// 0.6.0 §6.2 — judge panels, repeats, pinned temperature, target.
|
|
805
|
+
...judgePanelFields,
|
|
806
|
+
/**
|
|
807
|
+
* 0.6.0 §7.3 — the candidate a `retry_previous` re-run is FORCED onto:
|
|
808
|
+
* a tag or `$profile` from the gated step's/node's `model_pool` (the
|
|
809
|
+
* cascade's strong rung). Absent ⇒ the re-run keeps the gated block's
|
|
810
|
+
* own routing policy (today's behaviour).
|
|
811
|
+
*/
|
|
812
|
+
escalate_to: z
|
|
813
|
+
.string()
|
|
814
|
+
.min(1)
|
|
815
|
+
.optional()
|
|
816
|
+
.describe("0.6.0: pool tag or $profile the retry_previous re-run is forced onto (the gated step's/node's model_pool must declare it)"),
|
|
692
817
|
})
|
|
693
818
|
.strict()
|
|
819
|
+
.superRefine((j, ctx) => refineJudgePanel(j, ctx, "judge"))
|
|
694
820
|
.describe("judge gate config for kind: judge workflow steps and graph nodes");
|
|
695
821
|
/**
|
|
696
822
|
* Loop contract 0.4 (Batch A) — extended-thinking selector, carried on the
|
|
@@ -722,6 +848,21 @@ const thinkingBlock = z
|
|
|
722
848
|
}
|
|
723
849
|
})
|
|
724
850
|
.optional();
|
|
851
|
+
/**
|
|
852
|
+
* 0.6.0 §4.1 — the sampling-temperature knob, carried on the agent / step /
|
|
853
|
+
* node / role / sub-agent blocks and on every model profile and pool
|
|
854
|
+
* candidate. Mutually exclusive with `thinking` on the same block
|
|
855
|
+
* (`refineTemperatureThinking`). The range is the widest any in-tree
|
|
856
|
+
* provider accepts (OpenAI 0–2; Anthropic 0–1, and the Claude 5 family
|
|
857
|
+
* rejects the parameter outright — the adapter reports the drop through
|
|
858
|
+
* `effectiveParams`). OPTIMIZABLE at `agent.temperature` (0.6.0 §10.3, PR 19).
|
|
859
|
+
*/
|
|
860
|
+
const temperatureField = z
|
|
861
|
+
.number()
|
|
862
|
+
.min(0)
|
|
863
|
+
.max(2)
|
|
864
|
+
.optional()
|
|
865
|
+
.describe("sampling temperature (0–2); mutually exclusive with thinking on the same block");
|
|
725
866
|
/**
|
|
726
867
|
* Loop contract 0.4 (Batch A) — runaway-loop detection tuning inside the
|
|
727
868
|
* `limits:` block. `window` is the trailing tool-call window inspected;
|
|
@@ -840,6 +981,474 @@ const rateLimitsBlock = z
|
|
|
840
981
|
})
|
|
841
982
|
.strict())
|
|
842
983
|
.optional();
|
|
984
|
+
// ---------------------------------------------------------------------------
|
|
985
|
+
// 0.6.0 — per-model settings and hybrid setups (design plan §4, §5, §7).
|
|
986
|
+
//
|
|
987
|
+
// A `models:` registry declares everything that can differ per model ONCE;
|
|
988
|
+
// any model slot then references a profile as `$<name>`. A `model_pool`
|
|
989
|
+
// candidate may carry the same fields inline. The spec layer carries the
|
|
990
|
+
// declarations VERBATIM (no zod defaults on any new key, so a spec that omits
|
|
991
|
+
// them parses — and lowers — byte-identically to 0.5.x); the compiler resolves
|
|
992
|
+
// `$refs`, merges profile defaults field-by-field and validates the roster
|
|
993
|
+
// at lower time.
|
|
994
|
+
// ---------------------------------------------------------------------------
|
|
995
|
+
/**
|
|
996
|
+
* 0.6.0 §5.4 — per-profile permissions, a RESTRICTED schema: `deny` and `ask`
|
|
997
|
+
* ONLY. `permissionsBlock` admits `alwaysAllow` rules, a `mode` and an
|
|
998
|
+
* `ask_mode`, and a yaml-tier `alwaysAllow` outranks the builtin floor
|
|
999
|
+
* guards in the permission engine — exactly the escalation the sub-agent
|
|
1000
|
+
* permission inheritance was written to contain. A profile can therefore
|
|
1001
|
+
* NARROW the shape's permissions (decision-level meet: deny < ask < allow),
|
|
1002
|
+
* never widen them; the runtime proves the narrowing through `evaluate()`.
|
|
1003
|
+
*/
|
|
1004
|
+
const PROFILE_PERMISSION_FORBIDDEN_KEYS = [
|
|
1005
|
+
"alwaysAllow",
|
|
1006
|
+
"allow",
|
|
1007
|
+
"mode",
|
|
1008
|
+
"ask_mode",
|
|
1009
|
+
"rules",
|
|
1010
|
+
];
|
|
1011
|
+
const profilePermissionsBlock = z
|
|
1012
|
+
.object({
|
|
1013
|
+
deny: z.array(z.string().min(1)).optional(),
|
|
1014
|
+
ask: z.array(z.string().min(1)).optional(),
|
|
1015
|
+
})
|
|
1016
|
+
.strict(`a model profile's permissions may only NARROW the shape's: deny and ask lists only (${PROFILE_PERMISSION_FORBIDDEN_KEYS.join(", ")} are rejected — a profile can never widen what the shape allows)`)
|
|
1017
|
+
.optional();
|
|
1018
|
+
/**
|
|
1019
|
+
* 0.6.0 §7.11 (N1) — what a profile REQUIRES of its own model. The four
|
|
1020
|
+
* feature booleans mirror `ProviderFeatures` in `@crewhaus/adapter-anthropic`
|
|
1021
|
+
* (`tool_use`, `vision`, `thinking`, `web_search`); the two size floors are
|
|
1022
|
+
* spelled as floors because a requirement is unambiguously one
|
|
1023
|
+
* (`context_window_gte`, `max_output_tokens_gte` ↔ the capability table's
|
|
1024
|
+
* `contextWindowGte` / `maxOutputTokensGte`). Validated at compile time for
|
|
1025
|
+
* table-backed providers and re-checked per turn as an eligibility filter.
|
|
1026
|
+
*/
|
|
1027
|
+
const modelRequiresObject = z
|
|
1028
|
+
.object({
|
|
1029
|
+
tool_use: z.boolean().optional(),
|
|
1030
|
+
vision: z.boolean().optional(),
|
|
1031
|
+
thinking: z.boolean().optional(),
|
|
1032
|
+
web_search: z.boolean().optional(),
|
|
1033
|
+
context_window_gte: z
|
|
1034
|
+
.number()
|
|
1035
|
+
.int()
|
|
1036
|
+
.positive()
|
|
1037
|
+
.optional()
|
|
1038
|
+
.describe("the model's context window must be KNOWN and at least this many tokens"),
|
|
1039
|
+
max_output_tokens_gte: z
|
|
1040
|
+
.number()
|
|
1041
|
+
.int()
|
|
1042
|
+
.positive()
|
|
1043
|
+
.optional()
|
|
1044
|
+
.describe("the model's max output must be KNOWN and at least this many tokens"),
|
|
1045
|
+
})
|
|
1046
|
+
.strict();
|
|
1047
|
+
const modelRequiresBlock = modelRequiresObject.optional();
|
|
1048
|
+
/**
|
|
1049
|
+
* 0.6.0 §4.1 — a DECLARED capability override for models the capability
|
|
1050
|
+
* table does not know (local / azure / named hosts): the same feature
|
|
1051
|
+
* booleans plus the two size facts. Table-backed models never need it; a
|
|
1052
|
+
* non-table model without it gets a compile warning that `adapter.features`
|
|
1053
|
+
* is the only gate.
|
|
1054
|
+
*/
|
|
1055
|
+
const modelCapabilitiesBlock = z
|
|
1056
|
+
.object({
|
|
1057
|
+
tool_use: z.boolean().optional(),
|
|
1058
|
+
vision: z.boolean().optional(),
|
|
1059
|
+
thinking: z.boolean().optional(),
|
|
1060
|
+
web_search: z.boolean().optional(),
|
|
1061
|
+
caching: z.union([z.enum(["explicit", "automatic"]), z.literal(false)]).optional(),
|
|
1062
|
+
context_window: z.number().int().positive().optional(),
|
|
1063
|
+
max_output_tokens: z.number().int().positive().optional(),
|
|
1064
|
+
})
|
|
1065
|
+
.strict()
|
|
1066
|
+
.optional();
|
|
1067
|
+
/**
|
|
1068
|
+
* 0.6.0 §4.1 — the per-model settings a `models:` profile (and a pool
|
|
1069
|
+
* candidate, inline) may declare. Everything is optional; a profile supplies
|
|
1070
|
+
* DEFAULTS and slot-local fields override field-by-field at lower time
|
|
1071
|
+
* (`tags` REPLACE the profile's tags when the slot declares any — they are
|
|
1072
|
+
* the routing identity, not an accumulation).
|
|
1073
|
+
*/
|
|
1074
|
+
const modelProfileFields = {
|
|
1075
|
+
/** Routing identity tags (`cheap`, `strong`, …). */
|
|
1076
|
+
tags: z.array(z.string().min(1)).optional(),
|
|
1077
|
+
/** Model max OUTPUT tokens for one call on this model. */
|
|
1078
|
+
max_tokens: z.number().int().positive().optional(),
|
|
1079
|
+
thinking: thinkingBlock,
|
|
1080
|
+
temperature: temperatureField,
|
|
1081
|
+
/**
|
|
1082
|
+
* Per-model instructions OVERLAY: appended in the volatile region of the
|
|
1083
|
+
* system prompt when this candidate serves (never busts the cached
|
|
1084
|
+
* prefix); folded into `instructions` on a single-model slot.
|
|
1085
|
+
*/
|
|
1086
|
+
instructions: z
|
|
1087
|
+
.string()
|
|
1088
|
+
.min(1)
|
|
1089
|
+
.optional()
|
|
1090
|
+
.describe("per-model instructions overlay, appended when this model serves"),
|
|
1091
|
+
/**
|
|
1092
|
+
* SUBSET-ONLY tool selection — never additive. Builtin keys, server-scoped
|
|
1093
|
+
* MCP globs (`mcp__<server>__*`), `Consult` / `Escalate` (only with
|
|
1094
|
+
* `strategy.model_directed: true`). `[]` means ZERO shape tools (the
|
|
1095
|
+
* auto-registered loop tools survive unless denied via `permissions`).
|
|
1096
|
+
*/
|
|
1097
|
+
tools: z
|
|
1098
|
+
.array(z.string().min(1))
|
|
1099
|
+
.optional()
|
|
1100
|
+
.describe("subset of the shape's tools this model may see; [] = no shape tools"),
|
|
1101
|
+
/** Same `toolConfigBlock` superRefine ⇒ the sandbox-override guard applies per profile. */
|
|
1102
|
+
tool_config: toolConfigBlock,
|
|
1103
|
+
permissions: profilePermissionsBlock,
|
|
1104
|
+
rate_limits: rateLimitsBlock,
|
|
1105
|
+
limits: z
|
|
1106
|
+
.object({
|
|
1107
|
+
model_call_timeout_ms: z.number().int().positive().optional(),
|
|
1108
|
+
})
|
|
1109
|
+
.strict()
|
|
1110
|
+
.optional(),
|
|
1111
|
+
/** `prefer` (default) keeps prompt-cache markers; `off` strips them for this model. */
|
|
1112
|
+
caching: z.enum(["prefer", "off"]).optional(),
|
|
1113
|
+
/** Per-profile spend cap INSIDE a run: ineligible when spent, never ends the run. */
|
|
1114
|
+
cost: z
|
|
1115
|
+
.object({
|
|
1116
|
+
max_usd: z.number().positive(),
|
|
1117
|
+
})
|
|
1118
|
+
.strict()
|
|
1119
|
+
.optional(),
|
|
1120
|
+
requires: modelRequiresBlock,
|
|
1121
|
+
capabilities: modelCapabilitiesBlock,
|
|
1122
|
+
/** Per-profile failover chain (same grammar as `model_fallbacks`). */
|
|
1123
|
+
fallbacks: z.array(z.string().min(1)).min(1).optional(),
|
|
1124
|
+
circuit_breaker: circuitBreakerBlock,
|
|
1125
|
+
};
|
|
1126
|
+
/**
|
|
1127
|
+
* 0.6.0 §4.1 — one `models:` profile. `model` is required and is a
|
|
1128
|
+
* model-router grammar string or a sentinel (`cheapest` | `strongest`) —
|
|
1129
|
+
* NEVER a `$ref` (a profile referencing a profile would be circular; the
|
|
1130
|
+
* cross-field check rejects it). Sentinels inside a profile resolve by price
|
|
1131
|
+
* rank against the primary only.
|
|
1132
|
+
*/
|
|
1133
|
+
const modelProfileSchema = z
|
|
1134
|
+
.object({
|
|
1135
|
+
model: z
|
|
1136
|
+
.string()
|
|
1137
|
+
.min(1)
|
|
1138
|
+
.describe("model-router grammar string, or cheapest | strongest (never a $ref)"),
|
|
1139
|
+
...modelProfileFields,
|
|
1140
|
+
})
|
|
1141
|
+
.strict()
|
|
1142
|
+
.superRefine(refineTemperatureThinking)
|
|
1143
|
+
.describe("one model profile: everything that can differ per model, declared once");
|
|
1144
|
+
/**
|
|
1145
|
+
* 0.6.0 §4.1 — the top-level `models:` profile registry, attached to ALL 14
|
|
1146
|
+
* strict schemas (the `version` precedent: an optional field absent from any
|
|
1147
|
+
* union member would be rejected on that target). Keys are profile names
|
|
1148
|
+
* (`SPEC_PROFILE_NAME_RE`), values are profiles. Every model slot in the
|
|
1149
|
+
* spec accepts `$<name>` beside a grammar string; the compiler resolves the
|
|
1150
|
+
* reference at lower time. Absent ⇒ byte-identical bundles.
|
|
1151
|
+
*/
|
|
1152
|
+
const modelsBlock = z
|
|
1153
|
+
.record(profileName, modelProfileSchema)
|
|
1154
|
+
.optional()
|
|
1155
|
+
.describe("0.6.0: the model-profile registry; any model slot may reference a profile as $<name>");
|
|
1156
|
+
/**
|
|
1157
|
+
* 0.6.0 §7.1 — one pool candidate: `model` (grammar string or `$profile`),
|
|
1158
|
+
* `tags` (defaulted to `[]` for 0.5.x parity; on a `$profile` candidate an
|
|
1159
|
+
* empty list means "inherit the profile's tags"), `enabled: false` to
|
|
1160
|
+
* withdraw the candidate from routing without deleting its learned history,
|
|
1161
|
+
* plus every profile field inline.
|
|
1162
|
+
*/
|
|
1163
|
+
const modelPoolCandidateSchema = z
|
|
1164
|
+
.object({
|
|
1165
|
+
model: z.string().min(1),
|
|
1166
|
+
...modelProfileFields,
|
|
1167
|
+
tags: z.array(z.string().min(1)).default([]),
|
|
1168
|
+
/** `false` withdraws the candidate from routing; its arms survive. */
|
|
1169
|
+
enabled: z.literal(false).optional(),
|
|
1170
|
+
})
|
|
1171
|
+
.strict()
|
|
1172
|
+
.superRefine(refineTemperatureThinking);
|
|
1173
|
+
/**
|
|
1174
|
+
* 0.6.0 §7.2.2 — one rule-directed routing rule. Rules are pure, evaluated
|
|
1175
|
+
* first-match in `preRoute` before the policy; the matched `id` is
|
|
1176
|
+
* persisted on the `model_route` line. `when` must carry at least one
|
|
1177
|
+
* condition; `message_matches` must compile (the runtime additionally
|
|
1178
|
+
* validates it against catastrophic-backtracking shapes, length-caps the
|
|
1179
|
+
* input and time-budgets the evaluation, because on a channel shape the
|
|
1180
|
+
* regex runs against attacker-controlled text). `use` is a tag, a
|
|
1181
|
+
* `$profile` from the roster, or a capability requirement the turn's
|
|
1182
|
+
* candidates must satisfy. `enabled: false` disables the rule without
|
|
1183
|
+
* deleting it (the optimizer's switch).
|
|
1184
|
+
*/
|
|
1185
|
+
const modelPoolRuleWhenSchema = z
|
|
1186
|
+
.object({
|
|
1187
|
+
has_images: z.boolean().optional(),
|
|
1188
|
+
message_matches: z.string().min(1).optional(),
|
|
1189
|
+
user_text_chars_gt: z.number().int().nonnegative().optional(),
|
|
1190
|
+
context_tokens_gt: z.number().int().nonnegative().optional(),
|
|
1191
|
+
tool_in_play: z.boolean().optional(),
|
|
1192
|
+
channel: z.string().min(1).optional(),
|
|
1193
|
+
budget_spent_ratio_gt: z.number().min(0).max(1).optional(),
|
|
1194
|
+
turn_index_lt: z.number().int().positive().optional(),
|
|
1195
|
+
})
|
|
1196
|
+
.strict()
|
|
1197
|
+
.superRefine((w, ctx) => {
|
|
1198
|
+
if (Object.values(w).every((v) => v === undefined)) {
|
|
1199
|
+
ctx.addIssue({
|
|
1200
|
+
code: z.ZodIssueCode.custom,
|
|
1201
|
+
message: "rule when: requires at least one condition (has_images | message_matches | user_text_chars_gt | context_tokens_gt | tool_in_play | channel | budget_spent_ratio_gt | turn_index_lt)",
|
|
1202
|
+
});
|
|
1203
|
+
}
|
|
1204
|
+
if (w.message_matches !== undefined) {
|
|
1205
|
+
try {
|
|
1206
|
+
new RegExp(w.message_matches);
|
|
1207
|
+
}
|
|
1208
|
+
catch (err) {
|
|
1209
|
+
ctx.addIssue({
|
|
1210
|
+
code: z.ZodIssueCode.custom,
|
|
1211
|
+
path: ["message_matches"],
|
|
1212
|
+
message: `rule when.message_matches is not a valid regular expression: ${err instanceof Error ? err.message : String(err)}`,
|
|
1213
|
+
});
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
});
|
|
1217
|
+
const modelPoolRuleSchema = z
|
|
1218
|
+
.object({
|
|
1219
|
+
id: safeName,
|
|
1220
|
+
when: modelPoolRuleWhenSchema,
|
|
1221
|
+
use: z.union([
|
|
1222
|
+
z.string().min(1).describe("a candidate tag or a $profile from the roster"),
|
|
1223
|
+
z
|
|
1224
|
+
.object({ requires: modelRequiresObject })
|
|
1225
|
+
.strict()
|
|
1226
|
+
.describe("route to a candidate satisfying this capability requirement"),
|
|
1227
|
+
]),
|
|
1228
|
+
enabled: z.boolean().optional(),
|
|
1229
|
+
})
|
|
1230
|
+
.strict();
|
|
1231
|
+
/**
|
|
1232
|
+
* 0.6.0 §7.2.3 — `policy: classifier`: a forced-tool single call on
|
|
1233
|
+
* `model` whose label is constrained to the declared candidate tags (an
|
|
1234
|
+
* enum-constrained verdict — no free text, so no new content boundary).
|
|
1235
|
+
* `labels` maps each tag to the description the classifier chooses among;
|
|
1236
|
+
* `max_tokens` bounds the call (default 16). Falls back to `heuristic` on
|
|
1237
|
+
* any error.
|
|
1238
|
+
*/
|
|
1239
|
+
const modelPoolClassifierSchema = z
|
|
1240
|
+
.object({
|
|
1241
|
+
model: z.string().min(1),
|
|
1242
|
+
labels: z.record(z.string().min(1), z.string().min(1)),
|
|
1243
|
+
max_tokens: z.number().int().positive().optional(),
|
|
1244
|
+
})
|
|
1245
|
+
.strict()
|
|
1246
|
+
.superRefine((c, ctx) => {
|
|
1247
|
+
if (Object.keys(c.labels).length === 0) {
|
|
1248
|
+
ctx.addIssue({
|
|
1249
|
+
code: z.ZodIssueCode.custom,
|
|
1250
|
+
path: ["labels"],
|
|
1251
|
+
message: "classifier.labels requires at least one <tag>: <description> entry",
|
|
1252
|
+
});
|
|
1253
|
+
}
|
|
1254
|
+
});
|
|
1255
|
+
/**
|
|
1256
|
+
* 0.6.0 §7.3–§7.8 — the hybrid strategy block. Role slots (`draft`,
|
|
1257
|
+
* `escalate_to`, `members`, `escalate_on_disagreement`) name a candidate TAG
|
|
1258
|
+
* or a `$profile` in the roster; model slots (`guide.model`, `grade_with`,
|
|
1259
|
+
* `committee.judge`, `shadow.candidate`) accept a grammar string or a
|
|
1260
|
+
* `$profile`. `clean_prompt` is declared ONCE, here on `cascade` (the
|
|
1261
|
+
* `evaluation` block does not carry it). `committee` is legal on
|
|
1262
|
+
* single-turn hosts only (workflow steps, graph nodes, crew roles).
|
|
1263
|
+
* `model_directed: true` registers the `Escalate` + `Consult` tools.
|
|
1264
|
+
*/
|
|
1265
|
+
const modelPoolStrategySchema = z
|
|
1266
|
+
.object({
|
|
1267
|
+
cascade: z
|
|
1268
|
+
.object({
|
|
1269
|
+
draft: z.string().min(1),
|
|
1270
|
+
escalate_to: z.string().min(1),
|
|
1271
|
+
clean_prompt: z.boolean().optional(),
|
|
1272
|
+
})
|
|
1273
|
+
.strict()
|
|
1274
|
+
.optional(),
|
|
1275
|
+
guide: z
|
|
1276
|
+
.object({
|
|
1277
|
+
model: z.string().min(1),
|
|
1278
|
+
every: z.enum(["first_turn", "turn"]).optional(),
|
|
1279
|
+
max_tokens: z.number().int().positive().optional(),
|
|
1280
|
+
budget_usd: z.number().positive().optional(),
|
|
1281
|
+
})
|
|
1282
|
+
.strict()
|
|
1283
|
+
.optional(),
|
|
1284
|
+
shadow: z
|
|
1285
|
+
.object({
|
|
1286
|
+
candidate: z.string().min(1),
|
|
1287
|
+
sample_rate: z.number().min(0).max(1).optional(),
|
|
1288
|
+
grade_with: z.string().min(1).optional(),
|
|
1289
|
+
})
|
|
1290
|
+
.strict()
|
|
1291
|
+
.optional(),
|
|
1292
|
+
committee: z
|
|
1293
|
+
.object({
|
|
1294
|
+
members: z.array(z.string().min(1)).min(2),
|
|
1295
|
+
judge: z.string().min(1).optional(),
|
|
1296
|
+
escalate_on_disagreement: z.string().min(1).optional(),
|
|
1297
|
+
})
|
|
1298
|
+
.strict()
|
|
1299
|
+
.optional(),
|
|
1300
|
+
model_directed: z.boolean().optional(),
|
|
1301
|
+
max_escalations: z.number().int().nonnegative().optional(),
|
|
1302
|
+
})
|
|
1303
|
+
.strict();
|
|
1304
|
+
/**
|
|
1305
|
+
* 0.6.0 §6.3, §7.10 — the reward block, a SIBLING of `learning` (which the
|
|
1306
|
+
* optimizer whitelists wholesale; nothing here is optimizer-tunable).
|
|
1307
|
+
* `quality_source` (runtime default `none`) decides whether graded quality
|
|
1308
|
+
* reaches the live reward; `in_loop` REQUIRES a `floor` and an in-loop
|
|
1309
|
+
* grader on the same shape (cross-field). `floor` bounds learned
|
|
1310
|
+
* exploitation: an arm is exploitable only while its Wilson lower bound at
|
|
1311
|
+
* `confidence` stays within `tolerance` of the floor arm's judged mean.
|
|
1312
|
+
*/
|
|
1313
|
+
const modelPoolRewardSchema = z
|
|
1314
|
+
.object({
|
|
1315
|
+
quality_source: z.enum(["none", "in_loop", "shadow", "promoted"]).optional(),
|
|
1316
|
+
priors: z.enum(["none", "eval"]).optional(),
|
|
1317
|
+
floor: z
|
|
1318
|
+
.object({
|
|
1319
|
+
arm: z.string().min(1).optional(),
|
|
1320
|
+
confidence: z.number().gt(0).lt(1).optional(),
|
|
1321
|
+
tolerance: z.number().min(0).max(1).optional(),
|
|
1322
|
+
})
|
|
1323
|
+
.strict()
|
|
1324
|
+
.optional(),
|
|
1325
|
+
reset_on_profile_change: z.boolean().optional(),
|
|
1326
|
+
})
|
|
1327
|
+
.strict();
|
|
1328
|
+
const modelPoolBlock = z
|
|
1329
|
+
.object({
|
|
1330
|
+
candidates: z.array(modelPoolCandidateSchema).min(2),
|
|
1331
|
+
policy: z.enum(["static", "heuristic", "learned", "classifier"]).default("heuristic"),
|
|
1332
|
+
objective: z
|
|
1333
|
+
.object({
|
|
1334
|
+
quality: z.number().min(0).optional(),
|
|
1335
|
+
cost: z.number().min(0).optional(),
|
|
1336
|
+
latency: z.number().min(0).optional(),
|
|
1337
|
+
})
|
|
1338
|
+
.strict()
|
|
1339
|
+
.optional(),
|
|
1340
|
+
routing: z
|
|
1341
|
+
.object({
|
|
1342
|
+
contextTokenThreshold: z.number().int().positive().optional(),
|
|
1343
|
+
toolsToDefault: z.boolean().optional(),
|
|
1344
|
+
firstTurnToDefault: z.boolean().optional(),
|
|
1345
|
+
priorToolDensityThreshold: z.number().int().positive().optional(),
|
|
1346
|
+
strongTag: z.string().min(1).optional(),
|
|
1347
|
+
cheapTag: z.string().min(1).optional(),
|
|
1348
|
+
})
|
|
1349
|
+
.strict()
|
|
1350
|
+
.optional(),
|
|
1351
|
+
learning: z
|
|
1352
|
+
.object({
|
|
1353
|
+
minSamplesPerArm: z.number().int().positive().optional(),
|
|
1354
|
+
costRefUsd: z.number().positive().optional(),
|
|
1355
|
+
latencyRefMs: z.number().int().positive().optional(),
|
|
1356
|
+
// ε for ε-greedy online exploration once every arm clears the sample
|
|
1357
|
+
// floor (fraction of exploit-phase turns that try a non-best model).
|
|
1358
|
+
// Default 0 → deterministic explore-then-exploit, no RNG.
|
|
1359
|
+
explorationRate: z.number().min(0).max(1).optional(),
|
|
1360
|
+
// Fixed exploration seed for reproducible-across-runs behaviour (e.g.
|
|
1361
|
+
// tests). Omitted → the runtime seeds from the sessionId, so each run
|
|
1362
|
+
// explores differently while still replaying from its own transcript.
|
|
1363
|
+
seed: z.string().min(1).optional(),
|
|
1364
|
+
// Exploit-phase exploration strategy. "epsilon-greedy" (default) uses
|
|
1365
|
+
// explorationRate; "thompson" draws each arm from its reward posterior
|
|
1366
|
+
// and self-balances (explorationRate is then ignored).
|
|
1367
|
+
bandit: z.enum(["epsilon-greedy", "thompson"]).optional(),
|
|
1368
|
+
})
|
|
1369
|
+
.strict()
|
|
1370
|
+
.optional(),
|
|
1371
|
+
// ---- 0.6.0 §7.1 — the hybrid container. Optional, NO zod defaults:
|
|
1372
|
+
// an absent key stays absent on the parsed spec (byte-identical
|
|
1373
|
+
// lowering); the runtime owns `directives: false` and
|
|
1374
|
+
// `reward.quality_source: none`.
|
|
1375
|
+
/** Per-message `/model` steering; default OFF on every shape (§7.2.1). */
|
|
1376
|
+
directives: z
|
|
1377
|
+
.boolean()
|
|
1378
|
+
.optional()
|
|
1379
|
+
.describe("0.6.0: accept per-message /model directives (default false on every shape)"),
|
|
1380
|
+
rules: z.array(modelPoolRuleSchema).min(1).optional(),
|
|
1381
|
+
classifier: modelPoolClassifierSchema.optional(),
|
|
1382
|
+
strategy: modelPoolStrategySchema.optional(),
|
|
1383
|
+
reward: modelPoolRewardSchema.optional(),
|
|
1384
|
+
/** Scoped routeKey prefix (§7.9). Carried verbatim when declared; when absent the
|
|
1385
|
+
* compiler leaves it unstamped (pre-0.6.0 pool blobs stay byte-identical) and the
|
|
1386
|
+
* host defaults it at runtime — the crew orchestrator to the role name (PR 7b), the
|
|
1387
|
+
* composition root to the caller's toolset scope (PR 10). */
|
|
1388
|
+
scope: safeName.optional(),
|
|
1389
|
+
})
|
|
1390
|
+
.strict()
|
|
1391
|
+
.optional();
|
|
1392
|
+
// Section 13 — sub-agent definitions. Inline on the agent block (cli +
|
|
1393
|
+
// channel today; workflow has no agent block). The map's key is the
|
|
1394
|
+
// `subagent_type` users pass to the Task tool. Permissions field mirrors
|
|
1395
|
+
// the runtime's resolution shape.
|
|
1396
|
+
//
|
|
1397
|
+
// 0.6.0 §7.7 — a sub-agent carries the same routing blocks as an agent
|
|
1398
|
+
// (`model_pool` / `model_tiers` / `model_fallbacks` / `circuit_breaker`,
|
|
1399
|
+
// sharing `refineModelSelection`), its own `thinking` / `max_tokens` /
|
|
1400
|
+
// `temperature`, a `budget_share` of the parent's cap, `inherit_routing`
|
|
1401
|
+
// (children inherit the SERVED arm only behind it — default false keeps
|
|
1402
|
+
// today's declared-primary behaviour) and `allowed_profiles` (the
|
|
1403
|
+
// `$profile` allowlist the Task tool's model-filled `profile` argument is
|
|
1404
|
+
// validated against).
|
|
1405
|
+
const subAgentDefinitionSchema = z
|
|
1406
|
+
.object({
|
|
1407
|
+
description: z.string().min(1),
|
|
1408
|
+
instructions: z.string().min(1),
|
|
1409
|
+
tools: z.array(z.string().min(1)).optional(),
|
|
1410
|
+
model: z.string().min(1).optional(),
|
|
1411
|
+
permissions: z
|
|
1412
|
+
.union([
|
|
1413
|
+
z.enum(["inherit", "scoped"]),
|
|
1414
|
+
z
|
|
1415
|
+
.object({
|
|
1416
|
+
allow: z.array(z.string().min(1)),
|
|
1417
|
+
deny: z.array(z.string().min(1)),
|
|
1418
|
+
})
|
|
1419
|
+
.strict(),
|
|
1420
|
+
])
|
|
1421
|
+
.optional(),
|
|
1422
|
+
inherit_bypass: z.boolean().optional(),
|
|
1423
|
+
/**
|
|
1424
|
+
* Item 2 (G31 — A2A federation) — wire this sub-agent to a REMOTE peer
|
|
1425
|
+
* instead of spawning it locally. `url` is the peer deployment's base
|
|
1426
|
+
* URL; the spawner routes the Task call through `@crewhaus/federation-
|
|
1427
|
+
* router` to the peer's inbound A2A handler (whose Agent Card lives at
|
|
1428
|
+
* `<url>/.well-known/agent-card.json`), mapping the federation envelope
|
|
1429
|
+
* onto A2A message/task semantics. Present ⇒ the entry is a federated
|
|
1430
|
+
* peer reference; `description`/`instructions` still describe it to the
|
|
1431
|
+
* parent's Task tool (the remote peer owns its own prompt).
|
|
1432
|
+
*/
|
|
1433
|
+
federation: z.object({ url: z.string().url() }).strict().optional(),
|
|
1434
|
+
// 0.6.0 §7.7 — per-sub-agent routing and params.
|
|
1435
|
+
model_fallbacks: modelFallbacksBlock,
|
|
1436
|
+
circuit_breaker: circuitBreakerBlock,
|
|
1437
|
+
model_tiers: modelTiersBlock,
|
|
1438
|
+
model_pool: modelPoolBlock,
|
|
1439
|
+
thinking: thinkingBlock,
|
|
1440
|
+
max_tokens: z.number().int().positive().optional(),
|
|
1441
|
+
temperature: temperatureField,
|
|
1442
|
+
/** Fraction of the parent's `budget.usd` this child may spend. */
|
|
1443
|
+
budget_share: z.number().gt(0).max(1).optional(),
|
|
1444
|
+
/** Inherit the parent's SERVED arm instead of the declared primary (default false). */
|
|
1445
|
+
inherit_routing: z.boolean().optional(),
|
|
1446
|
+
/** `$profile` refs the Task tool's `profile` argument may name. */
|
|
1447
|
+
allowed_profiles: z.array(z.string().min(1)).min(1).optional(),
|
|
1448
|
+
})
|
|
1449
|
+
.strict()
|
|
1450
|
+
.superRefine(refineModelSelection);
|
|
1451
|
+
const subAgentsBlock = z.record(safeName, subAgentDefinitionSchema).optional();
|
|
843
1452
|
/**
|
|
844
1453
|
* Response-feedback block — declares that a harness collects human ratings on
|
|
845
1454
|
* agent responses (thumbs/stars/scale/comment) which `crewhaus distill` turns
|
|
@@ -1327,6 +1936,20 @@ const sloBlock = z
|
|
|
1327
1936
|
cost_per_hour_usd: z.number().positive().optional(),
|
|
1328
1937
|
/** Fractional egress-block rate ceiling (egress-blocked / external calls), e.g. 0.1. */
|
|
1329
1938
|
egress_block_rate: z.number().min(0).max(1).optional(),
|
|
1939
|
+
// ---- 0.6.0 (design §8.4) — hybrid-routing targets, each a 0..1 rate.
|
|
1940
|
+
// The runtime `SloTargets` / alert watchdog gained them in PR 18; these
|
|
1941
|
+
// are the spec keys that reach them (lowered by the compiler into
|
|
1942
|
+
// `IrSlo`, passed through by `crewhaus run` as `sloTargets`). Same shape
|
|
1943
|
+
// and validation as the existing rate targets. Like every other
|
|
1944
|
+
// `observability.slo.*` key they are NOT in the optimizer whitelist: an
|
|
1945
|
+
// SLO threshold decides when production is paused or rolled back, which
|
|
1946
|
+
// is an operator's call, never an eval loop's.
|
|
1947
|
+
/** Fractional escalation-rate ceiling (escalations / turns), e.g. 0.3. */
|
|
1948
|
+
escalation_rate: z.number().min(0).max(1).optional(),
|
|
1949
|
+
/** Fractional judge-fail-rate ceiling (failing verdicts / all in-loop + judge-gate verdicts), e.g. 0.5. */
|
|
1950
|
+
judge_fail_rate: z.number().min(0).max(1).optional(),
|
|
1951
|
+
/** Fractional floor-block-rate ceiling (floor-forced route decisions / all decisions), e.g. 0.2. */
|
|
1952
|
+
floor_block_rate: z.number().min(0).max(1).optional(),
|
|
1330
1953
|
/**
|
|
1331
1954
|
* Rolling window (seconds) a breach must persist before the ladder fires.
|
|
1332
1955
|
* A single blip never mitigates — the monitor only acts on a SUSTAINED
|
|
@@ -1348,7 +1971,10 @@ const sloBlock = z
|
|
|
1348
1971
|
s.p95_latency_ms !== undefined ||
|
|
1349
1972
|
s.ttft_ms !== undefined ||
|
|
1350
1973
|
s.cost_per_hour_usd !== undefined ||
|
|
1351
|
-
s.egress_block_rate !== undefined
|
|
1974
|
+
s.egress_block_rate !== undefined ||
|
|
1975
|
+
s.escalation_rate !== undefined ||
|
|
1976
|
+
s.judge_fail_rate !== undefined ||
|
|
1977
|
+
s.floor_block_rate !== undefined, { message: "observability.slo must declare at least one target threshold" });
|
|
1352
1978
|
/**
|
|
1353
1979
|
* Loop contract 0.4 (Batch C, G26) — the observability control sub-blocks.
|
|
1354
1980
|
* These declare which of the runtime's observability subscribers the emitted
|
|
@@ -1679,6 +2305,8 @@ const cliSchema = z
|
|
|
1679
2305
|
name: safeName,
|
|
1680
2306
|
version: versionField,
|
|
1681
2307
|
target: z.literal("cli"),
|
|
2308
|
+
// 0.6.0 §4.1 — the model-profile registry (all 14 shapes).
|
|
2309
|
+
models: modelsBlock,
|
|
1682
2310
|
agent: z
|
|
1683
2311
|
.object({
|
|
1684
2312
|
model: z.string().min(1),
|
|
@@ -1689,6 +2317,8 @@ const cliSchema = z
|
|
|
1689
2317
|
max_tokens: z.number().int().positive().optional(),
|
|
1690
2318
|
// Loop contract 0.4 (Batch A) — extended-thinking selector.
|
|
1691
2319
|
thinking: thinkingBlock,
|
|
2320
|
+
// 0.6.0 §4.1 — sampling temperature (exclusive with thinking).
|
|
2321
|
+
temperature: temperatureField,
|
|
1692
2322
|
// Loop contract 0.4 (Batch A) — stream partial output tokens.
|
|
1693
2323
|
// Optional; absent means false (the cli-shape default).
|
|
1694
2324
|
streaming: z.boolean().optional(),
|
|
@@ -1748,6 +2378,8 @@ const workflowStepSchema = z
|
|
|
1748
2378
|
max_tokens: z.number().int().positive().optional(),
|
|
1749
2379
|
// Loop contract 0.4 (Batch A) — per-step extended-thinking selector.
|
|
1750
2380
|
thinking: thinkingBlock,
|
|
2381
|
+
// 0.6.0 §4.1 — sampling temperature (exclusive with thinking).
|
|
2382
|
+
temperature: temperatureField,
|
|
1751
2383
|
tools: z.array(z.string().min(1)).optional(),
|
|
1752
2384
|
tool_config: toolConfigBlock,
|
|
1753
2385
|
// Item 9 (G37) — per-step model routing, adopting the cli agent block's
|
|
@@ -1785,6 +2417,7 @@ const workflowSchema = z
|
|
|
1785
2417
|
name: safeName,
|
|
1786
2418
|
version: versionField,
|
|
1787
2419
|
target: z.literal("workflow"),
|
|
2420
|
+
models: modelsBlock,
|
|
1788
2421
|
model: z.string().min(1),
|
|
1789
2422
|
steps: z.array(workflowAnyStepSchema).min(1),
|
|
1790
2423
|
mcp_servers: mcpServersBlock,
|
|
@@ -1886,6 +2519,8 @@ const channelAgentSchema = z
|
|
|
1886
2519
|
max_tokens: z.number().int().positive().optional(),
|
|
1887
2520
|
// Loop contract 0.4 (Batch A) — extended-thinking selector.
|
|
1888
2521
|
thinking: thinkingBlock,
|
|
2522
|
+
// 0.6.0 §4.1 — sampling temperature (exclusive with thinking).
|
|
2523
|
+
temperature: temperatureField,
|
|
1889
2524
|
// Loop contract 0.4 (Batch A) — per-tool rate limits.
|
|
1890
2525
|
rate_limits: rateLimitsBlock,
|
|
1891
2526
|
// Item 22 — provider failover chain (see modelFallbacksBlock docs).
|
|
@@ -1906,6 +2541,7 @@ const channelSchema = z
|
|
|
1906
2541
|
name: safeName,
|
|
1907
2542
|
version: versionField,
|
|
1908
2543
|
target: z.literal("channel"),
|
|
2544
|
+
models: modelsBlock,
|
|
1909
2545
|
agent: channelAgentSchema,
|
|
1910
2546
|
channels: channelsBlock,
|
|
1911
2547
|
routing: routingBlock,
|
|
@@ -1955,8 +2591,19 @@ const graphNodeSchema = z
|
|
|
1955
2591
|
max_tokens: z.number().int().positive().optional(),
|
|
1956
2592
|
// Loop contract 0.4 (Batch A) — per-node extended-thinking selector.
|
|
1957
2593
|
thinking: thinkingBlock,
|
|
2594
|
+
// 0.6.0 §4.1 — sampling temperature (exclusive with thinking).
|
|
2595
|
+
temperature: temperatureField,
|
|
1958
2596
|
tools: z.array(z.string().min(1)).optional(),
|
|
1959
2597
|
tool_config: toolConfigBlock,
|
|
2598
|
+
// 0.6.0 §7.7 — per-node model routing (graph nodes carried NO routing
|
|
2599
|
+
// before 0.6.0): the cli agent block's pooled pattern verbatim, sharing
|
|
2600
|
+
// the one mutual-exclusion rule via `refineModelSelection`. Omitted →
|
|
2601
|
+
// the node's single (`node.model ?? graph.model`) model, byte-identical
|
|
2602
|
+
// bundles.
|
|
2603
|
+
model_fallbacks: modelFallbacksBlock,
|
|
2604
|
+
circuit_breaker: circuitBreakerBlock,
|
|
2605
|
+
model_tiers: modelTiersBlock,
|
|
2606
|
+
model_pool: modelPoolBlock,
|
|
1960
2607
|
/**
|
|
1961
2608
|
* A human approval gate on this node, and a PRE-condition: the node
|
|
1962
2609
|
* calls `ctx.requestApproval(prompt)` BEFORE its model turn, so the
|
|
@@ -1988,7 +2635,8 @@ const graphNodeSchema = z
|
|
|
1988
2635
|
.strict()
|
|
1989
2636
|
.optional(),
|
|
1990
2637
|
})
|
|
1991
|
-
.strict()
|
|
2638
|
+
.strict()
|
|
2639
|
+
.superRefine(refineModelSelection);
|
|
1992
2640
|
/**
|
|
1993
2641
|
* Loop contract 0.4 (Batch B, G02) — the `kind: "judge"` graph-node
|
|
1994
2642
|
* variant: a gate over the node's UPSTREAM output (see
|
|
@@ -2060,6 +2708,7 @@ const graphSchema = z
|
|
|
2060
2708
|
name: safeName,
|
|
2061
2709
|
version: versionField,
|
|
2062
2710
|
target: z.literal("graph"),
|
|
2711
|
+
models: modelsBlock,
|
|
2063
2712
|
model: z.string().min(1),
|
|
2064
2713
|
entry: z.string().min(1),
|
|
2065
2714
|
nodes: z.record(safeName, graphAnyNodeSchema),
|
|
@@ -2109,6 +2758,8 @@ const managedAgentSchema = z
|
|
|
2109
2758
|
max_tokens: z.number().int().positive().optional(),
|
|
2110
2759
|
// Loop contract 0.4 (Batch A) — extended-thinking selector.
|
|
2111
2760
|
thinking: thinkingBlock,
|
|
2761
|
+
// 0.6.0 §4.1 — sampling temperature (exclusive with thinking).
|
|
2762
|
+
temperature: temperatureField,
|
|
2112
2763
|
// Loop contract 0.4 (Batch A) — per-tool rate limits.
|
|
2113
2764
|
rate_limits: rateLimitsBlock,
|
|
2114
2765
|
// Item 22 — provider failover chain (see modelFallbacksBlock docs).
|
|
@@ -2131,6 +2782,7 @@ const managedSchema = z
|
|
|
2131
2782
|
name: safeName,
|
|
2132
2783
|
version: versionField,
|
|
2133
2784
|
target: z.literal("managed"),
|
|
2785
|
+
models: modelsBlock,
|
|
2134
2786
|
agent: managedAgentSchema,
|
|
2135
2787
|
tenants: z.array(managedTenantSchema).min(1),
|
|
2136
2788
|
permissions: permissionsBlock,
|
|
@@ -2206,6 +2858,9 @@ const pooledSingleAgentObject = z
|
|
|
2206
2858
|
instructions: z.string().min(1),
|
|
2207
2859
|
// Adaptive model routing — N-candidate pool with a selection policy.
|
|
2208
2860
|
model_pool: modelPoolBlock,
|
|
2861
|
+
// 0.6.0 §4.1 — sampling temperature (no thinking on this block, so no
|
|
2862
|
+
// exclusivity case arises here).
|
|
2863
|
+
temperature: temperatureField,
|
|
2209
2864
|
})
|
|
2210
2865
|
.strict();
|
|
2211
2866
|
const pooledSingleAgentSchema = pooledSingleAgentObject.superRefine(refineModelSelection);
|
|
@@ -2227,6 +2882,7 @@ const pipelineSchema = z
|
|
|
2227
2882
|
name: safeName,
|
|
2228
2883
|
version: versionField,
|
|
2229
2884
|
target: z.literal("pipeline"),
|
|
2885
|
+
models: modelsBlock,
|
|
2230
2886
|
agent: pooledSingleAgentSchema,
|
|
2231
2887
|
retrieve: z
|
|
2232
2888
|
.object({
|
|
@@ -2271,6 +2927,8 @@ const crewRoleSchema = z
|
|
|
2271
2927
|
max_tokens: z.number().int().positive().optional(),
|
|
2272
2928
|
// Loop contract 0.4 (Batch A) — per-role extended-thinking selector.
|
|
2273
2929
|
thinking: thinkingBlock,
|
|
2930
|
+
// 0.6.0 §4.1 — sampling temperature (exclusive with thinking).
|
|
2931
|
+
temperature: temperatureField,
|
|
2274
2932
|
tools: z.array(z.string().min(1)).optional(),
|
|
2275
2933
|
tool_config: toolConfigBlock,
|
|
2276
2934
|
sub_agents: subAgentsBlock,
|
|
@@ -2297,6 +2955,17 @@ const crewRoutingSchema = z
|
|
|
2297
2955
|
.object({
|
|
2298
2956
|
kind: z.enum(["match", "llm"]),
|
|
2299
2957
|
match: z.record(z.string().min(1), z.array(crewRoutingMatchEntrySchema).min(1)).optional(),
|
|
2958
|
+
/**
|
|
2959
|
+
* 0.6.0 §7.7 — the model the `kind: llm` router runs on (grammar string
|
|
2960
|
+
* or `$profile`). Today the router is hard-wired to the entry role's
|
|
2961
|
+
* model; this slot makes it declarable. Only meaningful with
|
|
2962
|
+
* `kind: llm` (cross-field).
|
|
2963
|
+
*/
|
|
2964
|
+
model: z
|
|
2965
|
+
.string()
|
|
2966
|
+
.min(1)
|
|
2967
|
+
.optional()
|
|
2968
|
+
.describe("0.6.0: the model (or $profile) the kind: llm router runs on"),
|
|
2300
2969
|
})
|
|
2301
2970
|
.strict();
|
|
2302
2971
|
const crewSchema = z
|
|
@@ -2304,6 +2973,7 @@ const crewSchema = z
|
|
|
2304
2973
|
name: safeName,
|
|
2305
2974
|
version: versionField,
|
|
2306
2975
|
target: z.literal("crew"),
|
|
2976
|
+
models: modelsBlock,
|
|
2307
2977
|
/** Crew-wide model fallback used by any role that omits `role.model`. */
|
|
2308
2978
|
model: z.string().min(1),
|
|
2309
2979
|
entry: z.string().min(1),
|
|
@@ -2356,6 +3026,7 @@ const researchSchema = z
|
|
|
2356
3026
|
name: safeName,
|
|
2357
3027
|
version: versionField,
|
|
2358
3028
|
target: z.literal("research"),
|
|
3029
|
+
models: modelsBlock,
|
|
2359
3030
|
agent: pooledSingleAgentWithMaxTokensSchema,
|
|
2360
3031
|
goal: z.string().min(1),
|
|
2361
3032
|
branchingFactor: z.number().int().min(1).max(8).default(3),
|
|
@@ -2398,6 +3069,7 @@ const batchSchema = z
|
|
|
2398
3069
|
name: safeName,
|
|
2399
3070
|
version: versionField,
|
|
2400
3071
|
target: z.literal("batch"),
|
|
3072
|
+
models: modelsBlock,
|
|
2401
3073
|
agent: pooledSingleAgentWithMaxTokensSchema,
|
|
2402
3074
|
queue: batchQueueSchema,
|
|
2403
3075
|
concurrency: z.number().int().min(1).max(64).default(4),
|
|
@@ -2443,6 +3115,7 @@ const voiceSchema = z
|
|
|
2443
3115
|
name: safeName,
|
|
2444
3116
|
version: versionField,
|
|
2445
3117
|
target: z.literal("voice"),
|
|
3118
|
+
models: modelsBlock,
|
|
2446
3119
|
agent: z
|
|
2447
3120
|
.object({
|
|
2448
3121
|
model: z.string().min(1),
|
|
@@ -2497,6 +3170,7 @@ const browserSchema = z
|
|
|
2497
3170
|
name: safeName,
|
|
2498
3171
|
version: versionField,
|
|
2499
3172
|
target: z.literal("browser"),
|
|
3173
|
+
models: modelsBlock,
|
|
2500
3174
|
agent: pooledSingleAgentWithMaxTokensSchema,
|
|
2501
3175
|
driver: browserDriverSchema.default({}),
|
|
2502
3176
|
/** Vision-grounding model. Defaults to the agent's primary model. */
|
|
@@ -2528,6 +3202,7 @@ const evalSchema = z
|
|
|
2528
3202
|
name: safeName,
|
|
2529
3203
|
version: versionField,
|
|
2530
3204
|
target: z.literal("eval"),
|
|
3205
|
+
models: modelsBlock,
|
|
2531
3206
|
agent: z
|
|
2532
3207
|
.object({
|
|
2533
3208
|
model: z.string().min(1),
|
|
@@ -2593,6 +3268,7 @@ const onchainSchema = z
|
|
|
2593
3268
|
name: safeName,
|
|
2594
3269
|
version: versionField,
|
|
2595
3270
|
target: z.literal("onchain"),
|
|
3271
|
+
models: modelsBlock,
|
|
2596
3272
|
agent: z
|
|
2597
3273
|
.object({
|
|
2598
3274
|
model: z.string().min(1),
|
|
@@ -2628,6 +3304,7 @@ const onchainGameSchema = z
|
|
|
2628
3304
|
name: safeName,
|
|
2629
3305
|
version: versionField,
|
|
2630
3306
|
target: z.literal("onchain-game"),
|
|
3307
|
+
models: modelsBlock,
|
|
2631
3308
|
agent: z
|
|
2632
3309
|
.object({
|
|
2633
3310
|
model: z.string().min(1),
|
|
@@ -2857,8 +3534,535 @@ function crossFieldIssues(data) {
|
|
|
2857
3534
|
custom(["watchme", "share"], "watchme.share publishes co-learning articles; thredz.visibility: private blocks cross-agent sharing — set visibility: shared or drop watchme.share");
|
|
2858
3535
|
}
|
|
2859
3536
|
}
|
|
3537
|
+
// 0.6.0 §4.1 / §7.1 — the model-profile registry, `$profile` references
|
|
3538
|
+
// and the hybrid pool blocks (appended last: check order is load-bearing).
|
|
3539
|
+
modelSurfaceIssues(data, custom);
|
|
2860
3540
|
return issues;
|
|
2861
3541
|
}
|
|
3542
|
+
const MODEL_DIRECTED_TOOLS = new Set(["Consult", "Escalate"]);
|
|
3543
|
+
const MCP_TOOL_SELECTOR_RE = /^mcp__([^_].*?)__(.+)$/;
|
|
3544
|
+
/** Levenshtein distance — the did-you-mean helper for unknown `$refs` / tags. */
|
|
3545
|
+
function editDistance(a, b) {
|
|
3546
|
+
if (a === b)
|
|
3547
|
+
return 0;
|
|
3548
|
+
if (a.length === 0)
|
|
3549
|
+
return b.length;
|
|
3550
|
+
if (b.length === 0)
|
|
3551
|
+
return a.length;
|
|
3552
|
+
let prev = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
3553
|
+
for (let i = 1; i <= a.length; i++) {
|
|
3554
|
+
const cur = [i];
|
|
3555
|
+
for (let j = 1; j <= b.length; j++) {
|
|
3556
|
+
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
3557
|
+
cur[j] = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost);
|
|
3558
|
+
}
|
|
3559
|
+
prev = cur;
|
|
3560
|
+
}
|
|
3561
|
+
return prev[b.length];
|
|
3562
|
+
}
|
|
3563
|
+
function nearestOf(target, declared) {
|
|
3564
|
+
let best;
|
|
3565
|
+
let bestDistance = 4;
|
|
3566
|
+
for (const candidate of declared) {
|
|
3567
|
+
const d = editDistance(target, candidate);
|
|
3568
|
+
if (d < bestDistance) {
|
|
3569
|
+
best = candidate;
|
|
3570
|
+
bestDistance = d;
|
|
3571
|
+
}
|
|
3572
|
+
}
|
|
3573
|
+
return best;
|
|
3574
|
+
}
|
|
3575
|
+
/**
|
|
3576
|
+
* Does a spec-wide scan for `strategy.model_directed: true` on ANY pool —
|
|
3577
|
+
* the `Consult` / `Escalate` tools exist only when some pool registers them,
|
|
3578
|
+
* so a profile naming them elsewhere would advertise tools that never exist.
|
|
3579
|
+
*/
|
|
3580
|
+
function specDeclaresModelDirected(value) {
|
|
3581
|
+
if (Array.isArray(value))
|
|
3582
|
+
return value.some(specDeclaresModelDirected);
|
|
3583
|
+
if (typeof value !== "object" || value === null)
|
|
3584
|
+
return false;
|
|
3585
|
+
const record = value;
|
|
3586
|
+
const pool = record["model_pool"];
|
|
3587
|
+
if (typeof pool === "object" && pool !== null) {
|
|
3588
|
+
const strategy = pool.strategy;
|
|
3589
|
+
if (strategy?.model_directed === true)
|
|
3590
|
+
return true;
|
|
3591
|
+
}
|
|
3592
|
+
return Object.values(record).some(specDeclaresModelDirected);
|
|
3593
|
+
}
|
|
3594
|
+
/** A model slot: grammar string, sentinel, or `$profile` — the ref must resolve. */
|
|
3595
|
+
function checkModelSlot(ctx, path, value) {
|
|
3596
|
+
if (value === undefined)
|
|
3597
|
+
return;
|
|
3598
|
+
const name = profileRefName(value);
|
|
3599
|
+
if (name === undefined)
|
|
3600
|
+
return;
|
|
3601
|
+
const declared = Object.keys(ctx.registry);
|
|
3602
|
+
if (!SPEC_PROFILE_NAME_RE.test(name)) {
|
|
3603
|
+
ctx.custom(path, `${path.join(".")}: "${value}" is not a valid profile reference — profile names match /^[a-z][a-z0-9_-]{0,63}$/`);
|
|
3604
|
+
return;
|
|
3605
|
+
}
|
|
3606
|
+
if (ctx.registry[name] !== undefined)
|
|
3607
|
+
return;
|
|
3608
|
+
const nearest = nearestOf(name, declared);
|
|
3609
|
+
const hint = declared.length === 0
|
|
3610
|
+
? "this spec declares no models: block"
|
|
3611
|
+
: nearest !== undefined
|
|
3612
|
+
? `did you mean "$${nearest}"? declared: ${declared.map((d) => `$${d}`).join(", ")}`
|
|
3613
|
+
: `declared: ${declared.map((d) => `$${d}`).join(", ")}`;
|
|
3614
|
+
ctx.custom(path, `${path.join(".")}: unknown profile "${value}" — ${hint}`);
|
|
3615
|
+
}
|
|
3616
|
+
/**
|
|
3617
|
+
* The effective routing tags of one candidate: its own when it declares any,
|
|
3618
|
+
* else the referenced profile's (an empty list on a `$profile` candidate means
|
|
3619
|
+
* "inherit", since the spec defaults `tags` to `[]`).
|
|
3620
|
+
*/
|
|
3621
|
+
function candidateTags(ctx, candidate) {
|
|
3622
|
+
if (candidate.tags.length > 0)
|
|
3623
|
+
return [...candidate.tags];
|
|
3624
|
+
const name = profileRefName(candidate.model);
|
|
3625
|
+
const profile = name !== undefined ? ctx.registry[name] : undefined;
|
|
3626
|
+
return profile?.tags !== undefined ? [...profile.tags] : [];
|
|
3627
|
+
}
|
|
3628
|
+
/**
|
|
3629
|
+
* A role slot names a candidate TAG or a `$profile` that is a roster member
|
|
3630
|
+
* (a forced arm must be a roster `PoolCandidate`, §7.2.4; a strategy role
|
|
3631
|
+
* naming an absent candidate could never be served).
|
|
3632
|
+
*/
|
|
3633
|
+
function checkRoleSlot(ctx, path, value, pool, tags) {
|
|
3634
|
+
const name = profileRefName(value);
|
|
3635
|
+
if (name !== undefined) {
|
|
3636
|
+
checkModelSlot(ctx, path, value);
|
|
3637
|
+
if (ctx.registry[name] !== undefined && !pool.candidates.some((c) => c.model === value)) {
|
|
3638
|
+
ctx.custom(path, `${path.join(".")}: "${value}" is a declared profile but not one of this model_pool's candidates (${pool.candidates.map((c) => c.model).join(", ")}) — a strategy role must name a roster member`);
|
|
3639
|
+
}
|
|
3640
|
+
return;
|
|
3641
|
+
}
|
|
3642
|
+
if (tags.has(value))
|
|
3643
|
+
return;
|
|
3644
|
+
const declared = [...tags];
|
|
3645
|
+
const nearest = nearestOf(value, declared);
|
|
3646
|
+
const hint = declared.length === 0
|
|
3647
|
+
? "no candidate declares tags"
|
|
3648
|
+
: nearest !== undefined
|
|
3649
|
+
? `did you mean "${nearest}"? candidate tags: ${declared.join(", ")}`
|
|
3650
|
+
: `candidate tags: ${declared.join(", ")}`;
|
|
3651
|
+
ctx.custom(path, `${path.join(".")}: "${value}" is neither a candidate tag nor a $profile in this model_pool — ${hint}`);
|
|
3652
|
+
}
|
|
3653
|
+
/**
|
|
3654
|
+
* A profile's / candidate's `tools` is SUBSET-ONLY (§5.2): builtin keys must
|
|
3655
|
+
* be among the shape's declared tools; `mcp__<server>__<tool|*>` must name a
|
|
3656
|
+
* declared MCP server; `Consult` / `Escalate` exist only under
|
|
3657
|
+
* `strategy.model_directed: true`; and a tool-less shape admits no `tools` at
|
|
3658
|
+
* all.
|
|
3659
|
+
*/
|
|
3660
|
+
function checkProfileTools(ctx, path, tools, host) {
|
|
3661
|
+
if (tools === undefined)
|
|
3662
|
+
return;
|
|
3663
|
+
if (host.toolLess) {
|
|
3664
|
+
ctx.custom(path, `${path.join(".")}: this shape registers no tool catalog, so a per-model tools list has nothing to narrow — remove it`);
|
|
3665
|
+
return;
|
|
3666
|
+
}
|
|
3667
|
+
for (const [i, tool] of tools.entries()) {
|
|
3668
|
+
if (tool.startsWith("mcp__")) {
|
|
3669
|
+
const server = tool.match(MCP_TOOL_SELECTOR_RE)?.[1];
|
|
3670
|
+
if (server === undefined) {
|
|
3671
|
+
ctx.custom([...path, i], `${path.join(".")}[${i}]: "${tool}" is not a valid MCP tool selector — use mcp__<server>__<tool> or the server-scoped glob mcp__<server>__*`);
|
|
3672
|
+
}
|
|
3673
|
+
else if (!ctx.mcpServers.has(server)) {
|
|
3674
|
+
ctx.custom([...path, i], `${path.join(".")}[${i}]: "${tool}" names MCP server "${server}", which mcp_servers does not declare${ctx.mcpServers.size > 0 ? ` (declared: ${[...ctx.mcpServers].join(", ")})` : ""}`);
|
|
3675
|
+
}
|
|
3676
|
+
continue;
|
|
3677
|
+
}
|
|
3678
|
+
if (MODEL_DIRECTED_TOOLS.has(tool)) {
|
|
3679
|
+
if (!ctx.modelDirected) {
|
|
3680
|
+
ctx.custom([...path, i], `${path.join(".")}[${i}]: "${tool}" is registered only when a model_pool declares strategy.model_directed: true — no pool in this spec does`);
|
|
3681
|
+
}
|
|
3682
|
+
continue;
|
|
3683
|
+
}
|
|
3684
|
+
if (host.shapeTools !== undefined && !host.shapeTools.includes(tool)) {
|
|
3685
|
+
ctx.custom([...path, i], `${path.join(".")}[${i}]: "${tool}" is not one of the shape's tools (${host.shapeTools.join(", ")}) — a per-model tools list can only narrow the shape's toolset, never add to it`);
|
|
3686
|
+
}
|
|
3687
|
+
}
|
|
3688
|
+
}
|
|
3689
|
+
/** The checks a `models:` profile and an inline pool candidate share. */
|
|
3690
|
+
function checkProfileBody(ctx, path, body, host, opts) {
|
|
3691
|
+
if (opts.isProfile) {
|
|
3692
|
+
// A profile's own `model` / `fallbacks` are grammar strings or sentinels,
|
|
3693
|
+
// never `$refs` — a profile referencing a profile would be circular.
|
|
3694
|
+
if (profileRefName(body.model) !== undefined) {
|
|
3695
|
+
ctx.custom([...path, "model"], `${path.join(".")}.model: a profile's model must be a model string or a sentinel (cheapest | strongest), not another profile reference ("${body.model}") — profiles do not inherit from profiles`);
|
|
3696
|
+
}
|
|
3697
|
+
for (const [i, fallback] of (body.fallbacks ?? []).entries()) {
|
|
3698
|
+
if (profileRefName(fallback) !== undefined) {
|
|
3699
|
+
ctx.custom([...path, "fallbacks", i], `${path.join(".")}.fallbacks[${i}]: a profile's fallback chain names model strings, not profile references ("${fallback}")`);
|
|
3700
|
+
}
|
|
3701
|
+
}
|
|
3702
|
+
}
|
|
3703
|
+
else {
|
|
3704
|
+
for (const [i, fallback] of (body.fallbacks ?? []).entries()) {
|
|
3705
|
+
checkModelSlot(ctx, [...path, "fallbacks", i], fallback);
|
|
3706
|
+
}
|
|
3707
|
+
}
|
|
3708
|
+
checkProfileTools(ctx, [...path, "tools"], body.tools, host);
|
|
3709
|
+
}
|
|
3710
|
+
/** The whole `models:` registry against the shape it is declared on. */
|
|
3711
|
+
function checkProfiles(ctx, host) {
|
|
3712
|
+
for (const [name, profile] of Object.entries(ctx.registry)) {
|
|
3713
|
+
checkProfileBody(ctx, ["models", name], profile, host, { isProfile: true });
|
|
3714
|
+
}
|
|
3715
|
+
}
|
|
3716
|
+
/** One `model_pool` block — candidates, rules, classifier, strategy, reward. */
|
|
3717
|
+
function checkModelPool(ctx, path, pool, host) {
|
|
3718
|
+
const tags = new Set();
|
|
3719
|
+
for (const [i, candidate] of pool.candidates.entries()) {
|
|
3720
|
+
const cpath = [...path, "candidates", i];
|
|
3721
|
+
checkModelSlot(ctx, [...cpath, "model"], candidate.model);
|
|
3722
|
+
checkProfileBody(ctx, cpath, candidate, host, { isProfile: false });
|
|
3723
|
+
for (const tag of candidateTags(ctx, candidate))
|
|
3724
|
+
tags.add(tag);
|
|
3725
|
+
}
|
|
3726
|
+
if (pool.candidates.every((c) => c.enabled === false)) {
|
|
3727
|
+
ctx.custom([...path, "candidates"], `${path.join(".")}.candidates: every candidate is enabled: false — at least one must stay routable`);
|
|
3728
|
+
}
|
|
3729
|
+
// §7.2.3 — `policy: classifier` and the `classifier` block go together.
|
|
3730
|
+
if (pool.policy === "classifier" && pool.classifier === undefined) {
|
|
3731
|
+
ctx.custom([...path, "policy"], `${path.join(".")}.policy: classifier requires a classifier block (model + labels) on the same model_pool`);
|
|
3732
|
+
}
|
|
3733
|
+
if (pool.classifier !== undefined) {
|
|
3734
|
+
if (pool.policy !== "classifier") {
|
|
3735
|
+
ctx.custom([...path, "classifier"], `${path.join(".")}.classifier is declared but policy is "${pool.policy}" — the classifier runs only under policy: classifier (declare it, or drop the block)`);
|
|
3736
|
+
}
|
|
3737
|
+
checkModelSlot(ctx, [...path, "classifier", "model"], pool.classifier.model);
|
|
3738
|
+
for (const label of Object.keys(pool.classifier.labels)) {
|
|
3739
|
+
if (!tags.has(label)) {
|
|
3740
|
+
ctx.custom([...path, "classifier", "labels", label], `${path.join(".")}.classifier.labels["${label}"]: every label must be a candidate tag (the verdict is constrained to the roster's tags: ${[...tags].join(", ") || "none declared"})`);
|
|
3741
|
+
}
|
|
3742
|
+
}
|
|
3743
|
+
}
|
|
3744
|
+
// §7.2.2 — rules: unique ids; `use` is a tag / $profile / requirement.
|
|
3745
|
+
if (pool.rules !== undefined) {
|
|
3746
|
+
const seen = new Set();
|
|
3747
|
+
for (const [i, rule] of pool.rules.entries()) {
|
|
3748
|
+
if (seen.has(rule.id)) {
|
|
3749
|
+
ctx.custom([...path, "rules", i, "id"], `${path.join(".")}.rules[${i}].id "${rule.id}" is declared twice — rule ids are persisted on the model_route line and must be unique`);
|
|
3750
|
+
}
|
|
3751
|
+
seen.add(rule.id);
|
|
3752
|
+
if (typeof rule.use === "string") {
|
|
3753
|
+
checkRoleSlot(ctx, [...path, "rules", i, "use"], rule.use, pool, tags);
|
|
3754
|
+
}
|
|
3755
|
+
}
|
|
3756
|
+
}
|
|
3757
|
+
// §7.3–§7.8 — strategy role and model slots.
|
|
3758
|
+
const strategy = pool.strategy;
|
|
3759
|
+
if (strategy !== undefined) {
|
|
3760
|
+
const spath = [...path, "strategy"];
|
|
3761
|
+
if (strategy.cascade !== undefined) {
|
|
3762
|
+
checkRoleSlot(ctx, [...spath, "cascade", "draft"], strategy.cascade.draft, pool, tags);
|
|
3763
|
+
checkRoleSlot(ctx, [...spath, "cascade", "escalate_to"], strategy.cascade.escalate_to, pool, tags);
|
|
3764
|
+
if (strategy.cascade.draft === strategy.cascade.escalate_to) {
|
|
3765
|
+
ctx.custom([...spath, "cascade", "escalate_to"], `${spath.join(".")}.cascade: draft and escalate_to both name "${strategy.cascade.draft}" — a cascade escalates to a DIFFERENT rung`);
|
|
3766
|
+
}
|
|
3767
|
+
}
|
|
3768
|
+
if (strategy.guide !== undefined) {
|
|
3769
|
+
checkModelSlot(ctx, [...spath, "guide", "model"], strategy.guide.model);
|
|
3770
|
+
}
|
|
3771
|
+
if (strategy.shadow !== undefined) {
|
|
3772
|
+
checkModelSlot(ctx, [...spath, "shadow", "candidate"], strategy.shadow.candidate);
|
|
3773
|
+
checkModelSlot(ctx, [...spath, "shadow", "grade_with"], strategy.shadow.grade_with);
|
|
3774
|
+
}
|
|
3775
|
+
if (strategy.committee !== undefined) {
|
|
3776
|
+
if (!host.committee) {
|
|
3777
|
+
ctx.custom([...spath, "committee"], `${spath.join(".")}.committee is legal on single-turn hosts only (workflow steps, graph nodes, crew roles) — a REPL or per-message turn is one the user is waiting on`);
|
|
3778
|
+
}
|
|
3779
|
+
const members = new Set();
|
|
3780
|
+
for (const [i, member] of strategy.committee.members.entries()) {
|
|
3781
|
+
checkRoleSlot(ctx, [...spath, "committee", "members", i], member, pool, tags);
|
|
3782
|
+
if (members.has(member)) {
|
|
3783
|
+
ctx.custom([...spath, "committee", "members", i], `${spath.join(".")}.committee.members[${i}] "${member}" is listed twice`);
|
|
3784
|
+
}
|
|
3785
|
+
members.add(member);
|
|
3786
|
+
}
|
|
3787
|
+
checkModelSlot(ctx, [...spath, "committee", "judge"], strategy.committee.judge);
|
|
3788
|
+
if (strategy.committee.escalate_on_disagreement !== undefined) {
|
|
3789
|
+
checkRoleSlot(ctx, [...spath, "committee", "escalate_on_disagreement"], strategy.committee.escalate_on_disagreement, pool, tags);
|
|
3790
|
+
}
|
|
3791
|
+
}
|
|
3792
|
+
}
|
|
3793
|
+
// §6.3 — reward: `in_loop` needs a floor AND an in-loop grader.
|
|
3794
|
+
const reward = pool.reward;
|
|
3795
|
+
if (reward !== undefined) {
|
|
3796
|
+
if (reward.floor?.arm !== undefined) {
|
|
3797
|
+
checkRoleSlot(ctx, [...path, "reward", "floor", "arm"], reward.floor.arm, pool, tags);
|
|
3798
|
+
}
|
|
3799
|
+
if (reward.quality_source === "in_loop") {
|
|
3800
|
+
if (reward.floor === undefined) {
|
|
3801
|
+
ctx.custom([...path, "reward", "quality_source"], `${path.join(".")}.reward.quality_source: in_loop requires reward.floor — learned exploitation must be bounded by a floor arm when graded quality steers routing`);
|
|
3802
|
+
}
|
|
3803
|
+
if (!ctx.gradedInLoop) {
|
|
3804
|
+
ctx.custom([...path, "reward", "quality_source"], `${path.join(".")}.reward.quality_source: in_loop requires an in-loop grader on this shape (an evaluation: block, or a kind: judge step/node) — without one every observation would default to perfect quality`);
|
|
3805
|
+
}
|
|
3806
|
+
}
|
|
3807
|
+
}
|
|
3808
|
+
}
|
|
3809
|
+
/** One sub-agent definition (recurses through its own routing surface). */
|
|
3810
|
+
function checkSubAgent(ctx, path, def) {
|
|
3811
|
+
for (const [i, entry] of (def.allowed_profiles ?? []).entries()) {
|
|
3812
|
+
if (profileRefName(entry) === undefined) {
|
|
3813
|
+
ctx.custom([...path, "allowed_profiles", i], `${path.join(".")}.allowed_profiles[${i}]: "${entry}" must be a $profile reference — the Task tool's profile argument is validated against declared profiles`);
|
|
3814
|
+
continue;
|
|
3815
|
+
}
|
|
3816
|
+
checkModelSlot(ctx, [...path, "allowed_profiles", i], entry);
|
|
3817
|
+
}
|
|
3818
|
+
checkRoutedBlock(ctx, path, def, {
|
|
3819
|
+
committee: false,
|
|
3820
|
+
shapeTools: def.tools,
|
|
3821
|
+
toolLess: false,
|
|
3822
|
+
});
|
|
3823
|
+
}
|
|
3824
|
+
/** Every model slot and pool on an agent-like block, plus its sub-agents. */
|
|
3825
|
+
function checkRoutedBlock(ctx, path, block, host) {
|
|
3826
|
+
checkModelSlot(ctx, [...path, "model"], block.model);
|
|
3827
|
+
for (const [i, fallback] of (block.model_fallbacks ?? []).entries()) {
|
|
3828
|
+
checkModelSlot(ctx, [...path, "model_fallbacks", i], fallback);
|
|
3829
|
+
}
|
|
3830
|
+
if (block.model_tiers !== undefined) {
|
|
3831
|
+
checkModelSlot(ctx, [...path, "model_tiers", "fast"], block.model_tiers.fast);
|
|
3832
|
+
checkModelSlot(ctx, [...path, "model_tiers", "default"], block.model_tiers.default);
|
|
3833
|
+
}
|
|
3834
|
+
if (block.model_pool !== undefined) {
|
|
3835
|
+
checkModelPool(ctx, [...path, "model_pool"], block.model_pool, host);
|
|
3836
|
+
}
|
|
3837
|
+
if (block.sub_agents !== undefined) {
|
|
3838
|
+
for (const [name, def] of Object.entries(block.sub_agents)) {
|
|
3839
|
+
checkSubAgent(ctx, [...path, "sub_agents", name], def);
|
|
3840
|
+
}
|
|
3841
|
+
}
|
|
3842
|
+
}
|
|
3843
|
+
/** The judge-gate model slots plus `escalate_to` against the gated block's pool. */
|
|
3844
|
+
function checkJudgeGate(ctx, path, judge, gatedPool, gatedLabel) {
|
|
3845
|
+
checkModelSlot(ctx, [...path, "model"], judge.model);
|
|
3846
|
+
for (const [i, member] of (judge.judges ?? []).entries()) {
|
|
3847
|
+
checkModelSlot(ctx, [...path, "judges", i], member);
|
|
3848
|
+
}
|
|
3849
|
+
if (judge.escalate_to === undefined)
|
|
3850
|
+
return;
|
|
3851
|
+
if (gatedPool === undefined) {
|
|
3852
|
+
ctx.custom([...path, "escalate_to"], `${path.join(".")}.escalate_to forces the retry_previous re-run onto a pool candidate, but ${gatedLabel} declares no model_pool`);
|
|
3853
|
+
checkModelSlot(ctx, [...path, "escalate_to"], judge.escalate_to);
|
|
3854
|
+
return;
|
|
3855
|
+
}
|
|
3856
|
+
const tags = new Set();
|
|
3857
|
+
for (const candidate of gatedPool.candidates) {
|
|
3858
|
+
for (const tag of candidateTags(ctx, candidate))
|
|
3859
|
+
tags.add(tag);
|
|
3860
|
+
}
|
|
3861
|
+
checkRoleSlot(ctx, [...path, "escalate_to"], judge.escalate_to, gatedPool, tags);
|
|
3862
|
+
}
|
|
3863
|
+
/** The top-level auxiliary model slots the interactive shapes share. */
|
|
3864
|
+
function checkAuxSlots(ctx, spec) {
|
|
3865
|
+
checkModelSlot(ctx, ["compaction", "model"], spec.compaction?.model);
|
|
3866
|
+
checkModelSlot(ctx, ["security", "justification", "model"], spec.security?.justification?.model);
|
|
3867
|
+
if (spec.budget?.on_exceed?.action === "degrade") {
|
|
3868
|
+
checkModelSlot(ctx, ["budget", "on_exceed", "model"], spec.budget.on_exceed.model);
|
|
3869
|
+
}
|
|
3870
|
+
checkModelSlot(ctx, ["watchme", "judge", "model"], spec.watchme?.judge?.model);
|
|
3871
|
+
const evaluation = spec.evaluation;
|
|
3872
|
+
if (evaluation !== undefined) {
|
|
3873
|
+
if (evaluation.grader.type === "llm_judge") {
|
|
3874
|
+
checkModelSlot(ctx, ["evaluation", "grader", "model"], evaluation.grader.model);
|
|
3875
|
+
for (const [i, member] of (evaluation.grader.judges ?? []).entries()) {
|
|
3876
|
+
checkModelSlot(ctx, ["evaluation", "grader", "judges", i], member);
|
|
3877
|
+
}
|
|
3878
|
+
}
|
|
3879
|
+
// §7.3 — `escalate` re-runs the turn on a FORCED roster candidate, so it
|
|
3880
|
+
// needs a roster (forcing on a non-pool run is a compile-time error).
|
|
3881
|
+
if (evaluation.on_fail === "escalate" && spec.agent?.model_pool === undefined) {
|
|
3882
|
+
ctx.custom(["evaluation", "on_fail"], "evaluation.on_fail: escalate re-runs a failing turn on a stronger pool candidate (strategy.cascade.escalate_to, else the strongest candidate), so agent.model_pool must be declared");
|
|
3883
|
+
}
|
|
3884
|
+
}
|
|
3885
|
+
}
|
|
3886
|
+
function modelSurfaceIssues(data, custom) {
|
|
3887
|
+
const registry = data.models ?? {};
|
|
3888
|
+
const mcpServers = new Set(Object.keys(data.mcp_servers ?? {}));
|
|
3889
|
+
const modelDirected = specDeclaresModelDirected(data);
|
|
3890
|
+
switch (data.target) {
|
|
3891
|
+
case "cli":
|
|
3892
|
+
case "channel":
|
|
3893
|
+
case "managed": {
|
|
3894
|
+
const ctx = {
|
|
3895
|
+
custom,
|
|
3896
|
+
registry,
|
|
3897
|
+
mcpServers,
|
|
3898
|
+
modelDirected,
|
|
3899
|
+
gradedInLoop: data.evaluation !== undefined,
|
|
3900
|
+
};
|
|
3901
|
+
const shapeTools = data.target === "cli" ? data.tools : data.agent.tools;
|
|
3902
|
+
const host = { committee: false, shapeTools, toolLess: false };
|
|
3903
|
+
checkProfiles(ctx, host);
|
|
3904
|
+
checkRoutedBlock(ctx, ["agent"], data.agent, host);
|
|
3905
|
+
checkAuxSlots(ctx, data);
|
|
3906
|
+
return;
|
|
3907
|
+
}
|
|
3908
|
+
case "workflow": {
|
|
3909
|
+
const ctx = {
|
|
3910
|
+
custom,
|
|
3911
|
+
registry,
|
|
3912
|
+
mcpServers,
|
|
3913
|
+
modelDirected,
|
|
3914
|
+
gradedInLoop: data.steps.some((s) => "kind" in s && s.kind === "judge"),
|
|
3915
|
+
};
|
|
3916
|
+
const declaredTools = data.steps.flatMap((s) => ("tools" in s ? (s.tools ?? []) : []));
|
|
3917
|
+
const anyStepDeclaresTools = data.steps.some((s) => "tools" in s && s.tools !== undefined);
|
|
3918
|
+
checkProfiles(ctx, {
|
|
3919
|
+
committee: true,
|
|
3920
|
+
shapeTools: anyStepDeclaresTools ? declaredTools : undefined,
|
|
3921
|
+
toolLess: false,
|
|
3922
|
+
});
|
|
3923
|
+
checkModelSlot(ctx, ["model"], data.model);
|
|
3924
|
+
checkAuxSlots(ctx, data);
|
|
3925
|
+
for (const [i, step] of data.steps.entries()) {
|
|
3926
|
+
// `kind` exists ONLY on the judge variant, so the `in` check is the
|
|
3927
|
+
// discriminator in BOTH branches (a `=== "judge"` conjunct would
|
|
3928
|
+
// narrow the single remaining member's property, not the union).
|
|
3929
|
+
if ("kind" in step) {
|
|
3930
|
+
const previous = data.steps[i - 1];
|
|
3931
|
+
const gatedPool = previous !== undefined && "model_pool" in previous ? previous.model_pool : undefined;
|
|
3932
|
+
checkJudgeGate(ctx, ["steps", i, "judge"], step.judge, gatedPool, previous !== undefined ? `the gated step "${previous.name}"` : "the gated step");
|
|
3933
|
+
continue;
|
|
3934
|
+
}
|
|
3935
|
+
checkRoutedBlock(ctx, ["steps", i], step, {
|
|
3936
|
+
committee: true,
|
|
3937
|
+
shapeTools: step.tools,
|
|
3938
|
+
toolLess: false,
|
|
3939
|
+
});
|
|
3940
|
+
}
|
|
3941
|
+
return;
|
|
3942
|
+
}
|
|
3943
|
+
case "graph": {
|
|
3944
|
+
const nodes = Object.entries(data.nodes);
|
|
3945
|
+
const ctx = {
|
|
3946
|
+
custom,
|
|
3947
|
+
registry,
|
|
3948
|
+
mcpServers,
|
|
3949
|
+
modelDirected,
|
|
3950
|
+
gradedInLoop: nodes.some(([, n]) => "kind" in n && n.kind === "judge"),
|
|
3951
|
+
};
|
|
3952
|
+
const declaredTools = nodes.flatMap(([, n]) => ("tools" in n ? (n.tools ?? []) : []));
|
|
3953
|
+
const anyNodeDeclaresTools = nodes.some(([, n]) => "tools" in n && n.tools !== undefined);
|
|
3954
|
+
checkProfiles(ctx, {
|
|
3955
|
+
committee: true,
|
|
3956
|
+
shapeTools: anyNodeDeclaresTools ? declaredTools : undefined,
|
|
3957
|
+
toolLess: false,
|
|
3958
|
+
});
|
|
3959
|
+
checkModelSlot(ctx, ["model"], data.model);
|
|
3960
|
+
checkAuxSlots(ctx, data);
|
|
3961
|
+
for (const [name, node] of nodes) {
|
|
3962
|
+
if ("kind" in node) {
|
|
3963
|
+
// The upstream node is an edge-time fact, so `escalate_to` is
|
|
3964
|
+
// checked against the UNION of the graph's node pools: any tag or
|
|
3965
|
+
// roster $profile declared on some node is accepted here; the
|
|
3966
|
+
// ir-pass pins it to the actual upstream node once edges resolve.
|
|
3967
|
+
const pools = nodes.flatMap(([, n]) => "model_pool" in n && n.model_pool !== undefined ? [n.model_pool] : []);
|
|
3968
|
+
const union = pools.length > 0
|
|
3969
|
+
? { ...pools[0], candidates: pools.flatMap((p) => p.candidates) }
|
|
3970
|
+
: undefined;
|
|
3971
|
+
checkJudgeGate(ctx, ["nodes", name, "judge"], node.judge, union, "no graph node");
|
|
3972
|
+
continue;
|
|
3973
|
+
}
|
|
3974
|
+
checkRoutedBlock(ctx, ["nodes", name], node, {
|
|
3975
|
+
committee: true,
|
|
3976
|
+
shapeTools: node.tools,
|
|
3977
|
+
toolLess: false,
|
|
3978
|
+
});
|
|
3979
|
+
}
|
|
3980
|
+
return;
|
|
3981
|
+
}
|
|
3982
|
+
case "crew": {
|
|
3983
|
+
const ctx = {
|
|
3984
|
+
custom,
|
|
3985
|
+
registry,
|
|
3986
|
+
mcpServers,
|
|
3987
|
+
modelDirected,
|
|
3988
|
+
gradedInLoop: false,
|
|
3989
|
+
};
|
|
3990
|
+
const roles = Object.entries(data.roles);
|
|
3991
|
+
const declaredTools = roles.flatMap(([, r]) => r.tools ?? []);
|
|
3992
|
+
const anyRoleDeclaresTools = roles.some(([, r]) => r.tools !== undefined);
|
|
3993
|
+
checkProfiles(ctx, {
|
|
3994
|
+
committee: true,
|
|
3995
|
+
shapeTools: anyRoleDeclaresTools ? declaredTools : undefined,
|
|
3996
|
+
toolLess: false,
|
|
3997
|
+
});
|
|
3998
|
+
checkModelSlot(ctx, ["model"], data.model);
|
|
3999
|
+
checkAuxSlots(ctx, data);
|
|
4000
|
+
for (const [name, role] of roles) {
|
|
4001
|
+
checkRoutedBlock(ctx, ["roles", name], role, {
|
|
4002
|
+
committee: true,
|
|
4003
|
+
shapeTools: role.tools,
|
|
4004
|
+
toolLess: false,
|
|
4005
|
+
});
|
|
4006
|
+
}
|
|
4007
|
+
if (data.routing?.model !== undefined) {
|
|
4008
|
+
if (data.routing.kind !== "llm") {
|
|
4009
|
+
custom(["routing", "model"], `crew.routing.model is the model the kind: llm router runs on — routing.kind is "${data.routing.kind}", which never calls a model`);
|
|
4010
|
+
}
|
|
4011
|
+
checkModelSlot(ctx, ["routing", "model"], data.routing.model);
|
|
4012
|
+
}
|
|
4013
|
+
return;
|
|
4014
|
+
}
|
|
4015
|
+
case "pipeline":
|
|
4016
|
+
case "research":
|
|
4017
|
+
case "batch":
|
|
4018
|
+
case "browser": {
|
|
4019
|
+
const ctx = {
|
|
4020
|
+
custom,
|
|
4021
|
+
registry,
|
|
4022
|
+
mcpServers,
|
|
4023
|
+
modelDirected,
|
|
4024
|
+
gradedInLoop: false,
|
|
4025
|
+
};
|
|
4026
|
+
const toolLess = data.target === "pipeline";
|
|
4027
|
+
const host = {
|
|
4028
|
+
committee: false,
|
|
4029
|
+
shapeTools: toolLess ? undefined : data.tools,
|
|
4030
|
+
toolLess,
|
|
4031
|
+
};
|
|
4032
|
+
checkProfiles(ctx, host);
|
|
4033
|
+
checkRoutedBlock(ctx, ["agent"], data.agent, host);
|
|
4034
|
+
checkAuxSlots(ctx, data);
|
|
4035
|
+
if (data.target === "browser") {
|
|
4036
|
+
checkModelSlot(ctx, ["groundingModel"], data.groundingModel);
|
|
4037
|
+
}
|
|
4038
|
+
return;
|
|
4039
|
+
}
|
|
4040
|
+
case "voice":
|
|
4041
|
+
case "eval":
|
|
4042
|
+
case "onchain":
|
|
4043
|
+
case "onchain-game": {
|
|
4044
|
+
// Profile → model/params only on these shapes (§11.3); the other
|
|
4045
|
+
// profile fields are reported as field-precise warnings at lower time.
|
|
4046
|
+
const ctx = {
|
|
4047
|
+
custom,
|
|
4048
|
+
registry,
|
|
4049
|
+
mcpServers,
|
|
4050
|
+
modelDirected,
|
|
4051
|
+
gradedInLoop: false,
|
|
4052
|
+
};
|
|
4053
|
+
checkProfiles(ctx, {
|
|
4054
|
+
committee: false,
|
|
4055
|
+
shapeTools: data.target === "eval" ? data.agent.tools : data.tools,
|
|
4056
|
+
toolLess: false,
|
|
4057
|
+
});
|
|
4058
|
+
checkModelSlot(ctx, ["agent", "model"], data.agent.model);
|
|
4059
|
+
checkAuxSlots(ctx, data);
|
|
4060
|
+
return;
|
|
4061
|
+
}
|
|
4062
|
+
default:
|
|
4063
|
+
return;
|
|
4064
|
+
}
|
|
4065
|
+
}
|
|
2862
4066
|
export function parseSpec(yamlText) {
|
|
2863
4067
|
let raw;
|
|
2864
4068
|
try {
|