@crewhaus/evaluation-service 0.0.0-stage → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/skills.js ADDED
@@ -0,0 +1,200 @@
1
+ /**
2
+ * The skill a gated harness advertises (design/0.8.0-provability-release.md
3
+ * §11): `@crewhaus/default-skills`' `provability` skill — what the gate checks
4
+ * at each rung and how to cite so an answer passes — rendered from the lowered
5
+ * block. The `learning-loop` precedent: shipped text, templated from the spec
6
+ * at wiring time.
7
+ *
8
+ * The brief for the text is "say exactly what the gate checks", so every
9
+ * setting that changes what this gate accepts is a substitution
10
+ * ({@link provabilitySkillSubstitutions}), never a sentence that is true
11
+ * only under the defaults: the floor and the checker order, what a failing
12
+ * answer becomes (`on_fail`, `max_retries`), how far back evidence counts
13
+ * (`scope`), whether and how many assumptions are accepted (`assumptions`,
14
+ * `max_assumptions`), how many independent sources a `grounded` claim needs
15
+ * (`grounded.independent`) — and a section kept only where the gate wires
16
+ * `ProofCheckers` + `ProofCheck` (`wantsProofCheckTools`), so the skill never
17
+ * tells the model to call a tool its harness lacks. Settings that only make
18
+ * the gate MORE lenient than the text (a `threshold` below 1, `bibliographic`
19
+ * sources, `replay: false`) are left out: the text then asks for more than
20
+ * the gate needs, never less.
21
+ *
22
+ * Registered ONLY when the IR carries `provability:`, and through the seams
23
+ * every other builtin skill already uses: a surface that discovers its own
24
+ * skills passes `provabilitySkills(block)` as `discoverSkills({ builtinSkills })`,
25
+ * and one whose memory fabric owns the skill surface passes it to
26
+ * `wireMemory(…, { builtinSkills })`. Either way it merges at LOWEST
27
+ * precedence — a user or project skill named `provability` replaces it — and
28
+ * its frontmatter is classified at discovery and its body at load, at the
29
+ * `"skill"` TrustOrigin, like every builtin (Pillar 3). A spec without the
30
+ * block passes nothing: its prompt gains no bytes and its bundle none either
31
+ * (`renderGateSkills` renders the empty string).
32
+ *
33
+ * A pure function of the block rather than a field of `wireEvaluation`'s
34
+ * result, because every surface discovers its skills BEFORE it wires the
35
+ * gates — the cli bundle's extension boot, the channel daemon's boot, the
36
+ * interpreter's run setup — and the skill depends on nothing the wiring
37
+ * builds. The discovery site calls this directly; no boot is reordered.
38
+ *
39
+ * The rendering is the one Task children receive (`./child-skill`'s
40
+ * `provabilitySkillText`, the same substitutions over the same body — a test
41
+ * holds the two together), so a parent and its children are told one thing.
42
+ */
43
+ import { DEFAULT_SKILLS, renderSkill, skillTokens } from "@crewhaus/default-skills";
44
+ import { createLogger } from "@crewhaus/logging";
45
+ import { wantsProofCheckTools } from "./tools";
46
+ const logger = createLogger({ bindings: { module: "evaluation-service" } });
47
+ /** The skill's name — what a user or project skill overrides it by. */
48
+ export const PROVABILITY_SKILL_NAME = "provability";
49
+ const plural = (n, one, many) => (n === 1 ? one : many);
50
+ /**
51
+ * What a failing answer becomes — completes "An answer that falls short …".
52
+ *
53
+ * `retry` and `escalate` both re-run the turn and both count against
54
+ * `max_retries` (runtime-core's `runProvableTurn`: a pass at
55
+ * `retryIndex >= maxRetries` is spent and its answer stands with the failing
56
+ * verdict). So the cap belongs in both sentences: at 0 neither re-runs
57
+ * anything — note semantics — and past the cap the last answer, the stronger
58
+ * model's included, is served failing.
59
+ */
60
+ function onFailText(config) {
61
+ const anyway = "is served anyway, with a failing verdict";
62
+ switch (config.onFail) {
63
+ case "halt":
64
+ return "is never served: the run stops";
65
+ case "note":
66
+ return anyway;
67
+ case "retry":
68
+ case "escalate": {
69
+ // runtime-core's reading of the cap: a non-finite one is 0, and 0 is note.
70
+ const n = Number.isFinite(config.maxRetries) ? Math.max(0, Math.floor(config.maxRetries)) : 0;
71
+ if (n === 0)
72
+ return anyway;
73
+ const rerun = config.onFail === "retry"
74
+ ? "comes back to you with what failed"
75
+ : "goes, with what failed, to a stronger model to answer instead";
76
+ return `${rerun}, at most ${plural(n, "once", `${n} times`)}; after that it is served with a failing verdict`;
77
+ }
78
+ }
79
+ }
80
+ /** How many assumptions an answer may declare — completes "an assumption: …". */
81
+ function assumptionsText(config) {
82
+ const cap = config.maxAssumptions;
83
+ if (config.assumptions === "forbid" || cap === 0) {
84
+ return "refused here — declaring one fails the whole answer";
85
+ }
86
+ const keep = "so keep it for what no evidence can settle";
87
+ if (cap === undefined)
88
+ return `counted and shown to the reader, ${keep}`;
89
+ return `at most ${plural(cap, "one", String(cap))} per answer, each shown to the reader, ${keep}`;
90
+ }
91
+ /**
92
+ * Every substitution the `provability` skill's body can ask for, computed
93
+ * from `config` — the ONE place the skill's wording meets the block, shared
94
+ * by {@link provabilitySkills} and `./child-skill`'s rendering. A caller
95
+ * passes only the names its body uses (`skillTokens`): the renderer refuses
96
+ * the rest.
97
+ *
98
+ * - `level` — the floor; `checkers` — `formal.checkers`, comma-separated;
99
+ * - `on_fail` — what a failing answer becomes;
100
+ * - `window` — where evidence must come from: `in this session`, or under
101
+ * `scope: turn` `since the user's last message` (the gate's evidence
102
+ * index then holds this turn's calls only);
103
+ * - `assumptions` — accepted and counted, capped, or refused;
104
+ * - `independent` — at a `grounded` floor with `grounded.independent > 1`,
105
+ * the clause asking for that many distinct origins; else empty (the rule
106
+ * applies at no other floor);
107
+ * - `proofs` — the section holding the proof tools, non-empty exactly when
108
+ * the gate wires `ProofCheckers` + `ProofCheck` (`wantsProofCheckTools`):
109
+ * at `proven` every claim needs a proof; below it a proof meets any
110
+ * floor.
111
+ */
112
+ export function provabilitySkillSubstitutions(config) {
113
+ const independent = config.grounded.independent;
114
+ return {
115
+ level: config.level,
116
+ checkers: config.formal.checkers.join(", "),
117
+ on_fail: onFailText(config),
118
+ window: config.scope === "turn" ? "since the user's last message" : "in this session",
119
+ assumptions: assumptionsText(config),
120
+ independent: config.level === "grounded" && independent > 1
121
+ ? `; each claim needs ${independent} independent sources (different sites, publishers or files)`
122
+ : "",
123
+ proofs: !wantsProofCheckTools(config)
124
+ ? ""
125
+ : config.level === "proven"
126
+ ? "Every claim needs a proof:"
127
+ : "A proof meets any floor. To prove a claim,",
128
+ };
129
+ }
130
+ /** {@link provabilitySkillSubstitutions}, narrowed to the names `body` uses. */
131
+ export function substitutionsFor(config, body) {
132
+ const used = skillTokens(body);
133
+ return Object.fromEntries(Object.entries(provabilitySkillSubstitutions(config)).filter(([token]) => used.has(token)));
134
+ }
135
+ const DEFAULT_SOURCE = { skills: DEFAULT_SKILLS, render: renderSkill };
136
+ /**
137
+ * The gate's builtin skills for `config` — the rendered `provability` skill,
138
+ * or `[]` when there is no block (so a call site may pass the IR's optional
139
+ * block unconditionally). Also `[]`, with a warning, when this build ships no
140
+ * such skill or its body asks for a substitution the block cannot fill: a
141
+ * missing skill is a wiring gap the gate's retry nudge still covers, never a
142
+ * reason to fail a harness at boot.
143
+ */
144
+ export function provabilitySkills(config, source = DEFAULT_SOURCE) {
145
+ if (config === undefined)
146
+ return [];
147
+ const skill = source.skills.find((s) => s.name === PROVABILITY_SKILL_NAME);
148
+ if (skill === undefined) {
149
+ logger.warn("provability: this build ships no provability skill; none is registered");
150
+ return [];
151
+ }
152
+ try {
153
+ // Only the names the body uses: the renderer refuses a key the body lacks.
154
+ // Computed inside the try: a block missing a field (a bundle compiled
155
+ // against another shape) costs the skill, never the boot.
156
+ const substitutions = substitutionsFor(config, skill.body);
157
+ return [{ ...skill, body: source.render(PROVABILITY_SKILL_NAME, substitutions) }];
158
+ }
159
+ catch (err) {
160
+ logger.warn("provability: the provability skill could not be rendered; none is registered", {
161
+ message: err instanceof Error ? err.message : String(err),
162
+ });
163
+ return [];
164
+ }
165
+ }
166
+ const NO_GATE_SKILLS = { imports: [], expr: "" };
167
+ /**
168
+ * What an emitter splices into a skill-discovery site (design §11): the
169
+ * import and the `provabilitySkills(…)` expression, rendered from the lowered
170
+ * block. The emitter passes it as `builtinSkills` to the `discoverSkills` /
171
+ * `wireMemory` call that builds the run's skill list — the
172
+ * `renderEvaluationWiring` / `renderMemory` precedent: one rendering, owned
173
+ * by the composition root, so no emitter templates the skill itself. Both
174
+ * pieces empty without `provability:`, keeping those bundles byte-identical.
175
+ *
176
+ * The argument is `JSON.stringify` of the {@link ProvabilitySkillConfig}
177
+ * fields, in a fixed order: plain data (rung and mode names, counts, checker
178
+ * ids), so JSON's quoting is the escaping, and only what the skill reads
179
+ * rides along — the full block is already in the bundle's `wireEvaluation`
180
+ * fragment.
181
+ */
182
+ export function renderGateSkills(ir) {
183
+ const p = ir.provability;
184
+ if (p === undefined)
185
+ return NO_GATE_SKILLS;
186
+ const arg = {
187
+ level: p.level,
188
+ onFail: p.onFail,
189
+ maxRetries: p.maxRetries,
190
+ scope: p.scope,
191
+ assumptions: p.assumptions,
192
+ ...(p.maxAssumptions !== undefined ? { maxAssumptions: p.maxAssumptions } : {}),
193
+ grounded: { independent: p.grounded.independent },
194
+ formal: { checkers: [...p.formal.checkers] },
195
+ };
196
+ return {
197
+ imports: ['import { provabilitySkills } from "@crewhaus/evaluation-service";'],
198
+ expr: `provabilitySkills(${JSON.stringify(arg)})`,
199
+ };
200
+ }
@@ -0,0 +1,106 @@
1
+ /**
2
+ * The proof tools a gated harness gets (design §7): `@crewhaus/tool-proof`'s
3
+ * three tools, bound by `createProofTools` to the SAME configuration, proofs
4
+ * root and evidence the gate uses — so a `ProofCheck` artifact is filed where
5
+ * the gate looks for it, and `ProvabilityPreflight` grades a draft against
6
+ * exactly the index the gate will build.
7
+ *
8
+ * Which tools a level adds — decided here, once, for every shape:
9
+ *
10
+ * - `ProvabilityPreflight` always: at every rung the model should see what
11
+ * the gate will see before it finishes (§7);
12
+ * - `ProofCheckers` + `ProofCheck` when the floor is `proven` — the only
13
+ * rung where every claim needs a `proof:` artifact (design §2) — or when
14
+ * the spec chose its own `formal.checkers` (anything but the lowered
15
+ * default `builtin, smt`): declaring checkers is a spec asking for
16
+ * proofs, and a `proof:` binding meets ANY floor, so a `checked` spec
17
+ * that named `lean4` means the model to use it. Below `proven` with the
18
+ * default checkers the pair is not added: a tool advertised at every
19
+ * turn that no claim needs is prompt weight and a distraction.
20
+ *
21
+ * Which tools a level adds is a separate question from which it replaces.
22
+ * All three are always built, bound to this gate, and a tool the run already
23
+ * carries under one of their names — a spec that listed `proofCheck` (or
24
+ * `all-proof`) in `tools:` at `checked` — is swapped for the bound one
25
+ * whatever the level: the builtin instance files its artifacts under
26
+ * `.crewhaus/proofs/default/` (a run context carries no spec name), where
27
+ * this gate never looks, so every `proof:` citation it produced would read
28
+ * `proof_missing` — and a `proof:` binding meets any floor (design §2), so a
29
+ * spec that listed the tool means it to work. `registerGateTools` (a
30
+ * catalog) and `mergeGateTools` (a tool list) apply that rule; nothing else
31
+ * should.
32
+ *
33
+ * The checkers: with a sandbox handed in (`deps.sandbox`), they are built
34
+ * here over it, in `formal.checkers` order with the builtin always present
35
+ * (tool-proof's `buildDefaultCheckers`, i.e. proof-checker's
36
+ * `createDefaultCheckers` over the `formal` block). Without one, tool-proof
37
+ * builds the same defaults on the first call, choosing the sandbox from what
38
+ * the RUN says is wired (`sandboxAvailable` on the bridge, `CREWHAUS_SANDBOX`)
39
+ * — a choice a boot-time wiring cannot make for the run.
40
+ */
41
+ import type { IrProvability } from "@crewhaus/ir";
42
+ import type { SandboxLike } from "@crewhaus/proof-checker";
43
+ import type { Tenant } from "@crewhaus/tenancy";
44
+ import type { RegisteredTool, ToolExecuteContext } from "@crewhaus/tool-catalog";
45
+ import { type ProofToolEvidence } from "@crewhaus/tool-proof";
46
+ /** The `formal.checkers` the compiler lowers when a spec names none. */
47
+ export declare const DEFAULT_FORMAL_CHECKERS: ReadonlyArray<string>;
48
+ /**
49
+ * Whether a gated harness also gets `ProofCheckers` + `ProofCheck` (see the
50
+ * module doc). Reads the floor and the checkers only — the `provability`
51
+ * skill asks the same question of the fields a bundle hands it (`./skills`),
52
+ * so the skill teaches the proof tools exactly where they are wired.
53
+ */
54
+ export declare function wantsProofCheckTools(config: {
55
+ readonly level: IrProvability["level"];
56
+ readonly formal: {
57
+ readonly checkers: ReadonlyArray<string>;
58
+ };
59
+ }): boolean;
60
+ export type GateToolsOptions = {
61
+ readonly config: IrProvability;
62
+ readonly workspaceRoot: string;
63
+ /** The proofs root for the tenant a call runs under (the gate's own resolver). */
64
+ readonly proofsRoot: (tenant: Tenant | undefined) => string;
65
+ /**
66
+ * The evidence a preflight grades against: the gate's builder, without a
67
+ * transcript — under `scope: turn` the index with the turn it keeps.
68
+ */
69
+ readonly evidence: (ctx: ToolExecuteContext | undefined) => Promise<ProofToolEvidence>;
70
+ readonly sandbox?: SandboxLike;
71
+ readonly now?: () => Date;
72
+ };
73
+ /** The proof tools for one gate: what it adds, and everything it binds. */
74
+ export type GateProofTools = {
75
+ /** The tools the level adds to every run (see the module doc). */
76
+ readonly tools: ReadonlyArray<RegisteredTool>;
77
+ /**
78
+ * All three proof tools, bound to this gate, in tool-proof's order — the
79
+ * ones in `tools` included. A same-named tool already on the run is
80
+ * replaced by its member here even when the level does not add it.
81
+ */
82
+ readonly proofTools: ReadonlyArray<RegisteredTool>;
83
+ };
84
+ /** The tools for `config`, registration-ready, in tool-proof's order. */
85
+ export declare function gateTools(opts: GateToolsOptions): GateProofTools;
86
+ /**
87
+ * Put the gate's proof tools on `catalog` — the catalog form the cli and
88
+ * managed bundles render (`renderEvaluationWiring`): every tool the level
89
+ * adds, and the bound twin of any proof tool already registered (a builtin
90
+ * the spec listed), which is unregistered first. Call it after the builtin
91
+ * registrations and before the plugin ones, so first-party still wins every
92
+ * plugin collision.
93
+ */
94
+ export declare function registerGateTools(catalog: {
95
+ has(name: string): boolean;
96
+ unregister(name: string): unknown;
97
+ register(tool: RegisteredTool): void;
98
+ }, gate: GateProofTools): void;
99
+ /**
100
+ * `tools` with the gate's proof tools merged in by the same rule as
101
+ * {@link registerGateTools} — the list form, for a run that is handed a tool
102
+ * list (the channel bot's `runTurn`, `crewhaus run` / `serve`). Order is
103
+ * kept: the run's other tools first, then the gate's, in tool-proof's order.
104
+ * A copy, never the caller's array.
105
+ */
106
+ export declare function mergeGateTools(tools: ReadonlyArray<RegisteredTool>, gate: GateProofTools): RegisteredTool[];
package/dist/tools.js ADDED
@@ -0,0 +1,74 @@
1
+ import { buildDefaultCheckers, createProofTools, } from "@crewhaus/tool-proof";
2
+ /** The `formal.checkers` the compiler lowers when a spec names none. */
3
+ export const DEFAULT_FORMAL_CHECKERS = ["builtin", "smt"];
4
+ /**
5
+ * Whether a gated harness also gets `ProofCheckers` + `ProofCheck` (see the
6
+ * module doc). Reads the floor and the checkers only — the `provability`
7
+ * skill asks the same question of the fields a bundle hands it (`./skills`),
8
+ * so the skill teaches the proof tools exactly where they are wired.
9
+ */
10
+ export function wantsProofCheckTools(config) {
11
+ if (config.level === "proven")
12
+ return true;
13
+ const checkers = config.formal.checkers;
14
+ return (checkers.length !== DEFAULT_FORMAL_CHECKERS.length ||
15
+ checkers.some((id, i) => id !== DEFAULT_FORMAL_CHECKERS[i]));
16
+ }
17
+ /** The tools for `config`, registration-ready, in tool-proof's order. */
18
+ export function gateTools(opts) {
19
+ const sandbox = opts.sandbox;
20
+ const tools = createProofTools({
21
+ config: opts.config,
22
+ proofsRoot: opts.proofsRoot,
23
+ workspaceRoot: opts.workspaceRoot,
24
+ evidence: opts.evidence,
25
+ ...(sandbox !== undefined
26
+ ? {
27
+ checkers: buildDefaultCheckers(opts.config, {
28
+ sandboxAvailable: true,
29
+ createSandbox: () => sandbox,
30
+ }),
31
+ }
32
+ : {}),
33
+ ...(opts.now !== undefined ? { now: opts.now } : {}),
34
+ });
35
+ const proofTools = [tools.proofCheckers, tools.proofCheck, tools.provabilityPreflight];
36
+ return {
37
+ tools: wantsProofCheckTools(opts.config) ? proofTools : [tools.provabilityPreflight],
38
+ proofTools,
39
+ };
40
+ }
41
+ /** The proof tools a run ends up with: the added ones, plus a bound twin of any it already has. */
42
+ function chosen(gate, present) {
43
+ const added = new Set(gate.tools.map((tool) => tool.name));
44
+ return gate.proofTools.filter((tool) => added.has(tool.name) || present(tool.name));
45
+ }
46
+ /**
47
+ * Put the gate's proof tools on `catalog` — the catalog form the cli and
48
+ * managed bundles render (`renderEvaluationWiring`): every tool the level
49
+ * adds, and the bound twin of any proof tool already registered (a builtin
50
+ * the spec listed), which is unregistered first. Call it after the builtin
51
+ * registrations and before the plugin ones, so first-party still wins every
52
+ * plugin collision.
53
+ */
54
+ export function registerGateTools(catalog, gate) {
55
+ for (const tool of chosen(gate, (name) => catalog.has(name))) {
56
+ catalog.unregister(tool.name);
57
+ catalog.register(tool);
58
+ }
59
+ }
60
+ /**
61
+ * `tools` with the gate's proof tools merged in by the same rule as
62
+ * {@link registerGateTools} — the list form, for a run that is handed a tool
63
+ * list (the channel bot's `runTurn`, `crewhaus run` / `serve`). Order is
64
+ * kept: the run's other tools first, then the gate's, in tool-proof's order.
65
+ * A copy, never the caller's array.
66
+ */
67
+ export function mergeGateTools(tools, gate) {
68
+ if (gate.proofTools.length === 0)
69
+ return [...tools];
70
+ const present = new Set(tools.map((tool) => tool.name));
71
+ const merged = chosen(gate, (name) => present.has(name));
72
+ const names = new Set(merged.map((tool) => tool.name));
73
+ return [...tools.filter((tool) => !names.has(tool.name)), ...merged];
74
+ }
package/dist/wire.d.ts ADDED
@@ -0,0 +1,106 @@
1
+ /**
2
+ * `wireEvaluation` — THE composition root for the in-loop gates (design §4.4).
3
+ *
4
+ * Every emitter (cli, channel, managed) renders one call to it and spreads
5
+ * what it returns into `runChatLoop`; the `crewhaus run` / `serve`
6
+ * interpreter calls it too. It builds:
7
+ *
8
+ * - `evaluation` — the `evaluation:` judge (`./evaluation`), exactly the
9
+ * behaviour the three `renderEvaluation` copies and `buildRunEvaluation`
10
+ * had;
11
+ * - `provability` — the provability gate's closures (`./provability`);
12
+ * - `tools` / `proofTools` — the proof tools bound to the same
13
+ * configuration, proofs root and evidence as the gate (`./tools`): the
14
+ * ones the level adds, and all three (a same-named tool the run already
15
+ * has is replaced by its bound twin). The caller puts them on its run
16
+ * with `registerGateTools` (a catalog) or `mergeGateTools` (a list).
17
+ * Empty without a `provability:` block.
18
+ *
19
+ * The 0.6.0 rule now covers in-loop gates: a new gate lands here, never in an
20
+ * emitter's codegen, and runtime-core receives closures, never a dependency.
21
+ *
22
+ * Nothing is opened at wiring time — no directory, no log, no sandbox, no
23
+ * network: every root is resolved per turn (per call, for the tools) for the
24
+ * tenant the turn runs under, so one wiring at a managed daemon's boot serves
25
+ * every tenant. Any name the spec schema admits wires: the proofs directory
26
+ * is `proofsDirName(specName)` (`./roots`), which maps a name provability's
27
+ * store would refuse (a space, a `:`) instead of failing the bundle at boot.
28
+ */
29
+ import { CrewhausError } from "@crewhaus/errors";
30
+ import type { CategoricalJudgeOptions } from "@crewhaus/eval-judge";
31
+ import type { SandboxLike } from "@crewhaus/proof-checker";
32
+ import { type ProofBundle } from "@crewhaus/provability";
33
+ import type { RunEvaluation, RunProvability } from "@crewhaus/runtime-core";
34
+ import type { EvaluationFragment } from "./fragment";
35
+ import { type GateCatalog } from "./replay";
36
+ import { type GateProofTools } from "./tools";
37
+ export declare class EvaluationServiceError extends CrewhausError {
38
+ readonly name = "EvaluationServiceError";
39
+ constructor(message: string, cause?: unknown);
40
+ }
41
+ export type WireEvaluationDeps = {
42
+ /** The harness working directory: path sources resolve inside it, `.crewhaus/proofs/` lives under it. */
43
+ readonly cwd: string;
44
+ /**
45
+ * Where `checked` replay finds the tools it re-runs: the bundle's
46
+ * `defaultCatalog`, or the run's tool list. Read at replay time, so tools
47
+ * registered after wiring are found. Absent ⇒ no replay.
48
+ */
49
+ readonly catalog?: GateCatalog;
50
+ /**
51
+ * The session-log root the RUN was given (`runChatLoop({ sessionRootDir })`),
52
+ * when it was given one. Absent ⇒ the runtime's own order: the tenant's
53
+ * `sessionRoot`, `CREWHAUS_SESSION_DIR`, `.crewhaus/sessions`.
54
+ */
55
+ readonly sessionRootDir?: string;
56
+ /**
57
+ * Where the proof footer goes: `"stream"` (cli — written after the
58
+ * answer), `"reply"` (channel bots — appended to the text they send) or
59
+ * `"none"` (the managed gateway and the eval bridges — the bundle rides
60
+ * their own envelope through `onBundle`, and the returned text is the
61
+ * answer alone). Absent ⇒ the runtime default, `"stream"`.
62
+ */
63
+ readonly footerTarget?: "stream" | "reply" | "none";
64
+ /**
65
+ * The reader reads the turn as it streams though the footer rides the
66
+ * returned text (`RunProvability.streamed`): `crewhaus run --prompt`,
67
+ * which prints every attempt to stdout and then the result with its
68
+ * footer. Absent ⇒ the runtime reads it off `footerTarget`.
69
+ */
70
+ readonly streamed?: boolean;
71
+ /** A sandbox for the `smt` / `lean4` checkers; absent ⇒ chosen per run from what the run wires. */
72
+ readonly sandbox?: SandboxLike;
73
+ /**
74
+ * The HEAD request behind `sources.resolve: head` (default: tool-fetch's
75
+ * guarded HEAD); `signal` aborts at the verifier's HEAD deadline or when
76
+ * the turn ends.
77
+ */
78
+ readonly resolveHead?: (url: string, signal?: AbortSignal) => Promise<number | undefined>;
79
+ /** How long one verification waits on its HEAD requests in all (default 15 s). */
80
+ readonly headDeadlineMs?: number;
81
+ /** Observe every proof bundle (gateway envelopes, tests). */
82
+ readonly onBundle?: (bundle: ProofBundle, bundlePath?: string) => void;
83
+ /** Test seam: the adapter every judge call (evaluation and waiver) uses, bypassing the model-router. */
84
+ readonly judgeAdapter?: CategoricalJudgeOptions["adapter"];
85
+ /** Clock for bundles and artifacts (tests). */
86
+ readonly now?: () => Date;
87
+ /**
88
+ * 0.8.0 §9.1 — the proofs root, when it must not be the harness's
89
+ * `<cwd>/.crewhaus/proofs/<spec>/` (or the tenant's): where `ProofCheck`
90
+ * files its artifacts, where the gate resolves `proof:` citations and where
91
+ * bundles persist. The eval runner points it into each sample's own fabric
92
+ * root (Pillar 2 isolation: a sample neither writes into nor cites from the
93
+ * live proofs tree); `cwd` stays the workspace path citations resolve in.
94
+ * Fenced to the active tenant's root like every other root. Absent ⇒ the
95
+ * default, unchanged.
96
+ */
97
+ readonly proofsRoot?: string;
98
+ };
99
+ export type WiredGates = GateProofTools & {
100
+ /** Present iff the fragment carries `evaluation`. */
101
+ readonly evaluation?: RunEvaluation;
102
+ /** Present iff the fragment carries `provability`. */
103
+ readonly provability?: RunProvability;
104
+ };
105
+ /** Build the gates a fragment declares (see the module doc). */
106
+ export declare function wireEvaluation(fragment: EvaluationFragment, deps: WireEvaluationDeps): Promise<WiredGates>;
package/dist/wire.js ADDED
@@ -0,0 +1,133 @@
1
+ /**
2
+ * `wireEvaluation` — THE composition root for the in-loop gates (design §4.4).
3
+ *
4
+ * Every emitter (cli, channel, managed) renders one call to it and spreads
5
+ * what it returns into `runChatLoop`; the `crewhaus run` / `serve`
6
+ * interpreter calls it too. It builds:
7
+ *
8
+ * - `evaluation` — the `evaluation:` judge (`./evaluation`), exactly the
9
+ * behaviour the three `renderEvaluation` copies and `buildRunEvaluation`
10
+ * had;
11
+ * - `provability` — the provability gate's closures (`./provability`);
12
+ * - `tools` / `proofTools` — the proof tools bound to the same
13
+ * configuration, proofs root and evidence as the gate (`./tools`): the
14
+ * ones the level adds, and all three (a same-named tool the run already
15
+ * has is replaced by its bound twin). The caller puts them on its run
16
+ * with `registerGateTools` (a catalog) or `mergeGateTools` (a list).
17
+ * Empty without a `provability:` block.
18
+ *
19
+ * The 0.6.0 rule now covers in-loop gates: a new gate lands here, never in an
20
+ * emitter's codegen, and runtime-core receives closures, never a dependency.
21
+ *
22
+ * Nothing is opened at wiring time — no directory, no log, no sandbox, no
23
+ * network: every root is resolved per turn (per call, for the tools) for the
24
+ * tenant the turn runs under, so one wiring at a managed daemon's boot serves
25
+ * every tenant. Any name the spec schema admits wires: the proofs directory
26
+ * is `proofsDirName(specName)` (`./roots`), which maps a name provability's
27
+ * store would refuse (a space, a `:`) instead of failing the bundle at boot.
28
+ */
29
+ import { CrewhausError } from "@crewhaus/errors";
30
+ import { isProvabilityLevel } from "@crewhaus/provability";
31
+ import { buildRunEvaluation } from "./evaluation";
32
+ import { gateEvidenceScoped } from "./evidence";
33
+ import { buildRunProvability } from "./provability";
34
+ import { schemaLookup } from "./replay";
35
+ import { activeTenant, fencedDir, gateProofsRoot, gateSessionRoot } from "./roots";
36
+ import { gateTools } from "./tools";
37
+ export class EvaluationServiceError extends CrewhausError {
38
+ name = "EvaluationServiceError";
39
+ constructor(message, cause) {
40
+ super("config", message, cause);
41
+ }
42
+ }
43
+ const SESSION_ID_REGEX = /^sess_[0-9a-f]{16}$/;
44
+ /** The run's session id, read the way tool-proof reads it: the context's, else the bridge's. */
45
+ function sessionIdOfCall(ctx) {
46
+ const bridge = ctx?.bridge;
47
+ const id = ctx?.runContext?.sessionId ?? bridge?.runContext?.sessionId;
48
+ return typeof id === "string" && SESSION_ID_REGEX.test(id) ? id : undefined;
49
+ }
50
+ /** Where the runtime said its session logs live, when it said. */
51
+ function sessionRootOfCall(ctx) {
52
+ const bridge = ctx?.bridge;
53
+ const dir = typeof bridge === "object" && bridge !== null ? bridge.sessionRootDir : undefined;
54
+ return typeof dir === "string" && dir !== "" ? dir : undefined;
55
+ }
56
+ /** A malformed fragment is a build or a hand-edit gone wrong: refuse it at boot, by name. */
57
+ function assertProvability(config) {
58
+ if (typeof config !== "object" || config === null || !isProvabilityLevel(config.level)) {
59
+ throw new EvaluationServiceError(`evaluation-service: fragment.provability.level must be one of cited, grounded, checked, proven (got ${JSON.stringify(config?.level)})`);
60
+ }
61
+ }
62
+ /** Build the gates a fragment declares (see the module doc). */
63
+ export async function wireEvaluation(fragment, deps) {
64
+ if (typeof fragment !== "object" || fragment === null) {
65
+ throw new EvaluationServiceError("evaluation-service: wireEvaluation needs a fragment");
66
+ }
67
+ const evaluation = fragment.evaluation !== undefined
68
+ ? buildRunEvaluation(fragment.evaluation, fragment.primaryModel, {
69
+ ...(deps.judgeAdapter !== undefined ? { judgeAdapter: deps.judgeAdapter } : {}),
70
+ })
71
+ : undefined;
72
+ const config = fragment.provability;
73
+ if (config === undefined) {
74
+ return { ...(evaluation !== undefined ? { evaluation } : {}), tools: [], proofTools: [] };
75
+ }
76
+ assertProvability(config);
77
+ const { cwd } = deps;
78
+ const specName = fragment.specName;
79
+ if (typeof specName !== "string") {
80
+ throw new EvaluationServiceError("evaluation-service: fragment.specName must be a string");
81
+ }
82
+ const provability = buildRunProvability(config, {
83
+ specName,
84
+ cwd,
85
+ ...(deps.catalog !== undefined ? { catalog: deps.catalog } : {}),
86
+ ...(deps.sessionRootDir !== undefined ? { sessionRootDir: deps.sessionRootDir } : {}),
87
+ ...(deps.footerTarget !== undefined ? { footerTarget: deps.footerTarget } : {}),
88
+ ...(deps.streamed !== undefined ? { streamed: deps.streamed } : {}),
89
+ ...(deps.resolveHead !== undefined ? { resolveHead: deps.resolveHead } : {}),
90
+ ...(deps.headDeadlineMs !== undefined ? { headDeadlineMs: deps.headDeadlineMs } : {}),
91
+ ...(deps.onBundle !== undefined ? { onBundle: deps.onBundle } : {}),
92
+ ...(deps.judgeAdapter !== undefined ? { judgeAdapter: deps.judgeAdapter } : {}),
93
+ ...(deps.now !== undefined ? { now: deps.now } : {}),
94
+ ...(deps.proofsRoot !== undefined ? { proofsRoot: deps.proofsRoot } : {}),
95
+ });
96
+ // The preflight's evidence: the gate's own builder over the run's log
97
+ // (the transcript is not reachable from a tool call, and the log already
98
+ // holds every call the turn has finished), scoped the same way — under
99
+ // `scope: turn` the session's index with the turn it keeps, as the gate
100
+ // hands the verifier.
101
+ const evidence = async (ctx) => {
102
+ const tenant = activeTenant();
103
+ const callRoot = sessionRootOfCall(ctx);
104
+ const sessionRootDir = deps.sessionRootDir === undefined && callRoot !== undefined
105
+ ? fencedDir(callRoot, tenant)
106
+ : gateSessionRoot(deps.sessionRootDir, tenant);
107
+ const sessionId = sessionIdOfCall(ctx);
108
+ return gateEvidenceScoped({
109
+ scope: config.scope,
110
+ workspaceRoot: cwd,
111
+ ...(tenant !== undefined ? { toolResultsRoot: tenant.toolResultRoot } : {}),
112
+ ...(sessionId !== undefined ? { sessionId } : {}),
113
+ ...(sessionRootDir !== undefined ? { sessionRootDir } : {}),
114
+ // The schemas the gate reads MCP origins by, so the preflight sees what it sees.
115
+ inputSchemaOf: schemaLookup(deps.catalog),
116
+ });
117
+ };
118
+ const proofTools = gateTools({
119
+ config,
120
+ workspaceRoot: cwd,
121
+ proofsRoot: (tenant) => deps.proofsRoot !== undefined
122
+ ? fencedDir(deps.proofsRoot, tenant)
123
+ : gateProofsRoot(cwd, specName, tenant),
124
+ evidence,
125
+ ...(deps.sandbox !== undefined ? { sandbox: deps.sandbox } : {}),
126
+ ...(deps.now !== undefined ? { now: deps.now } : {}),
127
+ });
128
+ return {
129
+ ...(evaluation !== undefined ? { evaluation } : {}),
130
+ provability,
131
+ ...proofTools,
132
+ };
133
+ }