@crewhaus/evaluation-service 0.0.0-stage → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +9 -0
- package/README.md +278 -2
- package/dist/child-skill.d.ts +33 -0
- package/dist/child-skill.js +6 -0
- package/dist/evaluation.d.ts +40 -0
- package/dist/evaluation.js +128 -0
- package/dist/evidence.d.ts +97 -0
- package/dist/evidence.js +147 -0
- package/dist/fragment.d.ts +40 -0
- package/dist/fragment.js +17 -0
- package/dist/head.d.ts +115 -0
- package/dist/head.js +181 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +24 -0
- package/dist/judge.d.ts +62 -0
- package/dist/judge.js +103 -0
- package/dist/observe.d.ts +32 -0
- package/dist/observe.js +31 -0
- package/dist/provability.d.ts +47 -0
- package/dist/provability.js +261 -0
- package/dist/render.d.ts +95 -0
- package/dist/render.js +99 -0
- package/dist/replay.d.ts +53 -0
- package/dist/replay.js +309 -0
- package/dist/roots.d.ts +36 -0
- package/dist/roots.js +90 -0
- package/dist/skills.d.ts +83 -0
- package/dist/skills.js +200 -0
- package/dist/tools.d.ts +106 -0
- package/dist/tools.js +74 -0
- package/dist/wire.d.ts +106 -0
- package/dist/wire.js +133 -0
- package/package.json +65 -4
package/dist/skills.js
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The skill a gated harness advertises (design/0.8.0-provability-release.md
|
|
3
|
+
* §11): `@crewhaus/default-skills`' `provability` skill — what the gate checks
|
|
4
|
+
* at each rung and how to cite so an answer passes — rendered from the lowered
|
|
5
|
+
* block. The `learning-loop` precedent: shipped text, templated from the spec
|
|
6
|
+
* at wiring time.
|
|
7
|
+
*
|
|
8
|
+
* The brief for the text is "say exactly what the gate checks", so every
|
|
9
|
+
* setting that changes what this gate accepts is a substitution
|
|
10
|
+
* ({@link provabilitySkillSubstitutions}), never a sentence that is true
|
|
11
|
+
* only under the defaults: the floor and the checker order, what a failing
|
|
12
|
+
* answer becomes (`on_fail`, `max_retries`), how far back evidence counts
|
|
13
|
+
* (`scope`), whether and how many assumptions are accepted (`assumptions`,
|
|
14
|
+
* `max_assumptions`), how many independent sources a `grounded` claim needs
|
|
15
|
+
* (`grounded.independent`) — and a section kept only where the gate wires
|
|
16
|
+
* `ProofCheckers` + `ProofCheck` (`wantsProofCheckTools`), so the skill never
|
|
17
|
+
* tells the model to call a tool its harness lacks. Settings that only make
|
|
18
|
+
* the gate MORE lenient than the text (a `threshold` below 1, `bibliographic`
|
|
19
|
+
* sources, `replay: false`) are left out: the text then asks for more than
|
|
20
|
+
* the gate needs, never less.
|
|
21
|
+
*
|
|
22
|
+
* Registered ONLY when the IR carries `provability:`, and through the seams
|
|
23
|
+
* every other builtin skill already uses: a surface that discovers its own
|
|
24
|
+
* skills passes `provabilitySkills(block)` as `discoverSkills({ builtinSkills })`,
|
|
25
|
+
* and one whose memory fabric owns the skill surface passes it to
|
|
26
|
+
* `wireMemory(…, { builtinSkills })`. Either way it merges at LOWEST
|
|
27
|
+
* precedence — a user or project skill named `provability` replaces it — and
|
|
28
|
+
* its frontmatter is classified at discovery and its body at load, at the
|
|
29
|
+
* `"skill"` TrustOrigin, like every builtin (Pillar 3). A spec without the
|
|
30
|
+
* block passes nothing: its prompt gains no bytes and its bundle none either
|
|
31
|
+
* (`renderGateSkills` renders the empty string).
|
|
32
|
+
*
|
|
33
|
+
* A pure function of the block rather than a field of `wireEvaluation`'s
|
|
34
|
+
* result, because every surface discovers its skills BEFORE it wires the
|
|
35
|
+
* gates — the cli bundle's extension boot, the channel daemon's boot, the
|
|
36
|
+
* interpreter's run setup — and the skill depends on nothing the wiring
|
|
37
|
+
* builds. The discovery site calls this directly; no boot is reordered.
|
|
38
|
+
*
|
|
39
|
+
* The rendering is the one Task children receive (`./child-skill`'s
|
|
40
|
+
* `provabilitySkillText`, the same substitutions over the same body — a test
|
|
41
|
+
* holds the two together), so a parent and its children are told one thing.
|
|
42
|
+
*/
|
|
43
|
+
import { DEFAULT_SKILLS, renderSkill, skillTokens } from "@crewhaus/default-skills";
|
|
44
|
+
import { createLogger } from "@crewhaus/logging";
|
|
45
|
+
import { wantsProofCheckTools } from "./tools";
|
|
46
|
+
const logger = createLogger({ bindings: { module: "evaluation-service" } });
|
|
47
|
+
/** The skill's name — what a user or project skill overrides it by. */
|
|
48
|
+
export const PROVABILITY_SKILL_NAME = "provability";
|
|
49
|
+
const plural = (n, one, many) => (n === 1 ? one : many);
|
|
50
|
+
/**
|
|
51
|
+
* What a failing answer becomes — completes "An answer that falls short …".
|
|
52
|
+
*
|
|
53
|
+
* `retry` and `escalate` both re-run the turn and both count against
|
|
54
|
+
* `max_retries` (runtime-core's `runProvableTurn`: a pass at
|
|
55
|
+
* `retryIndex >= maxRetries` is spent and its answer stands with the failing
|
|
56
|
+
* verdict). So the cap belongs in both sentences: at 0 neither re-runs
|
|
57
|
+
* anything — note semantics — and past the cap the last answer, the stronger
|
|
58
|
+
* model's included, is served failing.
|
|
59
|
+
*/
|
|
60
|
+
function onFailText(config) {
|
|
61
|
+
const anyway = "is served anyway, with a failing verdict";
|
|
62
|
+
switch (config.onFail) {
|
|
63
|
+
case "halt":
|
|
64
|
+
return "is never served: the run stops";
|
|
65
|
+
case "note":
|
|
66
|
+
return anyway;
|
|
67
|
+
case "retry":
|
|
68
|
+
case "escalate": {
|
|
69
|
+
// runtime-core's reading of the cap: a non-finite one is 0, and 0 is note.
|
|
70
|
+
const n = Number.isFinite(config.maxRetries) ? Math.max(0, Math.floor(config.maxRetries)) : 0;
|
|
71
|
+
if (n === 0)
|
|
72
|
+
return anyway;
|
|
73
|
+
const rerun = config.onFail === "retry"
|
|
74
|
+
? "comes back to you with what failed"
|
|
75
|
+
: "goes, with what failed, to a stronger model to answer instead";
|
|
76
|
+
return `${rerun}, at most ${plural(n, "once", `${n} times`)}; after that it is served with a failing verdict`;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/** How many assumptions an answer may declare — completes "an assumption: …". */
|
|
81
|
+
function assumptionsText(config) {
|
|
82
|
+
const cap = config.maxAssumptions;
|
|
83
|
+
if (config.assumptions === "forbid" || cap === 0) {
|
|
84
|
+
return "refused here — declaring one fails the whole answer";
|
|
85
|
+
}
|
|
86
|
+
const keep = "so keep it for what no evidence can settle";
|
|
87
|
+
if (cap === undefined)
|
|
88
|
+
return `counted and shown to the reader, ${keep}`;
|
|
89
|
+
return `at most ${plural(cap, "one", String(cap))} per answer, each shown to the reader, ${keep}`;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Every substitution the `provability` skill's body can ask for, computed
|
|
93
|
+
* from `config` — the ONE place the skill's wording meets the block, shared
|
|
94
|
+
* by {@link provabilitySkills} and `./child-skill`'s rendering. A caller
|
|
95
|
+
* passes only the names its body uses (`skillTokens`): the renderer refuses
|
|
96
|
+
* the rest.
|
|
97
|
+
*
|
|
98
|
+
* - `level` — the floor; `checkers` — `formal.checkers`, comma-separated;
|
|
99
|
+
* - `on_fail` — what a failing answer becomes;
|
|
100
|
+
* - `window` — where evidence must come from: `in this session`, or under
|
|
101
|
+
* `scope: turn` `since the user's last message` (the gate's evidence
|
|
102
|
+
* index then holds this turn's calls only);
|
|
103
|
+
* - `assumptions` — accepted and counted, capped, or refused;
|
|
104
|
+
* - `independent` — at a `grounded` floor with `grounded.independent > 1`,
|
|
105
|
+
* the clause asking for that many distinct origins; else empty (the rule
|
|
106
|
+
* applies at no other floor);
|
|
107
|
+
* - `proofs` — the section holding the proof tools, non-empty exactly when
|
|
108
|
+
* the gate wires `ProofCheckers` + `ProofCheck` (`wantsProofCheckTools`):
|
|
109
|
+
* at `proven` every claim needs a proof; below it a proof meets any
|
|
110
|
+
* floor.
|
|
111
|
+
*/
|
|
112
|
+
export function provabilitySkillSubstitutions(config) {
|
|
113
|
+
const independent = config.grounded.independent;
|
|
114
|
+
return {
|
|
115
|
+
level: config.level,
|
|
116
|
+
checkers: config.formal.checkers.join(", "),
|
|
117
|
+
on_fail: onFailText(config),
|
|
118
|
+
window: config.scope === "turn" ? "since the user's last message" : "in this session",
|
|
119
|
+
assumptions: assumptionsText(config),
|
|
120
|
+
independent: config.level === "grounded" && independent > 1
|
|
121
|
+
? `; each claim needs ${independent} independent sources (different sites, publishers or files)`
|
|
122
|
+
: "",
|
|
123
|
+
proofs: !wantsProofCheckTools(config)
|
|
124
|
+
? ""
|
|
125
|
+
: config.level === "proven"
|
|
126
|
+
? "Every claim needs a proof:"
|
|
127
|
+
: "A proof meets any floor. To prove a claim,",
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
/** {@link provabilitySkillSubstitutions}, narrowed to the names `body` uses. */
|
|
131
|
+
export function substitutionsFor(config, body) {
|
|
132
|
+
const used = skillTokens(body);
|
|
133
|
+
return Object.fromEntries(Object.entries(provabilitySkillSubstitutions(config)).filter(([token]) => used.has(token)));
|
|
134
|
+
}
|
|
135
|
+
const DEFAULT_SOURCE = { skills: DEFAULT_SKILLS, render: renderSkill };
|
|
136
|
+
/**
|
|
137
|
+
* The gate's builtin skills for `config` — the rendered `provability` skill,
|
|
138
|
+
* or `[]` when there is no block (so a call site may pass the IR's optional
|
|
139
|
+
* block unconditionally). Also `[]`, with a warning, when this build ships no
|
|
140
|
+
* such skill or its body asks for a substitution the block cannot fill: a
|
|
141
|
+
* missing skill is a wiring gap the gate's retry nudge still covers, never a
|
|
142
|
+
* reason to fail a harness at boot.
|
|
143
|
+
*/
|
|
144
|
+
export function provabilitySkills(config, source = DEFAULT_SOURCE) {
|
|
145
|
+
if (config === undefined)
|
|
146
|
+
return [];
|
|
147
|
+
const skill = source.skills.find((s) => s.name === PROVABILITY_SKILL_NAME);
|
|
148
|
+
if (skill === undefined) {
|
|
149
|
+
logger.warn("provability: this build ships no provability skill; none is registered");
|
|
150
|
+
return [];
|
|
151
|
+
}
|
|
152
|
+
try {
|
|
153
|
+
// Only the names the body uses: the renderer refuses a key the body lacks.
|
|
154
|
+
// Computed inside the try: a block missing a field (a bundle compiled
|
|
155
|
+
// against another shape) costs the skill, never the boot.
|
|
156
|
+
const substitutions = substitutionsFor(config, skill.body);
|
|
157
|
+
return [{ ...skill, body: source.render(PROVABILITY_SKILL_NAME, substitutions) }];
|
|
158
|
+
}
|
|
159
|
+
catch (err) {
|
|
160
|
+
logger.warn("provability: the provability skill could not be rendered; none is registered", {
|
|
161
|
+
message: err instanceof Error ? err.message : String(err),
|
|
162
|
+
});
|
|
163
|
+
return [];
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
const NO_GATE_SKILLS = { imports: [], expr: "" };
|
|
167
|
+
/**
|
|
168
|
+
* What an emitter splices into a skill-discovery site (design §11): the
|
|
169
|
+
* import and the `provabilitySkills(…)` expression, rendered from the lowered
|
|
170
|
+
* block. The emitter passes it as `builtinSkills` to the `discoverSkills` /
|
|
171
|
+
* `wireMemory` call that builds the run's skill list — the
|
|
172
|
+
* `renderEvaluationWiring` / `renderMemory` precedent: one rendering, owned
|
|
173
|
+
* by the composition root, so no emitter templates the skill itself. Both
|
|
174
|
+
* pieces empty without `provability:`, keeping those bundles byte-identical.
|
|
175
|
+
*
|
|
176
|
+
* The argument is `JSON.stringify` of the {@link ProvabilitySkillConfig}
|
|
177
|
+
* fields, in a fixed order: plain data (rung and mode names, counts, checker
|
|
178
|
+
* ids), so JSON's quoting is the escaping, and only what the skill reads
|
|
179
|
+
* rides along — the full block is already in the bundle's `wireEvaluation`
|
|
180
|
+
* fragment.
|
|
181
|
+
*/
|
|
182
|
+
export function renderGateSkills(ir) {
|
|
183
|
+
const p = ir.provability;
|
|
184
|
+
if (p === undefined)
|
|
185
|
+
return NO_GATE_SKILLS;
|
|
186
|
+
const arg = {
|
|
187
|
+
level: p.level,
|
|
188
|
+
onFail: p.onFail,
|
|
189
|
+
maxRetries: p.maxRetries,
|
|
190
|
+
scope: p.scope,
|
|
191
|
+
assumptions: p.assumptions,
|
|
192
|
+
...(p.maxAssumptions !== undefined ? { maxAssumptions: p.maxAssumptions } : {}),
|
|
193
|
+
grounded: { independent: p.grounded.independent },
|
|
194
|
+
formal: { checkers: [...p.formal.checkers] },
|
|
195
|
+
};
|
|
196
|
+
return {
|
|
197
|
+
imports: ['import { provabilitySkills } from "@crewhaus/evaluation-service";'],
|
|
198
|
+
expr: `provabilitySkills(${JSON.stringify(arg)})`,
|
|
199
|
+
};
|
|
200
|
+
}
|
package/dist/tools.d.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The proof tools a gated harness gets (design §7): `@crewhaus/tool-proof`'s
|
|
3
|
+
* three tools, bound by `createProofTools` to the SAME configuration, proofs
|
|
4
|
+
* root and evidence the gate uses — so a `ProofCheck` artifact is filed where
|
|
5
|
+
* the gate looks for it, and `ProvabilityPreflight` grades a draft against
|
|
6
|
+
* exactly the index the gate will build.
|
|
7
|
+
*
|
|
8
|
+
* Which tools a level adds — decided here, once, for every shape:
|
|
9
|
+
*
|
|
10
|
+
* - `ProvabilityPreflight` always: at every rung the model should see what
|
|
11
|
+
* the gate will see before it finishes (§7);
|
|
12
|
+
* - `ProofCheckers` + `ProofCheck` when the floor is `proven` — the only
|
|
13
|
+
* rung where every claim needs a `proof:` artifact (design §2) — or when
|
|
14
|
+
* the spec chose its own `formal.checkers` (anything but the lowered
|
|
15
|
+
* default `builtin, smt`): declaring checkers is a spec asking for
|
|
16
|
+
* proofs, and a `proof:` binding meets ANY floor, so a `checked` spec
|
|
17
|
+
* that named `lean4` means the model to use it. Below `proven` with the
|
|
18
|
+
* default checkers the pair is not added: a tool advertised at every
|
|
19
|
+
* turn that no claim needs is prompt weight and a distraction.
|
|
20
|
+
*
|
|
21
|
+
* Which tools a level adds is a separate question from which it replaces.
|
|
22
|
+
* All three are always built, bound to this gate, and a tool the run already
|
|
23
|
+
* carries under one of their names — a spec that listed `proofCheck` (or
|
|
24
|
+
* `all-proof`) in `tools:` at `checked` — is swapped for the bound one
|
|
25
|
+
* whatever the level: the builtin instance files its artifacts under
|
|
26
|
+
* `.crewhaus/proofs/default/` (a run context carries no spec name), where
|
|
27
|
+
* this gate never looks, so every `proof:` citation it produced would read
|
|
28
|
+
* `proof_missing` — and a `proof:` binding meets any floor (design §2), so a
|
|
29
|
+
* spec that listed the tool means it to work. `registerGateTools` (a
|
|
30
|
+
* catalog) and `mergeGateTools` (a tool list) apply that rule; nothing else
|
|
31
|
+
* should.
|
|
32
|
+
*
|
|
33
|
+
* The checkers: with a sandbox handed in (`deps.sandbox`), they are built
|
|
34
|
+
* here over it, in `formal.checkers` order with the builtin always present
|
|
35
|
+
* (tool-proof's `buildDefaultCheckers`, i.e. proof-checker's
|
|
36
|
+
* `createDefaultCheckers` over the `formal` block). Without one, tool-proof
|
|
37
|
+
* builds the same defaults on the first call, choosing the sandbox from what
|
|
38
|
+
* the RUN says is wired (`sandboxAvailable` on the bridge, `CREWHAUS_SANDBOX`)
|
|
39
|
+
* — a choice a boot-time wiring cannot make for the run.
|
|
40
|
+
*/
|
|
41
|
+
import type { IrProvability } from "@crewhaus/ir";
|
|
42
|
+
import type { SandboxLike } from "@crewhaus/proof-checker";
|
|
43
|
+
import type { Tenant } from "@crewhaus/tenancy";
|
|
44
|
+
import type { RegisteredTool, ToolExecuteContext } from "@crewhaus/tool-catalog";
|
|
45
|
+
import { type ProofToolEvidence } from "@crewhaus/tool-proof";
|
|
46
|
+
/** The `formal.checkers` the compiler lowers when a spec names none. */
|
|
47
|
+
export declare const DEFAULT_FORMAL_CHECKERS: ReadonlyArray<string>;
|
|
48
|
+
/**
|
|
49
|
+
* Whether a gated harness also gets `ProofCheckers` + `ProofCheck` (see the
|
|
50
|
+
* module doc). Reads the floor and the checkers only — the `provability`
|
|
51
|
+
* skill asks the same question of the fields a bundle hands it (`./skills`),
|
|
52
|
+
* so the skill teaches the proof tools exactly where they are wired.
|
|
53
|
+
*/
|
|
54
|
+
export declare function wantsProofCheckTools(config: {
|
|
55
|
+
readonly level: IrProvability["level"];
|
|
56
|
+
readonly formal: {
|
|
57
|
+
readonly checkers: ReadonlyArray<string>;
|
|
58
|
+
};
|
|
59
|
+
}): boolean;
|
|
60
|
+
export type GateToolsOptions = {
|
|
61
|
+
readonly config: IrProvability;
|
|
62
|
+
readonly workspaceRoot: string;
|
|
63
|
+
/** The proofs root for the tenant a call runs under (the gate's own resolver). */
|
|
64
|
+
readonly proofsRoot: (tenant: Tenant | undefined) => string;
|
|
65
|
+
/**
|
|
66
|
+
* The evidence a preflight grades against: the gate's builder, without a
|
|
67
|
+
* transcript — under `scope: turn` the index with the turn it keeps.
|
|
68
|
+
*/
|
|
69
|
+
readonly evidence: (ctx: ToolExecuteContext | undefined) => Promise<ProofToolEvidence>;
|
|
70
|
+
readonly sandbox?: SandboxLike;
|
|
71
|
+
readonly now?: () => Date;
|
|
72
|
+
};
|
|
73
|
+
/** The proof tools for one gate: what it adds, and everything it binds. */
|
|
74
|
+
export type GateProofTools = {
|
|
75
|
+
/** The tools the level adds to every run (see the module doc). */
|
|
76
|
+
readonly tools: ReadonlyArray<RegisteredTool>;
|
|
77
|
+
/**
|
|
78
|
+
* All three proof tools, bound to this gate, in tool-proof's order — the
|
|
79
|
+
* ones in `tools` included. A same-named tool already on the run is
|
|
80
|
+
* replaced by its member here even when the level does not add it.
|
|
81
|
+
*/
|
|
82
|
+
readonly proofTools: ReadonlyArray<RegisteredTool>;
|
|
83
|
+
};
|
|
84
|
+
/** The tools for `config`, registration-ready, in tool-proof's order. */
|
|
85
|
+
export declare function gateTools(opts: GateToolsOptions): GateProofTools;
|
|
86
|
+
/**
|
|
87
|
+
* Put the gate's proof tools on `catalog` — the catalog form the cli and
|
|
88
|
+
* managed bundles render (`renderEvaluationWiring`): every tool the level
|
|
89
|
+
* adds, and the bound twin of any proof tool already registered (a builtin
|
|
90
|
+
* the spec listed), which is unregistered first. Call it after the builtin
|
|
91
|
+
* registrations and before the plugin ones, so first-party still wins every
|
|
92
|
+
* plugin collision.
|
|
93
|
+
*/
|
|
94
|
+
export declare function registerGateTools(catalog: {
|
|
95
|
+
has(name: string): boolean;
|
|
96
|
+
unregister(name: string): unknown;
|
|
97
|
+
register(tool: RegisteredTool): void;
|
|
98
|
+
}, gate: GateProofTools): void;
|
|
99
|
+
/**
|
|
100
|
+
* `tools` with the gate's proof tools merged in by the same rule as
|
|
101
|
+
* {@link registerGateTools} — the list form, for a run that is handed a tool
|
|
102
|
+
* list (the channel bot's `runTurn`, `crewhaus run` / `serve`). Order is
|
|
103
|
+
* kept: the run's other tools first, then the gate's, in tool-proof's order.
|
|
104
|
+
* A copy, never the caller's array.
|
|
105
|
+
*/
|
|
106
|
+
export declare function mergeGateTools(tools: ReadonlyArray<RegisteredTool>, gate: GateProofTools): RegisteredTool[];
|
package/dist/tools.js
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { buildDefaultCheckers, createProofTools, } from "@crewhaus/tool-proof";
|
|
2
|
+
/** The `formal.checkers` the compiler lowers when a spec names none. */
|
|
3
|
+
export const DEFAULT_FORMAL_CHECKERS = ["builtin", "smt"];
|
|
4
|
+
/**
|
|
5
|
+
* Whether a gated harness also gets `ProofCheckers` + `ProofCheck` (see the
|
|
6
|
+
* module doc). Reads the floor and the checkers only — the `provability`
|
|
7
|
+
* skill asks the same question of the fields a bundle hands it (`./skills`),
|
|
8
|
+
* so the skill teaches the proof tools exactly where they are wired.
|
|
9
|
+
*/
|
|
10
|
+
export function wantsProofCheckTools(config) {
|
|
11
|
+
if (config.level === "proven")
|
|
12
|
+
return true;
|
|
13
|
+
const checkers = config.formal.checkers;
|
|
14
|
+
return (checkers.length !== DEFAULT_FORMAL_CHECKERS.length ||
|
|
15
|
+
checkers.some((id, i) => id !== DEFAULT_FORMAL_CHECKERS[i]));
|
|
16
|
+
}
|
|
17
|
+
/** The tools for `config`, registration-ready, in tool-proof's order. */
|
|
18
|
+
export function gateTools(opts) {
|
|
19
|
+
const sandbox = opts.sandbox;
|
|
20
|
+
const tools = createProofTools({
|
|
21
|
+
config: opts.config,
|
|
22
|
+
proofsRoot: opts.proofsRoot,
|
|
23
|
+
workspaceRoot: opts.workspaceRoot,
|
|
24
|
+
evidence: opts.evidence,
|
|
25
|
+
...(sandbox !== undefined
|
|
26
|
+
? {
|
|
27
|
+
checkers: buildDefaultCheckers(opts.config, {
|
|
28
|
+
sandboxAvailable: true,
|
|
29
|
+
createSandbox: () => sandbox,
|
|
30
|
+
}),
|
|
31
|
+
}
|
|
32
|
+
: {}),
|
|
33
|
+
...(opts.now !== undefined ? { now: opts.now } : {}),
|
|
34
|
+
});
|
|
35
|
+
const proofTools = [tools.proofCheckers, tools.proofCheck, tools.provabilityPreflight];
|
|
36
|
+
return {
|
|
37
|
+
tools: wantsProofCheckTools(opts.config) ? proofTools : [tools.provabilityPreflight],
|
|
38
|
+
proofTools,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
/** The proof tools a run ends up with: the added ones, plus a bound twin of any it already has. */
|
|
42
|
+
function chosen(gate, present) {
|
|
43
|
+
const added = new Set(gate.tools.map((tool) => tool.name));
|
|
44
|
+
return gate.proofTools.filter((tool) => added.has(tool.name) || present(tool.name));
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Put the gate's proof tools on `catalog` — the catalog form the cli and
|
|
48
|
+
* managed bundles render (`renderEvaluationWiring`): every tool the level
|
|
49
|
+
* adds, and the bound twin of any proof tool already registered (a builtin
|
|
50
|
+
* the spec listed), which is unregistered first. Call it after the builtin
|
|
51
|
+
* registrations and before the plugin ones, so first-party still wins every
|
|
52
|
+
* plugin collision.
|
|
53
|
+
*/
|
|
54
|
+
export function registerGateTools(catalog, gate) {
|
|
55
|
+
for (const tool of chosen(gate, (name) => catalog.has(name))) {
|
|
56
|
+
catalog.unregister(tool.name);
|
|
57
|
+
catalog.register(tool);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* `tools` with the gate's proof tools merged in by the same rule as
|
|
62
|
+
* {@link registerGateTools} — the list form, for a run that is handed a tool
|
|
63
|
+
* list (the channel bot's `runTurn`, `crewhaus run` / `serve`). Order is
|
|
64
|
+
* kept: the run's other tools first, then the gate's, in tool-proof's order.
|
|
65
|
+
* A copy, never the caller's array.
|
|
66
|
+
*/
|
|
67
|
+
export function mergeGateTools(tools, gate) {
|
|
68
|
+
if (gate.proofTools.length === 0)
|
|
69
|
+
return [...tools];
|
|
70
|
+
const present = new Set(tools.map((tool) => tool.name));
|
|
71
|
+
const merged = chosen(gate, (name) => present.has(name));
|
|
72
|
+
const names = new Set(merged.map((tool) => tool.name));
|
|
73
|
+
return [...tools.filter((tool) => !names.has(tool.name)), ...merged];
|
|
74
|
+
}
|
package/dist/wire.d.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `wireEvaluation` — THE composition root for the in-loop gates (design §4.4).
|
|
3
|
+
*
|
|
4
|
+
* Every emitter (cli, channel, managed) renders one call to it and spreads
|
|
5
|
+
* what it returns into `runChatLoop`; the `crewhaus run` / `serve`
|
|
6
|
+
* interpreter calls it too. It builds:
|
|
7
|
+
*
|
|
8
|
+
* - `evaluation` — the `evaluation:` judge (`./evaluation`), exactly the
|
|
9
|
+
* behaviour the three `renderEvaluation` copies and `buildRunEvaluation`
|
|
10
|
+
* had;
|
|
11
|
+
* - `provability` — the provability gate's closures (`./provability`);
|
|
12
|
+
* - `tools` / `proofTools` — the proof tools bound to the same
|
|
13
|
+
* configuration, proofs root and evidence as the gate (`./tools`): the
|
|
14
|
+
* ones the level adds, and all three (a same-named tool the run already
|
|
15
|
+
* has is replaced by its bound twin). The caller puts them on its run
|
|
16
|
+
* with `registerGateTools` (a catalog) or `mergeGateTools` (a list).
|
|
17
|
+
* Empty without a `provability:` block.
|
|
18
|
+
*
|
|
19
|
+
* The 0.6.0 rule now covers in-loop gates: a new gate lands here, never in an
|
|
20
|
+
* emitter's codegen, and runtime-core receives closures, never a dependency.
|
|
21
|
+
*
|
|
22
|
+
* Nothing is opened at wiring time — no directory, no log, no sandbox, no
|
|
23
|
+
* network: every root is resolved per turn (per call, for the tools) for the
|
|
24
|
+
* tenant the turn runs under, so one wiring at a managed daemon's boot serves
|
|
25
|
+
* every tenant. Any name the spec schema admits wires: the proofs directory
|
|
26
|
+
* is `proofsDirName(specName)` (`./roots`), which maps a name provability's
|
|
27
|
+
* store would refuse (a space, a `:`) instead of failing the bundle at boot.
|
|
28
|
+
*/
|
|
29
|
+
import { CrewhausError } from "@crewhaus/errors";
|
|
30
|
+
import type { CategoricalJudgeOptions } from "@crewhaus/eval-judge";
|
|
31
|
+
import type { SandboxLike } from "@crewhaus/proof-checker";
|
|
32
|
+
import { type ProofBundle } from "@crewhaus/provability";
|
|
33
|
+
import type { RunEvaluation, RunProvability } from "@crewhaus/runtime-core";
|
|
34
|
+
import type { EvaluationFragment } from "./fragment";
|
|
35
|
+
import { type GateCatalog } from "./replay";
|
|
36
|
+
import { type GateProofTools } from "./tools";
|
|
37
|
+
export declare class EvaluationServiceError extends CrewhausError {
|
|
38
|
+
readonly name = "EvaluationServiceError";
|
|
39
|
+
constructor(message: string, cause?: unknown);
|
|
40
|
+
}
|
|
41
|
+
export type WireEvaluationDeps = {
|
|
42
|
+
/** The harness working directory: path sources resolve inside it, `.crewhaus/proofs/` lives under it. */
|
|
43
|
+
readonly cwd: string;
|
|
44
|
+
/**
|
|
45
|
+
* Where `checked` replay finds the tools it re-runs: the bundle's
|
|
46
|
+
* `defaultCatalog`, or the run's tool list. Read at replay time, so tools
|
|
47
|
+
* registered after wiring are found. Absent ⇒ no replay.
|
|
48
|
+
*/
|
|
49
|
+
readonly catalog?: GateCatalog;
|
|
50
|
+
/**
|
|
51
|
+
* The session-log root the RUN was given (`runChatLoop({ sessionRootDir })`),
|
|
52
|
+
* when it was given one. Absent ⇒ the runtime's own order: the tenant's
|
|
53
|
+
* `sessionRoot`, `CREWHAUS_SESSION_DIR`, `.crewhaus/sessions`.
|
|
54
|
+
*/
|
|
55
|
+
readonly sessionRootDir?: string;
|
|
56
|
+
/**
|
|
57
|
+
* Where the proof footer goes: `"stream"` (cli — written after the
|
|
58
|
+
* answer), `"reply"` (channel bots — appended to the text they send) or
|
|
59
|
+
* `"none"` (the managed gateway and the eval bridges — the bundle rides
|
|
60
|
+
* their own envelope through `onBundle`, and the returned text is the
|
|
61
|
+
* answer alone). Absent ⇒ the runtime default, `"stream"`.
|
|
62
|
+
*/
|
|
63
|
+
readonly footerTarget?: "stream" | "reply" | "none";
|
|
64
|
+
/**
|
|
65
|
+
* The reader reads the turn as it streams though the footer rides the
|
|
66
|
+
* returned text (`RunProvability.streamed`): `crewhaus run --prompt`,
|
|
67
|
+
* which prints every attempt to stdout and then the result with its
|
|
68
|
+
* footer. Absent ⇒ the runtime reads it off `footerTarget`.
|
|
69
|
+
*/
|
|
70
|
+
readonly streamed?: boolean;
|
|
71
|
+
/** A sandbox for the `smt` / `lean4` checkers; absent ⇒ chosen per run from what the run wires. */
|
|
72
|
+
readonly sandbox?: SandboxLike;
|
|
73
|
+
/**
|
|
74
|
+
* The HEAD request behind `sources.resolve: head` (default: tool-fetch's
|
|
75
|
+
* guarded HEAD); `signal` aborts at the verifier's HEAD deadline or when
|
|
76
|
+
* the turn ends.
|
|
77
|
+
*/
|
|
78
|
+
readonly resolveHead?: (url: string, signal?: AbortSignal) => Promise<number | undefined>;
|
|
79
|
+
/** How long one verification waits on its HEAD requests in all (default 15 s). */
|
|
80
|
+
readonly headDeadlineMs?: number;
|
|
81
|
+
/** Observe every proof bundle (gateway envelopes, tests). */
|
|
82
|
+
readonly onBundle?: (bundle: ProofBundle, bundlePath?: string) => void;
|
|
83
|
+
/** Test seam: the adapter every judge call (evaluation and waiver) uses, bypassing the model-router. */
|
|
84
|
+
readonly judgeAdapter?: CategoricalJudgeOptions["adapter"];
|
|
85
|
+
/** Clock for bundles and artifacts (tests). */
|
|
86
|
+
readonly now?: () => Date;
|
|
87
|
+
/**
|
|
88
|
+
* 0.8.0 §9.1 — the proofs root, when it must not be the harness's
|
|
89
|
+
* `<cwd>/.crewhaus/proofs/<spec>/` (or the tenant's): where `ProofCheck`
|
|
90
|
+
* files its artifacts, where the gate resolves `proof:` citations and where
|
|
91
|
+
* bundles persist. The eval runner points it into each sample's own fabric
|
|
92
|
+
* root (Pillar 2 isolation: a sample neither writes into nor cites from the
|
|
93
|
+
* live proofs tree); `cwd` stays the workspace path citations resolve in.
|
|
94
|
+
* Fenced to the active tenant's root like every other root. Absent ⇒ the
|
|
95
|
+
* default, unchanged.
|
|
96
|
+
*/
|
|
97
|
+
readonly proofsRoot?: string;
|
|
98
|
+
};
|
|
99
|
+
export type WiredGates = GateProofTools & {
|
|
100
|
+
/** Present iff the fragment carries `evaluation`. */
|
|
101
|
+
readonly evaluation?: RunEvaluation;
|
|
102
|
+
/** Present iff the fragment carries `provability`. */
|
|
103
|
+
readonly provability?: RunProvability;
|
|
104
|
+
};
|
|
105
|
+
/** Build the gates a fragment declares (see the module doc). */
|
|
106
|
+
export declare function wireEvaluation(fragment: EvaluationFragment, deps: WireEvaluationDeps): Promise<WiredGates>;
|
package/dist/wire.js
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `wireEvaluation` — THE composition root for the in-loop gates (design §4.4).
|
|
3
|
+
*
|
|
4
|
+
* Every emitter (cli, channel, managed) renders one call to it and spreads
|
|
5
|
+
* what it returns into `runChatLoop`; the `crewhaus run` / `serve`
|
|
6
|
+
* interpreter calls it too. It builds:
|
|
7
|
+
*
|
|
8
|
+
* - `evaluation` — the `evaluation:` judge (`./evaluation`), exactly the
|
|
9
|
+
* behaviour the three `renderEvaluation` copies and `buildRunEvaluation`
|
|
10
|
+
* had;
|
|
11
|
+
* - `provability` — the provability gate's closures (`./provability`);
|
|
12
|
+
* - `tools` / `proofTools` — the proof tools bound to the same
|
|
13
|
+
* configuration, proofs root and evidence as the gate (`./tools`): the
|
|
14
|
+
* ones the level adds, and all three (a same-named tool the run already
|
|
15
|
+
* has is replaced by its bound twin). The caller puts them on its run
|
|
16
|
+
* with `registerGateTools` (a catalog) or `mergeGateTools` (a list).
|
|
17
|
+
* Empty without a `provability:` block.
|
|
18
|
+
*
|
|
19
|
+
* The 0.6.0 rule now covers in-loop gates: a new gate lands here, never in an
|
|
20
|
+
* emitter's codegen, and runtime-core receives closures, never a dependency.
|
|
21
|
+
*
|
|
22
|
+
* Nothing is opened at wiring time — no directory, no log, no sandbox, no
|
|
23
|
+
* network: every root is resolved per turn (per call, for the tools) for the
|
|
24
|
+
* tenant the turn runs under, so one wiring at a managed daemon's boot serves
|
|
25
|
+
* every tenant. Any name the spec schema admits wires: the proofs directory
|
|
26
|
+
* is `proofsDirName(specName)` (`./roots`), which maps a name provability's
|
|
27
|
+
* store would refuse (a space, a `:`) instead of failing the bundle at boot.
|
|
28
|
+
*/
|
|
29
|
+
import { CrewhausError } from "@crewhaus/errors";
|
|
30
|
+
import { isProvabilityLevel } from "@crewhaus/provability";
|
|
31
|
+
import { buildRunEvaluation } from "./evaluation";
|
|
32
|
+
import { gateEvidenceScoped } from "./evidence";
|
|
33
|
+
import { buildRunProvability } from "./provability";
|
|
34
|
+
import { schemaLookup } from "./replay";
|
|
35
|
+
import { activeTenant, fencedDir, gateProofsRoot, gateSessionRoot } from "./roots";
|
|
36
|
+
import { gateTools } from "./tools";
|
|
37
|
+
export class EvaluationServiceError extends CrewhausError {
|
|
38
|
+
name = "EvaluationServiceError";
|
|
39
|
+
constructor(message, cause) {
|
|
40
|
+
super("config", message, cause);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const SESSION_ID_REGEX = /^sess_[0-9a-f]{16}$/;
|
|
44
|
+
/** The run's session id, read the way tool-proof reads it: the context's, else the bridge's. */
|
|
45
|
+
function sessionIdOfCall(ctx) {
|
|
46
|
+
const bridge = ctx?.bridge;
|
|
47
|
+
const id = ctx?.runContext?.sessionId ?? bridge?.runContext?.sessionId;
|
|
48
|
+
return typeof id === "string" && SESSION_ID_REGEX.test(id) ? id : undefined;
|
|
49
|
+
}
|
|
50
|
+
/** Where the runtime said its session logs live, when it said. */
|
|
51
|
+
function sessionRootOfCall(ctx) {
|
|
52
|
+
const bridge = ctx?.bridge;
|
|
53
|
+
const dir = typeof bridge === "object" && bridge !== null ? bridge.sessionRootDir : undefined;
|
|
54
|
+
return typeof dir === "string" && dir !== "" ? dir : undefined;
|
|
55
|
+
}
|
|
56
|
+
/** A malformed fragment is a build or a hand-edit gone wrong: refuse it at boot, by name. */
|
|
57
|
+
function assertProvability(config) {
|
|
58
|
+
if (typeof config !== "object" || config === null || !isProvabilityLevel(config.level)) {
|
|
59
|
+
throw new EvaluationServiceError(`evaluation-service: fragment.provability.level must be one of cited, grounded, checked, proven (got ${JSON.stringify(config?.level)})`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/** Build the gates a fragment declares (see the module doc). */
|
|
63
|
+
export async function wireEvaluation(fragment, deps) {
|
|
64
|
+
if (typeof fragment !== "object" || fragment === null) {
|
|
65
|
+
throw new EvaluationServiceError("evaluation-service: wireEvaluation needs a fragment");
|
|
66
|
+
}
|
|
67
|
+
const evaluation = fragment.evaluation !== undefined
|
|
68
|
+
? buildRunEvaluation(fragment.evaluation, fragment.primaryModel, {
|
|
69
|
+
...(deps.judgeAdapter !== undefined ? { judgeAdapter: deps.judgeAdapter } : {}),
|
|
70
|
+
})
|
|
71
|
+
: undefined;
|
|
72
|
+
const config = fragment.provability;
|
|
73
|
+
if (config === undefined) {
|
|
74
|
+
return { ...(evaluation !== undefined ? { evaluation } : {}), tools: [], proofTools: [] };
|
|
75
|
+
}
|
|
76
|
+
assertProvability(config);
|
|
77
|
+
const { cwd } = deps;
|
|
78
|
+
const specName = fragment.specName;
|
|
79
|
+
if (typeof specName !== "string") {
|
|
80
|
+
throw new EvaluationServiceError("evaluation-service: fragment.specName must be a string");
|
|
81
|
+
}
|
|
82
|
+
const provability = buildRunProvability(config, {
|
|
83
|
+
specName,
|
|
84
|
+
cwd,
|
|
85
|
+
...(deps.catalog !== undefined ? { catalog: deps.catalog } : {}),
|
|
86
|
+
...(deps.sessionRootDir !== undefined ? { sessionRootDir: deps.sessionRootDir } : {}),
|
|
87
|
+
...(deps.footerTarget !== undefined ? { footerTarget: deps.footerTarget } : {}),
|
|
88
|
+
...(deps.streamed !== undefined ? { streamed: deps.streamed } : {}),
|
|
89
|
+
...(deps.resolveHead !== undefined ? { resolveHead: deps.resolveHead } : {}),
|
|
90
|
+
...(deps.headDeadlineMs !== undefined ? { headDeadlineMs: deps.headDeadlineMs } : {}),
|
|
91
|
+
...(deps.onBundle !== undefined ? { onBundle: deps.onBundle } : {}),
|
|
92
|
+
...(deps.judgeAdapter !== undefined ? { judgeAdapter: deps.judgeAdapter } : {}),
|
|
93
|
+
...(deps.now !== undefined ? { now: deps.now } : {}),
|
|
94
|
+
...(deps.proofsRoot !== undefined ? { proofsRoot: deps.proofsRoot } : {}),
|
|
95
|
+
});
|
|
96
|
+
// The preflight's evidence: the gate's own builder over the run's log
|
|
97
|
+
// (the transcript is not reachable from a tool call, and the log already
|
|
98
|
+
// holds every call the turn has finished), scoped the same way — under
|
|
99
|
+
// `scope: turn` the session's index with the turn it keeps, as the gate
|
|
100
|
+
// hands the verifier.
|
|
101
|
+
const evidence = async (ctx) => {
|
|
102
|
+
const tenant = activeTenant();
|
|
103
|
+
const callRoot = sessionRootOfCall(ctx);
|
|
104
|
+
const sessionRootDir = deps.sessionRootDir === undefined && callRoot !== undefined
|
|
105
|
+
? fencedDir(callRoot, tenant)
|
|
106
|
+
: gateSessionRoot(deps.sessionRootDir, tenant);
|
|
107
|
+
const sessionId = sessionIdOfCall(ctx);
|
|
108
|
+
return gateEvidenceScoped({
|
|
109
|
+
scope: config.scope,
|
|
110
|
+
workspaceRoot: cwd,
|
|
111
|
+
...(tenant !== undefined ? { toolResultsRoot: tenant.toolResultRoot } : {}),
|
|
112
|
+
...(sessionId !== undefined ? { sessionId } : {}),
|
|
113
|
+
...(sessionRootDir !== undefined ? { sessionRootDir } : {}),
|
|
114
|
+
// The schemas the gate reads MCP origins by, so the preflight sees what it sees.
|
|
115
|
+
inputSchemaOf: schemaLookup(deps.catalog),
|
|
116
|
+
});
|
|
117
|
+
};
|
|
118
|
+
const proofTools = gateTools({
|
|
119
|
+
config,
|
|
120
|
+
workspaceRoot: cwd,
|
|
121
|
+
proofsRoot: (tenant) => deps.proofsRoot !== undefined
|
|
122
|
+
? fencedDir(deps.proofsRoot, tenant)
|
|
123
|
+
: gateProofsRoot(cwd, specName, tenant),
|
|
124
|
+
evidence,
|
|
125
|
+
...(deps.sandbox !== undefined ? { sandbox: deps.sandbox } : {}),
|
|
126
|
+
...(deps.now !== undefined ? { now: deps.now } : {}),
|
|
127
|
+
});
|
|
128
|
+
return {
|
|
129
|
+
...(evaluation !== undefined ? { evaluation } : {}),
|
|
130
|
+
provability,
|
|
131
|
+
...proofTools,
|
|
132
|
+
};
|
|
133
|
+
}
|