@crewhaus/evaluation-service 0.0.0-stage → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +9 -0
- package/README.md +278 -2
- package/dist/child-skill.d.ts +33 -0
- package/dist/child-skill.js +6 -0
- package/dist/evaluation.d.ts +40 -0
- package/dist/evaluation.js +128 -0
- package/dist/evidence.d.ts +97 -0
- package/dist/evidence.js +147 -0
- package/dist/fragment.d.ts +40 -0
- package/dist/fragment.js +17 -0
- package/dist/head.d.ts +115 -0
- package/dist/head.js +181 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +24 -0
- package/dist/judge.d.ts +62 -0
- package/dist/judge.js +103 -0
- package/dist/observe.d.ts +32 -0
- package/dist/observe.js +31 -0
- package/dist/provability.d.ts +47 -0
- package/dist/provability.js +261 -0
- package/dist/render.d.ts +95 -0
- package/dist/render.js +99 -0
- package/dist/replay.d.ts +53 -0
- package/dist/replay.js +309 -0
- package/dist/roots.d.ts +36 -0
- package/dist/roots.js +90 -0
- package/dist/skills.d.ts +83 -0
- package/dist/skills.js +200 -0
- package/dist/tools.d.ts +106 -0
- package/dist/tools.js +74 -0
- package/dist/wire.d.ts +106 -0
- package/dist/wire.js +133 -0
- package/package.json +65 -4
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The provability gate's closures (design §5.1–§5.4, §6), built from the
|
|
3
|
+
* lowered `IrProvability`. runtime-core owns the gate — when it runs, the
|
|
4
|
+
* retry/halt/note/escalate ladder, the trace and audit lines — and receives
|
|
5
|
+
* everything that touches evidence, files, the network or a model as these
|
|
6
|
+
* closures, so it stays dependency-free (the `opts.memory` /
|
|
7
|
+
* `opts.continuity` posture).
|
|
8
|
+
*
|
|
9
|
+
* - `verify(turn)` — `@crewhaus/provability`'s `verifyDocument` over the
|
|
10
|
+
* turn's final text, against the evidence index `gateEvidence` builds
|
|
11
|
+
* from the session log and the transcript (scoped by `config.scope`),
|
|
12
|
+
* the proof artifacts under the spec's proofs root, `checked` replay
|
|
13
|
+
* through the catalog (pure tools only, compared in the shape the log
|
|
14
|
+
* holds the call in, never a tool whose answer depends on the
|
|
15
|
+
* workspace — `./replay`), the guarded HEAD when `sources.resolve:
|
|
16
|
+
* head` (each URL screened by the run's egress fabric before it is sent,
|
|
17
|
+
* each request decided recorded as an `egress_decision` on the run's
|
|
18
|
+
* audit sink, when it has one), and the
|
|
19
|
+
* waiver judge when `provability.judge` is set — skipped, not guessed,
|
|
20
|
+
* once the run's `budget.judge_share` is spent. Every root is resolved
|
|
21
|
+
* per turn for the active tenant.
|
|
22
|
+
* - `renderBundle` — the one-line footer (`render: footer`) or the JSON
|
|
23
|
+
* (`render: json`); absent under `render: none`.
|
|
24
|
+
* - `displayPath` — the footer's path rule, for the run's live surfaces
|
|
25
|
+
* (the trace event a gateway client streams, a halt's detail).
|
|
26
|
+
* - `nudge` — provability's `renderNudge`: the model's own sentences, the
|
|
27
|
+
* evidence identifiers and fixed reasons, never fetched content.
|
|
28
|
+
* - `persist` — the bundle store under the proofs root, mode 0600.
|
|
29
|
+
* - `skill` — the rendered `provability` skill, which the runtime hands a
|
|
30
|
+
* Task child as read-only guidance (design §8, `./child-skill`); absent
|
|
31
|
+
* when this build ships none.
|
|
32
|
+
*
|
|
33
|
+
* `level`, `onFail`, `maxRetries`, `threshold`, `escalateTo` and `render`
|
|
34
|
+
* are the IR's resolved values, verbatim.
|
|
35
|
+
*/
|
|
36
|
+
import { isAbsolute, relative, sep } from "node:path";
|
|
37
|
+
import { createLogger } from "@crewhaus/logging";
|
|
38
|
+
import { createBundleStore, createFileArtifactStore, renderBundleJson, renderFooter, renderNudge, restrictToTurns, verifyDocument, } from "@crewhaus/provability";
|
|
39
|
+
import { proofRecheck, runtimeStateRoots, tenantRootOf } from "@crewhaus/tool-proof";
|
|
40
|
+
import { provabilitySkillText } from "./child-skill";
|
|
41
|
+
import { gateEvidenceScoped } from "./evidence";
|
|
42
|
+
import { RESOLVE_SINK_ID, SHARED_HEAD_CACHE, createHeadResolver, } from "./head";
|
|
43
|
+
import { createWaiverJudge } from "./judge";
|
|
44
|
+
import { catalogReplay, schemaLookup } from "./replay";
|
|
45
|
+
import { activeTenant, fencedDir, gateProofsRoot, gateSessionRoot } from "./roots";
|
|
46
|
+
const logger = createLogger({ bindings: { module: "evaluation-service" } });
|
|
47
|
+
/**
|
|
48
|
+
* A bundle path as the footer shows it: relative to `base` when it lies
|
|
49
|
+
* inside it (`.crewhaus/proofs/<spec>/<session>/turn-7.json`), else as given.
|
|
50
|
+
*/
|
|
51
|
+
export function displayPath(path, base) {
|
|
52
|
+
const rel = relative(base, path);
|
|
53
|
+
return rel !== "" && rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel) ? rel : path;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* The text(s) `evidence` holds for a call of `toolName` with `input` — what a
|
|
57
|
+
* replay is compared against. The verifier hands replay the `input` object
|
|
58
|
+
* of the very record it is checking, so identity finds it; an equal input
|
|
59
|
+
* logged by another call of the same tool counts too.
|
|
60
|
+
*/
|
|
61
|
+
function loggedResults(evidence) {
|
|
62
|
+
return (toolName, input) => {
|
|
63
|
+
let key;
|
|
64
|
+
const keyOf = () => {
|
|
65
|
+
try {
|
|
66
|
+
key ??= JSON.stringify(input);
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
// An input JSON cannot hold matches by identity only.
|
|
70
|
+
}
|
|
71
|
+
return key;
|
|
72
|
+
};
|
|
73
|
+
const found = [];
|
|
74
|
+
for (const record of evidence.all()) {
|
|
75
|
+
if (record.toolName !== toolName || record.isError)
|
|
76
|
+
continue;
|
|
77
|
+
let same = record.input === input;
|
|
78
|
+
if (!same) {
|
|
79
|
+
try {
|
|
80
|
+
same = keyOf() !== undefined && JSON.stringify(record.input) === keyOf();
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
same = false;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
if (same)
|
|
87
|
+
found.push(record.content);
|
|
88
|
+
}
|
|
89
|
+
return found;
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
/** One HEAD the verifier decided on, onto the run's audit sink as an `egress_decision`. */
|
|
93
|
+
function recordHead(sink) {
|
|
94
|
+
return async (decision) => {
|
|
95
|
+
try {
|
|
96
|
+
await sink.append({ kind: "egress_decision", payload: decision });
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
// As on every egress path: a full audit disk must not fail the answer.
|
|
100
|
+
logger.warn("provability: the HEAD's egress_decision could not be recorded", {
|
|
101
|
+
host: decision.host,
|
|
102
|
+
error: err instanceof Error ? err.message : String(err),
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
/** A signal that aborts when either does; `undefined` when neither is given. */
|
|
108
|
+
function eitherSignal(a, b) {
|
|
109
|
+
if (a === undefined)
|
|
110
|
+
return b;
|
|
111
|
+
if (b === undefined)
|
|
112
|
+
return a;
|
|
113
|
+
return AbortSignal.any([a, b]);
|
|
114
|
+
}
|
|
115
|
+
/** The run's live judge-share read; a read that throws cannot say the share is spent. */
|
|
116
|
+
function shareSpent(turn) {
|
|
117
|
+
try {
|
|
118
|
+
return turn.judgeShareExhausted?.() === true;
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* The base a footer path is shown against: the tenant's own root inside a
|
|
126
|
+
* tenant scope — the reply goes to that tenant's client, and the server's
|
|
127
|
+
* directory layout above the tenant's root is none of its business — else
|
|
128
|
+
* the workspace.
|
|
129
|
+
*/
|
|
130
|
+
function footerBase(cwd) {
|
|
131
|
+
const tenant = activeTenant();
|
|
132
|
+
return tenant !== undefined ? tenantRootOf(tenant) : cwd;
|
|
133
|
+
}
|
|
134
|
+
/** The `RunProvability` for a lowered `provability:` block (see the module doc). */
|
|
135
|
+
export function buildRunProvability(config, opts) {
|
|
136
|
+
const { cwd, specName } = opts;
|
|
137
|
+
const proofsRootOf = (tenant) => opts.proofsRoot !== undefined
|
|
138
|
+
? fencedDir(opts.proofsRoot, tenant)
|
|
139
|
+
: gateProofsRoot(cwd, specName, tenant);
|
|
140
|
+
const customHead = opts.resolveHead;
|
|
141
|
+
// The default guarded HEAD shares one process-wide cache, so a session
|
|
142
|
+
// asks once even where the gates are wired per turn (the channel bot).
|
|
143
|
+
const head = config.sources.resolve === "head"
|
|
144
|
+
? customHead !== undefined
|
|
145
|
+
? createHeadResolver(config.sources, (url, signal) => customHead(url, signal))
|
|
146
|
+
: createHeadResolver(config.sources, undefined, SHARED_HEAD_CACHE)
|
|
147
|
+
: undefined;
|
|
148
|
+
const verify = async (turn) => {
|
|
149
|
+
const tenant = activeTenant();
|
|
150
|
+
const proofs = proofsRootOf(tenant);
|
|
151
|
+
const sessionRootDir = gateSessionRoot(opts.sessionRootDir, tenant);
|
|
152
|
+
// Under `scope: turn` the index is the session's with the turn marked
|
|
153
|
+
// (`turnNumbers`): grounding reads the turn, the self-written trace the
|
|
154
|
+
// whole session (design §10).
|
|
155
|
+
const { index: evidence, turnNumbers } = await gateEvidenceScoped({
|
|
156
|
+
scope: config.scope,
|
|
157
|
+
workspaceRoot: cwd,
|
|
158
|
+
// A managed turn's runtime spills under the tenant's own root.
|
|
159
|
+
...(tenant !== undefined ? { toolResultsRoot: tenant.toolResultRoot } : {}),
|
|
160
|
+
messages: turn.messages,
|
|
161
|
+
...(turn.isSynthetic !== undefined ? { isSynthetic: turn.isSynthetic } : {}),
|
|
162
|
+
...(turn.sessionId !== undefined ? { sessionId: turn.sessionId } : {}),
|
|
163
|
+
...(sessionRootDir !== undefined ? { sessionRootDir } : {}),
|
|
164
|
+
// An MCP call's origin is a field its schema declares: the run's own catalog's.
|
|
165
|
+
inputSchemaOf: schemaLookup(opts.catalog),
|
|
166
|
+
onUnreadable: (info) => logger.warn("provability: a session log could not be read to its end", { ...info }),
|
|
167
|
+
});
|
|
168
|
+
const { replay, isPureTool } = catalogReplay(opts.catalog, {
|
|
169
|
+
logged: loggedResults(turnNumbers !== undefined ? restrictToTurns(evidence, turnNumbers) : evidence),
|
|
170
|
+
workspaceRoot: cwd,
|
|
171
|
+
...(turn.signal !== undefined ? { signal: turn.signal } : {}),
|
|
172
|
+
});
|
|
173
|
+
const record = turn.egressAuditSink !== undefined ? recordHead(turn.egressAuditSink) : undefined;
|
|
174
|
+
// Pillar 3 — the run's egress fabric screens every URL before its HEAD.
|
|
175
|
+
const screenEgress = turn.screenEgress;
|
|
176
|
+
const screen = screenEgress !== undefined ? (url) => screenEgress(url, RESOLVE_SINK_ID) : undefined;
|
|
177
|
+
const verifyOptions = {
|
|
178
|
+
config,
|
|
179
|
+
evidence,
|
|
180
|
+
...(turnNumbers !== undefined ? { turnNumbers } : {}),
|
|
181
|
+
workspaceRoot: cwd,
|
|
182
|
+
artifacts: createFileArtifactStore({ rootDir: proofs }),
|
|
183
|
+
// Where this run keeps its own record beyond the workspace's
|
|
184
|
+
// `.crewhaus/`: a relocated session root, the proofs root, the
|
|
185
|
+
// tenant's roots, the env relocations. A path under one is the model's
|
|
186
|
+
// own text, never a source; the preflight reads the same list.
|
|
187
|
+
stateRoots: runtimeStateRoots({
|
|
188
|
+
workspaceRoot: cwd,
|
|
189
|
+
proofsRoot: proofs,
|
|
190
|
+
...(sessionRootDir !== undefined ? { sessionRootDir } : {}),
|
|
191
|
+
...(tenant !== undefined ? { tenant } : {}),
|
|
192
|
+
}),
|
|
193
|
+
retryIndex: turn.retryIndex,
|
|
194
|
+
...(opts.headDeadlineMs !== undefined ? { headDeadlineMs: opts.headDeadlineMs } : {}),
|
|
195
|
+
...(opts.catalog !== undefined ? { replay, isPureTool } : {}),
|
|
196
|
+
...(head !== undefined
|
|
197
|
+
? {
|
|
198
|
+
// The verifier's own signal ends a HEAD at its deadline (and
|
|
199
|
+
// before the retry pass starts its own); the turn's ends it when
|
|
200
|
+
// the turn does. Either one stops the request.
|
|
201
|
+
resolveHead: (url, signal) => {
|
|
202
|
+
const stop = eitherSignal(signal, turn.signal);
|
|
203
|
+
return head(url, {
|
|
204
|
+
...(turn.sessionId !== undefined ? { sessionId: turn.sessionId } : {}),
|
|
205
|
+
...(stop !== undefined ? { signal: stop } : {}),
|
|
206
|
+
...(record !== undefined ? { record } : {}),
|
|
207
|
+
...(screen !== undefined ? { screen } : {}),
|
|
208
|
+
});
|
|
209
|
+
},
|
|
210
|
+
}
|
|
211
|
+
: {}),
|
|
212
|
+
// A stored builtin proof is checked again before it backs a claim
|
|
213
|
+
// (design §6.4): the file is in the workspace, the checker is not.
|
|
214
|
+
recheckProof: proofRecheck({
|
|
215
|
+
timeoutMs: config.formal.timeoutMs,
|
|
216
|
+
...(turn.signal !== undefined ? { signal: turn.signal } : {}),
|
|
217
|
+
}),
|
|
218
|
+
// Past the judge share the waiver is skipped, not guessed (§6.2): no
|
|
219
|
+
// judge at all when the share is already spent, and the judge itself
|
|
220
|
+
// re-reads it before every call, since its own spend can cross it.
|
|
221
|
+
...(config.judge !== undefined && !shareSpent(turn)
|
|
222
|
+
? {
|
|
223
|
+
judgeWaiver: createWaiverJudge({
|
|
224
|
+
judge: config.judge,
|
|
225
|
+
bus: turn.bus,
|
|
226
|
+
...(opts.judgeAdapter !== undefined ? { adapter: opts.judgeAdapter } : {}),
|
|
227
|
+
...(turn.judgeShareExhausted !== undefined
|
|
228
|
+
? { exhausted: turn.judgeShareExhausted }
|
|
229
|
+
: {}),
|
|
230
|
+
}),
|
|
231
|
+
}
|
|
232
|
+
: {}),
|
|
233
|
+
...(opts.now !== undefined ? { now: opts.now } : {}),
|
|
234
|
+
};
|
|
235
|
+
return verifyDocument(turn.finalText, verifyOptions);
|
|
236
|
+
};
|
|
237
|
+
// §8 — rendered once per wiring: read-only guidance for Task children.
|
|
238
|
+
const skill = provabilitySkillText(config);
|
|
239
|
+
const renderBundle = config.render === "none"
|
|
240
|
+
? undefined
|
|
241
|
+
: config.render === "json"
|
|
242
|
+
? (bundle) => renderBundleJson(bundle)
|
|
243
|
+
: (bundle, bundlePath) => renderFooter(bundle, bundlePath !== undefined ? { path: displayPath(bundlePath, footerBase(cwd)) } : {});
|
|
244
|
+
return {
|
|
245
|
+
level: config.level,
|
|
246
|
+
onFail: config.onFail,
|
|
247
|
+
displayPath: (bundlePath) => displayPath(bundlePath, footerBase(cwd)),
|
|
248
|
+
maxRetries: config.maxRetries,
|
|
249
|
+
threshold: config.threshold,
|
|
250
|
+
...(config.escalateTo !== undefined ? { escalateTo: config.escalateTo } : {}),
|
|
251
|
+
render: config.render,
|
|
252
|
+
...(opts.footerTarget !== undefined ? { footerTarget: opts.footerTarget } : {}),
|
|
253
|
+
...(opts.streamed !== undefined ? { streamed: opts.streamed } : {}),
|
|
254
|
+
verify,
|
|
255
|
+
...(renderBundle !== undefined ? { renderBundle } : {}),
|
|
256
|
+
nudge: renderNudge,
|
|
257
|
+
persist: async (bundle, at) => createBundleStore({ rootDir: proofsRootOf(activeTenant()) }).put(bundle, at),
|
|
258
|
+
...(opts.onBundle !== undefined ? { onBundle: opts.onBundle } : {}),
|
|
259
|
+
...(skill !== undefined ? { skill } : {}),
|
|
260
|
+
};
|
|
261
|
+
}
|
package/dist/render.d.ts
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The codegen twin of `wireEvaluation` (design §4.4): what every emitter
|
|
3
|
+
* renders INSTEAD of its own copy of the evaluation grader — the
|
|
4
|
+
* `renderModelWiringFields` / `renderMemory` precedent. The bundle then calls
|
|
5
|
+
* the same composition root the interpreter calls, so a compiled harness and
|
|
6
|
+
* `crewhaus run` cannot drift on what a gate does.
|
|
7
|
+
*
|
|
8
|
+
* Four pieces, each the EMPTY STRING when the IR declares neither
|
|
9
|
+
* `evaluation:` nor `provability:` — so every such bundle stays byte-identical
|
|
10
|
+
* to its pre-0.8.0 bytes (design §1 principle 3):
|
|
11
|
+
*
|
|
12
|
+
* - `imports` — `import { wireEvaluation } from "@crewhaus/evaluation-service";`
|
|
13
|
+
* (plus `registerGateTools` under `provability:`, or `mergeGateTools`
|
|
14
|
+
* when the emitter hands tools over as a list);
|
|
15
|
+
* - `bootBlock` — one line,
|
|
16
|
+
* `const __gates = await wireEvaluation(<fragment JSON>, { cwd, … });`.
|
|
17
|
+
* The deps name the catalog and the footer target only when the IR has
|
|
18
|
+
* `provability:` (an evaluation-only gate needs neither);
|
|
19
|
+
* - `field` — the `runChatLoop` fields, only for the keys the IR has:
|
|
20
|
+
* `evaluation: __gates.evaluation,` and/or `provability: __gates.provability,`;
|
|
21
|
+
* - `toolsRegistration` — under `provability:`, one
|
|
22
|
+
* `registerGateTools(<catalog>, __gates);` that puts the proof tools on
|
|
23
|
+
* the bundle's catalog, REPLACING a same-named builtin a spec also listed
|
|
24
|
+
* in `tools:` at any level (the builtin instances run under default
|
|
25
|
+
* settings and file artifacts where this spec's gate does not look —
|
|
26
|
+
* `./tools` has the rule). Emitters place it after their builtin
|
|
27
|
+
* registrations and before the plugin registration, so first-party still
|
|
28
|
+
* wins every plugin collision.
|
|
29
|
+
*
|
|
30
|
+
* An emitter that hands the run a tool LIST rather than a catalog snapshot
|
|
31
|
+
* (the channel bot's per-turn `runTurn`) asks for `toolsInto: "list"` and
|
|
32
|
+
* wraps its tools expression with `toolsExpr(…)` instead. One that serves
|
|
33
|
+
* many runs from one wiring and must return each run's bundle (the managed
|
|
34
|
+
* gateway) names a per-run observer with `bundleObserverExpr` (`./observe`).
|
|
35
|
+
*
|
|
36
|
+
* The fragment is `JSON.stringify` of `evaluationFragmentFromIr(ir)`: the
|
|
37
|
+
* lowered blocks are plain data (strings, numbers, booleans, arrays), so
|
|
38
|
+
* JSON's quoting is the escaping, and an inline literal argument is
|
|
39
|
+
* contextually typed by `wireEvaluation`'s parameter, so the bundle
|
|
40
|
+
* typechecks against the real contract.
|
|
41
|
+
*/
|
|
42
|
+
import { type EvaluationIrSlice } from "./fragment";
|
|
43
|
+
/** The variable the boot line binds. */
|
|
44
|
+
export declare const GATES_VAR = "__gates";
|
|
45
|
+
export type EvaluationWiringOptions = {
|
|
46
|
+
/**
|
|
47
|
+
* The catalog EXPRESSION: where replay looks tools up and (in `"catalog"`
|
|
48
|
+
* mode) where the proof tools register — `defaultCatalog`, or a tool-list
|
|
49
|
+
* expression the run sees (`config.tools`).
|
|
50
|
+
*/
|
|
51
|
+
readonly catalogVar: string;
|
|
52
|
+
/** The working-directory expression (`process.cwd()`, `__cwd`). */
|
|
53
|
+
readonly cwdExpr: string;
|
|
54
|
+
/**
|
|
55
|
+
* `"stream"` for the cli; `"reply"` where the returned text is what gets
|
|
56
|
+
* sent; `"none"` where the caller carries the bundle on its own envelope
|
|
57
|
+
* (the managed gateway's `proof`) and the returned text must be the answer
|
|
58
|
+
* alone.
|
|
59
|
+
*/
|
|
60
|
+
readonly footerTarget: "stream" | "reply" | "none";
|
|
61
|
+
/**
|
|
62
|
+
* An EXPRESSION for the footer target, in place of `footerTarget`'s
|
|
63
|
+
* literal, when one emitted file serves two callers that want different
|
|
64
|
+
* targets (the channel bot's `agent.ts`, shared by the daemon and the eval
|
|
65
|
+
* bridge entry). `footerTarget` stays the fallback the expression is
|
|
66
|
+
* written around. Ignored without `provability:`.
|
|
67
|
+
*/
|
|
68
|
+
readonly footerTargetExpr?: string;
|
|
69
|
+
/** The indent of a `runChatLoop` field line in this emitter. */
|
|
70
|
+
readonly indent: string;
|
|
71
|
+
/** How the proof tools reach the run (default `"catalog"`). */
|
|
72
|
+
readonly toolsInto?: "catalog" | "list";
|
|
73
|
+
/** An expression for the run's session-log root, when the bundle passes one to `runChatLoop`. */
|
|
74
|
+
readonly sessionRootDirExpr?: string;
|
|
75
|
+
/**
|
|
76
|
+
* An expression for a per-run bundle observer (`ProofBundleObserver |
|
|
77
|
+
* undefined`), when the emitter serves many runs from one wiring and must
|
|
78
|
+
* hand each run's caller the bundle of that run (the managed gateway's
|
|
79
|
+
* `proof`, design §5.4). Under `provability:` the field then reads
|
|
80
|
+
* `provability: observeProofBundles(__gates.provability, <expr>),` and the
|
|
81
|
+
* import names `observeProofBundles` and the `ProofBundleObserver` type.
|
|
82
|
+
* Ignored without `provability:`.
|
|
83
|
+
*/
|
|
84
|
+
readonly bundleObserverExpr?: string;
|
|
85
|
+
};
|
|
86
|
+
export type EvaluationWiring = {
|
|
87
|
+
readonly imports: readonly string[];
|
|
88
|
+
readonly bootBlock: string;
|
|
89
|
+
readonly field: string;
|
|
90
|
+
readonly toolsRegistration: string;
|
|
91
|
+
/** `"list"` mode: the run's tools with the proof tools merged in; identity otherwise. */
|
|
92
|
+
readonly toolsExpr: (base: string) => string;
|
|
93
|
+
};
|
|
94
|
+
/** The four pieces for `ir` (see the module doc). */
|
|
95
|
+
export declare function renderEvaluationWiring(ir: EvaluationIrSlice, opts: EvaluationWiringOptions): EvaluationWiring;
|
package/dist/render.js
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The codegen twin of `wireEvaluation` (design §4.4): what every emitter
|
|
3
|
+
* renders INSTEAD of its own copy of the evaluation grader — the
|
|
4
|
+
* `renderModelWiringFields` / `renderMemory` precedent. The bundle then calls
|
|
5
|
+
* the same composition root the interpreter calls, so a compiled harness and
|
|
6
|
+
* `crewhaus run` cannot drift on what a gate does.
|
|
7
|
+
*
|
|
8
|
+
* Four pieces, each the EMPTY STRING when the IR declares neither
|
|
9
|
+
* `evaluation:` nor `provability:` — so every such bundle stays byte-identical
|
|
10
|
+
* to its pre-0.8.0 bytes (design §1 principle 3):
|
|
11
|
+
*
|
|
12
|
+
* - `imports` — `import { wireEvaluation } from "@crewhaus/evaluation-service";`
|
|
13
|
+
* (plus `registerGateTools` under `provability:`, or `mergeGateTools`
|
|
14
|
+
* when the emitter hands tools over as a list);
|
|
15
|
+
* - `bootBlock` — one line,
|
|
16
|
+
* `const __gates = await wireEvaluation(<fragment JSON>, { cwd, … });`.
|
|
17
|
+
* The deps name the catalog and the footer target only when the IR has
|
|
18
|
+
* `provability:` (an evaluation-only gate needs neither);
|
|
19
|
+
* - `field` — the `runChatLoop` fields, only for the keys the IR has:
|
|
20
|
+
* `evaluation: __gates.evaluation,` and/or `provability: __gates.provability,`;
|
|
21
|
+
* - `toolsRegistration` — under `provability:`, one
|
|
22
|
+
* `registerGateTools(<catalog>, __gates);` that puts the proof tools on
|
|
23
|
+
* the bundle's catalog, REPLACING a same-named builtin a spec also listed
|
|
24
|
+
* in `tools:` at any level (the builtin instances run under default
|
|
25
|
+
* settings and file artifacts where this spec's gate does not look —
|
|
26
|
+
* `./tools` has the rule). Emitters place it after their builtin
|
|
27
|
+
* registrations and before the plugin registration, so first-party still
|
|
28
|
+
* wins every plugin collision.
|
|
29
|
+
*
|
|
30
|
+
* An emitter that hands the run a tool LIST rather than a catalog snapshot
|
|
31
|
+
* (the channel bot's per-turn `runTurn`) asks for `toolsInto: "list"` and
|
|
32
|
+
* wraps its tools expression with `toolsExpr(…)` instead. One that serves
|
|
33
|
+
* many runs from one wiring and must return each run's bundle (the managed
|
|
34
|
+
* gateway) names a per-run observer with `bundleObserverExpr` (`./observe`).
|
|
35
|
+
*
|
|
36
|
+
* The fragment is `JSON.stringify` of `evaluationFragmentFromIr(ir)`: the
|
|
37
|
+
* lowered blocks are plain data (strings, numbers, booleans, arrays), so
|
|
38
|
+
* JSON's quoting is the escaping, and an inline literal argument is
|
|
39
|
+
* contextually typed by `wireEvaluation`'s parameter, so the bundle
|
|
40
|
+
* typechecks against the real contract.
|
|
41
|
+
*/
|
|
42
|
+
import { evaluationFragmentFromIr } from "./fragment";
|
|
43
|
+
/** The variable the boot line binds. */
|
|
44
|
+
export const GATES_VAR = "__gates";
|
|
45
|
+
const EMPTY = {
|
|
46
|
+
imports: [],
|
|
47
|
+
bootBlock: "",
|
|
48
|
+
field: "",
|
|
49
|
+
toolsRegistration: "",
|
|
50
|
+
toolsExpr: (base) => base,
|
|
51
|
+
};
|
|
52
|
+
/** The four pieces for `ir` (see the module doc). */
|
|
53
|
+
export function renderEvaluationWiring(ir, opts) {
|
|
54
|
+
const fragment = evaluationFragmentFromIr(ir);
|
|
55
|
+
if (fragment === undefined)
|
|
56
|
+
return EMPTY;
|
|
57
|
+
const provable = fragment.provability !== undefined;
|
|
58
|
+
const intoList = provable && opts.toolsInto === "list";
|
|
59
|
+
const observed = provable && opts.bundleObserverExpr !== undefined;
|
|
60
|
+
const helper = !provable ? "" : intoList ? "mergeGateTools, " : "registerGateTools, ";
|
|
61
|
+
const observer = observed ? "type ProofBundleObserver, observeProofBundles, " : "";
|
|
62
|
+
const imports = [
|
|
63
|
+
`import { ${observer}${helper}wireEvaluation } from "@crewhaus/evaluation-service";`,
|
|
64
|
+
];
|
|
65
|
+
const sessionRootDep = provable && opts.sessionRootDirExpr !== undefined
|
|
66
|
+
? `, ...(${opts.sessionRootDirExpr} !== undefined ? { sessionRootDir: ${opts.sessionRootDirExpr} } : {})`
|
|
67
|
+
: "";
|
|
68
|
+
const footerTarget = opts.footerTargetExpr ?? JSON.stringify(opts.footerTarget);
|
|
69
|
+
const deps = provable
|
|
70
|
+
? `{ cwd: ${opts.cwdExpr}, catalog: ${opts.catalogVar}, footerTarget: ${footerTarget}${sessionRootDep} }`
|
|
71
|
+
: `{ cwd: ${opts.cwdExpr} }`;
|
|
72
|
+
const bootBlock = `const ${GATES_VAR} = await wireEvaluation(${JSON.stringify(fragment)}, ${deps});`;
|
|
73
|
+
const field = [
|
|
74
|
+
...(fragment.evaluation !== undefined
|
|
75
|
+
? [`\n${opts.indent}evaluation: ${GATES_VAR}.evaluation,`]
|
|
76
|
+
: []),
|
|
77
|
+
...(provable
|
|
78
|
+
? [
|
|
79
|
+
observed
|
|
80
|
+
? `\n${opts.indent}provability: observeProofBundles(${GATES_VAR}.provability, ${opts.bundleObserverExpr}),`
|
|
81
|
+
: `\n${opts.indent}provability: ${GATES_VAR}.provability,`,
|
|
82
|
+
]
|
|
83
|
+
: []),
|
|
84
|
+
].join("");
|
|
85
|
+
const toolsRegistration = provable && !intoList
|
|
86
|
+
? [
|
|
87
|
+
"// 0.8.0 — the proof tools, bound to this spec's provability block; a same-named",
|
|
88
|
+
"// builtin listed in tools: is replaced (it would file proofs where the gate never looks).",
|
|
89
|
+
`registerGateTools(${opts.catalogVar}, ${GATES_VAR});`,
|
|
90
|
+
].join("\n")
|
|
91
|
+
: "";
|
|
92
|
+
return {
|
|
93
|
+
imports,
|
|
94
|
+
bootBlock,
|
|
95
|
+
field,
|
|
96
|
+
toolsRegistration,
|
|
97
|
+
toolsExpr: intoList ? (base) => `mergeGateTools(${base}, ${GATES_VAR})` : (base) => base,
|
|
98
|
+
};
|
|
99
|
+
}
|
package/dist/replay.d.ts
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { InputSchemaLookup, ReplayResult } from "@crewhaus/provability";
|
|
2
|
+
import { type RegisteredTool, type ToolExecuteResult } from "@crewhaus/tool-catalog";
|
|
3
|
+
/** How long one replayed call may take. */
|
|
4
|
+
export declare const REPLAY_TIMEOUT_MS = 10000;
|
|
5
|
+
/**
|
|
6
|
+
* The pause before a disagreeing replay is run again: long enough that a
|
|
7
|
+
* tool reading a millisecond clock has moved on, short enough to be noise
|
|
8
|
+
* beside the replay itself.
|
|
9
|
+
*/
|
|
10
|
+
export declare const REPLAY_RECHECK_MS = 5;
|
|
11
|
+
/** Where replay looks tools up: a catalog (`defaultCatalog`) or the run's tool list. */
|
|
12
|
+
export type GateCatalog = {
|
|
13
|
+
get(name: string): RegisteredTool | undefined;
|
|
14
|
+
} | ReadonlyArray<RegisteredTool>;
|
|
15
|
+
/** The lookup over either catalog form. */
|
|
16
|
+
export declare function lookupTool(catalog: GateCatalog | undefined, name: string): RegisteredTool | undefined;
|
|
17
|
+
/**
|
|
18
|
+
* The input schema a tool of the run declares, for the evidence index
|
|
19
|
+
* (provability `mcpOrigins`: an MCP call's origin is a field its schema
|
|
20
|
+
* declares): the gate's own catalog first — the interpreter and the eval
|
|
21
|
+
* runner register their MCP tools there, not in `defaultCatalog` — then
|
|
22
|
+
* `defaultCatalog`, where a compiled harness registers them. A tool neither
|
|
23
|
+
* holds, or one registered with no JSON schema, reads as unknown.
|
|
24
|
+
*/
|
|
25
|
+
export declare function schemaLookup(catalog: GateCatalog | undefined): InputSchemaLookup;
|
|
26
|
+
/** A tool result's text, the way the evidence index reads one: text blocks joined by newlines. */
|
|
27
|
+
export declare function resultText(result: ToolExecuteResult): string;
|
|
28
|
+
export type CatalogReplayOptions = {
|
|
29
|
+
/** The turn's signal: an abort stops a running replay. */
|
|
30
|
+
readonly signal?: AbortSignal;
|
|
31
|
+
/** Bound on one replayed call (default {@link REPLAY_TIMEOUT_MS}). */
|
|
32
|
+
readonly timeoutMs?: number;
|
|
33
|
+
/**
|
|
34
|
+
* The text(s) the evidence holds for a call of `toolName` with `input` —
|
|
35
|
+
* the verifier hands replay the very `input` object of the record it is
|
|
36
|
+
* checking. Absent: a replay is handed over as it ran, and one over the
|
|
37
|
+
* spill threshold is not compared.
|
|
38
|
+
*/
|
|
39
|
+
readonly logged?: (toolName: string, input: unknown) => ReadonlyArray<string>;
|
|
40
|
+
/** The pause before a disagreeing replay runs again (default {@link REPLAY_RECHECK_MS}). */
|
|
41
|
+
readonly recheckMs?: number;
|
|
42
|
+
/**
|
|
43
|
+
* The workspace a logged input's names resolve in (the gate's `cwd`; the
|
|
44
|
+
* directory a workspace tool resolves against when it runs). Default
|
|
45
|
+
* `process.cwd()`.
|
|
46
|
+
*/
|
|
47
|
+
readonly workspaceRoot?: string;
|
|
48
|
+
};
|
|
49
|
+
/** The verifier's two replay hooks over `catalog`, bound to one turn's signal. */
|
|
50
|
+
export declare function catalogReplay(catalog: GateCatalog | undefined, opts?: CatalogReplayOptions): {
|
|
51
|
+
readonly isPureTool: (toolName: string) => boolean;
|
|
52
|
+
readonly replay: (toolName: string, input: unknown) => Promise<ReplayResult | undefined>;
|
|
53
|
+
};
|