@crewhaus/evaluation-service 0.0.0-stage → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +9 -0
- package/README.md +278 -2
- package/dist/child-skill.d.ts +33 -0
- package/dist/child-skill.js +6 -0
- package/dist/evaluation.d.ts +40 -0
- package/dist/evaluation.js +128 -0
- package/dist/evidence.d.ts +97 -0
- package/dist/evidence.js +147 -0
- package/dist/fragment.d.ts +40 -0
- package/dist/fragment.js +17 -0
- package/dist/head.d.ts +115 -0
- package/dist/head.js +181 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +24 -0
- package/dist/judge.d.ts +62 -0
- package/dist/judge.js +103 -0
- package/dist/observe.d.ts +32 -0
- package/dist/observe.js +31 -0
- package/dist/provability.d.ts +47 -0
- package/dist/provability.js +261 -0
- package/dist/render.d.ts +95 -0
- package/dist/render.js +99 -0
- package/dist/replay.d.ts +53 -0
- package/dist/replay.js +309 -0
- package/dist/roots.d.ts +36 -0
- package/dist/roots.js +90 -0
- package/dist/skills.d.ts +83 -0
- package/dist/skills.js +200 -0
- package/dist/tools.d.ts +106 -0
- package/dist/tools.js +74 -0
- package/dist/wire.d.ts +106 -0
- package/dist/wire.js +133 -0
- package/package.json +65 -4
package/dist/replay.js
ADDED
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `checked` replay (design §6.3): re-execute a PURE tool with the input the
|
|
3
|
+
* log recorded and hand the output back for the verifier's digest compare.
|
|
4
|
+
*
|
|
5
|
+
* Purity is `@crewhaus/provability`'s `isPureTool` over the tool's declared
|
|
6
|
+
* safety flags — read-only, concurrency-safe, not destructive, no sandbox,
|
|
7
|
+
* internal scope, no network or process capability — the one definition the
|
|
8
|
+
* preflight tool also reads, so `Bash`, `Python`, every fetch and every
|
|
9
|
+
* write are never re-run whatever a spec lists on `execution.allow`.
|
|
10
|
+
*
|
|
11
|
+
* A replay runs the tool the way the executor does: the logged input is
|
|
12
|
+
* parsed through the tool's own schema first (defaults and transforms apply,
|
|
13
|
+
* as they did on the original call), then `execute` gets a context carrying
|
|
14
|
+
* only an abort signal — a pure tool needs no bridge, no run context and no
|
|
15
|
+
* model. The call is bounded ({@link REPLAY_TIMEOUT_MS}) and follows the
|
|
16
|
+
* turn's own signal, so a hung tool cannot hold the answer.
|
|
17
|
+
*
|
|
18
|
+
* Pure is not the same as deterministic, and a replay is only a verdict on
|
|
19
|
+
* a tool that answers the same input the same way. Four things the flags
|
|
20
|
+
* cannot say are handled here, each as "could not replay" (`replayed:
|
|
21
|
+
* false`), never as a mismatch no retry could cure:
|
|
22
|
+
*
|
|
23
|
+
* - THE WORKSPACE. A read-only tool that reads files (GoldenCompare,
|
|
24
|
+
* ChecksumVerify, AcceptanceCheck, MarkdownLinkCheck, a `Read`) answers
|
|
25
|
+
* about the workspace AS IT WAS when it ran. Check, fix, report is the
|
|
26
|
+
* verify tools' whole workflow: the golden is updated, the missing file
|
|
27
|
+
* created, and a replay then disagrees with an honest record — a changed
|
|
28
|
+
* file is not a misquoted result, and re-running cannot tell the two
|
|
29
|
+
* apart. So a call is not re-run when the tool declares a path among its
|
|
30
|
+
* operative arguments (the catalog's own word that it acts on the
|
|
31
|
+
* workspace), or when its logged input names something that exists in
|
|
32
|
+
* the workspace now (a tool that reads a path it does not declare). Its
|
|
33
|
+
* binding says "it ran" without "it reproduces", as design §6.3 has
|
|
34
|
+
* `Bash` say (`replayed: false`).
|
|
35
|
+
* - THE CLOCK. A tool whose schema offers `now` ("overrides the real
|
|
36
|
+
* clock, for tests and replays" — DeadlineCheck, ErrorClassify,
|
|
37
|
+
* SequenceRun) read the real clock on a call that left it out, so that
|
|
38
|
+
* call is not re-run: its answer was about a moment that has passed.
|
|
39
|
+
* - ANY OTHER VARIANCE (the CSPRNG behind an unseeded Uuid v4, say). When
|
|
40
|
+
* a replay disagrees with the log it is run once more, after
|
|
41
|
+
* {@link REPLAY_RECHECK_MS}; two runs that disagree with EACH OTHER show
|
|
42
|
+
* the tool does not reproduce on this input, and only two that agree
|
|
43
|
+
* with each other and not with the log are handed over as the mismatch.
|
|
44
|
+
* - THE SPILL. runtime-core logs (and shows the model) only
|
|
45
|
+
* tool-result-store's preview of a string result over its threshold —
|
|
46
|
+
* the head plus its own `[truncated, full output at …]` markers. Such a
|
|
47
|
+
* replay is put in the shape it was logged in: when the logged text is
|
|
48
|
+
* this output's head followed by those markers, the logged text is what
|
|
49
|
+
* comes back (everything the evidence holds was reproduced); when it is
|
|
50
|
+
* a preview of another head, the replay's head does (a mismatch); when
|
|
51
|
+
* the logged text cannot be read as a preview at all, nothing does.
|
|
52
|
+
*
|
|
53
|
+
* Those last two need the text the log holds for the call: `logged` (the
|
|
54
|
+
* gate passes a lookup over the turn's evidence index). Without it, a
|
|
55
|
+
* result over the threshold is not compared at all.
|
|
56
|
+
*
|
|
57
|
+
* What comes back: `undefined` when there is nothing to re-run (the tool is
|
|
58
|
+
* not in the catalog any more, not pure, the logged input no longer parses,
|
|
59
|
+
* or one of the cases above); the output text when it ran; `isError: true`
|
|
60
|
+
* with the message when the tool threw on an input it once answered — a
|
|
61
|
+
* disagreement, so `replay_mismatch`. A timeout or an abort is not a verdict
|
|
62
|
+
* on the tool: it throws, and the verifier reads that as "could not run"
|
|
63
|
+
* (`replayed: false`).
|
|
64
|
+
*/
|
|
65
|
+
import { existsSync } from "node:fs";
|
|
66
|
+
import { resolve } from "node:path";
|
|
67
|
+
import { isPureTool } from "@crewhaus/provability";
|
|
68
|
+
import { defaultCatalog, } from "@crewhaus/tool-catalog";
|
|
69
|
+
import { DEFAULT_PREVIEW_LINES, DEFAULT_THRESHOLD_BYTES, previewHead, } from "@crewhaus/tool-result-store";
|
|
70
|
+
/** How long one replayed call may take. */
|
|
71
|
+
export const REPLAY_TIMEOUT_MS = 10_000;
|
|
72
|
+
/**
|
|
73
|
+
* The pause before a disagreeing replay is run again: long enough that a
|
|
74
|
+
* tool reading a millisecond clock has moved on, short enough to be noise
|
|
75
|
+
* beside the replay itself.
|
|
76
|
+
*/
|
|
77
|
+
export const REPLAY_RECHECK_MS = 5;
|
|
78
|
+
/** The lookup over either catalog form. */
|
|
79
|
+
export function lookupTool(catalog, name) {
|
|
80
|
+
if (catalog === undefined)
|
|
81
|
+
return undefined;
|
|
82
|
+
if (Array.isArray(catalog)) {
|
|
83
|
+
return catalog.find((tool) => tool.name === name);
|
|
84
|
+
}
|
|
85
|
+
return catalog.get(name);
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* The input schema a tool of the run declares, for the evidence index
|
|
89
|
+
* (provability `mcpOrigins`: an MCP call's origin is a field its schema
|
|
90
|
+
* declares): the gate's own catalog first — the interpreter and the eval
|
|
91
|
+
* runner register their MCP tools there, not in `defaultCatalog` — then
|
|
92
|
+
* `defaultCatalog`, where a compiled harness registers them. A tool neither
|
|
93
|
+
* holds, or one registered with no JSON schema, reads as unknown.
|
|
94
|
+
*/
|
|
95
|
+
export function schemaLookup(catalog) {
|
|
96
|
+
return (name) => lookupTool(catalog, name)?.jsonSchema ?? defaultCatalog.get(name)?.jsonSchema;
|
|
97
|
+
}
|
|
98
|
+
/** A tool result's text, the way the evidence index reads one: text blocks joined by newlines. */
|
|
99
|
+
export function resultText(result) {
|
|
100
|
+
if (typeof result === "string")
|
|
101
|
+
return result;
|
|
102
|
+
return result
|
|
103
|
+
.filter((block) => block.type === "text")
|
|
104
|
+
.map((block) => block.text)
|
|
105
|
+
.join("\n");
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* The verifier's comparison (provability's `outputDigest` hashes exactly
|
|
109
|
+
* this): whitespace runs collapsed to one space, the ends trimmed.
|
|
110
|
+
*/
|
|
111
|
+
function sameOutput(a, b) {
|
|
112
|
+
return a.replace(/\s+/g, " ").trim() === b.replace(/\s+/g, " ").trim();
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* What tool-result-store appends to a spilled result's head: the optional
|
|
116
|
+
* part line, then the truncation line (saved, or why it could not be). The
|
|
117
|
+
* whole tail, anchored at both ends — a preview is recognized by its own
|
|
118
|
+
* grammar, or not at all.
|
|
119
|
+
*/
|
|
120
|
+
const SPILL_TAIL = /^(?:\n\[(?:part 1 of \d+; Read [^\n]+ for part 2|the rest could not also be saved in parts to Read one at a time: [^\n]*)\])?\n\[truncated(?:, full output at [^\n]+|; the full output could not be saved: [^\n]*)\]$/;
|
|
121
|
+
/** The same tail, found at the END of a logged text whose head is unknown. */
|
|
122
|
+
const SPILL_TAIL_AT_END = new RegExp(`(?:${SPILL_TAIL.source.slice(1, -1)})$`);
|
|
123
|
+
/** Did the runtime log `text` as a spill preview whose head is `head`? */
|
|
124
|
+
function isPreviewOf(text, head) {
|
|
125
|
+
return text.startsWith(head) && SPILL_TAIL.test(text.slice(head.length));
|
|
126
|
+
}
|
|
127
|
+
function shapeAsLogged(result, logged) {
|
|
128
|
+
const text = resultText(result);
|
|
129
|
+
// runtime-core spills only string results (block content goes through whole).
|
|
130
|
+
if (typeof result !== "string" || Buffer.byteLength(text, "utf8") <= DEFAULT_THRESHOLD_BYTES) {
|
|
131
|
+
return { kind: "text", text };
|
|
132
|
+
}
|
|
133
|
+
if (logged === undefined || logged.length === 0)
|
|
134
|
+
return { kind: "incomparable" };
|
|
135
|
+
const head = previewHead(text, DEFAULT_PREVIEW_LINES, DEFAULT_THRESHOLD_BYTES);
|
|
136
|
+
const same = logged.find((entry) => isPreviewOf(entry, head));
|
|
137
|
+
if (same !== undefined)
|
|
138
|
+
return { kind: "text", text: same };
|
|
139
|
+
// A preview of some other head: the output differs where the evidence can see it.
|
|
140
|
+
if (logged.some((entry) => SPILL_TAIL_AT_END.test(entry)))
|
|
141
|
+
return { kind: "text", text: head };
|
|
142
|
+
return { kind: "incomparable" };
|
|
143
|
+
}
|
|
144
|
+
/** Does the parsed input leave out a `now` the tool's schema offers — so the call read the real clock? */
|
|
145
|
+
function readTheClock(tool, input) {
|
|
146
|
+
if (typeof input !== "object" || input === null || Array.isArray(input))
|
|
147
|
+
return false;
|
|
148
|
+
if (input["now"] !== undefined)
|
|
149
|
+
return false;
|
|
150
|
+
return offersField(tool, "now");
|
|
151
|
+
}
|
|
152
|
+
/** Whether a tool's input schema declares `field` (its JSON Schema, else its zod object shape). */
|
|
153
|
+
function offersField(tool, field) {
|
|
154
|
+
const json = tool.jsonSchema;
|
|
155
|
+
if (json?.properties !== undefined && typeof json.properties === "object") {
|
|
156
|
+
return Object.hasOwn(json.properties, field);
|
|
157
|
+
}
|
|
158
|
+
let schema = tool.inputSchema;
|
|
159
|
+
// Unwrap refinements, optionals, defaults and pipes down to the object, if any.
|
|
160
|
+
for (let depth = 0; depth < 8; depth += 1) {
|
|
161
|
+
const def = schema?._def;
|
|
162
|
+
if (def === undefined)
|
|
163
|
+
return false;
|
|
164
|
+
if (typeof def["shape"] === "function") {
|
|
165
|
+
const shape = def["shape"]();
|
|
166
|
+
return typeof shape === "object" && shape !== null && Object.hasOwn(shape, field);
|
|
167
|
+
}
|
|
168
|
+
schema = def["schema"] ?? def["innerType"] ?? def["in"] ?? def["type"];
|
|
169
|
+
}
|
|
170
|
+
return false;
|
|
171
|
+
}
|
|
172
|
+
/** How deep, and how many string leaves, {@link namesWorkspace} reads in one logged input. */
|
|
173
|
+
const INPUT_WALK_DEPTH = 8;
|
|
174
|
+
const INPUT_WALK_LEAVES = 256;
|
|
175
|
+
/** Longer than any path a filesystem opens (provability's `MAX_PATH_CHARS` reasoning). */
|
|
176
|
+
const MAX_PATH_CHARS = 4_096;
|
|
177
|
+
/** Whether `tool` declares a path among its operative arguments (`operativeArgs`). */
|
|
178
|
+
function declaresPath(tool) {
|
|
179
|
+
return (tool.operativeArgs ?? []).some((arg) => arg.kind === "path");
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Whether a string in the parsed `input` names something that exists under
|
|
183
|
+
* `root` (relative names resolve against it, absolute ones are themselves) —
|
|
184
|
+
* the mark of a call whose answer read the workspace. A URL, an empty
|
|
185
|
+
* string, and text with a line break or a NUL are not names.
|
|
186
|
+
*/
|
|
187
|
+
function namesWorkspace(input, root) {
|
|
188
|
+
let leaves = 0;
|
|
189
|
+
const walk = (value, depth) => {
|
|
190
|
+
if (typeof value === "string") {
|
|
191
|
+
leaves += 1;
|
|
192
|
+
if (value.length === 0 ||
|
|
193
|
+
value.length > MAX_PATH_CHARS ||
|
|
194
|
+
value.includes("\n") ||
|
|
195
|
+
value.includes("\r") ||
|
|
196
|
+
value.includes("\0") ||
|
|
197
|
+
/^[a-z][a-z0-9+.-]*:\/\//i.test(value)) {
|
|
198
|
+
return false;
|
|
199
|
+
}
|
|
200
|
+
try {
|
|
201
|
+
return existsSync(resolve(root, value));
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
if (depth >= INPUT_WALK_DEPTH || typeof value !== "object" || value === null)
|
|
208
|
+
return false;
|
|
209
|
+
const children = Array.isArray(value) ? value : Object.values(value);
|
|
210
|
+
for (const child of children) {
|
|
211
|
+
if (leaves >= INPUT_WALK_LEAVES)
|
|
212
|
+
return false;
|
|
213
|
+
if (walk(child, depth + 1))
|
|
214
|
+
return true;
|
|
215
|
+
}
|
|
216
|
+
return false;
|
|
217
|
+
};
|
|
218
|
+
return walk(input, 0);
|
|
219
|
+
}
|
|
220
|
+
class ReplayStopped extends Error {
|
|
221
|
+
name = "ReplayStopped";
|
|
222
|
+
}
|
|
223
|
+
/** The verifier's two replay hooks over `catalog`, bound to one turn's signal. */
|
|
224
|
+
export function catalogReplay(catalog, opts = {}) {
|
|
225
|
+
const pure = (toolName) => {
|
|
226
|
+
const tool = lookupTool(catalog, toolName);
|
|
227
|
+
return tool !== undefined && isPureTool(tool) ? tool : undefined;
|
|
228
|
+
};
|
|
229
|
+
const timeoutMs = opts.timeoutMs ?? REPLAY_TIMEOUT_MS;
|
|
230
|
+
const recheckMs = opts.recheckMs ?? REPLAY_RECHECK_MS;
|
|
231
|
+
const workspaceRoot = opts.workspaceRoot ?? process.cwd();
|
|
232
|
+
/** One bounded run: the result, a thrown error as `{ error }`, or a throw on timeout / abort. */
|
|
233
|
+
const runOnce = async (tool, input) => {
|
|
234
|
+
const ctrl = new AbortController();
|
|
235
|
+
const outer = opts.signal;
|
|
236
|
+
if (outer?.aborted === true)
|
|
237
|
+
throw new ReplayStopped("the turn was aborted");
|
|
238
|
+
let timer;
|
|
239
|
+
let onAbort;
|
|
240
|
+
const stop = new Promise((_resolve, reject) => {
|
|
241
|
+
const halt = (why) => {
|
|
242
|
+
ctrl.abort(why);
|
|
243
|
+
reject(why);
|
|
244
|
+
};
|
|
245
|
+
timer = setTimeout(() => halt(new ReplayStopped(`replaying ${tool.name} took longer than ${timeoutMs}ms`)), timeoutMs);
|
|
246
|
+
onAbort = () => halt(new ReplayStopped("the turn was aborted"));
|
|
247
|
+
outer?.addEventListener("abort", onAbort, { once: true });
|
|
248
|
+
});
|
|
249
|
+
try {
|
|
250
|
+
const result = await Promise.race([
|
|
251
|
+
Promise.resolve().then(() => tool.execute(input, { signal: ctrl.signal })),
|
|
252
|
+
stop,
|
|
253
|
+
]);
|
|
254
|
+
return { result };
|
|
255
|
+
}
|
|
256
|
+
catch (err) {
|
|
257
|
+
if (err instanceof ReplayStopped || ctrl.signal.aborted)
|
|
258
|
+
throw err;
|
|
259
|
+
return { error: err instanceof Error ? err.message : String(err) };
|
|
260
|
+
}
|
|
261
|
+
finally {
|
|
262
|
+
if (timer !== undefined)
|
|
263
|
+
clearTimeout(timer);
|
|
264
|
+
if (onAbort !== undefined)
|
|
265
|
+
outer?.removeEventListener("abort", onAbort);
|
|
266
|
+
}
|
|
267
|
+
};
|
|
268
|
+
/** A run as the verifier will compare it; `undefined` when it cannot be compared. */
|
|
269
|
+
const asCompared = (run, logged) => {
|
|
270
|
+
if ("error" in run)
|
|
271
|
+
return { content: run.error, isError: true };
|
|
272
|
+
const shaped = shapeAsLogged(run.result, logged);
|
|
273
|
+
return shaped.kind === "text" ? { content: shaped.text, isError: false } : undefined;
|
|
274
|
+
};
|
|
275
|
+
return {
|
|
276
|
+
isPureTool: (toolName) => pure(toolName) !== undefined,
|
|
277
|
+
replay: async (toolName, input) => {
|
|
278
|
+
const tool = pure(toolName);
|
|
279
|
+
if (tool === undefined)
|
|
280
|
+
return undefined;
|
|
281
|
+
const parsed = tool.inputSchema.safeParse(input);
|
|
282
|
+
if (!parsed.success)
|
|
283
|
+
return undefined;
|
|
284
|
+
if (readTheClock(tool, parsed.data))
|
|
285
|
+
return undefined;
|
|
286
|
+
// The workspace may have moved on since the call (see the module doc).
|
|
287
|
+
if (declaresPath(tool) || namesWorkspace(parsed.data, workspaceRoot))
|
|
288
|
+
return undefined;
|
|
289
|
+
const logged = opts.logged?.(toolName, input);
|
|
290
|
+
const first = asCompared(await runOnce(tool, parsed.data), logged);
|
|
291
|
+
if (first === undefined)
|
|
292
|
+
return undefined;
|
|
293
|
+
const agrees = !first.isError &&
|
|
294
|
+
logged !== undefined &&
|
|
295
|
+
logged.some((entry) => sameOutput(entry, first.content));
|
|
296
|
+
if (agrees)
|
|
297
|
+
return first;
|
|
298
|
+
// A disagreement — or nothing to check against. Is it the tool's answer,
|
|
299
|
+
// or does the tool not reproduce at all? Ask once more.
|
|
300
|
+
await new Promise((resolve) => setTimeout(resolve, recheckMs));
|
|
301
|
+
if (opts.signal?.aborted === true)
|
|
302
|
+
throw new ReplayStopped("the turn was aborted");
|
|
303
|
+
const second = asCompared(await runOnce(tool, parsed.data), logged);
|
|
304
|
+
if (second === undefined || second.isError !== first.isError)
|
|
305
|
+
return undefined;
|
|
306
|
+
return sameOutput(second.content, first.content) ? first : undefined;
|
|
307
|
+
},
|
|
308
|
+
};
|
|
309
|
+
}
|
package/dist/roots.d.ts
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { type Tenant } from "@crewhaus/tenancy";
|
|
2
|
+
/**
|
|
3
|
+
* The directory a spec's proofs live in: `.crewhaus/proofs/<proofsDirName>/`.
|
|
4
|
+
*
|
|
5
|
+
* The spec schema's `name:` admits spaces and `:` (`support bot`,
|
|
6
|
+
* `team:support`) and a dots-only name, while provability's store accepts
|
|
7
|
+
* `[A-Za-z0-9_.-]+`, not dots only — the stricter rule, because the name is
|
|
8
|
+
* a path segment there. Refusing such a name at wiring time would kill a
|
|
9
|
+
* bundle that compiled warning-free (at boot on cli and managed, on every
|
|
10
|
+
* turn on a channel bot), so it is mapped instead, here, in the one place
|
|
11
|
+
* every proofs path is built from:
|
|
12
|
+
*
|
|
13
|
+
* - a name the store accepts is its own directory, unchanged — every
|
|
14
|
+
* existing harness keeps its bundles where they are;
|
|
15
|
+
* - any other name becomes its safe characters (each run of others → `_`,
|
|
16
|
+
* `spec` if nothing is left) plus `-` and 12 hex digits of its sha256, so
|
|
17
|
+
* `support bot` and a spec literally named `support_bot` never share a
|
|
18
|
+
* directory, and the same name always maps to the same one.
|
|
19
|
+
*
|
|
20
|
+
* Exported for every reader of the proofs tree (`crewhaus provability`), so
|
|
21
|
+
* they find what the gate filed.
|
|
22
|
+
*/
|
|
23
|
+
export declare function proofsDirName(specName: string): string;
|
|
24
|
+
/** The tenant the current async context runs under, if any. */
|
|
25
|
+
export declare function activeTenant(): Tenant | undefined;
|
|
26
|
+
/** `dir`, resolved, after checking it lies inside the tenant's root (when there is one). */
|
|
27
|
+
export declare function fencedDir(dir: string, tenant: Tenant | undefined): string;
|
|
28
|
+
/** The proofs root for `specName` (as {@link proofsDirName} maps it) under the given tenant (or none). */
|
|
29
|
+
export declare function gateProofsRoot(cwd: string, specName: string, tenant: Tenant | undefined): string;
|
|
30
|
+
/**
|
|
31
|
+
* The session-log root: the caller's (`deps.sessionRootDir`, which the
|
|
32
|
+
* runtime was given too), else the tenant's / `CREWHAUS_SESSION_DIR`.
|
|
33
|
+
* `undefined` means event-log's default (`.crewhaus/sessions`, relative to
|
|
34
|
+
* the process cwd, exactly where runtime-core writes when given no root).
|
|
35
|
+
*/
|
|
36
|
+
export declare function gateSessionRoot(explicit: string | undefined, tenant: Tenant | undefined): string | undefined;
|
package/dist/roots.js
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where the gate reads and writes, resolved PER TURN for the tenant the turn
|
|
3
|
+
* runs under (design §5.3, §8: bundles are tenant-fenced on managed).
|
|
4
|
+
*
|
|
5
|
+
* A managed daemon wires its gates once at boot, when no tenant exists, and
|
|
6
|
+
* then serves every tenant from that one wiring — so nothing here is fixed at
|
|
7
|
+
* wiring time. Each turn asks `@crewhaus/tenancy` for the active tenant and
|
|
8
|
+
* resolves the proofs root (`<tenantRoot>/proofs/<spec>`, else
|
|
9
|
+
* `<cwd>/.crewhaus/proofs/<spec>`) and the session-log root (the tenant's
|
|
10
|
+
* `sessionRoot`, else `CREWHAUS_SESSION_DIR`, else event-log's default) with
|
|
11
|
+
* the same functions `@crewhaus/tool-proof` uses — so a `ProofCheck` artifact
|
|
12
|
+
* the tools file resolves at the gate, and the gate reads the logs the
|
|
13
|
+
* runtime wrote (runtime-core's `resolveSessionRootDir` order).
|
|
14
|
+
*
|
|
15
|
+
* Under a tenant every resolved root must lie inside that tenant's root, or
|
|
16
|
+
* the call fails closed (`TenancyError`, CWE-1230) — a caller-fixed root
|
|
17
|
+
* shared across tenants would otherwise pool them.
|
|
18
|
+
*
|
|
19
|
+
* The proofs directory is named by {@link proofsDirName}, never by the raw
|
|
20
|
+
* spec name: the spec schema admits names provability's store refuses.
|
|
21
|
+
*/
|
|
22
|
+
import { createHash } from "node:crypto";
|
|
23
|
+
import { resolve } from "node:path";
|
|
24
|
+
import { proofsRoot } from "@crewhaus/provability";
|
|
25
|
+
import { assertSamePath, currentTenantContext } from "@crewhaus/tenancy";
|
|
26
|
+
import { defaultProofsRoot, defaultSessionRoot, tenantRootOf } from "@crewhaus/tool-proof";
|
|
27
|
+
/** Whether provability's store files under `name` as it is (its own rule, asked, not copied). */
|
|
28
|
+
function filesAs(name) {
|
|
29
|
+
try {
|
|
30
|
+
proofsRoot(".", name);
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The directory a spec's proofs live in: `.crewhaus/proofs/<proofsDirName>/`.
|
|
39
|
+
*
|
|
40
|
+
* The spec schema's `name:` admits spaces and `:` (`support bot`,
|
|
41
|
+
* `team:support`) and a dots-only name, while provability's store accepts
|
|
42
|
+
* `[A-Za-z0-9_.-]+`, not dots only — the stricter rule, because the name is
|
|
43
|
+
* a path segment there. Refusing such a name at wiring time would kill a
|
|
44
|
+
* bundle that compiled warning-free (at boot on cli and managed, on every
|
|
45
|
+
* turn on a channel bot), so it is mapped instead, here, in the one place
|
|
46
|
+
* every proofs path is built from:
|
|
47
|
+
*
|
|
48
|
+
* - a name the store accepts is its own directory, unchanged — every
|
|
49
|
+
* existing harness keeps its bundles where they are;
|
|
50
|
+
* - any other name becomes its safe characters (each run of others → `_`,
|
|
51
|
+
* `spec` if nothing is left) plus `-` and 12 hex digits of its sha256, so
|
|
52
|
+
* `support bot` and a spec literally named `support_bot` never share a
|
|
53
|
+
* directory, and the same name always maps to the same one.
|
|
54
|
+
*
|
|
55
|
+
* Exported for every reader of the proofs tree (`crewhaus provability`), so
|
|
56
|
+
* they find what the gate filed.
|
|
57
|
+
*/
|
|
58
|
+
export function proofsDirName(specName) {
|
|
59
|
+
if (filesAs(specName))
|
|
60
|
+
return specName;
|
|
61
|
+
const digest = createHash("sha256").update(specName, "utf8").digest("hex").slice(0, 12);
|
|
62
|
+
const slug = specName.replace(/[^A-Za-z0-9_.-]+/g, "_");
|
|
63
|
+
const mapped = `${/^\.*$/.test(slug) ? "spec" : slug}-${digest}`;
|
|
64
|
+
return filesAs(mapped) ? mapped : `spec-${digest}`;
|
|
65
|
+
}
|
|
66
|
+
/** The tenant the current async context runs under, if any. */
|
|
67
|
+
export function activeTenant() {
|
|
68
|
+
return currentTenantContext()?.tenant;
|
|
69
|
+
}
|
|
70
|
+
/** `dir`, resolved, after checking it lies inside the tenant's root (when there is one). */
|
|
71
|
+
export function fencedDir(dir, tenant) {
|
|
72
|
+
const abs = resolve(dir);
|
|
73
|
+
if (tenant !== undefined)
|
|
74
|
+
assertSamePath(abs, tenantRootOf(tenant));
|
|
75
|
+
return abs;
|
|
76
|
+
}
|
|
77
|
+
/** The proofs root for `specName` (as {@link proofsDirName} maps it) under the given tenant (or none). */
|
|
78
|
+
export function gateProofsRoot(cwd, specName, tenant) {
|
|
79
|
+
return fencedDir(defaultProofsRoot(cwd, proofsDirName(specName), tenant), tenant);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* The session-log root: the caller's (`deps.sessionRootDir`, which the
|
|
83
|
+
* runtime was given too), else the tenant's / `CREWHAUS_SESSION_DIR`.
|
|
84
|
+
* `undefined` means event-log's default (`.crewhaus/sessions`, relative to
|
|
85
|
+
* the process cwd, exactly where runtime-core writes when given no root).
|
|
86
|
+
*/
|
|
87
|
+
export function gateSessionRoot(explicit, tenant) {
|
|
88
|
+
const chosen = explicit ?? defaultSessionRoot(tenant);
|
|
89
|
+
return chosen === undefined ? undefined : fencedDir(chosen, tenant);
|
|
90
|
+
}
|
package/dist/skills.d.ts
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import type { IrProvability } from "@crewhaus/ir";
|
|
2
|
+
import type { LoadedSkill } from "@crewhaus/skills-registry";
|
|
3
|
+
/** The skill's name — what a user or project skill overrides it by. */
|
|
4
|
+
export declare const PROVABILITY_SKILL_NAME = "provability";
|
|
5
|
+
/**
|
|
6
|
+
* The fields of the lowered block the skill is rendered from — everything
|
|
7
|
+
* that changes what the gate accepts at the floor (see the module doc).
|
|
8
|
+
* `IrProvability` is one; a compiled bundle passes exactly these
|
|
9
|
+
* ({@link renderGateSkills}).
|
|
10
|
+
*/
|
|
11
|
+
export type ProvabilitySkillConfig = Pick<IrProvability, "level" | "onFail" | "maxRetries" | "scope" | "assumptions" | "maxAssumptions"> & {
|
|
12
|
+
readonly grounded: Pick<IrProvability["grounded"], "independent">;
|
|
13
|
+
readonly formal: {
|
|
14
|
+
readonly checkers: ReadonlyArray<string>;
|
|
15
|
+
};
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Every substitution the `provability` skill's body can ask for, computed
|
|
19
|
+
* from `config` — the ONE place the skill's wording meets the block, shared
|
|
20
|
+
* by {@link provabilitySkills} and `./child-skill`'s rendering. A caller
|
|
21
|
+
* passes only the names its body uses (`skillTokens`): the renderer refuses
|
|
22
|
+
* the rest.
|
|
23
|
+
*
|
|
24
|
+
* - `level` — the floor; `checkers` — `formal.checkers`, comma-separated;
|
|
25
|
+
* - `on_fail` — what a failing answer becomes;
|
|
26
|
+
* - `window` — where evidence must come from: `in this session`, or under
|
|
27
|
+
* `scope: turn` `since the user's last message` (the gate's evidence
|
|
28
|
+
* index then holds this turn's calls only);
|
|
29
|
+
* - `assumptions` — accepted and counted, capped, or refused;
|
|
30
|
+
* - `independent` — at a `grounded` floor with `grounded.independent > 1`,
|
|
31
|
+
* the clause asking for that many distinct origins; else empty (the rule
|
|
32
|
+
* applies at no other floor);
|
|
33
|
+
* - `proofs` — the section holding the proof tools, non-empty exactly when
|
|
34
|
+
* the gate wires `ProofCheckers` + `ProofCheck` (`wantsProofCheckTools`):
|
|
35
|
+
* at `proven` every claim needs a proof; below it a proof meets any
|
|
36
|
+
* floor.
|
|
37
|
+
*/
|
|
38
|
+
export declare function provabilitySkillSubstitutions(config: ProvabilitySkillConfig): Record<string, string>;
|
|
39
|
+
/** {@link provabilitySkillSubstitutions}, narrowed to the names `body` uses. */
|
|
40
|
+
export declare function substitutionsFor(config: ProvabilitySkillConfig, body: string): Record<string, string>;
|
|
41
|
+
/** Where the skill comes from (default: the shipped default skills); a seam for tests. */
|
|
42
|
+
export type ProvabilitySkillSource = {
|
|
43
|
+
readonly skills: ReadonlyArray<LoadedSkill>;
|
|
44
|
+
readonly render: (name: string, substitutions: Record<string, string>) => string;
|
|
45
|
+
};
|
|
46
|
+
/**
|
|
47
|
+
* The gate's builtin skills for `config` — the rendered `provability` skill,
|
|
48
|
+
* or `[]` when there is no block (so a call site may pass the IR's optional
|
|
49
|
+
* block unconditionally). Also `[]`, with a warning, when this build ships no
|
|
50
|
+
* such skill or its body asks for a substitution the block cannot fill: a
|
|
51
|
+
* missing skill is a wiring gap the gate's retry nudge still covers, never a
|
|
52
|
+
* reason to fail a harness at boot.
|
|
53
|
+
*/
|
|
54
|
+
export declare function provabilitySkills(config: ProvabilitySkillConfig | undefined, source?: ProvabilitySkillSource): ReadonlyArray<LoadedSkill>;
|
|
55
|
+
/** The codegen twin of {@link provabilitySkills} (see {@link renderGateSkills}). */
|
|
56
|
+
export type GateSkillsWiring = {
|
|
57
|
+
/** `import { provabilitySkills } from "@crewhaus/evaluation-service";`, or none. */
|
|
58
|
+
readonly imports: readonly string[];
|
|
59
|
+
/**
|
|
60
|
+
* The expression for the gate's builtin skills — `provabilitySkills({…})`
|
|
61
|
+
* over the fields it reads — or `""` when the IR declares no
|
|
62
|
+
* `provability:`.
|
|
63
|
+
*/
|
|
64
|
+
readonly expr: string;
|
|
65
|
+
};
|
|
66
|
+
/**
|
|
67
|
+
* What an emitter splices into a skill-discovery site (design §11): the
|
|
68
|
+
* import and the `provabilitySkills(…)` expression, rendered from the lowered
|
|
69
|
+
* block. The emitter passes it as `builtinSkills` to the `discoverSkills` /
|
|
70
|
+
* `wireMemory` call that builds the run's skill list — the
|
|
71
|
+
* `renderEvaluationWiring` / `renderMemory` precedent: one rendering, owned
|
|
72
|
+
* by the composition root, so no emitter templates the skill itself. Both
|
|
73
|
+
* pieces empty without `provability:`, keeping those bundles byte-identical.
|
|
74
|
+
*
|
|
75
|
+
* The argument is `JSON.stringify` of the {@link ProvabilitySkillConfig}
|
|
76
|
+
* fields, in a fixed order: plain data (rung and mode names, counts, checker
|
|
77
|
+
* ids), so JSON's quoting is the escaping, and only what the skill reads
|
|
78
|
+
* rides along — the full block is already in the bundle's `wireEvaluation`
|
|
79
|
+
* fragment.
|
|
80
|
+
*/
|
|
81
|
+
export declare function renderGateSkills(ir: {
|
|
82
|
+
readonly provability?: IrProvability;
|
|
83
|
+
}): GateSkillsWiring;
|