@crewhaus/evaluation-service 0.0.0-stage → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +9 -0
- package/README.md +278 -2
- package/dist/child-skill.d.ts +33 -0
- package/dist/child-skill.js +6 -0
- package/dist/evaluation.d.ts +40 -0
- package/dist/evaluation.js +128 -0
- package/dist/evidence.d.ts +97 -0
- package/dist/evidence.js +147 -0
- package/dist/fragment.d.ts +40 -0
- package/dist/fragment.js +17 -0
- package/dist/head.d.ts +115 -0
- package/dist/head.js +181 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +24 -0
- package/dist/judge.d.ts +62 -0
- package/dist/judge.js +103 -0
- package/dist/observe.d.ts +32 -0
- package/dist/observe.js +31 -0
- package/dist/provability.d.ts +47 -0
- package/dist/provability.js +261 -0
- package/dist/render.d.ts +95 -0
- package/dist/render.js +99 -0
- package/dist/replay.d.ts +53 -0
- package/dist/replay.js +309 -0
- package/dist/roots.d.ts +36 -0
- package/dist/roots.js +90 -0
- package/dist/skills.d.ts +83 -0
- package/dist/skills.js +200 -0
- package/dist/tools.d.ts +106 -0
- package/dist/tools.js +74 -0
- package/dist/wire.d.ts +106 -0
- package/dist/wire.js +133 -0
- package/package.json +65 -4
package/dist/head.js
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { hostMatches, isPrivateHost } from "@crewhaus/provability";
|
|
2
|
+
import { FetchPermissionError, guardedHead } from "@crewhaus/tool-fetch";
|
|
3
|
+
/** How many (session, URL) answers one wiring remembers. */
|
|
4
|
+
export const HEAD_CACHE_ENTRIES = 1024;
|
|
5
|
+
/** The sink id a HEAD's `egress_decision` names (design §6.1). */
|
|
6
|
+
export const RESOLVE_SINK_ID = "provability.resolve";
|
|
7
|
+
/** The spec's source policy as a host test: private hosts never, deny wins, an allow list admits only its hosts. */
|
|
8
|
+
export function sourceHostPolicy(sources) {
|
|
9
|
+
return (raw) => {
|
|
10
|
+
const host = raw.replace(/^\[/, "").replace(/\]$/, "");
|
|
11
|
+
if (isPrivateHost(host))
|
|
12
|
+
return false;
|
|
13
|
+
if (sources.deny.length > 0 && hostMatches(host, sources.deny))
|
|
14
|
+
return false;
|
|
15
|
+
if (sources.allow.length > 0 && !hostMatches(host, sources.allow))
|
|
16
|
+
return false;
|
|
17
|
+
return true;
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The per-hop test for one request: the full policy on the cited URL, the
|
|
22
|
+
* private-host rule and `deny` on every hop a redirect names. `guardedHead`
|
|
23
|
+
* asks once per hop, in order, starting with the URL it was given.
|
|
24
|
+
*/
|
|
25
|
+
export function hopPolicy(sources) {
|
|
26
|
+
const cited = sourceHostPolicy(sources);
|
|
27
|
+
const redirected = sourceHostPolicy({ ...sources, allow: [] });
|
|
28
|
+
let first = true;
|
|
29
|
+
return (host) => {
|
|
30
|
+
const ok = first ? cited(host) : redirected(host);
|
|
31
|
+
first = false;
|
|
32
|
+
return ok;
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* The guarded HEAD's answers for the whole process, shared by every wiring
|
|
37
|
+
* that uses the default request: a channel bot wires its gates per turn, and
|
|
38
|
+
* a per-wiring cache would ask again on every turn of a session. Keyed by
|
|
39
|
+
* session, the source policy and the URL, so two specs in one process never
|
|
40
|
+
* read each other's answers under different policies. Bounded like any
|
|
41
|
+
* cache here ({@link HEAD_CACHE_ENTRIES}).
|
|
42
|
+
*/
|
|
43
|
+
export const SHARED_HEAD_CACHE = new Map();
|
|
44
|
+
/** A refusal the guard will make again for the same URL — an answer worth keeping. */
|
|
45
|
+
function isLastingRefusal(err) {
|
|
46
|
+
// An unresolvable name is reported as a refusal too, but DNS can come back.
|
|
47
|
+
return err instanceof FetchPermissionError && !/cannot resolve/i.test(err.message);
|
|
48
|
+
}
|
|
49
|
+
function hostOf(url) {
|
|
50
|
+
try {
|
|
51
|
+
return new URL(url).hostname.toLowerCase() || undefined;
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return undefined;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
/** The screen's findings, as a record carries them (none when it found none). */
|
|
58
|
+
function lineageOf(screened) {
|
|
59
|
+
return screened.originsFound.length > 0
|
|
60
|
+
? { originsFound: [...screened.originsFound], matchCount: screened.matchCount }
|
|
61
|
+
: {};
|
|
62
|
+
}
|
|
63
|
+
class Stopped extends Error {
|
|
64
|
+
name = "Stopped";
|
|
65
|
+
}
|
|
66
|
+
/** `work`, or a rejection as soon as `signal` aborts — whatever `work` is waiting on. */
|
|
67
|
+
function untilAborted(work, signal) {
|
|
68
|
+
if (signal === undefined)
|
|
69
|
+
return work;
|
|
70
|
+
if (signal.aborted) {
|
|
71
|
+
// Nobody waits on `work` any more; its own failure is not this turn's.
|
|
72
|
+
work.catch(() => { });
|
|
73
|
+
return Promise.reject(new Stopped("the turn was aborted"));
|
|
74
|
+
}
|
|
75
|
+
let onAbort;
|
|
76
|
+
const stop = new Promise((_resolve, reject) => {
|
|
77
|
+
onAbort = () => reject(new Stopped("the turn was aborted"));
|
|
78
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
79
|
+
});
|
|
80
|
+
return Promise.race([work, stop]).finally(() => {
|
|
81
|
+
if (onAbort !== undefined)
|
|
82
|
+
signal.removeEventListener("abort", onAbort);
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* The cached resolver. `head` is the request itself — `guardedHead` in
|
|
87
|
+
* production; a caller's `deps.resolveHead` (tests, a proxy-aware host)
|
|
88
|
+
* replaces it and is cached, bounded by the turn's signal and recorded the
|
|
89
|
+
* same way. `cache` is where answers are kept: a fresh one per resolver
|
|
90
|
+
* unless the caller passes one to share.
|
|
91
|
+
*/
|
|
92
|
+
export function createHeadResolver(sources, head, cache = new Map()) {
|
|
93
|
+
const request = head ??
|
|
94
|
+
((url, signal) => guardedHead(url, {
|
|
95
|
+
allowHost: hopPolicy(sources),
|
|
96
|
+
...(signal !== undefined ? { signal } : {}),
|
|
97
|
+
}));
|
|
98
|
+
const policy = JSON.stringify([sources.allow, sources.deny]);
|
|
99
|
+
return (url, ctx) => {
|
|
100
|
+
const key = `${ctx.sessionId ?? ""}\u0000${policy}\u0000${url}`;
|
|
101
|
+
const hit = cache.get(key);
|
|
102
|
+
if (hit !== undefined)
|
|
103
|
+
return hit;
|
|
104
|
+
const host = hostOf(url);
|
|
105
|
+
const record = async (decision) => {
|
|
106
|
+
try {
|
|
107
|
+
await ctx.record?.({
|
|
108
|
+
sinkId: RESOLVE_SINK_ID,
|
|
109
|
+
sinkScope: "external-dynamic",
|
|
110
|
+
method: "HEAD",
|
|
111
|
+
...(host !== undefined ? { host } : {}),
|
|
112
|
+
...decision,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
// The audit trail is best-effort here, as on every egress path.
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
const pending = (async () => {
|
|
120
|
+
let screened;
|
|
121
|
+
try {
|
|
122
|
+
// The sink side first: a URL carrying tagged content is never sent.
|
|
123
|
+
// A screen that cannot answer refuses — the request is an egress.
|
|
124
|
+
if (ctx.screen !== undefined) {
|
|
125
|
+
try {
|
|
126
|
+
screened = await untilAborted(ctx.screen(url), ctx.signal);
|
|
127
|
+
}
|
|
128
|
+
catch (err) {
|
|
129
|
+
if (ctx.signal?.aborted === true)
|
|
130
|
+
throw err;
|
|
131
|
+
screened = { verdict: "block", originsFound: [], matchCount: 0 };
|
|
132
|
+
}
|
|
133
|
+
if (screened.verdict === "block") {
|
|
134
|
+
cache.delete(key);
|
|
135
|
+
await record({ verdict: "block", reason: "egress", ...lineageOf(screened) });
|
|
136
|
+
// Not sent: the verifier says the egress fabric refused it,
|
|
137
|
+
// rather than that the URL gave no answer.
|
|
138
|
+
return { refused: "egress" };
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
const status = await untilAborted(request(url, ctx.signal), ctx.signal);
|
|
142
|
+
const answer = typeof status === "number" && Number.isFinite(status) ? status : undefined;
|
|
143
|
+
await record({
|
|
144
|
+
verdict: screened?.verdict ?? "pass",
|
|
145
|
+
...(answer !== undefined ? { status: answer } : { reason: "failed" }),
|
|
146
|
+
...(screened !== undefined ? lineageOf(screened) : {}),
|
|
147
|
+
});
|
|
148
|
+
if (answer === undefined)
|
|
149
|
+
cache.delete(key);
|
|
150
|
+
return answer;
|
|
151
|
+
}
|
|
152
|
+
catch (err) {
|
|
153
|
+
// The turn ending is not an answer about the page: forget it.
|
|
154
|
+
if (ctx.signal?.aborted === true) {
|
|
155
|
+
cache.delete(key);
|
|
156
|
+
throw err;
|
|
157
|
+
}
|
|
158
|
+
if (isLastingRefusal(err)) {
|
|
159
|
+
await record({ verdict: "block", reason: "refused" });
|
|
160
|
+
}
|
|
161
|
+
else {
|
|
162
|
+
await record({
|
|
163
|
+
verdict: screened?.verdict ?? "pass",
|
|
164
|
+
reason: "failed",
|
|
165
|
+
...(screened !== undefined ? lineageOf(screened) : {}),
|
|
166
|
+
});
|
|
167
|
+
cache.delete(key);
|
|
168
|
+
}
|
|
169
|
+
return undefined;
|
|
170
|
+
}
|
|
171
|
+
})();
|
|
172
|
+
cache.set(key, pending);
|
|
173
|
+
while (cache.size > HEAD_CACHE_ENTRIES) {
|
|
174
|
+
const oldest = cache.keys().next().value;
|
|
175
|
+
if (oldest === undefined)
|
|
176
|
+
break;
|
|
177
|
+
cache.delete(oldest);
|
|
178
|
+
}
|
|
179
|
+
return pending;
|
|
180
|
+
};
|
|
181
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @crewhaus/evaluation-service — the composition root for in-loop gates
|
|
3
|
+
* (design/0.8.0-provability-release.md §4.4).
|
|
4
|
+
*
|
|
5
|
+
* `wireEvaluation(fragment, deps)` builds the `evaluation:` judge, the
|
|
6
|
+
* `provability:` gate and the proof tools from the lowered IR; every emitter
|
|
7
|
+
* renders one call to it (`renderEvaluationWiring`) and the `crewhaus run` /
|
|
8
|
+
* `serve` interpreter calls it directly, so the gates are built in exactly
|
|
9
|
+
* one place. runtime-core receives the closures and stays dependency-free.
|
|
10
|
+
*/
|
|
11
|
+
export { type ChildSkillSource, provabilitySkillText } from "./child-skill";
|
|
12
|
+
export { buildRunEvaluation, type EvaluationGateOptions } from "./evaluation";
|
|
13
|
+
export { type GateEvidenceInput, currentTurnMessages, gateEvidence, gateEvidenceScoped, startsTurnMessage, } from "./evidence";
|
|
14
|
+
export { type EvaluationFragment, type EvaluationIrSlice, evaluationFragmentFromIr, } from "./fragment";
|
|
15
|
+
export { HEAD_CACHE_ENTRIES, type HeadCache, type HeadDecision, type HeadResolver, RESOLVE_SINK_ID, SHARED_HEAD_CACHE, createHeadResolver, hopPolicy, sourceHostPolicy, } from "./head";
|
|
16
|
+
export { WAIVER_NOT_SUPPORTED, WAIVER_RUBRIC, WAIVER_SUPPORTED, type WaiverJudgeOptions, createWaiverJudge, } from "./judge";
|
|
17
|
+
export { type ProofBundle, type ProofBundleObserver, observeProofBundles } from "./observe";
|
|
18
|
+
export { type ProvabilityGateOptions, buildRunProvability, displayPath } from "./provability";
|
|
19
|
+
export { type EvaluationWiring, type EvaluationWiringOptions, GATES_VAR, renderEvaluationWiring, } from "./render";
|
|
20
|
+
export { type CatalogReplayOptions, type GateCatalog, REPLAY_RECHECK_MS, REPLAY_TIMEOUT_MS, catalogReplay, lookupTool, resultText, } from "./replay";
|
|
21
|
+
export { activeTenant, fencedDir, gateProofsRoot, gateSessionRoot, proofsDirName, } from "./roots";
|
|
22
|
+
export { type GateSkillsWiring, PROVABILITY_SKILL_NAME, type ProvabilitySkillConfig, type ProvabilitySkillSource, provabilitySkillSubstitutions, provabilitySkills, renderGateSkills, } from "./skills";
|
|
23
|
+
export { DEFAULT_FORMAL_CHECKERS, type GateProofTools, type GateToolsOptions, gateTools, mergeGateTools, registerGateTools, wantsProofCheckTools, } from "./tools";
|
|
24
|
+
export { EvaluationServiceError, type WireEvaluationDeps, type WiredGates, wireEvaluation, } from "./wire";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @crewhaus/evaluation-service — the composition root for in-loop gates
|
|
3
|
+
* (design/0.8.0-provability-release.md §4.4).
|
|
4
|
+
*
|
|
5
|
+
* `wireEvaluation(fragment, deps)` builds the `evaluation:` judge, the
|
|
6
|
+
* `provability:` gate and the proof tools from the lowered IR; every emitter
|
|
7
|
+
* renders one call to it (`renderEvaluationWiring`) and the `crewhaus run` /
|
|
8
|
+
* `serve` interpreter calls it directly, so the gates are built in exactly
|
|
9
|
+
* one place. runtime-core receives the closures and stays dependency-free.
|
|
10
|
+
*/
|
|
11
|
+
export { provabilitySkillText } from "./child-skill";
|
|
12
|
+
export { buildRunEvaluation } from "./evaluation";
|
|
13
|
+
export { currentTurnMessages, gateEvidence, gateEvidenceScoped, startsTurnMessage, } from "./evidence";
|
|
14
|
+
export { evaluationFragmentFromIr, } from "./fragment";
|
|
15
|
+
export { HEAD_CACHE_ENTRIES, RESOLVE_SINK_ID, SHARED_HEAD_CACHE, createHeadResolver, hopPolicy, sourceHostPolicy, } from "./head";
|
|
16
|
+
export { WAIVER_NOT_SUPPORTED, WAIVER_RUBRIC, WAIVER_SUPPORTED, createWaiverJudge, } from "./judge";
|
|
17
|
+
export { observeProofBundles } from "./observe";
|
|
18
|
+
export { buildRunProvability, displayPath } from "./provability";
|
|
19
|
+
export { GATES_VAR, renderEvaluationWiring, } from "./render";
|
|
20
|
+
export { REPLAY_RECHECK_MS, REPLAY_TIMEOUT_MS, catalogReplay, lookupTool, resultText, } from "./replay";
|
|
21
|
+
export { activeTenant, fencedDir, gateProofsRoot, gateSessionRoot, proofsDirName, } from "./roots";
|
|
22
|
+
export { PROVABILITY_SKILL_NAME, provabilitySkillSubstitutions, provabilitySkills, renderGateSkills, } from "./skills";
|
|
23
|
+
export { DEFAULT_FORMAL_CHECKERS, gateTools, mergeGateTools, registerGateTools, wantsProofCheckTools, } from "./tools";
|
|
24
|
+
export { EvaluationServiceError, wireEvaluation, } from "./wire";
|
package/dist/judge.d.ts
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The opt-in paraphrase-waiver judge (design §6.2, `provability.judge`): the
|
|
3
|
+
* one model-backed step in the whole provability ladder, and the only one
|
|
4
|
+
* `@crewhaus/provability` cannot make itself — it takes this closure as
|
|
5
|
+
* `VerifyOptions.judgeWaiver` and calls it only for a `grounded` binding that
|
|
6
|
+
* failed on paraphrase coverage alone (the body was fetched, the words were
|
|
7
|
+
* partly found). It can flip that one binding, marked `judgeWaived: true`,
|
|
8
|
+
* and nothing else.
|
|
9
|
+
*
|
|
10
|
+
* One yes/no call through `@crewhaus/eval-judge`'s `judgeCategorical`: the
|
|
11
|
+
* passage and its source go in the sentinel-wrapped INPUT block, the claim
|
|
12
|
+
* in the sentinel-wrapped OUTPUT block — both are untrusted (the passage is
|
|
13
|
+
* fetched content, the claim is model output), and the judge's system
|
|
14
|
+
* framing reads both as data. The answer is a forced `submit_label` call
|
|
15
|
+
* over a closed two-label vocabulary, so the judge classifies and never
|
|
16
|
+
* writes prose the gate would have to read. An abstention is a "no".
|
|
17
|
+
*
|
|
18
|
+
* Decoding is pinned to temperature 0, over a judge profile's own
|
|
19
|
+
* `temperature` if it pins one: a waiver must be as reproducible as the
|
|
20
|
+
* deterministic checks around it. The profile's other params
|
|
21
|
+
* (`max_tokens`, `thinking`) apply as on every auxiliary slot (0.6.0 §4.2).
|
|
22
|
+
*
|
|
23
|
+
* Metering: the call publishes `model_request` / `model_response` with
|
|
24
|
+
* `role: "judge"` on the RUN bus the turn carries — the mechanism
|
|
25
|
+
* `gradeWithJudgePanel` uses — so it is priced, shown in `cost-summary` and
|
|
26
|
+
* counted under `budget.judge_share`; the wire model and priced spend come
|
|
27
|
+
* back on the verdict and land on the bundle (`judgeModel`, `spentUsdMicros`).
|
|
28
|
+
*
|
|
29
|
+
* Past the share the waiver is SKIPPED, not guessed (design §6.2): the
|
|
30
|
+
* runtime's live read of the share (`ProvabilityTurn.judgeShareExhausted`)
|
|
31
|
+
* is asked before every call, and once it says spent the closure answers
|
|
32
|
+
* "not supported" without a model call — the binding keeps the miss the
|
|
33
|
+
* deterministic check found, exactly as with no judge at all. (The
|
|
34
|
+
* runtime's total cap is the stop; the share is what keeps a gate's
|
|
35
|
+
* bookkeeping judge from spending the answer's budget.)
|
|
36
|
+
*/
|
|
37
|
+
import { type CategoricalJudgeOptions, type CategoricalRubric } from "@crewhaus/eval-judge";
|
|
38
|
+
import type { IrProvability } from "@crewhaus/ir";
|
|
39
|
+
import type { JudgeWaiverVerdict } from "@crewhaus/provability";
|
|
40
|
+
import type { TraceEventBus } from "@crewhaus/trace-event-bus";
|
|
41
|
+
/** The label that waives the binding. */
|
|
42
|
+
export declare const WAIVER_SUPPORTED = "supported";
|
|
43
|
+
/** The label that keeps it unmet. */
|
|
44
|
+
export declare const WAIVER_NOT_SUPPORTED = "not_supported";
|
|
45
|
+
/** The waiver's two-label vocabulary. Scores are the grader mapping, never shown to the judge. */
|
|
46
|
+
export declare const WAIVER_RUBRIC: CategoricalRubric;
|
|
47
|
+
export type WaiverJudgeOptions = {
|
|
48
|
+
/** The lowered `provability.judge` slot. */
|
|
49
|
+
readonly judge: NonNullable<IrProvability["judge"]>;
|
|
50
|
+
/** The run bus the call meters on (the turn's). */
|
|
51
|
+
readonly bus?: TraceEventBus;
|
|
52
|
+
/** Test seam: a pre-built adapter that bypasses the model-router. */
|
|
53
|
+
readonly adapter?: CategoricalJudgeOptions["adapter"];
|
|
54
|
+
/** The run's live `budget.judge_share` read; true ⇒ the call is skipped (see the module doc). */
|
|
55
|
+
readonly exhausted?: () => boolean;
|
|
56
|
+
};
|
|
57
|
+
/** The `VerifyOptions.judgeWaiver` closure for one turn. */
|
|
58
|
+
export declare function createWaiverJudge(opts: WaiverJudgeOptions): (input: {
|
|
59
|
+
readonly claim: string;
|
|
60
|
+
readonly excerpt: string;
|
|
61
|
+
readonly source: string;
|
|
62
|
+
}) => Promise<JudgeWaiverVerdict>;
|
package/dist/judge.js
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The opt-in paraphrase-waiver judge (design §6.2, `provability.judge`): the
|
|
3
|
+
* one model-backed step in the whole provability ladder, and the only one
|
|
4
|
+
* `@crewhaus/provability` cannot make itself — it takes this closure as
|
|
5
|
+
* `VerifyOptions.judgeWaiver` and calls it only for a `grounded` binding that
|
|
6
|
+
* failed on paraphrase coverage alone (the body was fetched, the words were
|
|
7
|
+
* partly found). It can flip that one binding, marked `judgeWaived: true`,
|
|
8
|
+
* and nothing else.
|
|
9
|
+
*
|
|
10
|
+
* One yes/no call through `@crewhaus/eval-judge`'s `judgeCategorical`: the
|
|
11
|
+
* passage and its source go in the sentinel-wrapped INPUT block, the claim
|
|
12
|
+
* in the sentinel-wrapped OUTPUT block — both are untrusted (the passage is
|
|
13
|
+
* fetched content, the claim is model output), and the judge's system
|
|
14
|
+
* framing reads both as data. The answer is a forced `submit_label` call
|
|
15
|
+
* over a closed two-label vocabulary, so the judge classifies and never
|
|
16
|
+
* writes prose the gate would have to read. An abstention is a "no".
|
|
17
|
+
*
|
|
18
|
+
* Decoding is pinned to temperature 0, over a judge profile's own
|
|
19
|
+
* `temperature` if it pins one: a waiver must be as reproducible as the
|
|
20
|
+
* deterministic checks around it. The profile's other params
|
|
21
|
+
* (`max_tokens`, `thinking`) apply as on every auxiliary slot (0.6.0 §4.2).
|
|
22
|
+
*
|
|
23
|
+
* Metering: the call publishes `model_request` / `model_response` with
|
|
24
|
+
* `role: "judge"` on the RUN bus the turn carries — the mechanism
|
|
25
|
+
* `gradeWithJudgePanel` uses — so it is priced, shown in `cost-summary` and
|
|
26
|
+
* counted under `budget.judge_share`; the wire model and priced spend come
|
|
27
|
+
* back on the verdict and land on the bundle (`judgeModel`, `spentUsdMicros`).
|
|
28
|
+
*
|
|
29
|
+
* Past the share the waiver is SKIPPED, not guessed (design §6.2): the
|
|
30
|
+
* runtime's live read of the share (`ProvabilityTurn.judgeShareExhausted`)
|
|
31
|
+
* is asked before every call, and once it says spent the closure answers
|
|
32
|
+
* "not supported" without a model call — the binding keeps the miss the
|
|
33
|
+
* deterministic check found, exactly as with no judge at all. (The
|
|
34
|
+
* runtime's total cap is the stop; the share is what keeps a gate's
|
|
35
|
+
* bookkeeping judge from spending the answer's budget.)
|
|
36
|
+
*/
|
|
37
|
+
import { judgeCategorical, } from "@crewhaus/eval-judge";
|
|
38
|
+
/** The label that waives the binding. */
|
|
39
|
+
export const WAIVER_SUPPORTED = "supported";
|
|
40
|
+
/** The label that keeps it unmet. */
|
|
41
|
+
export const WAIVER_NOT_SUPPORTED = "not_supported";
|
|
42
|
+
/** The waiver's two-label vocabulary. Scores are the grader mapping, never shown to the judge. */
|
|
43
|
+
export const WAIVER_RUBRIC = {
|
|
44
|
+
kind: "categorical",
|
|
45
|
+
labels: [
|
|
46
|
+
{
|
|
47
|
+
name: WAIVER_SUPPORTED,
|
|
48
|
+
score: 1,
|
|
49
|
+
description: "The source passage (the sample input) states everything the claim (the agent output) asserts, possibly in other words: every fact, figure and qualifier in the claim is in the passage.",
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
name: WAIVER_NOT_SUPPORTED,
|
|
53
|
+
score: 0,
|
|
54
|
+
description: "The passage does not state the claim: it states only part of it, says something different or the opposite, or is about something else.",
|
|
55
|
+
},
|
|
56
|
+
],
|
|
57
|
+
passing_labels: [WAIVER_SUPPORTED],
|
|
58
|
+
};
|
|
59
|
+
/** Whether the share is spent; a read that throws cannot say it is. */
|
|
60
|
+
function shareSpent(exhausted) {
|
|
61
|
+
try {
|
|
62
|
+
return exhausted?.() === true;
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/** The judge-slot params with the temperature pin restored (see the module doc). */
|
|
69
|
+
function waiverParams(params) {
|
|
70
|
+
if (params === undefined)
|
|
71
|
+
return undefined;
|
|
72
|
+
const { temperature: _pinned, ...rest } = params;
|
|
73
|
+
return Object.keys(rest).length > 0 ? rest : undefined;
|
|
74
|
+
}
|
|
75
|
+
/** The `VerifyOptions.judgeWaiver` closure for one turn. */
|
|
76
|
+
export function createWaiverJudge(opts) {
|
|
77
|
+
const params = waiverParams(opts.judge.params);
|
|
78
|
+
return async ({ claim, excerpt, source }) => {
|
|
79
|
+
if (shareSpent(opts.exhausted))
|
|
80
|
+
return { supported: false };
|
|
81
|
+
const verdict = await judgeCategorical({
|
|
82
|
+
rubric: WAIVER_RUBRIC,
|
|
83
|
+
sample: {
|
|
84
|
+
id: "provability-waiver",
|
|
85
|
+
input: `Source: ${source}\n\nPassage:\n${excerpt}`,
|
|
86
|
+
},
|
|
87
|
+
agentOutput: claim,
|
|
88
|
+
model: opts.judge.model,
|
|
89
|
+
temperature: 0,
|
|
90
|
+
...(params !== undefined ? { params } : {}),
|
|
91
|
+
...(opts.adapter !== undefined ? { adapter: opts.adapter } : {}),
|
|
92
|
+
...(opts.bus !== undefined ? { bus: opts.bus } : {}),
|
|
93
|
+
role: "judge",
|
|
94
|
+
});
|
|
95
|
+
return {
|
|
96
|
+
supported: verdict.label === WAIVER_SUPPORTED && !verdict.abstain,
|
|
97
|
+
model: verdict.usage.model,
|
|
98
|
+
...(verdict.usage.costUsdMicros !== undefined
|
|
99
|
+
? { costUsdMicros: verdict.usage.costUsdMicros }
|
|
100
|
+
: {}),
|
|
101
|
+
};
|
|
102
|
+
};
|
|
103
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-run observers of the proof gate (design/0.8.0-provability-release.md
|
|
3
|
+
* §5.4): how a surface that serves many runs from one wiring sees the bundle
|
|
4
|
+
* of ONE of them.
|
|
5
|
+
*
|
|
6
|
+
* `wireEvaluation`'s `onBundle` dep observes every bundle the wiring ever
|
|
7
|
+
* grades, which is the right seam for a bundle that wires its gates per
|
|
8
|
+
* process and serves one run at a time. The managed daemon does not: it
|
|
9
|
+
* wires once at module load and serves concurrent runs for many tenants, and
|
|
10
|
+
* the gateway must answer each `runs.create` with the bundle of THAT run's
|
|
11
|
+
* final turn. So the daemon passes an observer per run, and this composes it
|
|
12
|
+
* onto the shared gate's own `onBundle` for that run's `runChatLoop` only.
|
|
13
|
+
*
|
|
14
|
+
* The runtime calls `onBundle` once per graded pass (the original answer and
|
|
15
|
+
* each gate-triggered retry), in order, so the LAST call a run sees is the
|
|
16
|
+
* standing verdict — the one the footer renders and the one to return.
|
|
17
|
+
*/
|
|
18
|
+
import type { ProofBundle } from "@crewhaus/provability";
|
|
19
|
+
import type { RunProvability } from "@crewhaus/runtime-core";
|
|
20
|
+
export type { ProofBundle } from "@crewhaus/provability";
|
|
21
|
+
/** Sees a graded bundle and, when it was persisted, where. */
|
|
22
|
+
export type ProofBundleObserver = (bundle: ProofBundle, bundlePath?: string) => void;
|
|
23
|
+
/**
|
|
24
|
+
* `provability` with `observer` added beside its own `onBundle`, for one run.
|
|
25
|
+
* Identity when either is absent, so a call site can thread an optional
|
|
26
|
+
* observer unconditionally and a run without one (or a spec without the
|
|
27
|
+
* block) is handed exactly the shared gate. The gate's own observer runs
|
|
28
|
+
* first; one that throws does not keep the run's from seeing the bundle (the
|
|
29
|
+
* runtime already isolates a throwing `onBundle` from the verdict, and this
|
|
30
|
+
* isolates the two observers from each other the same way).
|
|
31
|
+
*/
|
|
32
|
+
export declare function observeProofBundles(provability: RunProvability | undefined, observer: ProofBundleObserver | undefined): RunProvability | undefined;
|
package/dist/observe.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `provability` with `observer` added beside its own `onBundle`, for one run.
|
|
3
|
+
* Identity when either is absent, so a call site can thread an optional
|
|
4
|
+
* observer unconditionally and a run without one (or a spec without the
|
|
5
|
+
* block) is handed exactly the shared gate. The gate's own observer runs
|
|
6
|
+
* first; one that throws does not keep the run's from seeing the bundle (the
|
|
7
|
+
* runtime already isolates a throwing `onBundle` from the verdict, and this
|
|
8
|
+
* isolates the two observers from each other the same way).
|
|
9
|
+
*/
|
|
10
|
+
export function observeProofBundles(provability, observer) {
|
|
11
|
+
if (provability === undefined || observer === undefined)
|
|
12
|
+
return provability;
|
|
13
|
+
const own = provability.onBundle;
|
|
14
|
+
return {
|
|
15
|
+
...provability,
|
|
16
|
+
onBundle: (bundle, bundlePath) => {
|
|
17
|
+
let failure;
|
|
18
|
+
if (own !== undefined) {
|
|
19
|
+
try {
|
|
20
|
+
own(bundle, bundlePath);
|
|
21
|
+
}
|
|
22
|
+
catch (error) {
|
|
23
|
+
failure = { error };
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
observer(bundle, bundlePath);
|
|
27
|
+
if (failure !== undefined)
|
|
28
|
+
throw failure.error;
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { CategoricalJudgeOptions } from "@crewhaus/eval-judge";
|
|
2
|
+
import type { IrProvability } from "@crewhaus/ir";
|
|
3
|
+
import { type ProofBundle } from "@crewhaus/provability";
|
|
4
|
+
import type { RunProvability } from "@crewhaus/runtime-core";
|
|
5
|
+
import { type GateCatalog } from "./replay";
|
|
6
|
+
export type ProvabilityGateOptions = {
|
|
7
|
+
readonly specName: string;
|
|
8
|
+
/** Workspace root: path sources resolve inside it and `.crewhaus/proofs/` lives under it. */
|
|
9
|
+
readonly cwd: string;
|
|
10
|
+
/** Where `checked` replay finds the tools it re-runs. Absent ⇒ no replay (`replayed: false`). */
|
|
11
|
+
readonly catalog?: GateCatalog;
|
|
12
|
+
/** Where session logs live, when the run was given a root (else the runtime's default order). */
|
|
13
|
+
readonly sessionRootDir?: string;
|
|
14
|
+
/** Where the rendered bundle goes (`RunProvability.footerTarget`). */
|
|
15
|
+
readonly footerTarget?: "stream" | "reply" | "none";
|
|
16
|
+
/** The turn streams to its reader whatever `footerTarget` says (`RunProvability.streamed`). */
|
|
17
|
+
readonly streamed?: boolean;
|
|
18
|
+
/**
|
|
19
|
+
* The HEAD request behind `sources.resolve: head` (default: tool-fetch's
|
|
20
|
+
* `guardedHead`). `signal` aborts when the verifier's HEAD deadline
|
|
21
|
+
* passes or the turn ends; a request that honours it stops then.
|
|
22
|
+
*/
|
|
23
|
+
readonly resolveHead?: (url: string, signal?: AbortSignal) => Promise<number | undefined>;
|
|
24
|
+
/**
|
|
25
|
+
* How long one verification waits on its HEAD requests in all
|
|
26
|
+
* (`VerifyOptions.headDeadlineMs`; default 15 s).
|
|
27
|
+
*/
|
|
28
|
+
readonly headDeadlineMs?: number;
|
|
29
|
+
/** Observe every bundle (the gateway envelope, the research report, tests). */
|
|
30
|
+
readonly onBundle?: (bundle: ProofBundle, bundlePath?: string) => void;
|
|
31
|
+
/** Test seam: the waiver judge's adapter. */
|
|
32
|
+
readonly judgeAdapter?: CategoricalJudgeOptions["adapter"];
|
|
33
|
+
readonly now?: () => Date;
|
|
34
|
+
/**
|
|
35
|
+
* The proofs root, when it is not the harness's (see
|
|
36
|
+
* `WireEvaluationDeps.proofsRoot`): where `proof:` citations resolve and
|
|
37
|
+
* bundles persist. Fenced to the active tenant like the default.
|
|
38
|
+
*/
|
|
39
|
+
readonly proofsRoot?: string;
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* A bundle path as the footer shows it: relative to `base` when it lies
|
|
43
|
+
* inside it (`.crewhaus/proofs/<spec>/<session>/turn-7.json`), else as given.
|
|
44
|
+
*/
|
|
45
|
+
export declare function displayPath(path: string, base: string): string;
|
|
46
|
+
/** The `RunProvability` for a lowered `provability:` block (see the module doc). */
|
|
47
|
+
export declare function buildRunProvability(config: IrProvability, opts: ProvabilityGateOptions): RunProvability;
|