@crewhaus/evaluation-service 0.0.0-stage → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +9 -0
- package/README.md +278 -2
- package/dist/child-skill.d.ts +33 -0
- package/dist/child-skill.js +6 -0
- package/dist/evaluation.d.ts +40 -0
- package/dist/evaluation.js +128 -0
- package/dist/evidence.d.ts +97 -0
- package/dist/evidence.js +147 -0
- package/dist/fragment.d.ts +40 -0
- package/dist/fragment.js +17 -0
- package/dist/head.d.ts +115 -0
- package/dist/head.js +181 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +24 -0
- package/dist/judge.d.ts +62 -0
- package/dist/judge.js +103 -0
- package/dist/observe.d.ts +32 -0
- package/dist/observe.js +31 -0
- package/dist/provability.d.ts +47 -0
- package/dist/provability.js +261 -0
- package/dist/render.d.ts +95 -0
- package/dist/render.js +99 -0
- package/dist/replay.d.ts +53 -0
- package/dist/replay.js +309 -0
- package/dist/roots.d.ts +36 -0
- package/dist/roots.js +90 -0
- package/dist/skills.d.ts +83 -0
- package/dist/skills.js +200 -0
- package/dist/tools.d.ts +106 -0
- package/dist/tools.js +74 -0
- package/dist/wire.d.ts +106 -0
- package/dist/wire.js +133 -0
- package/package.json +65 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
package/NOTICE
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
CrewHaus Factory
|
|
2
|
+
Copyright 2026 Max Meier
|
|
3
|
+
|
|
4
|
+
This product includes software developed by Max Meier
|
|
5
|
+
(operating as StudioMax — https://studiomax.io) and contributors.
|
|
6
|
+
|
|
7
|
+
The CrewHaus name and logo are trademarks of Max Meier. See
|
|
8
|
+
https://github.com/crewhaus/factory/blob/main/TRADEMARK.md for the
|
|
9
|
+
use-of-marks policy.
|
package/README.md
CHANGED
|
@@ -1,3 +1,279 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @crewhaus/evaluation-service
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
The **composition root for in-loop gates**, the `wireMemory` / `wireModels` precedent applied to
|
|
4
|
+
the turn-completion gates. One call builds the `evaluation:` judge, the `provability:` gate and
|
|
5
|
+
the proof tools from the lowered IR; every emitter that carries the gates renders that call, and
|
|
6
|
+
the `crewhaus run` / `serve` interpreter and `crewhaus eval` make it directly:
|
|
7
|
+
|
|
8
|
+
```ts
|
|
9
|
+
import {
|
|
10
|
+
evaluationFragmentFromIr,
|
|
11
|
+
registerGateTools,
|
|
12
|
+
wireEvaluation,
|
|
13
|
+
} from "@crewhaus/evaluation-service";
|
|
14
|
+
import { runChatLoop } from "@crewhaus/runtime-core";
|
|
15
|
+
import { defaultCatalog } from "@crewhaus/tool-catalog";
|
|
16
|
+
|
|
17
|
+
const fragment = evaluationFragmentFromIr(ir); // undefined: neither block declared
|
|
18
|
+
const gates = fragment
|
|
19
|
+
? await wireEvaluation(fragment, { cwd: process.cwd(), catalog: defaultCatalog, footerTarget: "stream" })
|
|
20
|
+
: { tools: [], proofTools: [] };
|
|
21
|
+
registerGateTools(defaultCatalog, gates); // or: tools = mergeGateTools(tools, gates)
|
|
22
|
+
|
|
23
|
+
await runChatLoop({
|
|
24
|
+
model, instructions, tools,
|
|
25
|
+
evaluation: gates.evaluation, // the quality judge (when declared)
|
|
26
|
+
provability: gates.provability, // the proof gate (when declared)
|
|
27
|
+
});
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Before this package the in-loop grader was built in four places that had to agree knob for
|
|
31
|
+
knob: `renderEvaluation` in target-cli, target-channel-bot and target-managed, and
|
|
32
|
+
`buildRunEvaluation` in `apps/cli`. The provability gate would have been the fifth. Now there is
|
|
33
|
+
one, and the rule from 0.6.0 covers gates too: a new in-loop gate lands here, never in an
|
|
34
|
+
emitter's codegen. runtime-core still receives closures (`RunEvaluation`, `RunProvability`) and
|
|
35
|
+
stays dependency-free.
|
|
36
|
+
|
|
37
|
+
## The fragment
|
|
38
|
+
|
|
39
|
+
```ts
|
|
40
|
+
type EvaluationFragment = {
|
|
41
|
+
specName: string; // where proofs are filed (see proofsDirName)
|
|
42
|
+
primaryModel: string; // the llm_judge default
|
|
43
|
+
evaluation?: IrEvaluation; // lowered `evaluation:`
|
|
44
|
+
provability?: IrProvability; // lowered `provability:` (absent at `off`)
|
|
45
|
+
};
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
`evaluationFragmentFromIr(ir)` returns `undefined` when the IR has neither block, the signal for
|
|
49
|
+
an emitter to render nothing, so those bundles keep their bytes. The fragment is plain data: an
|
|
50
|
+
emitter serializes it into the boot line and the interpreter builds the same object, so a
|
|
51
|
+
compiled bundle and `crewhaus run` hand the root the identical input.
|
|
52
|
+
|
|
53
|
+
Any name the spec schema admits wires. The proofs directory is `proofsDirName(specName)`: the
|
|
54
|
+
name itself when provability's store accepts it (`[A-Za-z0-9_.-]+`, not dots only), else its safe
|
|
55
|
+
characters plus 12 hex digits of its sha256 (`support bot` → `support_bot-<12 hex>`). Anything
|
|
56
|
+
that reads the proofs tree (`crewhaus provability`) uses the same function.
|
|
57
|
+
|
|
58
|
+
## `wireEvaluation(fragment, deps): Promise<WiredGates>`
|
|
59
|
+
|
|
60
|
+
```ts
|
|
61
|
+
type WireEvaluationDeps = {
|
|
62
|
+
cwd: string; // the workspace: path sources resolve here, proofs live under it
|
|
63
|
+
catalog?: GateCatalog; // where `checked` replay finds tools (a catalog or a tool list); absent ⇒ no replay
|
|
64
|
+
sessionRootDir?: string; // the root the run was given, if any
|
|
65
|
+
footerTarget?: "stream" | "reply" | "none";
|
|
66
|
+
streamed?: boolean; // the reader watches every attempt print (`crewhaus run --prompt`)
|
|
67
|
+
sandbox?: SandboxLike; // for `smt` / `lean4`; absent ⇒ chosen per run from what the run wires
|
|
68
|
+
resolveHead?: (url, signal?) => Promise<number | undefined>; // default: tool-fetch's guardedHead
|
|
69
|
+
headDeadlineMs?: number; // one verification's HEAD budget (default 15 s)
|
|
70
|
+
onBundle?: (bundle, bundlePath?) => void;
|
|
71
|
+
proofsRoot?: string; // the eval runner's per-sample proofs tree
|
|
72
|
+
judgeAdapter?; now?; // test seams
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
type WiredGates = {
|
|
76
|
+
evaluation?: RunEvaluation; // present iff the fragment carries `evaluation`
|
|
77
|
+
provability?: RunProvability; // present iff it carries `provability`
|
|
78
|
+
tools: RegisteredTool[]; // what the level adds to every run
|
|
79
|
+
proofTools: RegisteredTool[]; // every proof tool, bound to this gate
|
|
80
|
+
};
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Nothing is opened at wiring time: no directory, log, sandbox or socket. A malformed fragment (no
|
|
84
|
+
valid `provability.level`, a non-string `specName`) is refused at boot with
|
|
85
|
+
`EvaluationServiceError`.
|
|
86
|
+
|
|
87
|
+
**`evaluation`** is exactly the judge the four retired copies built: `llm_judge` through
|
|
88
|
+
`gradeWithJudgePanel` (panel, repeats, temperature, target, profile params; judge spend on the run
|
|
89
|
+
bus as `role: "judge"`; `judgeModel` names the instrument), `contains` / `regex` pure.
|
|
90
|
+
|
|
91
|
+
**`provability`** carries the IR's resolved `level`, `onFail`, `maxRetries`, `threshold`,
|
|
92
|
+
`escalateTo` and `render`, plus the closures runtime-core calls. runtime-core owns the gate (when
|
|
93
|
+
it runs, the retry, halt, note and escalate ladder, the trace and audit lines) and runs it before
|
|
94
|
+
the `evaluation:` judge, each gate with its own `on_fail` and `max_retries`:
|
|
95
|
+
|
|
96
|
+
- `verify(turn)` runs provability's `verifyDocument` over the turn's final text against an
|
|
97
|
+
evidence index built from the session event log (children included) merged with the
|
|
98
|
+
transcript. Under `scope: turn` the turn is everything after the last real user message (a
|
|
99
|
+
retry nudge does not start one), and the index holds the whole session with the turn marked
|
|
100
|
+
(`turnNumbers`), so grounding reads the turn while the self-written trace still sees a file an
|
|
101
|
+
earlier turn wrote. It also wires: proof artifacts from the spec's proofs root, with the
|
|
102
|
+
builtin checker re-checking a stored builtin proof (tool-proof's `proofRecheck`); the run's
|
|
103
|
+
own state directories as never-a-source (`runtimeStateRoots`); MCP origins read by the input
|
|
104
|
+
schemas of the run's own catalog; `checked` replay; the HEAD; and the waiver judge.
|
|
105
|
+
- `renderBundle` (the footer, or the JSON under `render: json`; absent under `render: none`),
|
|
106
|
+
`displayPath`, `nudge` (provability's `renderNudge`), `persist` (the bundle store, mode 0600),
|
|
107
|
+
`onBundle`, and `skill` (below).
|
|
108
|
+
|
|
109
|
+
**Replay** (`catalogReplay`) re-runs only a PURE tool (provability's `isPureTool`), parsed through
|
|
110
|
+
its own schema, bounded by `REPLAY_TIMEOUT_MS` (10 s) and the turn's signal, and compared in the
|
|
111
|
+
shape the log holds it in: a result over the tool-result-store threshold against its logged
|
|
112
|
+
preview. It is never read as a mismatch when the tool does not reproduce: a call that left out a
|
|
113
|
+
`now` its schema offers read the real clock, so it is not re-run; two re-runs that disagree with
|
|
114
|
+
each other mean "could not replay"; and a tool that declares a path among its operative
|
|
115
|
+
arguments, or whose logged input names something in the workspace, is not re-run either,
|
|
116
|
+
because its answer was about the workspace as it was.
|
|
117
|
+
|
|
118
|
+
**The HEAD** behind `sources.resolve: head` (`createHeadResolver`) goes through tool-fetch's
|
|
119
|
+
`guardedHead`: the spec's allow list on the cited URL, its deny list and the private-host rule on
|
|
120
|
+
every redirect hop, DNS bounded. One per URL per session; only answers are cached (a timeout or
|
|
121
|
+
network failure is asked again on the next verification), process-wide for the default request
|
|
122
|
+
(`SHARED_HEAD_CACHE`), so a per-turn wiring asks once. Before it is sent, each URL goes through
|
|
123
|
+
the run's egress fabric (`ProvabilityTurn.screenEgress`, the egress classifier over the run's
|
|
124
|
+
data lineage at `external-dynamic`): a URL carrying content a tool read is never sent, unless an
|
|
125
|
+
outside reader (a network-reading tool's search hit or fetched page) handed the run that URL
|
|
126
|
+
verbatim and nothing it carries reached the run any other way. A refused URL is `unresolved`.
|
|
127
|
+
Each request decided is recorded as an `egress_decision` (sink `provability.resolve`, host and
|
|
128
|
+
outcome only, never the URL) on the run's `egressAuditSink`.
|
|
129
|
+
|
|
130
|
+
**The waiver judge** (`createWaiverJudge`), when `provability.judge` is set: one sentinel-wrapped,
|
|
131
|
+
two-label categorical call per waivable paraphrase miss, at temperature 0, metered on the turn's
|
|
132
|
+
bus as `role: "judge"`, and skipped (not guessed) once the run's `budget.judge_share` is spent.
|
|
133
|
+
|
|
134
|
+
## Footer targets
|
|
135
|
+
|
|
136
|
+
`"stream"` on the cli (written after the answer), `"reply"` where the returned text is what gets
|
|
137
|
+
sent (channel bots), `"none"` where the caller carries the bundle on its own envelope and the
|
|
138
|
+
returned text must be the answer alone (the managed gateway's `proof`, the eval bridges). On
|
|
139
|
+
`"stream"`, and wherever `streamed: true` says the reader watches every attempt print, the
|
|
140
|
+
runtime marks a rejected attempt ✗ before it re-runs the turn, and when the final message states
|
|
141
|
+
nothing ("Done.") it grades the turn's printed text instead: everything since the last attempt a
|
|
142
|
+
pass rejected, with narration of what the agent is about to do left out ("Let me fetch the
|
|
143
|
+
policy page first.") unless it states something past its announcement or brings a figure of its
|
|
144
|
+
own. On every target, a line of the answer shaped like the footer is served with
|
|
145
|
+
`(model-written, not the verdict)` in front of it.
|
|
146
|
+
|
|
147
|
+
## The proof tools
|
|
148
|
+
|
|
149
|
+
`tools` is what the level adds: `ProvabilityPreflight` always; `ProofCheckers` + `ProofCheck` at
|
|
150
|
+
`proven`, or when the lowered `formal.checkers` is not the default `[builtin, smt]`
|
|
151
|
+
(`wantsProofCheckTools`).
|
|
152
|
+
`proofTools` is every proof tool, built by `@crewhaus/tool-proof`'s `createProofTools` over the
|
|
153
|
+
gate's configuration, proofs root and evidence builder. A proof tool the run already has (a spec that
|
|
154
|
+
listed `proofCheck` at `checked`) is replaced by its bound twin, since the builtin instance files
|
|
155
|
+
its artifacts where the gate never looks. `registerGateTools(catalog, gates)` (call it after the
|
|
156
|
+
builtin registrations and before the plugin ones) and `mergeGateTools(tools, gates)` apply that
|
|
157
|
+
rule.
|
|
158
|
+
|
|
159
|
+
`ProvabilityPreflight` and `ProofCheckers` are read-only and carry a builtin `alwaysAllow`
|
|
160
|
+
(permission-engine's bookkeeping rules), so the gate's own nudge never parks a headless turn. So
|
|
161
|
+
does `ProofCheck(builtin)`: the sandbox-free checker writes one artifact and starts nothing.
|
|
162
|
+
`ProofCheck(smt)` and `ProofCheck(lean4)` start a container and keep the mode's default; on a
|
|
163
|
+
channel or managed harness `crewhaus lint` says so (`provability-proofcheck-asks`), with the rule
|
|
164
|
+
that allows them.
|
|
165
|
+
|
|
166
|
+
## Tenancy
|
|
167
|
+
|
|
168
|
+
Every root (proofs, session logs, spilled results) is resolved per turn, and per call for the
|
|
169
|
+
tools, for the active tenant (`@crewhaus/tenancy`), and fenced: a root outside the tenant's fails
|
|
170
|
+
closed. One wiring at a managed daemon's boot serves every tenant, and the footer shows a bundle
|
|
171
|
+
path from the tenant's own root, never the server's layout. `gateProofsRoot`, `gateSessionRoot`,
|
|
172
|
+
`fencedDir` and `activeTenant` resolve them through the same tool-proof functions the tools use
|
|
173
|
+
(`defaultProofsRoot`, `defaultSessionRoot`), so an artifact the tools file resolves at the gate.
|
|
174
|
+
|
|
175
|
+
## One wiring, many runs: `observeProofBundles`
|
|
176
|
+
|
|
177
|
+
`deps.onBundle` sees every bundle the wiring grades. A surface that wires once and serves
|
|
178
|
+
concurrent runs (the managed gateway) needs the bundle of one run, so it composes a per-run
|
|
179
|
+
observer onto the shared gate for that run's `runChatLoop`:
|
|
180
|
+
|
|
181
|
+
```ts
|
|
182
|
+
runChatLoop({ …, provability: observeProofBundles(gates.provability, (bundle) => { proof = bundle; }) });
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
Identity without an observer; the gate's own `onBundle` runs first; the runtime calls `onBundle`
|
|
186
|
+
once per graded pass, so the last call a run sees is the standing verdict.
|
|
187
|
+
|
|
188
|
+
## `renderEvaluationWiring(ir, opts)`
|
|
189
|
+
|
|
190
|
+
The codegen twin: `{ imports, bootBlock, field, toolsRegistration, toolsExpr }`, all empty when
|
|
191
|
+
the IR has neither block. The boot line is `const __gates = await wireEvaluation(<fragment JSON>,
|
|
192
|
+
{ cwd, … });`; the fields are `evaluation: __gates.evaluation,` / `provability:
|
|
193
|
+
__gates.provability,` for the keys the IR has; `registerGateTools(<catalog>, __gates);` puts the
|
|
194
|
+
proof tools on the bundle's catalog, or in `toolsInto: "list"` mode `toolsExpr` wraps the run's
|
|
195
|
+
tool list with `mergeGateTools(…, __gates)`. `bundleObserverExpr` (the managed emitter's
|
|
196
|
+
per-run observer) renders the field as `provability: observeProofBundles(__gates.provability,
|
|
197
|
+
<expr>),` instead. The cli, channel and managed emitters render it; the research shape imports
|
|
198
|
+
`createHeadResolver` and `createWaiverJudge` for its report gate.
|
|
199
|
+
|
|
200
|
+
## The `provability` skill: `provabilitySkills(block)` / `renderGateSkills(ir)`
|
|
201
|
+
|
|
202
|
+
A gated harness advertises `@crewhaus/default-skills`' `provability` skill: what the gate checks
|
|
203
|
+
at each rung and how to cite so an answer passes. `provabilitySkillSubstitutions` renders it from
|
|
204
|
+
the block: the floor and checker order, and every setting that changes what this gate accepts
|
|
205
|
+
(`on_fail` and `max_retries`, `scope`, `assumptions` and `max_assumptions`, how many independent
|
|
206
|
+
sources a `grounded` claim needs), plus the proof-tool section exactly where
|
|
207
|
+
`wantsProofCheckTools` wires the pair. Settings that only make the gate more lenient than the
|
|
208
|
+
text (`threshold` below 1, `bibliographic`, `replay: false`) are left out.
|
|
209
|
+
|
|
210
|
+
`provabilitySkills(ir.provability)` returns it as a `LoadedSkill` (or `[]` without the block),
|
|
211
|
+
and every surface passes it as a builtin to whatever builds its skill list:
|
|
212
|
+
`discoverSkills({ builtinSkills })`, or `wireMemory(…, { builtinSkills })` where continuity owns
|
|
213
|
+
the surface. Lowest precedence, so a user or project skill named `provability` replaces it, and
|
|
214
|
+
classified like every builtin. It is a pure function of the block rather than part of
|
|
215
|
+
`wireEvaluation`'s result because every surface discovers its skills before it wires the gates.
|
|
216
|
+
`renderGateSkills(ir)` is its codegen twin, `{ imports, expr }`, both empty without the block.
|
|
217
|
+
|
|
218
|
+
`RunProvability.skill` is the same rendering as text (`provabilitySkillText`). The gate never
|
|
219
|
+
reads it: runtime-core projects it, with the floor, onto the bridge, and the sub-agent spawner
|
|
220
|
+
puts it in a Task child's instructions, so a researcher child cites the way the parent's gate
|
|
221
|
+
reads. Only the parent's final text is gated.
|
|
222
|
+
|
|
223
|
+
## Granular entry points
|
|
224
|
+
|
|
225
|
+
`wireEvaluation` composes these, and each is exported for a caller that needs one piece:
|
|
226
|
+
|
|
227
|
+
- `buildRunEvaluation(evaluation, primaryModel, opts?)`: the `RunEvaluation` for a lowered
|
|
228
|
+
`evaluation:` block alone.
|
|
229
|
+
- `buildRunProvability(provability, opts)`: the `RunProvability` alone, without the proof tools.
|
|
230
|
+
Its options (`ProvabilityGateOptions`) are `WireEvaluationDeps` without `sandbox`, plus the
|
|
231
|
+
`specName`.
|
|
232
|
+
- `gateEvidenceScoped(input)`: the evidence the gate and the preflight both verify against, as
|
|
233
|
+
`{ index, turnNumbers? }` (`turnNumbers` under `scope: turn` only). `gateEvidence(input)` is the
|
|
234
|
+
index alone. The input (`GateEvidenceInput`) names the scope, the session and its log root, the
|
|
235
|
+
workspace, the spill root, the transcript when there is one, and the input-schema lookup MCP
|
|
236
|
+
origins are read by. `currentTurnMessages` and `startsTurnMessage` are the turn rule over a
|
|
237
|
+
transcript: a turn starts at a user message the human sent, never one the runtime injected or
|
|
238
|
+
one carrying only tool results.
|
|
239
|
+
- `gateTools(opts)`: the bound proof tools for one configuration, `{ tools, proofTools }` as
|
|
240
|
+
`WiredGates` carries them; `wantsProofCheckTools(config)` and `DEFAULT_FORMAL_CHECKERS`
|
|
241
|
+
(`builtin`, `smt`) decide what `tools` holds.
|
|
242
|
+
- Replay: `catalogReplay(catalog, opts?)`, `lookupTool(catalog, name)` over either catalog form,
|
|
243
|
+
`resultText(result)` (a tool result's text blocks joined, as the evidence index reads one),
|
|
244
|
+
`REPLAY_TIMEOUT_MS` and `REPLAY_RECHECK_MS` (the pause before a disagreeing replay runs again).
|
|
245
|
+
- The HEAD: `createHeadResolver(sources, head?, cache?)`, `sourceHostPolicy(sources)` (the spec's
|
|
246
|
+
source policy as a host test) and `hopPolicy(sources)` (that policy on the cited URL, the
|
|
247
|
+
private-host rule and `deny` on every redirect hop), `SHARED_HEAD_CACHE`,
|
|
248
|
+
`HEAD_CACHE_ENTRIES` (the most answers one cache keeps) and `RESOLVE_SINK_ID`
|
|
249
|
+
(`provability.resolve`).
|
|
250
|
+
- The waiver: `createWaiverJudge(opts)`, its rubric `WAIVER_RUBRIC` and its two labels,
|
|
251
|
+
`WAIVER_SUPPORTED` and `WAIVER_NOT_SUPPORTED`.
|
|
252
|
+
- `GATES_VAR` (`__gates`), the variable `renderEvaluationWiring`'s boot line binds.
|
|
253
|
+
|
|
254
|
+
## Known limits
|
|
255
|
+
|
|
256
|
+
- A HEAD sent through a resolver the caller built itself, without a `screen`, is recorded but not
|
|
257
|
+
screened. The research report gate screens its own HEADs against every branch's lineage
|
|
258
|
+
(runtime-core's `lineageEgressScreen`), letting through only a URL its crawler already
|
|
259
|
+
requested; a resumed run screens against the branches it ran itself.
|
|
260
|
+
- Under a managed tenant, `cited` path sources resolve against the daemon's workspace (where the
|
|
261
|
+
managed file tools read too), so whether a path exists there shows in the verdict.
|
|
262
|
+
|
|
263
|
+
## Tests
|
|
264
|
+
|
|
265
|
+
```sh
|
|
266
|
+
bun test src
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
`acceptance.test.ts` runs the design's three scenarios through the live loop (`runChatLoop` with
|
|
270
|
+
a scripted model, `wireEvaluation`, the real verifier and session event log), the model
|
|
271
|
+
misbehaving each time: the fabricated citation at `grounded`, the hand-computed total at
|
|
272
|
+
`checked` with the real `MoneyAdd`, and the overlap claim at `proven` with the builtin checker.
|
|
273
|
+
`loop.test.ts` runs the honest and self-correcting paths, `acceptance-text.test.ts` holds the
|
|
274
|
+
scenarios against the extractor's harder shapes (a claim joined to a follow-up question, written
|
|
275
|
+
as a heading), and `provenance-loop.test.ts` holds the self-written and MCP-origin rules in the
|
|
276
|
+
loop. No test needs a network, a model or Docker.
|
|
277
|
+
|
|
278
|
+
Design: [design/0.8.0-provability-release.md](../../design/0.8.0-provability-release.md) §4.4,
|
|
279
|
+
§5, §8.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `provability` skill text a gated run's Task children receive
|
|
3
|
+
* (design/0.8.0-provability-release.md §8, §11).
|
|
4
|
+
*
|
|
5
|
+
* A child is never gated — only the parent's final text is — but it is the
|
|
6
|
+
* one fetching sources for the parent to cite, so it is told how the parent's
|
|
7
|
+
* gate reads a citation. runtime-core projects `RunProvability.skill` (with
|
|
8
|
+
* the floor) onto the bridge as the read-only provability seam, and the
|
|
9
|
+
* spawner puts it in the child's instructions after its own framing
|
|
10
|
+
* (`@crewhaus/sub-agent-spawner`'s `childProvabilityGuidance`).
|
|
11
|
+
*
|
|
12
|
+
* The text is the skill the parent itself advertises — `./skills`'
|
|
13
|
+
* `provabilitySkills`, the ONE renderer of `@crewhaus/default-skills`'
|
|
14
|
+
* `provability` skill (the floor, the checkers, the gate's settings; the
|
|
15
|
+
* `learning-loop` precedent) — so a parent and its children never read two
|
|
16
|
+
* renderings of one block. It is written to the gated model, so the spawner
|
|
17
|
+
* shows a child only the parts true for it: the sections after the skill's
|
|
18
|
+
* opening (what happens to a failing answer is the parent's alone), and a
|
|
19
|
+
* list item naming a provability tool only when the child's own tool set has
|
|
20
|
+
* it. The layout that rests on — the opening before the first `## ` section,
|
|
21
|
+
* each tool named in a list item — is pinned by this module's test.
|
|
22
|
+
*
|
|
23
|
+
* Undefined — and the children get the spawner's framing alone — when this
|
|
24
|
+
* build ships no such skill, or when its body asks for a substitution the
|
|
25
|
+
* block cannot fill (warned by the renderer: a wiring gap, not a reason to
|
|
26
|
+
* fail a run).
|
|
27
|
+
*/
|
|
28
|
+
import type { IrProvability } from "@crewhaus/ir";
|
|
29
|
+
import { type ProvabilitySkillSource } from "./skills";
|
|
30
|
+
/** Where the skill comes from (default: the shipped default skills); a seam for tests. */
|
|
31
|
+
export type ChildSkillSource = ProvabilitySkillSource;
|
|
32
|
+
/** The rendered `provability` skill for `config`, or undefined (see the module doc). */
|
|
33
|
+
export declare function provabilitySkillText(config: IrProvability, source?: ChildSkillSource): string | undefined;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { provabilitySkills } from "./skills";
|
|
2
|
+
/** The rendered `provability` skill for `config`, or undefined (see the module doc). */
|
|
3
|
+
export function provabilitySkillText(config, source) {
|
|
4
|
+
const text = provabilitySkills(config, source)[0]?.body.trim() ?? "";
|
|
5
|
+
return text === "" ? undefined : text;
|
|
6
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The in-loop `evaluation:` gate (loop contract 0.4, G02; 0.6.0 §6.2 / §7.3),
|
|
3
|
+
* built ONCE, here, from the lowered `IrEvaluation`.
|
|
4
|
+
*
|
|
5
|
+
* Until 0.8.0 this closure existed four times: the three `renderEvaluation`
|
|
6
|
+
* copies in target-cli / target-channel-bot / target-managed rendered it as
|
|
7
|
+
* inline source, and `apps/cli`'s `buildRunEvaluation` built it for `crewhaus
|
|
8
|
+
* run`. They had to agree knob for knob and were kept in step by hand. Design
|
|
9
|
+
* §4.4 collapses them into this function — the `wireMemory` / `wireModels`
|
|
10
|
+
* rule applied to in-loop gates — and the emitters render one
|
|
11
|
+
* `wireEvaluation(…)` call instead. The behaviour is exactly the one the four
|
|
12
|
+
* copies had:
|
|
13
|
+
*
|
|
14
|
+
* - `llm_judge` grades through `@crewhaus/eval-judge`'s `gradeWithJudgePanel`
|
|
15
|
+
* (one `createJudgeGrader` fan-out), honouring a declared `judges` panel,
|
|
16
|
+
* `repeats`, `temperature`, `target` and the judge profile's pinned
|
|
17
|
+
* `params`; the judge model defaults to the shape's primary when the spec
|
|
18
|
+
* omitted `grader.model`. Every judge call publishes on the RUN bus the
|
|
19
|
+
* runtime hands the grader (`role: "judge"`), so it is priced and counted
|
|
20
|
+
* under `budget.judge_share`; the verdict's wire model and priced spend
|
|
21
|
+
* come back as `EvaluationResult.judge`. `judgeModel` names the
|
|
22
|
+
* INSTRUMENT up front (a panel is its members joined with `+`), so the
|
|
23
|
+
* pool's per-arm quality lineage folds the judge's identity. An
|
|
24
|
+
* abstaining judge scores 0 (`gradeWithJudgePanel` already says so).
|
|
25
|
+
* - `contains` / `regex` are pure (score 1|0, no model spend, threshold 1);
|
|
26
|
+
* the regex's `lastIndex` is reset per call so a global/sticky flag can
|
|
27
|
+
* never flip a verdict between turns.
|
|
28
|
+
*
|
|
29
|
+
* `threshold`, `onFail` and `maxRetries` are the IR's resolved values, and the
|
|
30
|
+
* cascade's lower-time `escalateTo` is threaded only under `on_fail: escalate`.
|
|
31
|
+
*/
|
|
32
|
+
import { type JudgePanelKnobs } from "@crewhaus/eval-judge";
|
|
33
|
+
import type { IrEvaluation } from "@crewhaus/ir";
|
|
34
|
+
import type { RunEvaluation } from "@crewhaus/runtime-core";
|
|
35
|
+
export type EvaluationGateOptions = {
|
|
36
|
+
/** Test seam: a pre-built judge adapter that bypasses the model-router. */
|
|
37
|
+
readonly judgeAdapter?: JudgePanelKnobs["adapter"];
|
|
38
|
+
};
|
|
39
|
+
/** The `RunEvaluation` for a lowered `evaluation:` block (see the module doc). */
|
|
40
|
+
export declare function buildRunEvaluation(ev: IrEvaluation, primaryModel: string, opts?: EvaluationGateOptions): RunEvaluation;
|