@crewhaus/provability 0.0.0-stage → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +9 -0
- package/README.md +300 -2
- package/dist/artifact.d.ts +9 -0
- package/dist/artifact.js +38 -0
- package/dist/builtins.d.ts +32 -0
- package/dist/builtins.js +46 -0
- package/dist/evidence.d.ts +326 -0
- package/dist/evidence.js +1391 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.js +22 -0
- package/dist/levels.d.ts +19 -0
- package/dist/levels.js +42 -0
- package/dist/origin.d.ts +83 -0
- package/dist/origin.js +296 -0
- package/dist/private-address.d.ts +34 -0
- package/dist/private-address.js +209 -0
- package/dist/purity.d.ts +18 -0
- package/dist/purity.js +15 -0
- package/dist/render.d.ts +47 -0
- package/dist/render.js +315 -0
- package/dist/store.d.ts +59 -0
- package/dist/store.js +319 -0
- package/dist/summary.d.ts +8 -0
- package/dist/summary.js +22 -0
- package/dist/types.d.ts +459 -0
- package/dist/types.js +2 -0
- package/dist/vacuous.d.ts +50 -0
- package/dist/vacuous.js +423 -0
- package/dist/verify.d.ts +176 -0
- package/dist/verify.js +3608 -0
- package/package.json +47 -4
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The private-address classifier, for the `cited` rung's `private_address`
|
|
3
|
+
* check (design §6.1): a citation of `http://169.254.169.254/` is not a
|
|
4
|
+
* source, however well-formed it is.
|
|
5
|
+
*
|
|
6
|
+
* The block below is the repo's ONE classifier, copied byte for byte from
|
|
7
|
+
* `@crewhaus/tool-fetch` — `apps/cli/src/tool-registry.test.ts` lists this
|
|
8
|
+
* package among the copies it sweeps, proves every copy identical and
|
|
9
|
+
* EXECUTES one. Never edit it here: change every copy at once, or none.
|
|
10
|
+
*/
|
|
11
|
+
// BEGIN SYNCHRONISED BLOCK — private-address classifier
|
|
12
|
+
//
|
|
13
|
+
// This block is BYTE-IDENTICAL across every package that guards an outbound
|
|
14
|
+
// request. Do not edit one copy: `apps/cli/src/tool-registry.test.ts` hashes
|
|
15
|
+
// them all and fails if any differs, and it asserts how many it found, because
|
|
16
|
+
// a copy-scanning guard that matches nothing reports green.
|
|
17
|
+
//
|
|
18
|
+
// It exists in copies rather than a shared package because these files are
|
|
19
|
+
// otherwise independent per-package networking layers; a guard proving the
|
|
20
|
+
// copies are identical is cheaper and safer than the import graph a shared
|
|
21
|
+
// package would need across `crawler`, `computer-use-driver` and ten tools.
|
|
22
|
+
//
|
|
23
|
+
// WHY IT PARSES INSTEAD OF MATCHING TEXT. Six copies were confirmed
|
|
24
|
+
// exploitable on 2026-09-18 because they compared address STRINGS. The WHATWG
|
|
25
|
+
// URL parser rewrites `[::ffff:169.254.169.254]` to `[::ffff:a9fe:a9fe]`, so a
|
|
26
|
+
// text check never sees the spelling it was written for; and `64:ff9b::a9fe:a9fe`
|
|
27
|
+
// IS 169.254.169.254 on any network running DNS64/NAT64. Parsing numerically and
|
|
28
|
+
// recursing into the embedded IPv4 is the only form that holds.
|
|
29
|
+
//
|
|
30
|
+
// WHAT IT CANNOT DO. RFC 6052 lets an operator choose any Network-Specific
|
|
31
|
+
// Prefix for NAT64, so an embedded IPv4 behind an arbitrary NSP is undecidable
|
|
32
|
+
// from the address alone. That case is configuration, and the callers that need
|
|
33
|
+
// it resolve the host and re-check the ANSWER before dialling.
|
|
34
|
+
// ---------------------------------------------------------------------------
|
|
35
|
+
/**
|
|
36
|
+
* Canonicalise an IPv4 literal.
|
|
37
|
+
*
|
|
38
|
+
* `inet_aton` forms are the classic allow-list bypass: `0177.0.0.1`,
|
|
39
|
+
* `0x7f.0.0.1`, `2130706433` and `127.1` are all 127.0.0.1, and a check that
|
|
40
|
+
* only understands dotted-decimal waves every one of them through. Returns
|
|
41
|
+
* `null` when the string is not an IPv4 literal at all.
|
|
42
|
+
*/
|
|
43
|
+
export function normalizeIpv4(raw) {
|
|
44
|
+
const trimmed = raw.trim();
|
|
45
|
+
if (trimmed === "" || /[^0-9a-fA-FxX.]/.test(trimmed))
|
|
46
|
+
return null;
|
|
47
|
+
const parts = trimmed.split(".");
|
|
48
|
+
if (parts.length === 0 || parts.length > 4)
|
|
49
|
+
return null;
|
|
50
|
+
const values = [];
|
|
51
|
+
for (const part of parts) {
|
|
52
|
+
if (part === "")
|
|
53
|
+
return null;
|
|
54
|
+
let value;
|
|
55
|
+
if (/^0[xX][0-9a-fA-F]+$/.test(part))
|
|
56
|
+
value = Number.parseInt(part.slice(2), 16);
|
|
57
|
+
else if (/^0[0-7]+$/.test(part))
|
|
58
|
+
value = Number.parseInt(part.slice(1), 8);
|
|
59
|
+
else if (/^\d+$/.test(part))
|
|
60
|
+
value = Number.parseInt(part, 10);
|
|
61
|
+
else
|
|
62
|
+
return null;
|
|
63
|
+
if (!Number.isFinite(value) || value < 0)
|
|
64
|
+
return null;
|
|
65
|
+
values.push(value);
|
|
66
|
+
}
|
|
67
|
+
// The short forms pack the remaining octets into the last part: `127.1` is
|
|
68
|
+
// 127.0.0.1, not 127.1.0.0. Getting this backwards is how a bypass survives.
|
|
69
|
+
const last = values[values.length - 1];
|
|
70
|
+
const leading = values.slice(0, -1);
|
|
71
|
+
if (leading.some((v) => v > 255))
|
|
72
|
+
return null;
|
|
73
|
+
const remaining = 4 - leading.length;
|
|
74
|
+
if (last >= 2 ** (8 * remaining))
|
|
75
|
+
return null;
|
|
76
|
+
const octets = [...leading];
|
|
77
|
+
for (let i = remaining - 1; i >= 0; i--)
|
|
78
|
+
octets.push((last >>> (8 * i)) & 0xff);
|
|
79
|
+
return octets.join(".");
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Expand an IPv6 literal to its eight 16-bit groups, or `null` when the string
|
|
83
|
+
* is not one.
|
|
84
|
+
*
|
|
85
|
+
* Classifying IPv6 by its TEXT is where the bypasses live, because one address
|
|
86
|
+
* has many spellings and the one a check was written against is rarely the one
|
|
87
|
+
* that arrives. `http://[::ffff:169.254.169.254]/` never reaches a guard in
|
|
88
|
+
* that form: the WHATWG URL parser re-serialises the embedded quad as hex
|
|
89
|
+
* pieces, so what the guard sees is `::ffff:a9fe:a9fe`. Normalising to numbers
|
|
90
|
+
* first means the prefix tests below are arithmetic, and spelling stops
|
|
91
|
+
* mattering.
|
|
92
|
+
*/
|
|
93
|
+
export function parseIpv6(raw) {
|
|
94
|
+
let text = raw.trim().toLowerCase();
|
|
95
|
+
if (text.startsWith("["))
|
|
96
|
+
text = text.slice(1);
|
|
97
|
+
if (text.endsWith("]"))
|
|
98
|
+
text = text.slice(0, -1);
|
|
99
|
+
const zone = text.indexOf("%"); // fe80::1%eth0
|
|
100
|
+
if (zone !== -1)
|
|
101
|
+
text = text.slice(0, zone);
|
|
102
|
+
if (!text.includes(":"))
|
|
103
|
+
return null;
|
|
104
|
+
// A trailing dotted quad is the last two groups written in IPv4. Require all
|
|
105
|
+
// three dots: without that, `::1` parses as a one-part inet_aton address and
|
|
106
|
+
// takes a path that has nothing to do with what was written.
|
|
107
|
+
const dotted = /^(.*:)(\d+(?:\.\d+){3})$/.exec(text);
|
|
108
|
+
if (dotted !== null) {
|
|
109
|
+
const quad = normalizeIpv4(dotted[2]);
|
|
110
|
+
if (quad === null)
|
|
111
|
+
return null;
|
|
112
|
+
const o = quad.split(".").map((n) => Number.parseInt(n, 10));
|
|
113
|
+
const hi = (o[0] << 8) | o[1];
|
|
114
|
+
const lo = (o[2] << 8) | o[3];
|
|
115
|
+
text = `${dotted[1]}${hi.toString(16)}:${lo.toString(16)}`;
|
|
116
|
+
}
|
|
117
|
+
const halves = text.split("::");
|
|
118
|
+
if (halves.length > 2)
|
|
119
|
+
return null;
|
|
120
|
+
const pieces = (part) => (part === "" ? [] : part.split(":"));
|
|
121
|
+
const head = pieces(halves[0]);
|
|
122
|
+
const tail = halves.length === 2 ? pieces(halves[1]) : [];
|
|
123
|
+
if (halves.length === 1 && head.length !== 8)
|
|
124
|
+
return null;
|
|
125
|
+
if (head.length + tail.length > 8)
|
|
126
|
+
return null;
|
|
127
|
+
const groups = [];
|
|
128
|
+
for (const piece of head) {
|
|
129
|
+
if (!/^[0-9a-f]{1,4}$/.test(piece))
|
|
130
|
+
return null;
|
|
131
|
+
groups.push(Number.parseInt(piece, 16));
|
|
132
|
+
}
|
|
133
|
+
for (let i = head.length + tail.length; i < 8; i++)
|
|
134
|
+
groups.push(0);
|
|
135
|
+
for (const piece of tail) {
|
|
136
|
+
if (!/^[0-9a-f]{1,4}$/.test(piece))
|
|
137
|
+
return null;
|
|
138
|
+
groups.push(Number.parseInt(piece, 16));
|
|
139
|
+
}
|
|
140
|
+
return groups.length === 8 ? groups : null;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* The IPv4 address an IPv6 address carries, when it carries one.
|
|
144
|
+
*
|
|
145
|
+
* Every transition mechanism embeds a v4 address somewhere, and every one of
|
|
146
|
+
* them is a way to reach a v4 destination while wearing a v6 spelling that no
|
|
147
|
+
* v4 range check looks at. The NAT64 well-known prefix is the sharpest: a
|
|
148
|
+
* DNS64 resolver answers an IPv4-only name with `64:ff9b::<the v4>`, so
|
|
149
|
+
* `64:ff9b::a9fe:a9fe` IS the metadata service on any network that runs one.
|
|
150
|
+
*/
|
|
151
|
+
function embeddedIpv4(g) {
|
|
152
|
+
const quad = (hi, lo) => `${(hi >>> 8) & 0xff}.${hi & 0xff}.${(lo >>> 8) & 0xff}.${lo & 0xff}`;
|
|
153
|
+
const zeros = (from, to) => g.slice(from, to).every((x) => x === 0);
|
|
154
|
+
const last = quad(g[6], g[7]);
|
|
155
|
+
if (zeros(0, 5) && g[5] === 0xffff)
|
|
156
|
+
return last; // ::ffff:0:0/96, IPv4-mapped
|
|
157
|
+
if (zeros(0, 4) && g[4] === 0xffff && g[5] === 0)
|
|
158
|
+
return last; // ::ffff:0:0:0/96, translated
|
|
159
|
+
if (g[0] === 0x64 && g[1] === 0xff9b)
|
|
160
|
+
return last; // 64:ff9b::/96 and 64:ff9b:1::/48, NAT64
|
|
161
|
+
if (g[0] === 0x2002)
|
|
162
|
+
return quad(g[1], g[2]); // 2002::/16, 6to4
|
|
163
|
+
// ::a.b.c.d, IPv4-compatible: deprecated, still routed by some stacks, and
|
|
164
|
+
// not to be confused with `::` or `::1`, which are handled before this.
|
|
165
|
+
if (zeros(0, 6))
|
|
166
|
+
return last;
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
/** Ranges that are never a public RPC endpoint. */
|
|
170
|
+
export function isPrivateIp(address) {
|
|
171
|
+
const v4 = normalizeIpv4(address);
|
|
172
|
+
if (v4 !== null) {
|
|
173
|
+
const [a = 0, b = 0, c = 0, d = 0] = v4.split(".").map((n) => Number.parseInt(n, 10));
|
|
174
|
+
if (a === 0 || a === 10 || a === 127)
|
|
175
|
+
return true;
|
|
176
|
+
if (a === 169 && b === 254)
|
|
177
|
+
return true; // link-local, and the metadata service
|
|
178
|
+
if (a === 172 && b >= 16 && b <= 31)
|
|
179
|
+
return true;
|
|
180
|
+
if (a === 192 && b === 168)
|
|
181
|
+
return true;
|
|
182
|
+
if (a === 192 && b === 0 && c === 0)
|
|
183
|
+
return true;
|
|
184
|
+
if (a === 100 && b >= 64 && b <= 127)
|
|
185
|
+
return true; // carrier-grade NAT
|
|
186
|
+
if (a === 198 && (b === 18 || b === 19))
|
|
187
|
+
return true; // benchmarking
|
|
188
|
+
if (a >= 224)
|
|
189
|
+
return true; // multicast, reserved, broadcast
|
|
190
|
+
return a === 255 && b === 255 && c === 255 && d === 255;
|
|
191
|
+
}
|
|
192
|
+
const groups = parseIpv6(address);
|
|
193
|
+
if (groups === null)
|
|
194
|
+
return false;
|
|
195
|
+
if (groups.every((g) => g === 0))
|
|
196
|
+
return true; // ::, the unspecified address
|
|
197
|
+
if (groups.slice(0, 7).every((g) => g === 0) && groups[7] === 1)
|
|
198
|
+
return true; // ::1
|
|
199
|
+
const carried = embeddedIpv4(groups);
|
|
200
|
+
if (carried !== null)
|
|
201
|
+
return isPrivateIp(carried);
|
|
202
|
+
const head = groups[0];
|
|
203
|
+
if ((head & 0xfe00) === 0xfc00)
|
|
204
|
+
return true; // fc00::/7, unique-local
|
|
205
|
+
if ((head & 0xffc0) === 0xfe80)
|
|
206
|
+
return true; // fe80::/10, link-local
|
|
207
|
+
return (head & 0xff00) === 0xff00; // ff00::/8, multicast
|
|
208
|
+
}
|
|
209
|
+
// END SYNCHRONISED BLOCK
|
package/dist/purity.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which tools the `checked` rung may re-run (design §6.3, §10).
|
|
3
|
+
*
|
|
4
|
+
* Replay re-executes a tool with the input the log recorded and compares
|
|
5
|
+
* output digests, so it must only ever run a tool that cannot reach a
|
|
6
|
+
* network, a process or a filesystem write: the safety flags every
|
|
7
|
+
* registered tool declares already say which those are. One definition,
|
|
8
|
+
* here, so the gate's composition root and the preflight tool agree.
|
|
9
|
+
*/
|
|
10
|
+
import type { RegisteredTool } from "@crewhaus/tool-catalog";
|
|
11
|
+
/**
|
|
12
|
+
* Pure: read-only, safe to run concurrently, not destructive, no sandbox,
|
|
13
|
+
* internal scope and no declared network or process capability. `Bash`,
|
|
14
|
+
* `Python`, every fetch and every write fail at least one of these, so none
|
|
15
|
+
* is ever replayed — a spec that lists one on `execution.allow` opts into
|
|
16
|
+
* "it ran" without "it reproduces", and the binding says `replayed: false`.
|
|
17
|
+
*/
|
|
18
|
+
export declare function isPureTool(tool: Pick<RegisteredTool, "readOnly" | "concurrencySafe" | "destructive" | "requiresSandbox" | "scope" | "ioCapability">): boolean;
|
package/dist/purity.js
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure: read-only, safe to run concurrently, not destructive, no sandbox,
|
|
3
|
+
* internal scope and no declared network or process capability. `Bash`,
|
|
4
|
+
* `Python`, every fetch and every write fail at least one of these, so none
|
|
5
|
+
* is ever replayed — a spec that lists one on `execution.allow` opts into
|
|
6
|
+
* "it ran" without "it reproduces", and the binding says `replayed: false`.
|
|
7
|
+
*/
|
|
8
|
+
export function isPureTool(tool) {
|
|
9
|
+
return (tool.readOnly &&
|
|
10
|
+
tool.concurrencySafe &&
|
|
11
|
+
!tool.destructive &&
|
|
12
|
+
!tool.requiresSandbox &&
|
|
13
|
+
tool.scope === "internal" &&
|
|
14
|
+
tool.ioCapability === undefined);
|
|
15
|
+
}
|
package/dist/render.d.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The bundle beside the answer (design §5.4): one footer line, the JSON, the
|
|
3
|
+
* research report's `## Provability` section, and the retry nudge. Every
|
|
4
|
+
* surface renders the SAME bundle; these are its four spellings.
|
|
5
|
+
*
|
|
6
|
+
* The nudge is the one rendering the model reads, so it is the one with a
|
|
7
|
+
* trust rule (§5.1): it is composed from the model's own sentences, the
|
|
8
|
+
* evidence identifiers it cited, and fixed plain-words reasons — never from
|
|
9
|
+
* a fetched body (no excerpt) and never from judge text — so it crosses no
|
|
10
|
+
* trust boundary and needs no classification.
|
|
11
|
+
*/
|
|
12
|
+
import type { ProofBundle } from "./types";
|
|
13
|
+
export type FooterOptions = {
|
|
14
|
+
/** Where the bundle was persisted, shown last (`.crewhaus/proofs/…/turn-7.json`). */
|
|
15
|
+
readonly path?: string;
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* One line, after the answer (design §5.4):
|
|
19
|
+
*
|
|
20
|
+
* ── provability · grounded ✓ · 6/6 claims · 3 sources · 0 assumptions · <path>
|
|
21
|
+
* ── provability · grounded ✗ · 4/6 claims · unmet: [2] [5] · 3 sources · 0 assumptions · <path>
|
|
22
|
+
* ── provability · grounded ? unverified — <reason>
|
|
23
|
+
*
|
|
24
|
+
* `unmet:` lists the markers of the unmet claims' failing bindings — the
|
|
25
|
+
* reader finds them in the answer — and `+N uncited` for claims with none.
|
|
26
|
+
* Judge-waived bindings are counted separately: `6/6 claims (1 judge-waived)`,
|
|
27
|
+
* and so are refused assumptions: `3 assumptions (2 refused)`, and sources
|
|
28
|
+
* the run wrote itself: `3 sources (1 written by this run)`.
|
|
29
|
+
*/
|
|
30
|
+
export declare function renderFooter(bundle: ProofBundle, opts?: FooterOptions): string;
|
|
31
|
+
/** The bundle as pretty-printed JSON (`render: json`, `--json`). */
|
|
32
|
+
export declare function renderBundleJson(bundle: ProofBundle): string;
|
|
33
|
+
/**
|
|
34
|
+
* The research report's `## Provability` section (design §5.4, §8): the
|
|
35
|
+
* floor, the verdict, the score, a table of every claim with its status,
|
|
36
|
+
* best rung and evidence, then the assumptions and the checkers whose
|
|
37
|
+
* proofs back `proven` claims, each with its formal statement beside the
|
|
38
|
+
* prose — the formalization gap shown, not hidden (§6.4).
|
|
39
|
+
*/
|
|
40
|
+
export declare function renderMarkdownSection(bundle: ProofBundle): string;
|
|
41
|
+
/**
|
|
42
|
+
* The retry nudge (design §5.1): which claims fell short, in the model's own
|
|
43
|
+
* words, what each cited and why it did not count — and the standing
|
|
44
|
+
* instruction to run `ProvabilityPreflight` on the next draft. No excerpt,
|
|
45
|
+
* no fetched text, no judge text.
|
|
46
|
+
*/
|
|
47
|
+
export declare function renderNudge(bundle: ProofBundle): string;
|
package/dist/render.js
ADDED
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
import { claimReasons, clip } from "./verify";
|
|
2
|
+
const plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
|
|
3
|
+
/** An allowed assumption binding: never `ok` (it is not evidence), and not a failure either. */
|
|
4
|
+
const allowedAssumption = (b) => b.kind === "assumption" && b.reason === undefined;
|
|
5
|
+
/**
|
|
6
|
+
* The refs that failed in a failing answer's unmet claims (each once), and
|
|
7
|
+
* how many claims cite nothing. An allowed assumption beside failing
|
|
8
|
+
* evidence is not one of them: "[1] [A1]" with [1] wrong lists [1].
|
|
9
|
+
*/
|
|
10
|
+
function unmetRefs(bundle) {
|
|
11
|
+
const refs = [];
|
|
12
|
+
let uncited = 0;
|
|
13
|
+
for (const claim of bundle.claims) {
|
|
14
|
+
if (claim.status !== "unmet")
|
|
15
|
+
continue;
|
|
16
|
+
if (claim.bindings.length === 0)
|
|
17
|
+
uncited += 1;
|
|
18
|
+
for (const b of claim.bindings) {
|
|
19
|
+
if (!b.ok && !allowedAssumption(b) && !refs.includes(b.ref))
|
|
20
|
+
refs.push(b.ref);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
return { refs, uncited };
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Every assumption the answer rests on, accepted or refused — design §2:
|
|
27
|
+
* assumptions are always counted, so one refused (`forbid`, or past
|
|
28
|
+
* `max_assumptions`) still reads as an assumption, with how many were
|
|
29
|
+
* refused beside it: `3 assumptions (2 refused)`. The hypotheses an
|
|
30
|
+
* accepted proof rests on are assumptions too (§6.4, `hypotheses`): a
|
|
31
|
+
* proof that assumes its own conclusion does not read "0 assumptions".
|
|
32
|
+
*/
|
|
33
|
+
function assumptionsCount(bundle) {
|
|
34
|
+
return bundle.assumptions + (bundle.hypotheses ?? 0) + (bundle.assumptionsRefused ?? 0);
|
|
35
|
+
}
|
|
36
|
+
function assumptionsSegment(bundle) {
|
|
37
|
+
const refused = bundle.assumptionsRefused ?? 0;
|
|
38
|
+
const declared = plural(assumptionsCount(bundle), "assumption");
|
|
39
|
+
return refused > 0 ? `${declared} (${refused} refused)` : declared;
|
|
40
|
+
}
|
|
41
|
+
function claimsSegment(bundle) {
|
|
42
|
+
const counted = bundle.claimCount - bundle.assumptions;
|
|
43
|
+
let text = `${bundle.met}/${counted} ${counted === 1 ? "claim" : "claims"}`;
|
|
44
|
+
const notes = [];
|
|
45
|
+
if (bundle.vacuous !== undefined)
|
|
46
|
+
notes.push(bundle.vacuous);
|
|
47
|
+
if (bundle.judgeWaived > 0)
|
|
48
|
+
notes.push(`${bundle.judgeWaived} judge-waived`);
|
|
49
|
+
if (notes.length > 0)
|
|
50
|
+
text += ` (${notes.join(", ")})`;
|
|
51
|
+
return text;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* One line, after the answer (design §5.4):
|
|
55
|
+
*
|
|
56
|
+
* ── provability · grounded ✓ · 6/6 claims · 3 sources · 0 assumptions · <path>
|
|
57
|
+
* ── provability · grounded ✗ · 4/6 claims · unmet: [2] [5] · 3 sources · 0 assumptions · <path>
|
|
58
|
+
* ── provability · grounded ? unverified — <reason>
|
|
59
|
+
*
|
|
60
|
+
* `unmet:` lists the markers of the unmet claims' failing bindings — the
|
|
61
|
+
* reader finds them in the answer — and `+N uncited` for claims with none.
|
|
62
|
+
* Judge-waived bindings are counted separately: `6/6 claims (1 judge-waived)`,
|
|
63
|
+
* and so are refused assumptions: `3 assumptions (2 refused)`, and sources
|
|
64
|
+
* the run wrote itself: `3 sources (1 written by this run)`.
|
|
65
|
+
*/
|
|
66
|
+
export function renderFooter(bundle, opts = {}) {
|
|
67
|
+
const head = `── provability · ${bundle.level}`;
|
|
68
|
+
const tail = opts.path !== undefined && opts.path !== "" ? ` · ${opts.path}` : "";
|
|
69
|
+
if (bundle.verdict === "unverified") {
|
|
70
|
+
return `${head} ? unverified — ${clip(bundle.error ?? "the verifier failed", 200)}${tail}`;
|
|
71
|
+
}
|
|
72
|
+
const parts = [`${head} ${bundle.verdict === "pass" ? "✓" : "✗"}`, claimsSegment(bundle)];
|
|
73
|
+
if (bundle.verdict === "fail" || bundle.unmet > 0) {
|
|
74
|
+
const { refs, uncited } = unmetRefs(bundle);
|
|
75
|
+
const shown = refs.slice(0, 8);
|
|
76
|
+
const listed = [...shown, ...(refs.length > shown.length ? ["…"] : [])];
|
|
77
|
+
if (uncited > 0)
|
|
78
|
+
listed.push(listed.length > 0 ? `+${uncited} uncited` : `${uncited} uncited`);
|
|
79
|
+
if (listed.length > 0)
|
|
80
|
+
parts.push(`unmet: ${listed.join(" ")}`);
|
|
81
|
+
}
|
|
82
|
+
parts.push(sourcesSegment(bundle), assumptionsSegment(bundle));
|
|
83
|
+
return `${parts.join(" · ")}${tail}`;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* "N sources", and how many of them the run wrote itself before it read
|
|
87
|
+
* them back (`selfWritten`, design §10): `2 sources (1 written by this run)`.
|
|
88
|
+
*/
|
|
89
|
+
function sourcesSegment(bundle) {
|
|
90
|
+
const sources = plural(bundle.sources, "source");
|
|
91
|
+
const own = bundle.selfWritten ?? 0;
|
|
92
|
+
return own > 0 ? `${sources} (${own} written by this run)` : sources;
|
|
93
|
+
}
|
|
94
|
+
/** The bundle as pretty-printed JSON (`render: json`, `--json`). */
|
|
95
|
+
export function renderBundleJson(bundle) {
|
|
96
|
+
return JSON.stringify(bundle, null, 2);
|
|
97
|
+
}
|
|
98
|
+
/** A rung or reason in words: `not_fetched` → `not fetched`. */
|
|
99
|
+
const words = (token) => token.replace(/_/g, " ");
|
|
100
|
+
const cell = (text) => text.replace(/\|/g, "\\|").replace(/\s+/g, " ").trim();
|
|
101
|
+
function bindingCell(b) {
|
|
102
|
+
if (allowedAssumption(b))
|
|
103
|
+
return `${b.ref} assumption`;
|
|
104
|
+
if (b.ok) {
|
|
105
|
+
const notes = [
|
|
106
|
+
...(b.judgeWaived === true ? ["judge-waived"] : []),
|
|
107
|
+
...(b.writtenBy !== undefined ? ["written by this run"] : []),
|
|
108
|
+
];
|
|
109
|
+
return `${b.ref} ✓ ${b.rung}${notes.length > 0 ? ` (${notes.join(", ")})` : ""}`;
|
|
110
|
+
}
|
|
111
|
+
return `${b.ref} ✗ ${words(b.reason ?? "below_floor")}`;
|
|
112
|
+
}
|
|
113
|
+
function evidenceCell(claim) {
|
|
114
|
+
if (claim.bindings.length === 0)
|
|
115
|
+
return "✗ no citation";
|
|
116
|
+
const cells = claim.bindings.map(bindingCell).join("; ");
|
|
117
|
+
return claim.definition !== undefined
|
|
118
|
+
? `${cells} (text of definition ${claim.definition})`
|
|
119
|
+
: cells;
|
|
120
|
+
}
|
|
121
|
+
const percent = (n) => `${Math.round(n * 100)}%`;
|
|
122
|
+
/**
|
|
123
|
+
* The research report's `## Provability` section (design §5.4, §8): the
|
|
124
|
+
* floor, the verdict, the score, a table of every claim with its status,
|
|
125
|
+
* best rung and evidence, then the assumptions and the checkers whose
|
|
126
|
+
* proofs back `proven` claims, each with its formal statement beside the
|
|
127
|
+
* prose — the formalization gap shown, not hidden (§6.4).
|
|
128
|
+
*/
|
|
129
|
+
export function renderMarkdownSection(bundle) {
|
|
130
|
+
const lines = ["## Provability", ""];
|
|
131
|
+
const counted = bundle.claimCount - bundle.assumptions;
|
|
132
|
+
if (bundle.verdict === "unverified") {
|
|
133
|
+
lines.push(`**Floor:** \`${bundle.level}\` · **Verdict:** unverified — ${cell(bundle.error ?? "the verifier failed")}`, "", "The claims of this report were not checked: the verifier failed, so no verdict is given.");
|
|
134
|
+
return `${lines.join("\n")}\n`;
|
|
135
|
+
}
|
|
136
|
+
lines.push(`**Floor:** \`${bundle.level}\` · **Verdict:** ${bundle.verdict === "pass" ? "pass ✓" : "fail ✗"} · **Score:** ${bundle.met}/${counted} ${counted === 1 ? "claim" : "claims"} (${percent(bundle.score)}, threshold ${percent(bundle.threshold)}) · **Achieved:** \`${bundle.achieved}\``);
|
|
137
|
+
if (bundle.vacuous !== undefined)
|
|
138
|
+
lines.push("", `Vacuous pass: ${bundle.vacuous}.`);
|
|
139
|
+
if (bundle.judgeWaived > 0) {
|
|
140
|
+
lines.push("", `${plural(bundle.judgeWaived, "binding was", "bindings were")} accepted by the paraphrase-waiver judge${bundle.judgeModel !== undefined ? ` (${bundle.judgeModel})` : ""} — an opinion, not a deterministic check.`);
|
|
141
|
+
}
|
|
142
|
+
if ((bundle.selfWritten ?? 0) > 0) {
|
|
143
|
+
lines.push("", `${plural(bundle.selfWritten ?? 0, "source grounds", "sources ground")} only in text this run wrote there itself before reading it back — the run is its own source there.`);
|
|
144
|
+
}
|
|
145
|
+
if (bundle.claims.length > 0) {
|
|
146
|
+
lines.push("", "| # | Claim | Status | Best | Evidence |", "| --- | --- | --- | --- | --- |");
|
|
147
|
+
for (const claim of bundle.claims) {
|
|
148
|
+
lines.push(`| ${claim.index + 1} | ${cell(clip(claim.text, 160))} | ${claim.status} | ${claim.best} | ${cell(evidenceCell(claim))} |`);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
// Declared assumptions, then the hypotheses each cited proof rests on.
|
|
152
|
+
const assumed = [];
|
|
153
|
+
for (const claim of bundle.claims) {
|
|
154
|
+
for (const b of claim.bindings) {
|
|
155
|
+
const refusal = b.reason !== undefined ? words(b.reason) : undefined;
|
|
156
|
+
if (b.kind === "assumption" && b.origin !== undefined) {
|
|
157
|
+
assumed.push({
|
|
158
|
+
ref: b.ref,
|
|
159
|
+
text: b.origin,
|
|
160
|
+
flag: refusal !== undefined ? ` (${refusal})` : "",
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
else if (b.kind === "formal" && b.assumptions !== undefined) {
|
|
164
|
+
const of = `a hypothesis of ${b.origin ?? "the proof"}`;
|
|
165
|
+
for (const hypothesis of b.assumptions) {
|
|
166
|
+
assumed.push({
|
|
167
|
+
ref: b.ref,
|
|
168
|
+
text: hypothesis,
|
|
169
|
+
flag: ` (${refusal !== undefined ? `${of}; ${refusal}` : of})`,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
const refused = bundle.assumptionsRefused ?? 0;
|
|
176
|
+
lines.push("", `**Assumptions (${assumptionsCount(bundle)}${refused > 0 ? `, ${refused} refused` : ""}):**${assumed.length === 0 ? " none" : ""}`);
|
|
177
|
+
const seen = new Set();
|
|
178
|
+
for (const { ref, text, flag } of assumed) {
|
|
179
|
+
const key = `${ref} ${text}`;
|
|
180
|
+
if (seen.has(key))
|
|
181
|
+
continue;
|
|
182
|
+
seen.add(key);
|
|
183
|
+
lines.push(`- ${ref} ${cell(text)}${flag}`);
|
|
184
|
+
}
|
|
185
|
+
if (bundle.checkers !== undefined && bundle.checkers.length > 0) {
|
|
186
|
+
lines.push("", "**Checkers:**");
|
|
187
|
+
const statements = new Map();
|
|
188
|
+
for (const claim of bundle.claims) {
|
|
189
|
+
for (const b of claim.bindings) {
|
|
190
|
+
if (b.kind === "formal" && b.origin !== undefined && b.statement !== undefined) {
|
|
191
|
+
statements.set(b.origin, b.statement);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
for (const checker of bundle.checkers) {
|
|
196
|
+
const statement = statements.get(checker.artifact);
|
|
197
|
+
// What the proof rests on, beside its statement (§6.4): the
|
|
198
|
+
// hypotheses, when any, then the axioms.
|
|
199
|
+
const hypotheses = checker.assumptions ?? [];
|
|
200
|
+
const assuming = hypotheses.length > 0
|
|
201
|
+
? `assuming: ${hypotheses.map((h) => cell(clip(h, 120))).join("; ")}; `
|
|
202
|
+
: "";
|
|
203
|
+
lines.push(`- \`${checker.id}\` — ${checker.artifact}${statement !== undefined ? `: \`${cell(clip(statement, 200)).replace(/`/g, "'")}\`` : ""} (${assuming}axioms: ${checker.axioms.length === 0 ? "none" : checker.axioms.join(", ")})`);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return `${lines.join("\n")}\n`;
|
|
207
|
+
}
|
|
208
|
+
/** Each reason in plain words, for the nudge. ASCII only. */
|
|
209
|
+
const REASON_WORDS = {
|
|
210
|
+
no_citation: "no citation - add a marker like [1] and a definition line [1]: <source> at the end of the answer",
|
|
211
|
+
no_definition: "the marker has no definition line - add one such as [1]: <source>",
|
|
212
|
+
not_locatable: "this is not a URL, a doi: or a workspace path, so it cannot be checked",
|
|
213
|
+
host_denied: "this host is not accepted as a source here (provability.sources)",
|
|
214
|
+
private_address: "a loopback, private or link-local address is not a source",
|
|
215
|
+
path_outside_workspace: "this path is outside the workspace",
|
|
216
|
+
path_missing: "this file does not exist in the workspace",
|
|
217
|
+
unresolved: "the URL did not answer a HEAD request with 2xx or 3xx",
|
|
218
|
+
not_fetched: "this run never fetched or read it - fetch or read it first, then cite it",
|
|
219
|
+
not_found_in_source: "the source was read, but the claim's words are not in it together - quote the source, or restate the claim in its words",
|
|
220
|
+
polarity_conflict: "the source has these words with a negation the claim lacks - check what it actually says",
|
|
221
|
+
not_independent: "needs more independent origins - also cite a different site that says the same",
|
|
222
|
+
tool_missing: "no tool call with this id was recorded - run the tool and cite its toolUseId",
|
|
223
|
+
tool_errored: "that tool call returned an error - rerun it and cite the successful call",
|
|
224
|
+
tool_not_allowed: "results of this tool do not count as evidence here (provability.execution.allow)",
|
|
225
|
+
value_absent: "the tool result does not contain the values the claim states - restate them exactly as the result gives them",
|
|
226
|
+
replay_mismatch: "re-running the tool with the logged input gave a different output - rerun it and cite the new call",
|
|
227
|
+
proof_missing: "no proof artifact has this id - run ProofCheck and cite the id it returns",
|
|
228
|
+
proof_rejected: "the checker rejected the proof - fix the proof or the claim",
|
|
229
|
+
claim_mismatch: "the proof is of a different statement than this sentence makes",
|
|
230
|
+
assumption_forbidden: "this assumption is not accepted (assumptions are forbidden, or there are too many) - prove the claim or drop it",
|
|
231
|
+
below_floor: "this evidence is below the floor",
|
|
232
|
+
};
|
|
233
|
+
/**
|
|
234
|
+
* Reasons whose `detail` the nudge repeats. Every detail the verifier writes
|
|
235
|
+
* is built from identifiers, the model's own words and checker verdicts —
|
|
236
|
+
* never a fetched body — but the nudge repeats only the ones that tell the
|
|
237
|
+
* model what to change, so what it reads stays short.
|
|
238
|
+
*/
|
|
239
|
+
const NUDGE_DETAIL = new Set([
|
|
240
|
+
"value_absent",
|
|
241
|
+
"not_found_in_source",
|
|
242
|
+
"proof_rejected",
|
|
243
|
+
"claim_mismatch",
|
|
244
|
+
"not_independent",
|
|
245
|
+
"below_floor",
|
|
246
|
+
"assumption_forbidden",
|
|
247
|
+
"unresolved",
|
|
248
|
+
"host_denied",
|
|
249
|
+
"tool_not_allowed",
|
|
250
|
+
]);
|
|
251
|
+
const MAX_NUDGE_CLAIMS = 20;
|
|
252
|
+
/**
|
|
253
|
+
* Typography folded to ASCII — quotes, dashes and the ellipsis `clip` cuts
|
|
254
|
+
* with — applied to everything the nudge repeats (the model's sentences,
|
|
255
|
+
* the identifiers it cited, the verifier's details), so the nudge's own
|
|
256
|
+
* text is ASCII. Other characters of the model's own words are its own and
|
|
257
|
+
* are repeated as written.
|
|
258
|
+
*/
|
|
259
|
+
const ascii = (text) => text
|
|
260
|
+
.replace(/[‘’‛]/g, "'")
|
|
261
|
+
.replace(/[“”„‟]/g, '"')
|
|
262
|
+
.replace(/[—–−]/g, "-")
|
|
263
|
+
.replace(/…/g, "...");
|
|
264
|
+
function nudgeLine(b) {
|
|
265
|
+
const reason = b.reason ?? "below_floor";
|
|
266
|
+
const at = b.origin !== undefined ? ` ${ascii(clip(b.origin, 160))}` : "";
|
|
267
|
+
const detail = NUDGE_DETAIL.has(reason) && b.detail !== undefined ? ` (${ascii(clip(b.detail, 300))})` : "";
|
|
268
|
+
return ` - ${b.ref}${at}: ${REASON_WORDS[reason]}${detail}`;
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* The retry nudge (design §5.1): which claims fell short, in the model's own
|
|
272
|
+
* words, what each cited and why it did not count — and the standing
|
|
273
|
+
* instruction to run `ProvabilityPreflight` on the next draft. No excerpt,
|
|
274
|
+
* no fetched text, no judge text.
|
|
275
|
+
*/
|
|
276
|
+
export function renderNudge(bundle) {
|
|
277
|
+
const counted = bundle.claimCount - bundle.assumptions;
|
|
278
|
+
if (bundle.verdict === "unverified") {
|
|
279
|
+
return "The provability gate could not verify your last answer (its verifier failed), so it was not graded. Keep citing every claim; run ProvabilityPreflight on your next draft.";
|
|
280
|
+
}
|
|
281
|
+
const lines = [];
|
|
282
|
+
lines.push(bundle.verdict === "pass"
|
|
283
|
+
? `The provability gate accepted your last answer at the ${bundle.level} floor: ${bundle.met} of ${counted} claims meet it.`
|
|
284
|
+
: `The provability gate did not accept your last answer. Floor: ${bundle.level}. ${bundle.met} of ${counted} claims meet it; at least ${percent(bundle.threshold)} must.`);
|
|
285
|
+
const refused = bundle.assumptionsRefused ?? 0;
|
|
286
|
+
if (refused > 0) {
|
|
287
|
+
lines.push("", `${refused === 1 ? "1 declared assumption was" : `${refused} declared assumptions were`} refused (provability.assumptions is forbid, or there are more than provability.max_assumptions allows). A refused assumption fails the answer whatever the score: prove those claims, cite evidence for them, or drop them.`);
|
|
288
|
+
}
|
|
289
|
+
const unmet = bundle.claims.filter((claim) => claim.status === "unmet");
|
|
290
|
+
if (unmet.length > 0) {
|
|
291
|
+
lines.push("", "Claims that do not meet the floor:");
|
|
292
|
+
unmet.slice(0, MAX_NUDGE_CLAIMS).forEach((claim, i) => {
|
|
293
|
+
const where = claim.definition !== undefined
|
|
294
|
+
? ` (the text after the source on your definition line ${claim.definition})`
|
|
295
|
+
: "";
|
|
296
|
+
lines.push(`${i + 1}. "${ascii(clip(claim.text, 200))}"${where}`);
|
|
297
|
+
if (claim.bindings.length === 0) {
|
|
298
|
+
lines.push(` - ${REASON_WORDS[claimReasons(claim)[0] ?? "no_citation"]}`);
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
301
|
+
for (const b of claim.bindings)
|
|
302
|
+
if (!b.ok && b.reason !== undefined)
|
|
303
|
+
lines.push(nudgeLine(b));
|
|
304
|
+
});
|
|
305
|
+
if (unmet.length > MAX_NUDGE_CLAIMS) {
|
|
306
|
+
lines.push(`... and ${unmet.length - MAX_NUDGE_CLAIMS} more.`);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
lines.push("", "Cite every claim with a marker such as [1] and close the answer with one definition line per marker: [1]: https://... (a page you fetched), ./path (a file you read), doi:..., tool:<toolUseId> (a computation you ran), proof:prf_... (a ProofCheck artifact), or assume: <text> (a declared assumption).");
|
|
310
|
+
if (unmet.some((claim) => claim.definition !== undefined)) {
|
|
311
|
+
lines.push('Text after the source on a definition line is shown to the reader, so it is checked as a claim citing that source: put a page title in quotes ([1]: https://... "Title") or leave it out.');
|
|
312
|
+
}
|
|
313
|
+
lines.push("Before you finish your next draft, run ProvabilityPreflight on it and fix every claim it reports.");
|
|
314
|
+
return lines.join("\n");
|
|
315
|
+
}
|
package/dist/store.d.ts
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { ProofArtifactStore, ProofBundle } from "./types";
|
|
2
|
+
/** The directory bundles of a run with no session go under. */
|
|
3
|
+
export declare const NO_SESSION = "no-session";
|
|
4
|
+
/** `<cwd>/.crewhaus/proofs/<specName>`: one spec's bundles and artifacts. */
|
|
5
|
+
export declare function proofsRoot(cwd: string, specName: string): string;
|
|
6
|
+
export type BundleListing = {
|
|
7
|
+
readonly sessionId: string;
|
|
8
|
+
readonly turnNumber: number;
|
|
9
|
+
/** Which pass of the turn: 1 for `turn-<n>.json`, k for `turn-<n>.<k>.json`. (0.8.0 additive, fix pass.) */
|
|
10
|
+
readonly pass: number;
|
|
11
|
+
readonly path: string;
|
|
12
|
+
readonly mtimeMs: number;
|
|
13
|
+
};
|
|
14
|
+
export interface BundleStore {
|
|
15
|
+
/** The proofs root this store writes under. */
|
|
16
|
+
readonly rootDir: string;
|
|
17
|
+
/**
|
|
18
|
+
* Persist a bundle as `<root>/<sessionId|no-session>/turn-<n>.json` — or,
|
|
19
|
+
* when a pass of that turn is already stored (a retry, a turn a resumed
|
|
20
|
+
* session numbers again), the next free `turn-<n>.<k>.json` — and return
|
|
21
|
+
* the path written. A stored bundle is never replaced: the `bundleHash`
|
|
22
|
+
* each pass's audit record and trace line carry names a bundle that is
|
|
23
|
+
* still there (design §5.3), and `provability audit` can read the
|
|
24
|
+
* failing pass a retry fixed.
|
|
25
|
+
*/
|
|
26
|
+
put(bundle: ProofBundle, at: {
|
|
27
|
+
readonly sessionId?: string;
|
|
28
|
+
readonly turnNumber: number;
|
|
29
|
+
}): Promise<string>;
|
|
30
|
+
/** One pass of a turn — by default its last, the turn's standing bundle — or `undefined`. */
|
|
31
|
+
get(sessionId: string | undefined, turnNumber: number, pass?: number): Promise<ProofBundle | undefined>;
|
|
32
|
+
/** Each turn's last pass, newest first; `allPasses` lists every pass. */
|
|
33
|
+
list(opts?: {
|
|
34
|
+
readonly allPasses?: boolean;
|
|
35
|
+
}): Promise<ReadonlyArray<BundleListing>>;
|
|
36
|
+
/** Delete bundles (every pass) last written before `now - olderThanMs`; returns how many. */
|
|
37
|
+
prune(olderThanMs: number, now?: Date): Promise<number>;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* The bundle store over a proofs root (`proofsRoot(cwd, specName)`).
|
|
41
|
+
* Tenant fencing is the caller's: on managed, pass the tenant's root.
|
|
42
|
+
*/
|
|
43
|
+
export declare function createBundleStore(opts: {
|
|
44
|
+
readonly rootDir: string;
|
|
45
|
+
}): BundleStore;
|
|
46
|
+
/**
|
|
47
|
+
* Proof artifacts under `<rootDir>/artifacts/<id>.json`, where `rootDir` is
|
|
48
|
+
* the proofs root (`proofsRoot(cwd, specName)`). An id is checked against
|
|
49
|
+
* `prf_<16 hex>` BEFORE it is put into a path: `get` answers `undefined` for
|
|
50
|
+
* any other string (a `proof:../../x` citation reads nothing), and `put`
|
|
51
|
+
* refuses one. Artifacts are immutable: `put` never replaces an existing id
|
|
52
|
+
* — a later check is a new artifact, so the one a bundle cited stays the
|
|
53
|
+
* one it was graded against.
|
|
54
|
+
*/
|
|
55
|
+
export declare function createFileArtifactStore(opts: {
|
|
56
|
+
readonly rootDir: string;
|
|
57
|
+
}): ProofArtifactStore & {
|
|
58
|
+
readonly rootDir: string;
|
|
59
|
+
};
|