@gonrocca/nodd 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +350 -0
- package/extensions/nodd-agents.test.ts +129 -0
- package/extensions/nodd-agents.ts +185 -0
- package/extensions/nodd-allow.test.ts +75 -0
- package/extensions/nodd-allow.ts +76 -0
- package/extensions/nodd-enforcement.test.ts +676 -0
- package/extensions/nodd-gates.test.ts +108 -0
- package/extensions/nodd-gates.ts +121 -0
- package/extensions/nodd-kernel.test.ts +114 -0
- package/extensions/nodd-kernel.ts +593 -0
- package/extensions/nodd-models.test.ts +174 -0
- package/extensions/nodd-models.ts +253 -0
- package/extensions/nodd-promote.test.ts +150 -0
- package/extensions/nodd-promote.ts +96 -0
- package/extensions/nodd-prompt.test.ts +87 -0
- package/extensions/nodd-tools.test.ts +211 -0
- package/package.json +44 -0
- package/src/bash-classifier.test.ts +114 -0
- package/src/bash-classifier.ts +69 -0
- package/src/change-acceptance.test.ts +175 -0
- package/src/change-acceptance.ts +98 -0
- package/src/config.test.ts +61 -0
- package/src/config.ts +103 -0
- package/src/delivery.test.ts +156 -0
- package/src/delivery.ts +151 -0
- package/src/feature-doc.test.ts +120 -0
- package/src/feature-doc.ts +292 -0
- package/src/gates/authorize.test.ts +62 -0
- package/src/gates/authorize.ts +32 -0
- package/src/gates/classify.test.ts +54 -0
- package/src/gates/classify.ts +45 -0
- package/src/gates/delegate.test.ts +127 -0
- package/src/gates/delegate.ts +85 -0
- package/src/gates/evidence.test.ts +281 -0
- package/src/gates/evidence.ts +209 -0
- package/src/gates/policy.test.ts +77 -0
- package/src/gates/policy.ts +90 -0
- package/src/gates/promotion.test.ts +133 -0
- package/src/gates/promotion.ts +81 -0
- package/src/gates/registry.ts +21 -0
- package/src/gates/request.ts +41 -0
- package/src/gates/track.test.ts +80 -0
- package/src/gates/track.ts +58 -0
- package/src/io.test.ts +81 -0
- package/src/io.ts +94 -0
- package/src/ledger.test.ts +122 -0
- package/src/ledger.ts +133 -0
- package/src/manifest.test.ts +53 -0
- package/src/manifest.ts +61 -0
- package/src/models/assign.test.ts +125 -0
- package/src/models/assign.ts +138 -0
- package/src/models/picker.test.ts +141 -0
- package/src/models/picker.ts +98 -0
- package/src/models/profiles.test.ts +186 -0
- package/src/models/profiles.ts +162 -0
- package/src/models/slots.ts +48 -0
- package/src/observations.test.ts +61 -0
- package/src/observations.ts +51 -0
- package/src/odd-prose.test.ts +125 -0
- package/src/odd-prose.ts +198 -0
- package/src/outcome.test.ts +75 -0
- package/src/outcome.ts +63 -0
- package/src/promote.test.ts +129 -0
- package/src/promote.ts +64 -0
- package/src/prompt.test.ts +193 -0
- package/src/prompt.ts +136 -0
- package/src/review-candidate.test.ts +118 -0
- package/src/review-candidate.ts +81 -0
- package/src/state.test.ts +153 -0
- package/src/state.ts +163 -0
- package/test/package-invariants.test.ts +66 -0
- package/test/parity-matrix.test.ts +272 -0
- package/test/readme-contract.test.ts +182 -0
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { emptyCommitted, fold } from "../state.ts";
|
|
4
|
+
import { observation, pendingCall } from "../observations.ts";
|
|
5
|
+
import { emptyPolicy } from "./policy.ts";
|
|
6
|
+
import { classifyGate } from "./classify.ts";
|
|
7
|
+
|
|
8
|
+
const write = { toolName: "write", input: { path: "src/a.ts" } };
|
|
9
|
+
const noPending = new Map();
|
|
10
|
+
|
|
11
|
+
test("an undeclared write is blocked with a nodd_declare remedy", () => {
|
|
12
|
+
const decision = classifyGate(emptyCommitted(), write, emptyPolicy(), noPending);
|
|
13
|
+
assert.equal(decision.allow, false);
|
|
14
|
+
assert.ok(decision.allow === false && decision.remedy.action.includes("nodd_declare"));
|
|
15
|
+
assert.ok(decision.allow === false && decision.reason.includes("classify"));
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
test("edit and mutating bash block the same way; non-mutating bash does not", () => {
|
|
19
|
+
const p = emptyPolicy();
|
|
20
|
+
assert.equal(classifyGate(emptyCommitted(), { toolName: "edit", input: { path: "a.ts" } }, p, noPending).allow, false);
|
|
21
|
+
assert.equal(classifyGate(emptyCommitted(), { toolName: "bash", input: { command: "rm -rf x" } }, p, noPending).allow, false);
|
|
22
|
+
assert.equal(classifyGate(emptyCommitted(), { toolName: "bash", input: { command: "npm test" } }, p, noPending).allow, true);
|
|
23
|
+
assert.equal(classifyGate(emptyCommitted(), { toolName: "read", input: { path: "a.ts" } }, p, noPending).allow, true);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
test("a committed declaration allows the write", () => {
|
|
27
|
+
const committed = fold(emptyCommitted(), observation({
|
|
28
|
+
toolCallId: "d1", toolName: "nodd_declare",
|
|
29
|
+
input: { intent: "change", route: "inline", slug: "demo" },
|
|
30
|
+
isError: false, resultText: "", at: "t",
|
|
31
|
+
}));
|
|
32
|
+
assert.equal(classifyGate(committed, write, emptyPolicy(), noPending).allow, true);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
// extensions.md:757-758 — siblings are preflighted sequentially and executed
|
|
36
|
+
// concurrently, so a declaration in the same batch has not happened yet.
|
|
37
|
+
test("a declaration seen only in pending does not unblock, and the reason cites the batch rule", () => {
|
|
38
|
+
const pending = new Map([["d1", pendingCall({
|
|
39
|
+
toolCallId: "d1", toolName: "nodd_declare",
|
|
40
|
+
input: { intent: "change", route: "tracked", slug: "demo" },
|
|
41
|
+
})]]);
|
|
42
|
+
|
|
43
|
+
const decision = classifyGate(emptyCommitted(), write, emptyPolicy(), pending);
|
|
44
|
+
assert.equal(decision.allow, false);
|
|
45
|
+
assert.ok(decision.allow === false && /same (assistant )?(message|batch)/i.test(decision.reason),
|
|
46
|
+
`the reason must cite the batch rule: ${decision.allow === false ? decision.reason : ""}`);
|
|
47
|
+
assert.ok(decision.allow === false && /next turn|reissue/i.test(decision.reason),
|
|
48
|
+
"the reason must say how to resolve it");
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("the flag off allows the undeclared write", () => {
|
|
52
|
+
const policy = { ...emptyPolicy(), config: { classify: { enabled: false } } };
|
|
53
|
+
assert.equal(classifyGate(emptyCommitted(), write, policy, noPending).allow, true);
|
|
54
|
+
});
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// gate-classify — no first write without a declared route.
|
|
2
|
+
//
|
|
3
|
+
// This is the gate aimed squarely at the hole gentle admits in its own task
|
|
4
|
+
// file: "ODD routing is pure model judgment: nothing requires the route
|
|
5
|
+
// decision to be stated, recorded, or checked, so non-delegation is invisible."
|
|
6
|
+
//
|
|
7
|
+
// NODD does not mechanize the *judgement* — whether work is "substantial" is
|
|
8
|
+
// not observable from tool events, and pretending otherwise would be inventing
|
|
9
|
+
// enforcement. It mechanizes the *declaration*: until a `nodd_declare` result
|
|
10
|
+
// is committed, the first write does not happen. A skipped classification stops
|
|
11
|
+
// being invisible, which is all a mechanism can honestly do here.
|
|
12
|
+
|
|
13
|
+
import type { Committed } from "../state.ts";
|
|
14
|
+
import type { PendingCall } from "../observations.ts";
|
|
15
|
+
import { allow, refuse, resolveFlag, type GateDecision, type Policy } from "./policy.ts";
|
|
16
|
+
import { isMutation, type GateRequest } from "./request.ts";
|
|
17
|
+
|
|
18
|
+
export function classifyGate(
|
|
19
|
+
committed: Committed,
|
|
20
|
+
request: GateRequest,
|
|
21
|
+
policy: Policy,
|
|
22
|
+
pending: Map<string, PendingCall>,
|
|
23
|
+
): GateDecision {
|
|
24
|
+
if (!resolveFlag("classify", policy).enabled) return allow();
|
|
25
|
+
if (committed.declaration) return allow();
|
|
26
|
+
if (!isMutation(request)) return allow();
|
|
27
|
+
|
|
28
|
+
// A declaration preflighted in this same batch has not run yet: pi does not
|
|
29
|
+
// guarantee sibling results are visible here, so treating it as done would be
|
|
30
|
+
// accepting evidence that does not exist. Say so, and say what to do.
|
|
31
|
+
const declaringSibling = [...pending.values()].some((call) => call.toolName === "nodd_declare");
|
|
32
|
+
if (declaringSibling) {
|
|
33
|
+
return refuse(
|
|
34
|
+
"classify",
|
|
35
|
+
"a `nodd_declare` call is in this same assistant message, but sibling results are not observable yet, so no route is recorded",
|
|
36
|
+
"let this batch finish, then reissue the write on the next turn",
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return refuse(
|
|
41
|
+
"classify",
|
|
42
|
+
`no route has been declared for this session, and ${request.toolName} would be the first change`,
|
|
43
|
+
"call `nodd_declare` with an explicit intent and route (`inline`, `tracked` or `forge`)",
|
|
44
|
+
);
|
|
45
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { emptyCommitted, foldAll, type Committed } from "../state.ts";
|
|
4
|
+
import { observation, pendingCall, type Observation } from "../observations.ts";
|
|
5
|
+
import { THRESHOLDS } from "../manifest.ts";
|
|
6
|
+
import { emptyPolicy } from "./policy.ts";
|
|
7
|
+
import { delegateGate } from "./delegate.ts";
|
|
8
|
+
|
|
9
|
+
let n = 0;
|
|
10
|
+
function obs(toolName: string, input: Record<string, unknown>, isError = false): Observation {
|
|
11
|
+
return observation({ toolCallId: `c${++n}`, toolName, input, isError, resultText: "", at: "t" });
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function reads(count: number): Committed {
|
|
15
|
+
n = 0;
|
|
16
|
+
return foldAll(emptyCommitted(), Array.from({ length: count }, (_, i) => obs("read", { path: `f${i}.ts` })));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const write = { toolName: "write", input: { path: "out.ts" } };
|
|
20
|
+
const noPending = new Map();
|
|
21
|
+
|
|
22
|
+
test("4 distinct reads with no delegation blocks, quoting count and threshold", () => {
|
|
23
|
+
const decision = delegateGate(reads(4), write, emptyPolicy(), noPending);
|
|
24
|
+
assert.equal(decision.allow, false);
|
|
25
|
+
assert.ok(decision.allow === false && decision.reason.includes("4"));
|
|
26
|
+
assert.ok(decision.allow === false && decision.reason.includes(String(THRESHOLDS.mappingMinUnderstandingFiles)));
|
|
27
|
+
assert.ok(decision.allow === false && decision.remedy.action.toLowerCase().includes("delegat"));
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
test("the mapping boundary is asserted at both 3 and 4", () => {
|
|
31
|
+
assert.equal(delegateGate(reads(3), write, emptyPolicy(), noPending).allow, true, "3 files stays inline");
|
|
32
|
+
assert.equal(delegateGate(reads(4), write, emptyPolicy(), noPending).allow, false, "4 files must delegate");
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("one file read six times does not trip the mapping trigger", () => {
|
|
36
|
+
n = 0;
|
|
37
|
+
const committed = foldAll(emptyCommitted(), Array.from({ length: 6 }, () => obs("read", { path: "same.ts" })));
|
|
38
|
+
assert.equal(delegateGate(committed, write, emptyPolicy(), noPending).allow, true);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("a committed subagent result clears the block", () => {
|
|
42
|
+
n = 0;
|
|
43
|
+
const committed = foldAll(emptyCommitted(), [
|
|
44
|
+
...Array.from({ length: 5 }, (_, i) => obs("read", { path: `f${i}.ts` })),
|
|
45
|
+
obs("subagent", { agent: "nodd-explore" }),
|
|
46
|
+
]);
|
|
47
|
+
assert.equal(delegateGate(committed, write, emptyPolicy(), noPending).allow, true);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("the writer trigger fires at 2 distinct written files", () => {
|
|
51
|
+
n = 0;
|
|
52
|
+
const one = foldAll(emptyCommitted(), [obs("write", { path: "a.ts" })]);
|
|
53
|
+
assert.equal(delegateGate(one, { toolName: "write", input: { path: "a.ts" } }, emptyPolicy(), noPending).allow, true,
|
|
54
|
+
"rewriting the same file is still one file");
|
|
55
|
+
assert.equal(delegateGate(one, { toolName: "write", input: { path: "b.ts" } }, emptyPolicy(), noPending).allow, false,
|
|
56
|
+
"a second distinct file crosses the writer threshold");
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
// Intent is known at preflight, so counting a sibling's write *intent* is
|
|
60
|
+
// sound. Counting a sibling's *result* would not be, and is never done.
|
|
61
|
+
test("same-batch pending write intent counts toward the writer trigger", () => {
|
|
62
|
+
n = 0;
|
|
63
|
+
const committed = foldAll(emptyCommitted(), [obs("write", { path: "a.ts" })]);
|
|
64
|
+
const pending = new Map([["p1", pendingCall({ toolCallId: "p1", toolName: "write", input: { path: "b.ts" } })]]);
|
|
65
|
+
assert.equal(delegateGate(committed, { toolName: "write", input: { path: "c.ts" } }, emptyPolicy(), pending).allow, false);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
// routing.go:70 — tests, builds, installs and review actors may use fresh
|
|
69
|
+
// workers without changing the route. They are not writer files, so they never
|
|
70
|
+
// reach the mapping or writer trigger. The long-session backstop is a separate
|
|
71
|
+
// clause (routing.go:82) and it counts every tool call, these included — see
|
|
72
|
+
// the test below it.
|
|
73
|
+
test("test, build and install commands never trip the mapping or writer trigger", () => {
|
|
74
|
+
n = 0;
|
|
75
|
+
const committed = foldAll(emptyCommitted(), [
|
|
76
|
+
obs("bash", { command: "npm test" }),
|
|
77
|
+
obs("bash", { command: "npm run build" }),
|
|
78
|
+
obs("bash", { command: "npm install" }),
|
|
79
|
+
obs("bash", { command: "git status" }),
|
|
80
|
+
]);
|
|
81
|
+
assert.equal(delegateGate(committed, write, emptyPolicy(), noPending).allow, true);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
// The claim "test/build/install never trip it" was covered by four commands,
|
|
85
|
+
// which is under the backstop threshold, so the case that contradicts it was
|
|
86
|
+
// never exercised. Run the declared runner past the threshold and the backstop
|
|
87
|
+
// does fire — as ODD specifies. That is the behaviour; the requirement text now
|
|
88
|
+
// says so.
|
|
89
|
+
test("repeated runs of the declared runner do reach the long-session backstop", () => {
|
|
90
|
+
n = 0;
|
|
91
|
+
const committed = foldAll(
|
|
92
|
+
emptyCommitted(),
|
|
93
|
+
Array.from({ length: 25 }, () => obs("bash", { command: "npm test" })),
|
|
94
|
+
);
|
|
95
|
+
const decision = delegateGate(committed, write, emptyPolicy(), noPending);
|
|
96
|
+
assert.equal(decision.allow, false, "25 tool calls without delegating must reach the backstop");
|
|
97
|
+
// Asserting only `allow === false` let a combined mutation -- backstop off,
|
|
98
|
+
// writer threshold at 1 -- keep this green through the wrong trigger. The
|
|
99
|
+
// row is about the backstop, so the reason has to name it.
|
|
100
|
+
assert.match(
|
|
101
|
+
"reason" in decision ? decision.reason : "",
|
|
102
|
+
/25 tool calls/,
|
|
103
|
+
"the refusal must come from the backstop, not from another trigger firing by accident",
|
|
104
|
+
);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("the long-session backstop fires at 20 tool calls with no delegation", () => {
|
|
108
|
+
n = 0;
|
|
109
|
+
const under = foldAll(emptyCommitted(), Array.from({ length: 19 }, () => obs("bash", { command: "git status" })));
|
|
110
|
+
assert.equal(under.toolCalls, 19);
|
|
111
|
+
assert.equal(delegateGate(under, write, emptyPolicy(), noPending).allow, true);
|
|
112
|
+
|
|
113
|
+
const at = foldAll(under, [obs("bash", { command: "git log" })]);
|
|
114
|
+
assert.equal(at.toolCalls, THRESHOLDS.longSessionToolCalls);
|
|
115
|
+
const decision = delegateGate(at, write, emptyPolicy(), noPending);
|
|
116
|
+
assert.equal(decision.allow, false);
|
|
117
|
+
assert.ok(decision.allow === false && decision.reason.includes("20"));
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
test("the flag off allows everything", () => {
|
|
121
|
+
const policy = { ...emptyPolicy(), config: { delegate: { enabled: false } } };
|
|
122
|
+
assert.equal(delegateGate(reads(40), write, policy, noPending).allow, true);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test("the gate only speaks on writes, not on reads", () => {
|
|
126
|
+
assert.equal(delegateGate(reads(10), { toolName: "read", input: { path: "z.ts" } }, emptyPolicy(), noPending).allow, true);
|
|
127
|
+
});
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// gate-delegate — the clause ODD states and cannot enforce.
|
|
2
|
+
//
|
|
3
|
+
// ODD's non-delegation is invisible: the manifest triggers
|
|
4
|
+
// (`manifest.go:203-216`) and the long-session backstop (`routing.go:82`) are
|
|
5
|
+
// prose the model may simply not follow. Here they are counted from observed
|
|
6
|
+
// tool results and the write is refused.
|
|
7
|
+
//
|
|
8
|
+
// Three triggers, in ODD's own numbers (`src/manifest.ts`):
|
|
9
|
+
// - mapping: >= 4 distinct files read with no delegation yet
|
|
10
|
+
// - writer: >= 2 distinct files written
|
|
11
|
+
// - long session: >= 20 tool calls with no delegation yet
|
|
12
|
+
//
|
|
13
|
+
// Distinct *files*, never call counts: reading one file six times is one file
|
|
14
|
+
// in context, and a gate that could not tell those apart would fire on a
|
|
15
|
+
// careful re-read of the same module.
|
|
16
|
+
//
|
|
17
|
+
// The counters are per process. That is the intended semantics, not a gap: the
|
|
18
|
+
// triggers exist to keep *this* agent's context thin enough to orchestrate, and
|
|
19
|
+
// a delegated child has its own context window. There is no aggregate
|
|
20
|
+
// whole-session total across parent and children, and NODD does not claim one.
|
|
21
|
+
|
|
22
|
+
import type { Committed } from "../state.ts";
|
|
23
|
+
import type { PendingCall } from "../observations.ts";
|
|
24
|
+
import { THRESHOLDS } from "../manifest.ts";
|
|
25
|
+
import { classifyBash } from "../bash-classifier.ts";
|
|
26
|
+
import { allow, refuse, resolveFlag, type GateDecision, type Policy } from "./policy.ts";
|
|
27
|
+
import { isFileWrite, targetPath, type GateRequest } from "./request.ts";
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Files this session wrote, plus the ones it is about to. Write *intent* is
|
|
31
|
+
* known at preflight, so counting a sibling's intent is sound; a sibling's
|
|
32
|
+
* result is not observable yet and is never counted.
|
|
33
|
+
*/
|
|
34
|
+
function writtenFiles(committed: Committed, request: GateRequest, pending: Map<string, PendingCall>): Set<string> {
|
|
35
|
+
const files = new Set(committed.filesWritten.keys());
|
|
36
|
+
for (const call of pending.values()) {
|
|
37
|
+
if (isFileWrite(call)) {
|
|
38
|
+
const path = targetPath(call);
|
|
39
|
+
if (path) files.add(path);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
const path = targetPath(request);
|
|
43
|
+
if (path) files.add(path);
|
|
44
|
+
return files;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function delegateGate(
|
|
48
|
+
committed: Committed,
|
|
49
|
+
request: GateRequest,
|
|
50
|
+
policy: Policy,
|
|
51
|
+
pending: Map<string, PendingCall>,
|
|
52
|
+
): GateDecision {
|
|
53
|
+
if (!resolveFlag("delegate", policy).enabled) return allow();
|
|
54
|
+
if (!isFileWrite(request) && classifyBash(String(request.input?.command ?? "")) !== "mutating") return allow();
|
|
55
|
+
if (committed.delegations > 0) return allow();
|
|
56
|
+
|
|
57
|
+
const remedy = "delegate the work with the `subagent` tool, or declare this as small inline work";
|
|
58
|
+
|
|
59
|
+
const written = writtenFiles(committed, request, pending);
|
|
60
|
+
if (written.size >= THRESHOLDS.writerMinNonTrivialFiles) {
|
|
61
|
+
return refuse(
|
|
62
|
+
"delegate",
|
|
63
|
+
`this session has written ${written.size} distinct files (threshold ${THRESHOLDS.writerMinNonTrivialFiles}) without delegating`,
|
|
64
|
+
remedy,
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (committed.filesRead.size >= THRESHOLDS.mappingMinUnderstandingFiles) {
|
|
69
|
+
return refuse(
|
|
70
|
+
"delegate",
|
|
71
|
+
`this session has read ${committed.filesRead.size} distinct files (threshold ${THRESHOLDS.mappingMinUnderstandingFiles}) without delegating`,
|
|
72
|
+
remedy,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (committed.toolCalls >= THRESHOLDS.longSessionToolCalls) {
|
|
77
|
+
return refuse(
|
|
78
|
+
"delegate",
|
|
79
|
+
`this session has run ${committed.toolCalls} tool calls (threshold ${THRESHOLDS.longSessionToolCalls}) without delegating`,
|
|
80
|
+
remedy,
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
return allow();
|
|
85
|
+
}
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { emptyCommitted, fold, type Committed } from "../state.ts";
|
|
4
|
+
import { observation } from "../observations.ts";
|
|
5
|
+
import type { LedgerRecord } from "../ledger.ts";
|
|
6
|
+
import { parseOutcome } from "../outcome.ts";
|
|
7
|
+
import { emptyPolicy } from "./policy.ts";
|
|
8
|
+
import { evidenceGate, renderObserved } from "./evidence.ts";
|
|
9
|
+
|
|
10
|
+
function afterCommand(command: string, isError: boolean, resultText = "", id = "b1"): Committed {
|
|
11
|
+
return fold(emptyCommitted(), observation({
|
|
12
|
+
toolCallId: id, toolName: "bash", input: { command }, isError, resultText, at: "2026-09-19T10:05:00.000Z",
|
|
13
|
+
}));
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* The ledger the kernel would have written for these observations.
|
|
18
|
+
*
|
|
19
|
+
* The outcome must come from `parseOutcome`, exactly as `appendRecord`'s caller
|
|
20
|
+
* derives it. An earlier version of this helper wrote `unknown` for every error
|
|
21
|
+
* result, which is not what production records for a failing run — once the
|
|
22
|
+
* gate started comparing the ledger against the session, that invented value
|
|
23
|
+
* read as tampering. The fixture was wrong, not the check.
|
|
24
|
+
*/
|
|
25
|
+
function ledgerFor(committed: Committed): LedgerRecord[] {
|
|
26
|
+
return committed.commandResults.map((r) => ({
|
|
27
|
+
toolCallId: r.toolCallId,
|
|
28
|
+
tool: "bash",
|
|
29
|
+
command: r.command,
|
|
30
|
+
outcome: parseOutcome(r.isError, r.resultText),
|
|
31
|
+
at: r.at,
|
|
32
|
+
}));
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const check = { task: "T1", lastWriteAt: "2026-09-19T10:00:00.000Z", runner: "npm test" };
|
|
36
|
+
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// The mandated provenance test: the recorded value comes from the observed
|
|
39
|
+
// tool result, never from the model saying so.
|
|
40
|
+
// ---------------------------------------------------------------------------
|
|
41
|
+
test("only a real successful tool_result satisfies a checkoff", () => {
|
|
42
|
+
const policy = emptyPolicy();
|
|
43
|
+
|
|
44
|
+
// (a) the model asserts "tests pass" and no command ever ran.
|
|
45
|
+
const claimed = fold(emptyCommitted(), observation({
|
|
46
|
+
toolCallId: "a1", toolName: "assistant_message",
|
|
47
|
+
input: { text: "I ran the tests and they pass. Evidence: all 42 tests pass." },
|
|
48
|
+
isError: false, resultText: "", at: "2026-09-19T10:05:00.000Z",
|
|
49
|
+
}));
|
|
50
|
+
const a = evidenceGate(claimed, [], check, policy);
|
|
51
|
+
assert.equal(a.allow, false, "a model claim is not evidence");
|
|
52
|
+
assert.ok(a.allow === false && /no command/i.test(a.reason));
|
|
53
|
+
|
|
54
|
+
// (b) a real command that really failed.
|
|
55
|
+
const failed = afterCommand("npm test", true, "FAIL 3 tests\nCommand exited with code 1");
|
|
56
|
+
const b = evidenceGate(failed, ledgerFor(failed), check, policy);
|
|
57
|
+
assert.equal(b.allow, false, "a failing result is not evidence");
|
|
58
|
+
assert.ok(b.allow === false && b.reason.includes("exit 1"), "the refusal quotes the real code");
|
|
59
|
+
|
|
60
|
+
// (c) a real command that really succeeded.
|
|
61
|
+
const passed = afterCommand("npm test", false, "42 passing");
|
|
62
|
+
const c = evidenceGate(passed, ledgerFor(passed), check, policy);
|
|
63
|
+
assert.equal(c.allow, true);
|
|
64
|
+
assert.equal(c.allow === true && c.observed.command, "npm test");
|
|
65
|
+
assert.equal(c.allow === true && c.observed.outcome, "success");
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
// The property the whole product sells, isolated: the model's prose is not an
|
|
69
|
+
// input. Same real success, once with an assistant message asserting the
|
|
70
|
+
// result and once without — the recorded evidence must be byte-identical.
|
|
71
|
+
// Without this, the test above only proves "there was no evidence", which is a
|
|
72
|
+
// weaker and different claim.
|
|
73
|
+
test("an assistant message asserting the result changes nothing about what is recorded", () => {
|
|
74
|
+
const passed = afterCommand("npm test", false, "42 passing");
|
|
75
|
+
const withClaim = fold(passed, observation({
|
|
76
|
+
toolCallId: "a1", toolName: "assistant_message",
|
|
77
|
+
input: { text: "Tests pass. Evidence: 42/42 green, everything verified." },
|
|
78
|
+
isError: false, resultText: "", at: "2026-09-19T10:06:00.000Z",
|
|
79
|
+
}));
|
|
80
|
+
|
|
81
|
+
const silent = evidenceGate(passed, ledgerFor(passed), check, emptyPolicy());
|
|
82
|
+
const claiming = evidenceGate(withClaim, ledgerFor(withClaim), check, emptyPolicy());
|
|
83
|
+
|
|
84
|
+
assert.equal(silent.allow, true);
|
|
85
|
+
assert.equal(claiming.allow, true);
|
|
86
|
+
assert.deepEqual(
|
|
87
|
+
claiming.allow === true ? claiming.observed : null,
|
|
88
|
+
silent.allow === true ? silent.observed : null,
|
|
89
|
+
"the model's assertion must not alter the recorded evidence",
|
|
90
|
+
);
|
|
91
|
+
assert.equal(
|
|
92
|
+
renderObserved(claiming.allow === true ? claiming.observed : null),
|
|
93
|
+
renderObserved(silent.allow === true ? silent.observed : null),
|
|
94
|
+
"the rendered line must be byte-identical with and without the claim",
|
|
95
|
+
);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test("an empty ledger and no commands refuses", () => {
|
|
99
|
+
const decision = evidenceGate(emptyCommitted(), [], check, emptyPolicy());
|
|
100
|
+
assert.equal(decision.allow, false);
|
|
101
|
+
assert.ok(decision.allow === false && decision.remedy.action.length > 0);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test("aborted, timeout and unknown never satisfy a checkoff", () => {
|
|
105
|
+
for (const [text, quoted] of [
|
|
106
|
+
["Command aborted", "aborted"],
|
|
107
|
+
["Command timed out after 120 seconds", "timed out"],
|
|
108
|
+
["ENOENT: no such file", "unknown"],
|
|
109
|
+
]) {
|
|
110
|
+
const state = afterCommand("npm test", true, text);
|
|
111
|
+
const decision = evidenceGate(state, ledgerFor(state), check, emptyPolicy());
|
|
112
|
+
assert.equal(decision.allow, false, `${text} must refuse`);
|
|
113
|
+
assert.ok(decision.allow === false && decision.reason.includes(quoted), `the refusal must say "${quoted}"`);
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test("the success must come after the task's last write", () => {
|
|
118
|
+
const state = afterCommand("npm test", false, "ok");
|
|
119
|
+
const stale = { ...check, lastWriteAt: "2026-09-19T23:00:00.000Z" };
|
|
120
|
+
const decision = evidenceGate(state, ledgerFor(state), stale, emptyPolicy());
|
|
121
|
+
assert.equal(decision.allow, false, "a run predating the edit proves nothing about the edit");
|
|
122
|
+
assert.ok(decision.allow === false && /before/i.test(decision.reason));
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
// ---------------------------------------------------------------------------
|
|
126
|
+
// The two attacks the round-1 verdict landed. Both produced a green checkoff.
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
test("the stale-green attack is refused: a run before the edit never certifies the edit", () => {
|
|
129
|
+
// Green at 10:05, then the source is edited at 10:30. In round 1 this checked
|
|
130
|
+
// the task off, because `lastWriteAt` read `commandResults` (bash only) and
|
|
131
|
+
// `filesWritten` carried no timestamps at all.
|
|
132
|
+
const green = fold(emptyCommitted(), observation({
|
|
133
|
+
toolCallId: "b1", toolName: "bash", input: { command: "npm test" },
|
|
134
|
+
isError: false, resultText: "ok", at: "2026-09-19T10:05:00.000Z",
|
|
135
|
+
}));
|
|
136
|
+
const edited = fold(green, observation({
|
|
137
|
+
toolCallId: "w1", toolName: "edit", input: { path: "/repo/src/login.ts" },
|
|
138
|
+
isError: false, resultText: "", at: "2026-09-19T10:30:00.000Z",
|
|
139
|
+
}));
|
|
140
|
+
|
|
141
|
+
const decision = evidenceGate(
|
|
142
|
+
edited,
|
|
143
|
+
ledgerFor(edited),
|
|
144
|
+
{ task: "T1", lastWriteAt: "2026-09-19T10:30:00.000Z", runner: "npm test" },
|
|
145
|
+
emptyPolicy(),
|
|
146
|
+
);
|
|
147
|
+
assert.equal(decision.allow, false, "the edit is newer than the only green run");
|
|
148
|
+
assert.ok(decision.allow === false && /before the last write/i.test(decision.reason), decision.allow === false ? decision.reason : "");
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
test("the echo attack is refused: an exit-0 string the model chose is not the declared check", () => {
|
|
152
|
+
const state = afterCommand("echo 'I have verified that all tests pass'", false, "I have verified that all tests pass");
|
|
153
|
+
const decision = evidenceGate(state, ledgerFor(state), check, emptyPolicy());
|
|
154
|
+
assert.equal(decision.allow, false, "exit 0 on a sentence is not evidence of anything");
|
|
155
|
+
assert.ok(decision.allow === false && /npm test/.test(decision.reason), "the refusal names the declared runner");
|
|
156
|
+
assert.ok(decision.allow === false && /runner|declared/i.test(decision.reason), decision.allow === false ? decision.reason : "");
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
test("a command that merely contains the runner as a substring is not the runner", () => {
|
|
160
|
+
const state = afterCommand("echo npm test", false, "npm test");
|
|
161
|
+
assert.equal(evidenceGate(state, ledgerFor(state), check, emptyPolicy()).allow, false);
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
test("the declared runner with its own arguments is still the declared runner", () => {
|
|
165
|
+
// `npm test -- src/login.test.ts` is the runner scoped to a file, which is the
|
|
166
|
+
// ordinary way a task verifies itself. Refusing it would make the gate a
|
|
167
|
+
// nuisance and get it switched off.
|
|
168
|
+
const state = afterCommand("npm test -- src/login.test.ts", false, "ok");
|
|
169
|
+
assert.equal(evidenceGate(state, ledgerFor(state), check, emptyPolicy()).allow, true);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
test("with no runner declared, any observed success still has to postdate the write", () => {
|
|
173
|
+
// Honest limit, stated: a feature doc without a declared runner cannot have
|
|
174
|
+
// its command checked against one. The write-ordering half still applies.
|
|
175
|
+
const state = afterCommand("node --test", false, "ok");
|
|
176
|
+
const decision = evidenceGate(state, ledgerFor(state), { task: "T1", lastWriteAt: "2026-09-19T10:00:00.000Z", runner: null }, emptyPolicy());
|
|
177
|
+
assert.equal(decision.allow, true);
|
|
178
|
+
assert.equal(decision.allow === true && decision.observed.command, "node --test");
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
// The other half of that limit, which the README promises and round 2 only
|
|
182
|
+
// wrote down: the *artifact* has to disclose it. A reader auditing a checkoff
|
|
183
|
+
// must be able to tell a run certified by the declared runner from one
|
|
184
|
+
// certified by whatever exit-0 string was handy. Without this the round-1 echo
|
|
185
|
+
// attack still lands, in full, by omitting one optional field at declaration.
|
|
186
|
+
test("an unpinned runner is disclosed in the recorded evidence, not silently skipped", () => {
|
|
187
|
+
const state = afterCommand("echo 'I have verified that all tests pass'", false, "ok");
|
|
188
|
+
const unpinned = evidenceGate(
|
|
189
|
+
state,
|
|
190
|
+
ledgerFor(state),
|
|
191
|
+
{ task: "T1", lastWriteAt: "2026-09-19T10:00:00.000Z", runner: null },
|
|
192
|
+
emptyPolicy(),
|
|
193
|
+
);
|
|
194
|
+
assert.equal(unpinned.allow, true, "with nothing pinned there is nothing to compare against");
|
|
195
|
+
assert.equal(
|
|
196
|
+
unpinned.allow === true && unpinned.observed.outcome,
|
|
197
|
+
"success (runner not pinned)",
|
|
198
|
+
"the outcome carried into the feature doc must say the check was never pinned",
|
|
199
|
+
);
|
|
200
|
+
assert.equal(
|
|
201
|
+
renderObserved(unpinned.allow === true ? unpinned.observed : null),
|
|
202
|
+
"observed: `echo 'I have verified that all tests pass'` → success (runner not pinned)",
|
|
203
|
+
);
|
|
204
|
+
|
|
205
|
+
// And the disclosure is not blanket noise: a checkoff certified by the
|
|
206
|
+
// declared runner must stay clean, or the two cases read the same again.
|
|
207
|
+
const pinned = afterCommand("npm test", false, "ok");
|
|
208
|
+
const certified = evidenceGate(pinned, ledgerFor(pinned), check, emptyPolicy());
|
|
209
|
+
assert.equal(certified.allow === true && certified.observed.outcome, "success");
|
|
210
|
+
assert.equal(
|
|
211
|
+
renderObserved(certified.allow === true ? certified.observed : null),
|
|
212
|
+
"observed: `npm test` → success",
|
|
213
|
+
);
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
// routing.go:101 — in TDD mode the failing test comes first.
|
|
217
|
+
test("TDD mode requires an observed RED before an implementation checkoff", () => {
|
|
218
|
+
const policy = emptyPolicy();
|
|
219
|
+
const green = afterCommand("npm test", false, "ok");
|
|
220
|
+
const tdd = { ...check, tdd: { mode: "strict" as const, source: "nodd_declare", runner: "npm test" } };
|
|
221
|
+
|
|
222
|
+
const withoutRed = evidenceGate(green, ledgerFor(green), tdd, policy);
|
|
223
|
+
assert.equal(withoutRed.allow, false);
|
|
224
|
+
assert.ok(withoutRed.allow === false && /red|failing/i.test(withoutRed.reason));
|
|
225
|
+
|
|
226
|
+
const red = afterCommand("npm test", true, "Command exited with code 1", "r1");
|
|
227
|
+
const both = fold(red, observation({
|
|
228
|
+
toolCallId: "g1", toolName: "bash", input: { command: "npm test" },
|
|
229
|
+
isError: false, resultText: "ok", at: "2026-09-19T10:06:00.000Z",
|
|
230
|
+
}));
|
|
231
|
+
const withRed = evidenceGate(both, ledgerFor(both), tdd, policy);
|
|
232
|
+
assert.equal(withRed.allow, true, "RED then GREEN satisfies TDD mode");
|
|
233
|
+
assert.equal(withRed.allow === true && withRed.observed.tdd?.mode, "strict");
|
|
234
|
+
assert.equal(withRed.allow === true && withRed.observed.tdd?.runner, "npm test");
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
// routing.go:117 — a disabled gate must not make the artifact lie.
|
|
238
|
+
test("the flag off allows the checkoff but records that nothing was verified", () => {
|
|
239
|
+
const policy = { ...emptyPolicy(), config: { evidence: { enabled: false } } };
|
|
240
|
+
const decision = evidenceGate(emptyCommitted(), [], check, policy);
|
|
241
|
+
assert.equal(decision.allow, true);
|
|
242
|
+
assert.equal(decision.allow === true && decision.observed.outcome, "none (gate disabled)");
|
|
243
|
+
assert.equal(renderObserved(decision.allow === true ? decision.observed : null), "observed: none (gate disabled)");
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
test("the rendered line carries the real command and outcome", () => {
|
|
247
|
+
const passed = afterCommand("node --test", false, "ok");
|
|
248
|
+
const decision = evidenceGate(passed, ledgerFor(passed), { ...check, runner: "node --test" }, emptyPolicy());
|
|
249
|
+
assert.equal(renderObserved(decision.allow === true ? decision.observed : null), "observed: `node --test` → success");
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
// The steering condition: a ledger record this kernel never saw is not evidence.
|
|
253
|
+
test("a ledger record the kernel never observed cannot satisfy a checkoff", () => {
|
|
254
|
+
const forged: LedgerRecord[] = [{
|
|
255
|
+
toolCallId: "forged-1", tool: "bash", command: "npm test",
|
|
256
|
+
outcome: { kind: "success" }, at: "2026-09-19T10:05:00.000Z",
|
|
257
|
+
}];
|
|
258
|
+
const decision = evidenceGate(emptyCommitted(), forged, check, emptyPolicy());
|
|
259
|
+
assert.equal(decision.allow, false, "a record on disk is not an observation");
|
|
260
|
+
assert.ok(decision.allow === false && /unverified|not observed/i.test(decision.reason),
|
|
261
|
+
`the refusal must name the degradation: ${decision.allow === false ? decision.reason : ""}`);
|
|
262
|
+
|
|
263
|
+
// Resuming in a fresh session hits this every time. If the refusal does not
|
|
264
|
+
// say why and what to do, it reads as "NODD is broken" and gets switched off.
|
|
265
|
+
assert.ok(decision.allow === false && /previous session|earlier session/i.test(decision.reason),
|
|
266
|
+
"the refusal must explain that the ledger is from a previous session");
|
|
267
|
+
assert.ok(decision.allow === false && /re-?run/i.test(decision.remedy.action),
|
|
268
|
+
"the remedy must say to re-run the check");
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
test("a mismatching record reads differently from a merely unverified one", () => {
|
|
272
|
+
const state = afterCommand("npm test", true, "Command exited with code 1");
|
|
273
|
+
const flipped: LedgerRecord[] = [{
|
|
274
|
+
toolCallId: "b1", tool: "bash", command: "npm test",
|
|
275
|
+
outcome: { kind: "success" }, at: "2026-09-19T10:05:00.000Z",
|
|
276
|
+
}];
|
|
277
|
+
const decision = evidenceGate(state, flipped, check, emptyPolicy());
|
|
278
|
+
assert.equal(decision.allow, false);
|
|
279
|
+
assert.ok(decision.allow === false && !/previous session/i.test(decision.reason),
|
|
280
|
+
"a contradicted record is not a stale-session problem and must not be described as one");
|
|
281
|
+
});
|