@openagentsinc/conformance-kit 0.2.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +111 -0
- package/package.json +39 -0
- package/src/adapter-laws.test.ts +36 -0
- package/src/adapter-laws.ts +184 -0
- package/src/event-log-laws.test.ts +10 -0
- package/src/event-log-laws.ts +273 -0
- package/src/fixtures.ts +106 -0
- package/src/index.ts +24 -0
- package/src/recall-laws.test.ts +10 -0
- package/src/recall-laws.ts +270 -0
- package/src/reducer-laws.test.ts +17 -0
- package/src/reducer-laws.ts +277 -0
- package/src/rlm-cap-laws.test.ts +11 -0
- package/src/rlm-cap-laws.ts +207 -0
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
import { Deferred, Effect, Fiber, Ref, Stream } from "effect";
|
|
2
|
+
import {
|
|
3
|
+
buildTextDelta,
|
|
4
|
+
buildTurnFinished,
|
|
5
|
+
buildTurnStarted,
|
|
6
|
+
type HarnessEventLogStore,
|
|
7
|
+
makeHarnessEventLog,
|
|
8
|
+
} from "@openagentsinc/agent-harness-contract";
|
|
9
|
+
import type { KhalaRuntimeSource } from "@openagentsinc/agent-runtime-schema";
|
|
10
|
+
import { describe, expect, test } from "vite-plus/test";
|
|
11
|
+
|
|
12
|
+
import { scriptTurn, sequencesOf, TEST_SOURCE } from "./fixtures.ts";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Configuration for {@link runEventLogLaws}. The implementation under test is a
|
|
16
|
+
* factory that produces a fresh {@link HarnessEventLogStore} — the persistence
|
|
17
|
+
* port a real backend (the desktop local-turn journal, the managed-sandbox
|
|
18
|
+
* event store, or a third party's store) implements. The kit builds the
|
|
19
|
+
* {@link makeHarnessEventLog} runtime over the store, so a store that satisfies
|
|
20
|
+
* the port passes both the store-level and the live-attach laws.
|
|
21
|
+
*/
|
|
22
|
+
export interface EventLogLawsConfig {
|
|
23
|
+
/** A short label naming the implementation under test, used in test titles. */
|
|
24
|
+
readonly label: string;
|
|
25
|
+
/** Produce a fresh, empty store. Called once per law so laws never share state. */
|
|
26
|
+
readonly makeStore: () => Effect.Effect<HarnessEventLogStore>;
|
|
27
|
+
/** The event source label. Defaults to the shared `test_fixture` source. */
|
|
28
|
+
readonly source?: KhalaRuntimeSource;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* The durable event-log laws, parameterized over any store.
|
|
33
|
+
*
|
|
34
|
+
* Promoted, published form of `agent-harness-contract/src/event-log.test.ts`:
|
|
35
|
+
*
|
|
36
|
+
* - **Append monotonicity / dup-free.** A non-increasing sequence for a turn
|
|
37
|
+
* (duplicate or out-of-order) is rejected with a typed error — this is what
|
|
38
|
+
* makes replay duplicate-free.
|
|
39
|
+
* - **Gap-free, dup-free replay from a cursor.** `read` / `replay` return
|
|
40
|
+
* exactly the events after the cursor, ascending, no gap and no duplicate.
|
|
41
|
+
* - **Durable replay after process death.** A fresh log runtime over the SAME
|
|
42
|
+
* store replays everything a dead producer persisted.
|
|
43
|
+
* - **Rerun boundaries are recorded and reported** so a recomputed tail is
|
|
44
|
+
* distinguishable from an exact attach.
|
|
45
|
+
* - **Live attach** replays the persisted tail then follows new events with no
|
|
46
|
+
* gap or duplicate at the seam.
|
|
47
|
+
* - **Single-flight attach.** A newer attach for the same `(turn, class)`
|
|
48
|
+
* supersedes the older so a reconnecting consumer never double-runs.
|
|
49
|
+
*/
|
|
50
|
+
export const runEventLogLaws = (config: EventLogLawsConfig): void => {
|
|
51
|
+
const { label, makeStore } = config;
|
|
52
|
+
const source = config.source ?? TEST_SOURCE;
|
|
53
|
+
|
|
54
|
+
const turn = (turnId: string, words: ReadonlyArray<string>, startSeq = 0) =>
|
|
55
|
+
scriptTurn({ turnId, threadId: "s1", words, startSeq, source });
|
|
56
|
+
|
|
57
|
+
describe(`[${label}] event log — durable replay`, () => {
|
|
58
|
+
test("rejects a duplicate or out-of-order sequence for a turn", async () => {
|
|
59
|
+
const exit = await Effect.runPromiseExit(
|
|
60
|
+
Effect.gen(function* () {
|
|
61
|
+
const store = yield* makeStore();
|
|
62
|
+
const [first] = turn("t1", []); // just turn.started at seq 0
|
|
63
|
+
yield* store.append(first!);
|
|
64
|
+
// Re-append the same sequence 0 — must be rejected.
|
|
65
|
+
yield* store.append(first!);
|
|
66
|
+
}),
|
|
67
|
+
);
|
|
68
|
+
expect(exit._tag).toBe("Failure");
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test("read returns exactly the tail after a cursor, ascending, gap-free and dup-free", async () => {
|
|
72
|
+
const tail = await Effect.runPromise(
|
|
73
|
+
Effect.gen(function* () {
|
|
74
|
+
const store = yield* makeStore();
|
|
75
|
+
yield* Effect.forEach(turn("t1", ["a", "b", "c"]), (event) => store.append(event));
|
|
76
|
+
return yield* store.read({ turnId: "t1", fromCursor: 1 });
|
|
77
|
+
}),
|
|
78
|
+
);
|
|
79
|
+
expect(sequencesOf(tail)).toEqual([2, 3, 4]);
|
|
80
|
+
expect(new Set(sequencesOf(tail)).size).toBe(tail.length);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("lastCursor is -1 when empty and the greatest stored sequence otherwise", async () => {
|
|
84
|
+
const cursors = await Effect.runPromise(
|
|
85
|
+
Effect.gen(function* () {
|
|
86
|
+
const store = yield* makeStore();
|
|
87
|
+
const empty = yield* store.lastCursor({ turnId: "t1" });
|
|
88
|
+
yield* Effect.forEach(turn("t1", ["a", "b"]), (event) => store.append(event));
|
|
89
|
+
const filled = yield* store.lastCursor({ turnId: "t1" });
|
|
90
|
+
return { empty, filled };
|
|
91
|
+
}),
|
|
92
|
+
);
|
|
93
|
+
expect(cursors.empty).toBe(-1);
|
|
94
|
+
expect(cursors.filled).toBe(3);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
test("replays the tail from a cursor after process death (fresh runtime, same store)", async () => {
|
|
98
|
+
const outcome = await Effect.runPromise(
|
|
99
|
+
Effect.gen(function* () {
|
|
100
|
+
const store = yield* makeStore();
|
|
101
|
+
const script = turn("t1", ["a", "b", "c"]); // sequences 0..4
|
|
102
|
+
|
|
103
|
+
// Producer runtime appends the whole turn, then "dies".
|
|
104
|
+
const producer = yield* makeHarnessEventLog(store);
|
|
105
|
+
yield* Effect.forEach(script, (event) => producer.appendEvent(event));
|
|
106
|
+
|
|
107
|
+
// A different runtime rehydrates over the SAME store and replays from 1.
|
|
108
|
+
const recovered = yield* makeHarnessEventLog(store);
|
|
109
|
+
const tail = yield* Stream.runCollect(recovered.replay({ turnId: "t1", fromCursor: 1 }));
|
|
110
|
+
const last = yield* recovered.lastCursor({ turnId: "t1" });
|
|
111
|
+
const all = yield* Stream.runCollect(recovered.replay({ turnId: "t1", fromCursor: -1 }));
|
|
112
|
+
return { tail, last, all };
|
|
113
|
+
}),
|
|
114
|
+
);
|
|
115
|
+
expect(sequencesOf(outcome.tail)).toEqual([2, 3, 4]);
|
|
116
|
+
expect(new Set(sequencesOf(outcome.tail)).size).toBe(outcome.tail.length);
|
|
117
|
+
expect(outcome.last).toBe(4);
|
|
118
|
+
// A full replay from -1 returns the entire turn exactly once.
|
|
119
|
+
expect(sequencesOf(outcome.all)).toEqual([0, 1, 2, 3, 4]);
|
|
120
|
+
});
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
describe(`[${label}] event log — rerun boundary visibility`, () => {
|
|
124
|
+
test("records and reports a rerun boundary so a recomputed tail is distinguishable", async () => {
|
|
125
|
+
const boundaries = await Effect.runPromise(
|
|
126
|
+
Effect.gen(function* () {
|
|
127
|
+
const store = yield* makeStore();
|
|
128
|
+
const log = yield* makeHarnessEventLog(store);
|
|
129
|
+
yield* Effect.forEach(turn("t1", ["a"]), (event) => log.appendEvent(event));
|
|
130
|
+
yield* log.markRerunBoundary({ turnId: "t1", atCursor: 1 });
|
|
131
|
+
return yield* log.rerunBoundaries({ turnId: "t1" });
|
|
132
|
+
}),
|
|
133
|
+
);
|
|
134
|
+
expect(boundaries).toEqual([1]);
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
describe(`[${label}] event log — live attach`, () => {
|
|
139
|
+
test("attach replays the persisted tail then follows new events with no gap or duplicate", async () => {
|
|
140
|
+
const collected = await Effect.runPromise(
|
|
141
|
+
Effect.gen(function* () {
|
|
142
|
+
const store = yield* makeStore();
|
|
143
|
+
const log = yield* makeHarnessEventLog(store);
|
|
144
|
+
|
|
145
|
+
// Persist the first two events (seq 0, 1).
|
|
146
|
+
yield* log.appendEvent(
|
|
147
|
+
buildTurnStarted({ turnId: "t1", threadId: "s1", sequence: 0, source }),
|
|
148
|
+
);
|
|
149
|
+
yield* log.appendEvent(
|
|
150
|
+
buildTextDelta({
|
|
151
|
+
turnId: "t1",
|
|
152
|
+
threadId: "s1",
|
|
153
|
+
sequence: 1,
|
|
154
|
+
source,
|
|
155
|
+
messageId: "m",
|
|
156
|
+
text: "a",
|
|
157
|
+
}),
|
|
158
|
+
);
|
|
159
|
+
|
|
160
|
+
// Attach from cursor 0 (want seq 1 onward): replay 1, then live 2..3.
|
|
161
|
+
const sink = yield* Ref.make<ReadonlyArray<number>>([]);
|
|
162
|
+
const wantCount = 3; // seq 1, 2, 3
|
|
163
|
+
const done = yield* Deferred.make<void>();
|
|
164
|
+
const fiber = yield* Effect.forkChild(
|
|
165
|
+
log.attach({ turnId: "t1", fromCursor: 0, consumerClass: "renderer" }).pipe(
|
|
166
|
+
Stream.runForEach((event) =>
|
|
167
|
+
Effect.gen(function* () {
|
|
168
|
+
const next = yield* Ref.updateAndGet(sink, (xs) => [...xs, event.sequence]);
|
|
169
|
+
if (next.length >= wantCount) {
|
|
170
|
+
yield* Deferred.succeed(done, undefined);
|
|
171
|
+
}
|
|
172
|
+
}),
|
|
173
|
+
),
|
|
174
|
+
),
|
|
175
|
+
);
|
|
176
|
+
|
|
177
|
+
// Append two more live events after the attach is running.
|
|
178
|
+
yield* log.appendEvent(
|
|
179
|
+
buildTextDelta({
|
|
180
|
+
turnId: "t1",
|
|
181
|
+
threadId: "s1",
|
|
182
|
+
sequence: 2,
|
|
183
|
+
source,
|
|
184
|
+
messageId: "m",
|
|
185
|
+
text: "b",
|
|
186
|
+
}),
|
|
187
|
+
);
|
|
188
|
+
yield* log.appendEvent(
|
|
189
|
+
buildTurnFinished({
|
|
190
|
+
turnId: "t1",
|
|
191
|
+
threadId: "s1",
|
|
192
|
+
sequence: 3,
|
|
193
|
+
source,
|
|
194
|
+
finishReason: "stop",
|
|
195
|
+
}),
|
|
196
|
+
);
|
|
197
|
+
|
|
198
|
+
yield* Deferred.await(done);
|
|
199
|
+
yield* Fiber.interrupt(fiber);
|
|
200
|
+
return yield* Ref.get(sink);
|
|
201
|
+
}),
|
|
202
|
+
);
|
|
203
|
+
|
|
204
|
+
expect(collected).toEqual([1, 2, 3]);
|
|
205
|
+
expect(new Set(collected).size).toBe(collected.length);
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
test("single-flight: a newer attach for the same (turn, class) supersedes the older", async () => {
|
|
209
|
+
const result = await Effect.runPromise(
|
|
210
|
+
Effect.gen(function* () {
|
|
211
|
+
const store = yield* makeStore();
|
|
212
|
+
const log = yield* makeHarnessEventLog(store);
|
|
213
|
+
yield* log.appendEvent(
|
|
214
|
+
buildTurnStarted({ turnId: "t1", threadId: "s1", sequence: 0, source }),
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
const sinkA = yield* Ref.make<ReadonlyArray<number>>([]);
|
|
218
|
+
const sinkB = yield* Ref.make<ReadonlyArray<number>>([]);
|
|
219
|
+
const bGot = yield* Deferred.make<void>();
|
|
220
|
+
const aReplayed = yield* Deferred.make<void>();
|
|
221
|
+
|
|
222
|
+
// Attach A replays seq 0 then follows. Receiving seq 0 proves A is the
|
|
223
|
+
// active subscriber — deterministic, no sleep needed.
|
|
224
|
+
const fiberA = yield* Effect.forkChild(
|
|
225
|
+
log.attach({ turnId: "t1", fromCursor: -1, consumerClass: "renderer" }).pipe(
|
|
226
|
+
Stream.runForEach((event) =>
|
|
227
|
+
Effect.gen(function* () {
|
|
228
|
+
yield* Ref.update(sinkA, (xs) => [...xs, event.sequence]);
|
|
229
|
+
yield* Deferred.succeed(aReplayed, undefined);
|
|
230
|
+
}),
|
|
231
|
+
),
|
|
232
|
+
),
|
|
233
|
+
);
|
|
234
|
+
yield* Deferred.await(aReplayed);
|
|
235
|
+
|
|
236
|
+
// Attach B for the SAME (turn, class) — supersedes A.
|
|
237
|
+
const fiberB = yield* Effect.forkChild(
|
|
238
|
+
log.attach({ turnId: "t1", fromCursor: 0, consumerClass: "renderer" }).pipe(
|
|
239
|
+
Stream.runForEach((event) =>
|
|
240
|
+
Effect.gen(function* () {
|
|
241
|
+
yield* Ref.update(sinkB, (xs) => [...xs, event.sequence]);
|
|
242
|
+
yield* Deferred.succeed(bGot, undefined);
|
|
243
|
+
}),
|
|
244
|
+
),
|
|
245
|
+
),
|
|
246
|
+
);
|
|
247
|
+
|
|
248
|
+
// A should be interrupted by the supersede; await its fiber.
|
|
249
|
+
yield* Fiber.await(fiberA);
|
|
250
|
+
|
|
251
|
+
// Publish a new event; only B (the live subscriber) should receive it.
|
|
252
|
+
yield* log.appendEvent(
|
|
253
|
+
buildTextDelta({
|
|
254
|
+
turnId: "t1",
|
|
255
|
+
threadId: "s1",
|
|
256
|
+
sequence: 1,
|
|
257
|
+
source,
|
|
258
|
+
messageId: "m",
|
|
259
|
+
text: "b",
|
|
260
|
+
}),
|
|
261
|
+
);
|
|
262
|
+
yield* Deferred.await(bGot);
|
|
263
|
+
yield* Fiber.interrupt(fiberB);
|
|
264
|
+
|
|
265
|
+
return { a: yield* Ref.get(sinkA), b: yield* Ref.get(sinkB) };
|
|
266
|
+
}),
|
|
267
|
+
);
|
|
268
|
+
|
|
269
|
+
expect(result.a).toEqual([0]);
|
|
270
|
+
expect(result.b).toContain(1);
|
|
271
|
+
});
|
|
272
|
+
});
|
|
273
|
+
};
|
package/src/fixtures.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { Effect, Stream } from "effect";
|
|
2
|
+
import {
|
|
3
|
+
buildTextDelta,
|
|
4
|
+
buildTurnFinished,
|
|
5
|
+
buildTurnStarted,
|
|
6
|
+
type HarnessStreamEvent,
|
|
7
|
+
} from "@openagentsinc/agent-harness-contract";
|
|
8
|
+
import type { KhalaRuntimeSource } from "@openagentsinc/agent-runtime-schema";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Shared, deterministic fixtures for the conformance kit's law suites. Every
|
|
12
|
+
* builder here is pure and clock-free: identical inputs give identical events,
|
|
13
|
+
* so a law suite is reproducible against any implementation under test.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/** The canonical source label the kit stamps on every fixture event. */
|
|
17
|
+
export const TEST_SOURCE: KhalaRuntimeSource = { lane: "test_fixture" };
|
|
18
|
+
|
|
19
|
+
/** The `sequence` list of a run of events — the shape most laws assert over. */
|
|
20
|
+
export const sequencesOf = (events: ReadonlyArray<HarnessStreamEvent>): ReadonlyArray<number> =>
|
|
21
|
+
events.map((event) => event.sequence);
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* A full scripted turn: `turn.started`, one `text.delta` per word, then
|
|
25
|
+
* `turn.finished`. Sequences are contiguous from `startSeq`. This is the same
|
|
26
|
+
* scripted turn shape the in-repo reference suites use, hoisted so a consumer
|
|
27
|
+
* building its own store/log fixtures reuses the exact event vocabulary.
|
|
28
|
+
*/
|
|
29
|
+
export const scriptTurn = (params: {
|
|
30
|
+
readonly turnId: string;
|
|
31
|
+
readonly threadId: string;
|
|
32
|
+
readonly words: ReadonlyArray<string>;
|
|
33
|
+
readonly startSeq?: number;
|
|
34
|
+
readonly source?: KhalaRuntimeSource;
|
|
35
|
+
}): ReadonlyArray<HarnessStreamEvent> => {
|
|
36
|
+
const source = params.source ?? TEST_SOURCE;
|
|
37
|
+
const events: Array<HarnessStreamEvent> = [];
|
|
38
|
+
let seq = params.startSeq ?? 0;
|
|
39
|
+
events.push(
|
|
40
|
+
buildTurnStarted({ turnId: params.turnId, threadId: params.threadId, sequence: seq++, source }),
|
|
41
|
+
);
|
|
42
|
+
for (const word of params.words) {
|
|
43
|
+
events.push(
|
|
44
|
+
buildTextDelta({
|
|
45
|
+
turnId: params.turnId,
|
|
46
|
+
threadId: params.threadId,
|
|
47
|
+
sequence: seq++,
|
|
48
|
+
source,
|
|
49
|
+
messageId: `msg.${params.turnId}`,
|
|
50
|
+
text: word,
|
|
51
|
+
}),
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
events.push(
|
|
55
|
+
buildTurnFinished({
|
|
56
|
+
turnId: params.turnId,
|
|
57
|
+
threadId: params.threadId,
|
|
58
|
+
sequence: seq++,
|
|
59
|
+
source,
|
|
60
|
+
finishReason: "stop",
|
|
61
|
+
}),
|
|
62
|
+
);
|
|
63
|
+
return events;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
/** Collect a finite harness stream into an array. */
|
|
67
|
+
export const collect = <E>(
|
|
68
|
+
stream: Stream.Stream<HarnessStreamEvent, E>,
|
|
69
|
+
): Effect.Effect<ReadonlyArray<HarnessStreamEvent>, E> => Stream.runCollect(stream);
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Run an effect and report whether it succeeded or failed, without letting the
|
|
73
|
+
* typed failure escape. Law suites use this to probe optional behavior: a
|
|
74
|
+
* refused capability must fail with a specific typed error, and the suite must
|
|
75
|
+
* be able to observe that failure and assert on it rather than aborting.
|
|
76
|
+
*/
|
|
77
|
+
export type Attempt<A, E> =
|
|
78
|
+
| { readonly _tag: "ok"; readonly value: A }
|
|
79
|
+
| { readonly _tag: "failed"; readonly error: E };
|
|
80
|
+
|
|
81
|
+
export const attempt = <A, E>(effect: Effect.Effect<A, E>): Effect.Effect<Attempt<A, E>> =>
|
|
82
|
+
effect.pipe(
|
|
83
|
+
Effect.map((value): Attempt<A, E> => ({ _tag: "ok", value })),
|
|
84
|
+
Effect.catch((error: E) => Effect.succeed<Attempt<A, E>>({ _tag: "failed", error })),
|
|
85
|
+
);
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Assert a run of events is a contiguous, gap-free, duplicate-free ascending
|
|
89
|
+
* sequence. Returns the sequence list so a caller can chain further checks.
|
|
90
|
+
*/
|
|
91
|
+
export const assertContiguous = (
|
|
92
|
+
events: ReadonlyArray<HarnessStreamEvent>,
|
|
93
|
+
params: { readonly from: number },
|
|
94
|
+
): ReadonlyArray<number> => {
|
|
95
|
+
const seqs = sequencesOf(events);
|
|
96
|
+
const expected = seqs.map((_, index) => params.from + index);
|
|
97
|
+
if (JSON.stringify(seqs) !== JSON.stringify(expected)) {
|
|
98
|
+
throw new Error(
|
|
99
|
+
`expected contiguous sequences from ${params.from}: got ${JSON.stringify(seqs)}`,
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
if (new Set(seqs).size !== seqs.length) {
|
|
103
|
+
throw new Error(`expected no duplicate sequences: got ${JSON.stringify(seqs)}`);
|
|
104
|
+
}
|
|
105
|
+
return seqs;
|
|
106
|
+
};
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @openagentsinc/conformance-kit — published law suites for the
|
|
3
|
+
* `@openagentsinc/ai` contracts.
|
|
4
|
+
*
|
|
5
|
+
* Each law suite is a runner a consumer calls in their own test file,
|
|
6
|
+
* parameterized over the implementation under test. Point one at your adapter,
|
|
7
|
+
* store, reducer, recall source, or RLM engine and it either passes the kit or
|
|
8
|
+
* it is not conformant. The kit's own tests run every suite against the in-repo
|
|
9
|
+
* reference implementations, so the kit is proven to work.
|
|
10
|
+
*/
|
|
11
|
+
export { runAdapterLaws, type AdapterLawsConfig } from "./adapter-laws.ts";
|
|
12
|
+
export { runEventLogLaws, type EventLogLawsConfig } from "./event-log-laws.ts";
|
|
13
|
+
export { runReducerLaws, type ReducerLawsConfig } from "./reducer-laws.ts";
|
|
14
|
+
export { runRecallLaws, type RecallLawsConfig, type RecallSource } from "./recall-laws.ts";
|
|
15
|
+
export { runRlmCapLaws, type RlmCapLawsConfig } from "./rlm-cap-laws.ts";
|
|
16
|
+
export {
|
|
17
|
+
assertContiguous,
|
|
18
|
+
attempt,
|
|
19
|
+
type Attempt,
|
|
20
|
+
collect,
|
|
21
|
+
scriptTurn,
|
|
22
|
+
sequencesOf,
|
|
23
|
+
TEST_SOURCE,
|
|
24
|
+
} from "./fixtures.ts";
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { recallTierD } from "@openagentsinc/history-corpus";
|
|
2
|
+
|
|
3
|
+
import { runRecallLaws } from "./recall-laws.ts";
|
|
4
|
+
|
|
5
|
+
// The kit run against the in-repo reference recall backend (Tier D) proves the
|
|
6
|
+
// kit itself works.
|
|
7
|
+
runRecallLaws({
|
|
8
|
+
label: "recall-tier-d",
|
|
9
|
+
recall: (params) => recallTierD(params),
|
|
10
|
+
});
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import {
|
|
3
|
+
buildTextDelta,
|
|
4
|
+
buildTurnFinished,
|
|
5
|
+
buildTurnStarted,
|
|
6
|
+
type HarnessStreamEvent,
|
|
7
|
+
makeInMemoryEventLogStore,
|
|
8
|
+
} from "@openagentsinc/agent-harness-contract";
|
|
9
|
+
import {
|
|
10
|
+
buildHistoryCorpus,
|
|
11
|
+
type HistoryCorpusEntry,
|
|
12
|
+
type HistoryCorpusPolicy,
|
|
13
|
+
HistoryRecallError,
|
|
14
|
+
historyRecallDefaultCaps,
|
|
15
|
+
type HistoryRecallCaps,
|
|
16
|
+
type HistoryRecallQuestion,
|
|
17
|
+
type HistoryRecallResponse,
|
|
18
|
+
} from "@openagentsinc/history-corpus";
|
|
19
|
+
import type { KhalaRuntimeSource } from "@openagentsinc/agent-runtime-schema";
|
|
20
|
+
import { describe, expect, test } from "vite-plus/test";
|
|
21
|
+
|
|
22
|
+
const SOURCE: KhalaRuntimeSource = { lane: "test_fixture" };
|
|
23
|
+
const BUILT_AT = "2026-07-21T12:00:00.000Z";
|
|
24
|
+
const THREAD_ID = "t1";
|
|
25
|
+
const TURN_COUNT = 120;
|
|
26
|
+
const PLANTED_TURN = 80;
|
|
27
|
+
const LONG_TURN = 100;
|
|
28
|
+
const PLANTED_TEXT = "DECISION: adopt the blue protocol";
|
|
29
|
+
const LONG_TEXT = `LONGSPAN ${"x".repeat(600)}`;
|
|
30
|
+
|
|
31
|
+
const ownerPolicy: HistoryCorpusPolicy = {
|
|
32
|
+
includeVisibilities: ["public", "operator", "private"],
|
|
33
|
+
includeRedactionClasses: ["public_ref", "redacted_summary", "operator_summary", "private_ref"],
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
const turnIdOf = (turn: number): string => `t1.turn-${String(turn).padStart(3, "0")}`;
|
|
37
|
+
|
|
38
|
+
const scriptTurn = (turn: number, words: ReadonlyArray<string>): Array<HarnessStreamEvent> => {
|
|
39
|
+
const turnId = turnIdOf(turn);
|
|
40
|
+
const events: Array<HarnessStreamEvent> = [];
|
|
41
|
+
let seq = 0;
|
|
42
|
+
events.push(buildTurnStarted({ turnId, threadId: THREAD_ID, sequence: seq++, source: SOURCE }));
|
|
43
|
+
for (const word of words) {
|
|
44
|
+
events.push(
|
|
45
|
+
buildTextDelta({
|
|
46
|
+
turnId,
|
|
47
|
+
threadId: THREAD_ID,
|
|
48
|
+
sequence: seq++,
|
|
49
|
+
source: SOURCE,
|
|
50
|
+
messageId: `msg.${turnId}`,
|
|
51
|
+
text: word,
|
|
52
|
+
}),
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
events.push(
|
|
56
|
+
buildTurnFinished({
|
|
57
|
+
turnId,
|
|
58
|
+
threadId: THREAD_ID,
|
|
59
|
+
sequence: seq++,
|
|
60
|
+
source: SOURCE,
|
|
61
|
+
finishReason: "stop",
|
|
62
|
+
}),
|
|
63
|
+
);
|
|
64
|
+
return events;
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
interface RecallFixture {
|
|
68
|
+
readonly entries: ReadonlyArray<HistoryCorpusEntry>;
|
|
69
|
+
readonly coverageNote: string;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* A deep, deterministic corpus: many turns on one thread, with a decision
|
|
74
|
+
* planted hundreds of sequences in, and a 600-plus-character text that
|
|
75
|
+
* overflows the default per-span character cap.
|
|
76
|
+
*/
|
|
77
|
+
const buildRecallFixture = async (): Promise<RecallFixture> => {
|
|
78
|
+
const store = makeInMemoryEventLogStore();
|
|
79
|
+
const turnIds: Array<string> = [];
|
|
80
|
+
const events: Array<HarnessStreamEvent> = [];
|
|
81
|
+
for (let turn = 0; turn < TURN_COUNT; turn++) {
|
|
82
|
+
turnIds.push(turnIdOf(turn));
|
|
83
|
+
if (turn === PLANTED_TURN) {
|
|
84
|
+
events.push(...scriptTurn(turn, ["alpha", PLANTED_TEXT, "beta"]));
|
|
85
|
+
} else if (turn === LONG_TURN) {
|
|
86
|
+
events.push(...scriptTurn(turn, [LONG_TEXT]));
|
|
87
|
+
} else {
|
|
88
|
+
events.push(...scriptTurn(turn, ["alpha", "beta"]));
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
await Effect.runPromise(Effect.forEach(events, (event) => store.append(event)));
|
|
92
|
+
const corpus = await Effect.runPromise(
|
|
93
|
+
buildHistoryCorpus({
|
|
94
|
+
scope: { _tag: "Thread", threadId: THREAD_ID },
|
|
95
|
+
eventLog: store,
|
|
96
|
+
turnIds,
|
|
97
|
+
policy: ownerPolicy,
|
|
98
|
+
builtAt: BUILT_AT,
|
|
99
|
+
}),
|
|
100
|
+
);
|
|
101
|
+
return { entries: corpus.entries, coverageNote: corpus.manifest.coverage.note };
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
let cachedFixture: Promise<RecallFixture> | undefined;
|
|
105
|
+
const recallFixture = (): Promise<RecallFixture> => (cachedFixture ??= buildRecallFixture());
|
|
106
|
+
|
|
107
|
+
/** The recall query surface under test — matches the Tier D input signature. */
|
|
108
|
+
export interface RecallSource {
|
|
109
|
+
readonly recall: (params: {
|
|
110
|
+
readonly entries: ReadonlyArray<HistoryCorpusEntry>;
|
|
111
|
+
readonly coverageNote: string;
|
|
112
|
+
readonly question: HistoryRecallQuestion;
|
|
113
|
+
readonly caps?: HistoryRecallCaps | undefined;
|
|
114
|
+
}) => Effect.Effect<HistoryRecallResponse, HistoryRecallError>;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Configuration for {@link runRecallLaws}. */
|
|
118
|
+
export interface RecallLawsConfig extends RecallSource {
|
|
119
|
+
/** A short label naming the implementation under test, used in test titles. */
|
|
120
|
+
readonly label: string;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* The recall honesty laws, parameterized over any recall source.
|
|
125
|
+
*
|
|
126
|
+
* Promoted, published form of `history-corpus/src/recall.test.ts`. Recall is a
|
|
127
|
+
* deterministic query over a durable corpus; the laws hold the source to the
|
|
128
|
+
* honesty contract: caps TRUNCATE (they never fail), and the response must
|
|
129
|
+
* state exactly what it scanned, which caps it hit, and the corpus coverage
|
|
130
|
+
* bound. A source that answers correctly but lies about coverage — or that
|
|
131
|
+
* fabricates a model call behind a "deterministic" answer — is not conformant.
|
|
132
|
+
*
|
|
133
|
+
* - **Correctness anchor.** A decision planted deep is found with its exact
|
|
134
|
+
* cursor span and zero model calls.
|
|
135
|
+
* - **Caps truncate and report.** `maxEntriesScanned`, `maxSpans`, and
|
|
136
|
+
* `maxCharsPerSpan` each truncate; `truncated` is true exactly when `capsHit`
|
|
137
|
+
* is non-empty, and `capsHit` names the cap.
|
|
138
|
+
* - **`cost.modelCalls` is always 0** for every deterministic question kind.
|
|
139
|
+
* - **Coverage note carry-through.** The corpus coverage bound rides on every
|
|
140
|
+
* response's `honesty.coverageNote`.
|
|
141
|
+
* - **Invalid input fails typed.** An uncompilable grep pattern is a typed
|
|
142
|
+
* `invalid_pattern` error, never a crash.
|
|
143
|
+
*/
|
|
144
|
+
export const runRecallLaws = (config: RecallLawsConfig): void => {
|
|
145
|
+
const { label, recall } = config;
|
|
146
|
+
|
|
147
|
+
const ask = async (question: HistoryRecallQuestion, caps?: HistoryRecallCaps) => {
|
|
148
|
+
const { entries, coverageNote } = await recallFixture();
|
|
149
|
+
return Effect.runPromise(recall({ entries, coverageNote, question, caps }));
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
describe(`[${label}] recall — correctness and cost`, () => {
|
|
153
|
+
test("finds a decision planted deep with its exact cursor span and zero model calls", async () => {
|
|
154
|
+
const { entries } = await recallFixture();
|
|
155
|
+
const response = await ask({ _tag: "Grep", pattern: "DECISION: adopt" });
|
|
156
|
+
expect(response.answers.length).toBe(1);
|
|
157
|
+
expect(response.answers[0]).toMatchObject({
|
|
158
|
+
turnId: turnIdOf(PLANTED_TURN),
|
|
159
|
+
sequenceStart: 2,
|
|
160
|
+
sequenceEnd: 2,
|
|
161
|
+
excerpt: PLANTED_TEXT,
|
|
162
|
+
});
|
|
163
|
+
expect(response.honesty.tier).toBe("deterministic");
|
|
164
|
+
expect(response.honesty.entriesScanned).toBe(entries.length);
|
|
165
|
+
expect(response.honesty.entriesTotal).toBe(entries.length);
|
|
166
|
+
expect(response.honesty.truncated).toBe(false);
|
|
167
|
+
expect(response.honesty.capsHit).toEqual([]);
|
|
168
|
+
expect(response.cost.modelCalls).toBe(0);
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
test("every question kind answers with zero model calls", async () => {
|
|
172
|
+
const questions: ReadonlyArray<HistoryRecallQuestion> = [
|
|
173
|
+
{ _tag: "Grep", pattern: "alpha" },
|
|
174
|
+
{ _tag: "CursorSlice", fromSequence: 0, toSequence: 1 },
|
|
175
|
+
{ _tag: "KeyTurns", limit: 2 },
|
|
176
|
+
{ _tag: "TurnSummary", turnId: turnIdOf(0) },
|
|
177
|
+
];
|
|
178
|
+
for (const question of questions) {
|
|
179
|
+
const response = await ask(question, { maxSpans: 3 });
|
|
180
|
+
expect(response.cost.modelCalls).toBe(0);
|
|
181
|
+
expect(response.honesty.tier).toBe("deterministic");
|
|
182
|
+
}
|
|
183
|
+
});
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
describe(`[${label}] recall — caps truncate and report honestly`, () => {
|
|
187
|
+
test("maxEntriesScanned stops the scan and honesty names the cap, across cap values", async () => {
|
|
188
|
+
const { entries } = await recallFixture();
|
|
189
|
+
const total = entries.length;
|
|
190
|
+
for (const cap of [1, 5, 137, total, total + 1000]) {
|
|
191
|
+
const response = await ask(
|
|
192
|
+
{ _tag: "Grep", pattern: "zzz-no-such-text" },
|
|
193
|
+
{
|
|
194
|
+
maxEntriesScanned: cap,
|
|
195
|
+
},
|
|
196
|
+
);
|
|
197
|
+
expect(response.honesty.entriesScanned).toBe(Math.min(cap, total));
|
|
198
|
+
expect(response.honesty.entriesTotal).toBe(total);
|
|
199
|
+
const expectTruncated = cap < total;
|
|
200
|
+
expect(response.honesty.truncated).toBe(expectTruncated);
|
|
201
|
+
expect(response.honesty.capsHit).toEqual(expectTruncated ? ["maxEntriesScanned"] : []);
|
|
202
|
+
}
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
test("maxSpans caps the answers and honesty names the cap", async () => {
|
|
206
|
+
// Every turn matches sequence 0, so there are TURN_COUNT candidate spans.
|
|
207
|
+
const question: HistoryRecallQuestion = {
|
|
208
|
+
_tag: "CursorSlice",
|
|
209
|
+
fromSequence: 0,
|
|
210
|
+
toSequence: 0,
|
|
211
|
+
};
|
|
212
|
+
const capped = await ask(question, { maxSpans: 7 });
|
|
213
|
+
expect(capped.answers.length).toBe(7);
|
|
214
|
+
expect(capped.honesty.truncated).toBe(true);
|
|
215
|
+
expect(capped.honesty.capsHit).toEqual(["maxSpans"]);
|
|
216
|
+
|
|
217
|
+
const defaulted = await ask(question);
|
|
218
|
+
expect(defaulted.answers.length).toBe(historyRecallDefaultCaps.maxSpans);
|
|
219
|
+
expect(defaulted.honesty.capsHit).toEqual(["maxSpans"]);
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
test("maxCharsPerSpan truncates the excerpt and honesty names the cap", async () => {
|
|
223
|
+
const defaulted = await ask({ _tag: "Grep", pattern: "LONGSPAN" });
|
|
224
|
+
expect(defaulted.answers.length).toBe(1);
|
|
225
|
+
expect(defaulted.answers[0]!.excerpt.length).toBe(historyRecallDefaultCaps.maxCharsPerSpan);
|
|
226
|
+
expect(defaulted.honesty.truncated).toBe(true);
|
|
227
|
+
expect(defaulted.honesty.capsHit).toEqual(["maxCharsPerSpan"]);
|
|
228
|
+
|
|
229
|
+
const tight = await ask({ _tag: "Grep", pattern: "LONGSPAN" }, { maxCharsPerSpan: 100 });
|
|
230
|
+
expect(tight.answers[0]!.excerpt.length).toBe(100);
|
|
231
|
+
expect(tight.honesty.capsHit).toEqual(["maxCharsPerSpan"]);
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
test("truncated is true exactly when capsHit is non-empty", async () => {
|
|
235
|
+
const complete = await ask({ _tag: "Grep", pattern: "DECISION: adopt" });
|
|
236
|
+
expect(complete.honesty.capsHit).toEqual([]);
|
|
237
|
+
expect(complete.honesty.truncated).toBe(false);
|
|
238
|
+
|
|
239
|
+
const truncated = await ask(
|
|
240
|
+
{ _tag: "CursorSlice", fromSequence: 0, toSequence: 0 },
|
|
241
|
+
{
|
|
242
|
+
maxSpans: 2,
|
|
243
|
+
},
|
|
244
|
+
);
|
|
245
|
+
expect(truncated.honesty.capsHit.length).toBeGreaterThan(0);
|
|
246
|
+
expect(truncated.honesty.truncated).toBe(true);
|
|
247
|
+
});
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
describe(`[${label}] recall — coverage and typed failure`, () => {
|
|
251
|
+
test("honesty carries the corpus coverage note through every response", async () => {
|
|
252
|
+
const { coverageNote } = await recallFixture();
|
|
253
|
+
const response = await ask({ _tag: "Grep", pattern: "alpha" }, { maxSpans: 1 });
|
|
254
|
+
expect(response.honesty.coverageNote).toBe(coverageNote);
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
test("an invalid grep pattern is a typed invalid_pattern error, never a crash", async () => {
|
|
258
|
+
const { entries, coverageNote } = await recallFixture();
|
|
259
|
+
const error = await Effect.runPromise(
|
|
260
|
+
recall({
|
|
261
|
+
entries,
|
|
262
|
+
coverageNote,
|
|
263
|
+
question: { _tag: "Grep", pattern: "(unclosed" },
|
|
264
|
+
}).pipe(Effect.flip),
|
|
265
|
+
);
|
|
266
|
+
expect(error).toBeInstanceOf(HistoryRecallError);
|
|
267
|
+
expect(error.reason).toBe("invalid_pattern");
|
|
268
|
+
});
|
|
269
|
+
});
|
|
270
|
+
};
|