@anureo/conformance 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/examples/standalone.d.ts +1 -0
- package/examples/standalone.js +16 -0
- package/package.json +20 -21
- package/src/fake-claude.d.ts +78 -0
- package/src/fake-claude.js +68 -0
- package/src/fake.d.ts +122 -0
- package/src/fake.js +284 -0
- package/src/fixtures.d.ts +38 -0
- package/src/index.d.ts +6 -0
- package/src/index.js +6 -0
- package/src/native.d.ts +44 -0
- package/src/native.js +96 -0
- package/src/suite.d.ts +60 -0
- package/src/suite.js +289 -0
- package/dist/fake-claude.d.ts +0 -40
- package/dist/fake.d.ts +0 -44
- package/dist/fake.js +0 -199
- package/dist/fixtures.d.ts +0 -21
- package/dist/index.d.ts +0 -4
- package/dist/index.js +0 -409
- package/dist/suite.d.ts +0 -28
- package/examples/standalone.mjs +0 -53
- /package/{dist → src}/fixtures.js +0 -0
package/dist/index.js
DELETED
|
@@ -1,409 +0,0 @@
|
|
|
1
|
-
// packages/conformance/src/fake.ts
|
|
2
|
-
import {
|
|
3
|
-
negotiateVersion
|
|
4
|
-
} from "@anureo/protocol";
|
|
5
|
-
import { HarnessSdkError } from "@anureo/sdk";
|
|
6
|
-
var FAKE_CAPABILITIES = {
|
|
7
|
-
session: { resume: true, fork: ["before", "through"], revert: ["conversation", "files", "both"], history: true },
|
|
8
|
-
turn: { steer: true, interrupt: true, outputSchema: true, inputModalities: ["text", "image", "file"], overrides: ["model"] },
|
|
9
|
-
goal: { actions: ["set", "clear"] },
|
|
10
|
-
compact: true,
|
|
11
|
-
review: true,
|
|
12
|
-
interactions: ["approval", "userInput", "elicitation", "dynamicTool"],
|
|
13
|
-
config: { options: [{ id: "model", scope: "session", schema: { type: "string" } }] }
|
|
14
|
-
};
|
|
15
|
-
|
|
16
|
-
class ScriptedHarnessAdapter {
|
|
17
|
-
options;
|
|
18
|
-
calls = [];
|
|
19
|
-
opened = [];
|
|
20
|
-
sinks = new Map;
|
|
21
|
-
backends = new Map;
|
|
22
|
-
disposeCount = 0;
|
|
23
|
-
constructor(options = {}) {
|
|
24
|
-
this.options = options;
|
|
25
|
-
}
|
|
26
|
-
get capabilities() {
|
|
27
|
-
return structuredClone(this.options.capabilities ?? FAKE_CAPABILITIES);
|
|
28
|
-
}
|
|
29
|
-
async initialize(params) {
|
|
30
|
-
negotiateVersion(params.protocolVersions);
|
|
31
|
-
return { protocolVersion: 1, runtime: { name: "scripted", version: "1.0.0", adapter: "conformance/0.1.0" }, capabilities: this.capabilities };
|
|
32
|
-
}
|
|
33
|
-
async openSession(params, sink) {
|
|
34
|
-
this.opened.push(structuredClone(params));
|
|
35
|
-
this.sinks.set(params.binding.sessionId, sink);
|
|
36
|
-
const backend = new ScriptedBackendSession(this, params, sink);
|
|
37
|
-
this.backends.set(params.binding.sessionId, backend);
|
|
38
|
-
if (params.mode === "fork" && params.fork) {
|
|
39
|
-
const source = this.backends.get(params.fork.sourceSessionId);
|
|
40
|
-
if (!source)
|
|
41
|
-
throw new HarnessSdkError("SESSION_NOT_FOUND");
|
|
42
|
-
const boundaryIndex = source.history.findIndex((entry) => entry.turnId === params.fork.boundary.turnId);
|
|
43
|
-
if (boundaryIndex < 0)
|
|
44
|
-
throw new HarnessSdkError("TURN_NOT_FOUND");
|
|
45
|
-
const retained = params.fork.boundary.relation === "before" ? boundaryIndex : boundaryIndex + 1;
|
|
46
|
-
backend.history.push(...structuredClone(source.history.slice(0, retained)));
|
|
47
|
-
}
|
|
48
|
-
await this.options.onOpen?.(params, sink);
|
|
49
|
-
return backend;
|
|
50
|
-
}
|
|
51
|
-
async dispose() {
|
|
52
|
-
this.disposeCount++;
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
class ScriptedBackendSession {
|
|
57
|
-
adapter;
|
|
58
|
-
sink;
|
|
59
|
-
binding;
|
|
60
|
-
capabilities;
|
|
61
|
-
history = [];
|
|
62
|
-
disposeCount = 0;
|
|
63
|
-
state;
|
|
64
|
-
constructor(adapter, params, sink) {
|
|
65
|
-
this.adapter = adapter;
|
|
66
|
-
this.sink = sink;
|
|
67
|
-
this.binding = { ...params.binding, nativeRef: { provider: "scripted", value: `native:${params.binding.sessionId}` } };
|
|
68
|
-
this.capabilities = adapter.capabilities;
|
|
69
|
-
this.state = { binding: this.binding, capabilities: this.capabilities, status: "ready", config: params.config ?? {}, capabilityRevision: 1 };
|
|
70
|
-
}
|
|
71
|
-
async emit(payload, turnId) {
|
|
72
|
-
await this.sink.emit(payload, { turnId });
|
|
73
|
-
}
|
|
74
|
-
async execute(method, params) {
|
|
75
|
-
const call = { method, params };
|
|
76
|
-
this.adapter.calls.push(call);
|
|
77
|
-
const custom = await this.adapter.options.execute?.(call, this.sink);
|
|
78
|
-
if (custom !== undefined)
|
|
79
|
-
return custom;
|
|
80
|
-
let result;
|
|
81
|
-
switch (method) {
|
|
82
|
-
case "turn/start": {
|
|
83
|
-
const start = params;
|
|
84
|
-
await this.emit({ type: "turn/started", payload: { turnId: start.turnId } });
|
|
85
|
-
const item = { id: "native-reused", type: "agentMessage", text: start.input.filter((part) => part.type === "text").map((part) => part.text).join(`
|
|
86
|
-
`), phase: "final" };
|
|
87
|
-
await this.emit({ type: "item/started", payload: { item: { ...item, text: "" } } }, start.turnId);
|
|
88
|
-
await this.emit({ type: "item/delta", payload: { itemId: item.id, kind: "text", text: "preview " } }, start.turnId);
|
|
89
|
-
await this.emit({ type: "item/completed", payload: { item } }, start.turnId);
|
|
90
|
-
this.history.push({ turnId: start.turnId, item });
|
|
91
|
-
await this.emit({ type: "turn/completed", payload: { turnId: start.turnId, status: "completed" } });
|
|
92
|
-
result = { turnId: start.turnId, accepted: true };
|
|
93
|
-
break;
|
|
94
|
-
}
|
|
95
|
-
case "turn/steer":
|
|
96
|
-
result = { turnId: params.expectedTurnId, accepted: true };
|
|
97
|
-
break;
|
|
98
|
-
case "turn/interrupt": {
|
|
99
|
-
const id = params.turnId;
|
|
100
|
-
await this.emit({ type: "turn/completed", payload: { turnId: id, status: "interrupted" } });
|
|
101
|
-
result = { accepted: true, turnId: id };
|
|
102
|
-
break;
|
|
103
|
-
}
|
|
104
|
-
case "session/config/set":
|
|
105
|
-
this.state = { ...this.state, config: { ...this.state.config, ...params.patch }, capabilityRevision: this.state.capabilityRevision + 1 };
|
|
106
|
-
result = this.state;
|
|
107
|
-
break;
|
|
108
|
-
case "session/revert": {
|
|
109
|
-
const request = params;
|
|
110
|
-
if (request.scope !== "files") {
|
|
111
|
-
const index = this.history.findIndex((entry) => entry.turnId === request.boundary.turnId);
|
|
112
|
-
if (index < 0)
|
|
113
|
-
throw new HarnessSdkError("TURN_NOT_FOUND");
|
|
114
|
-
this.history.splice(request.boundary.relation === "before" ? index : index + 1);
|
|
115
|
-
}
|
|
116
|
-
result = { appliedScope: request.scope, boundary: request.boundary };
|
|
117
|
-
break;
|
|
118
|
-
}
|
|
119
|
-
case "history/read":
|
|
120
|
-
result = { items: this.history, snapshotToken: "fake-snapshot", source: this.binding.nativeRef };
|
|
121
|
-
break;
|
|
122
|
-
case "goal/set":
|
|
123
|
-
result = { goal: params.goal, status: "active" };
|
|
124
|
-
break;
|
|
125
|
-
case "goal/clear":
|
|
126
|
-
result = { goal: null, status: "cleared" };
|
|
127
|
-
break;
|
|
128
|
-
case "compact/start":
|
|
129
|
-
result = { maintenanceId: "fake-maintenance", status: "completed" };
|
|
130
|
-
break;
|
|
131
|
-
case "review/start": {
|
|
132
|
-
const id = params.turnId;
|
|
133
|
-
await this.emit({ type: "turn/started", payload: { turnId: id } });
|
|
134
|
-
await this.emit({ type: "turn/completed", payload: { turnId: id, status: "completed" } });
|
|
135
|
-
result = { accepted: true, turnId: id };
|
|
136
|
-
break;
|
|
137
|
-
}
|
|
138
|
-
case "session/close":
|
|
139
|
-
result = { closed: true };
|
|
140
|
-
break;
|
|
141
|
-
case "interaction/respond":
|
|
142
|
-
case "interaction/cancel":
|
|
143
|
-
throw new Error("Interactions must be handled by the SDK registry");
|
|
144
|
-
}
|
|
145
|
-
return result;
|
|
146
|
-
}
|
|
147
|
-
async dispose() {
|
|
148
|
-
this.disposeCount++;
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
function createScriptedAdapter(options) {
|
|
152
|
-
return new ScriptedHarnessAdapter(options);
|
|
153
|
-
}
|
|
154
|
-
function createScriptedConformanceFactory(capabilities) {
|
|
155
|
-
return (scenario) => {
|
|
156
|
-
const adapter = createScriptedAdapter({ capabilities, execute: async (call, sink) => {
|
|
157
|
-
if (call.method !== "turn/start")
|
|
158
|
-
return;
|
|
159
|
-
const { turnId } = call.params;
|
|
160
|
-
if (["interrupt", "steer", "disconnect", "backpressure"].includes(scenario)) {
|
|
161
|
-
await sink.emit({ type: "turn/started", payload: { turnId } });
|
|
162
|
-
if (scenario === "backpressure")
|
|
163
|
-
sink.tryEmit({ type: "notice", payload: { severity: "info", code: "FLOOD", message: "Conformance overflow" } });
|
|
164
|
-
return { accepted: true, turnId };
|
|
165
|
-
}
|
|
166
|
-
const interaction = {
|
|
167
|
-
approval: { interactionId: "conformance-approval", kind: "approval", turnId, options: [{ id: "deny", label: "Deny", semantic: "deny" }] },
|
|
168
|
-
userInput: { interactionId: "conformance-input", kind: "userInput", turnId, questions: [{ id: "q", prompt: "Value" }] },
|
|
169
|
-
elicitation: { interactionId: "conformance-elicitation", kind: "elicitation", turnId, mode: "form" },
|
|
170
|
-
dynamicTool: { interactionId: "conformance-tool", kind: "dynamicTool", turnId, toolName: "conformance", arguments: {} }
|
|
171
|
-
};
|
|
172
|
-
if (interaction[scenario] || scenario === "unknown-item") {
|
|
173
|
-
await sink.emit({ type: "turn/started", payload: { turnId } });
|
|
174
|
-
if (interaction[scenario])
|
|
175
|
-
await sink.requestInteraction(interaction[scenario], async () => {});
|
|
176
|
-
else
|
|
177
|
-
await sink.emit({ type: "item/completed", payload: { item: { id: "unknown", type: "toolCall", kind: "other", title: "Unknown", nativeType: "futureNativeType", raw: { safe: "value" } } } }, { turnId });
|
|
178
|
-
await sink.emit({ type: "turn/completed", payload: { turnId, status: "completed" } });
|
|
179
|
-
return { accepted: true, turnId };
|
|
180
|
-
}
|
|
181
|
-
return;
|
|
182
|
-
} });
|
|
183
|
-
return {
|
|
184
|
-
adapter,
|
|
185
|
-
openParams: { requestId: "conformance-open", mode: "new", binding: { sessionId: "conformance-session", adapterId: "scripted", runtimeConfigId: "conformance-runtime" }, cwd: "/tmp" },
|
|
186
|
-
...scenario === "backpressure" ? { runtimeOptions: { queue: { maxEvents: 2 } } } : {},
|
|
187
|
-
...scenario === "disconnect" ? { afterStart: async () => {
|
|
188
|
-
await adapter.sinks.get("conformance-session").fail(new HarnessSdkError("CONNECTION_CLOSED", "Conformance disconnection", false, true));
|
|
189
|
-
} } : {}
|
|
190
|
-
};
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
// packages/conformance/src/suite.ts
|
|
194
|
-
import { createHarnessRuntime } from "@anureo/sdk";
|
|
195
|
-
import { supportsCapability } from "@anureo/protocol";
|
|
196
|
-
var CONFORMANCE_SCENARIOS = ["lifecycle", "immediate", "identity", "approval", "userInput", "elicitation", "dynamicTool", "interrupt", "steer", "fork", "revert", "history", "unknown-item", "disconnect", "backpressure"];
|
|
197
|
-
function requireCondition(value, message) {
|
|
198
|
-
if (!value)
|
|
199
|
-
throw new Error(message);
|
|
200
|
-
}
|
|
201
|
-
function deadline(promise, timeoutMs) {
|
|
202
|
-
return new Promise((resolve, reject) => {
|
|
203
|
-
const timer = setTimeout(() => reject(new Error("Conformance timed out")), timeoutMs);
|
|
204
|
-
promise.then((result) => {
|
|
205
|
-
clearTimeout(timer);
|
|
206
|
-
resolve(result);
|
|
207
|
-
}, (cause) => {
|
|
208
|
-
clearTimeout(timer);
|
|
209
|
-
reject(cause);
|
|
210
|
-
});
|
|
211
|
-
});
|
|
212
|
-
}
|
|
213
|
-
function applicable(scenario, capabilities) {
|
|
214
|
-
if (["approval", "userInput", "elicitation", "dynamicTool"].includes(scenario))
|
|
215
|
-
return capabilities.interactions?.includes(scenario) ?? false;
|
|
216
|
-
if (scenario === "interrupt" || scenario === "steer")
|
|
217
|
-
return supportsCapability(capabilities, `turn.${scenario}`);
|
|
218
|
-
if (scenario === "fork")
|
|
219
|
-
return !!capabilities.session?.fork?.length;
|
|
220
|
-
if (scenario === "revert")
|
|
221
|
-
return !!capabilities.session?.revert?.length;
|
|
222
|
-
if (scenario === "history")
|
|
223
|
-
return !!capabilities.session?.history;
|
|
224
|
-
return true;
|
|
225
|
-
}
|
|
226
|
-
function defaultAnswer(request) {
|
|
227
|
-
switch (request.kind) {
|
|
228
|
-
case "approval":
|
|
229
|
-
return { kind: "approval", optionId: request.options[0].id };
|
|
230
|
-
case "userInput":
|
|
231
|
-
return { kind: "userInput", answers: Object.fromEntries(request.questions.map((question) => [question.id, question.options?.[0]?.id ?? "conformance"])) };
|
|
232
|
-
case "elicitation":
|
|
233
|
-
return { kind: "elicitation", action: "decline" };
|
|
234
|
-
case "dynamicTool":
|
|
235
|
-
return { kind: "dynamicTool", content: [{ type: "text", text: "conformance" }] };
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
async function runConformance(factory, options = {}) {
|
|
239
|
-
const cases = [];
|
|
240
|
-
for (const scenario of options.scenarios ?? CONFORMANCE_SCENARIOS) {
|
|
241
|
-
let runtime;
|
|
242
|
-
let reader;
|
|
243
|
-
try {
|
|
244
|
-
const fixture = await factory(scenario);
|
|
245
|
-
runtime = createHarnessRuntime(fixture.adapter, fixture.runtimeOptions);
|
|
246
|
-
const initialized = await runtime.initialize({ protocolVersions: [1], clientInfo: { name: "harness-conformance", version: "0.1.0" }, clientCapabilities: { interactions: ["approval", "userInput", "elicitation", "dynamicTool"] } });
|
|
247
|
-
if (!applicable(scenario, initialized.capabilities)) {
|
|
248
|
-
cases.push({ scenario, status: "not-applicable" });
|
|
249
|
-
continue;
|
|
250
|
-
}
|
|
251
|
-
const session = await runtime.openSession(fixture.openParams);
|
|
252
|
-
const sessionId = session.binding.sessionId;
|
|
253
|
-
reader = session.events()[Symbol.asyncIterator]();
|
|
254
|
-
const events = [];
|
|
255
|
-
let interactionSeen = false;
|
|
256
|
-
const collect = async (turnId) => {
|
|
257
|
-
for (let count = 0;count < 1e4; count++) {
|
|
258
|
-
const entry = await reader.next();
|
|
259
|
-
requireCondition(!entry.done, "Source ended before a terminal event");
|
|
260
|
-
const event = entry.value;
|
|
261
|
-
events.push(event);
|
|
262
|
-
if (event.type === "interaction/requested") {
|
|
263
|
-
interactionSeen = true;
|
|
264
|
-
await session.respond({ requestId: `respond:${event.payload.interactionId}`, sessionId, interactionId: event.payload.interactionId, response: (fixture.answer ?? defaultAnswer)(event.payload) });
|
|
265
|
-
}
|
|
266
|
-
if (event.type === "turn/completed" && event.payload.turnId === turnId)
|
|
267
|
-
return event;
|
|
268
|
-
}
|
|
269
|
-
throw new Error("Too many events without completion");
|
|
270
|
-
};
|
|
271
|
-
const run = async (turnId) => {
|
|
272
|
-
const request = session.startTurn({ requestId: `start:${turnId}`, sessionId, turnId, input: [{ type: "text", text: "conformance" }] });
|
|
273
|
-
request.catch(() => {});
|
|
274
|
-
if (scenario === "backpressure") {
|
|
275
|
-
await request;
|
|
276
|
-
await fixture.afterStart?.(session);
|
|
277
|
-
} else if (scenario === "interrupt" || scenario === "steer" || scenario === "disconnect") {
|
|
278
|
-
await request;
|
|
279
|
-
if (scenario === "steer")
|
|
280
|
-
await session.steer({ requestId: "steer", sessionId, expectedTurnId: turnId, input: [{ type: "text", text: "redirect" }] });
|
|
281
|
-
if (scenario !== "disconnect")
|
|
282
|
-
await session.interrupt({ requestId: "interrupt", sessionId, turnId });
|
|
283
|
-
await fixture.afterStart?.(session);
|
|
284
|
-
}
|
|
285
|
-
const terminal = await collect(turnId);
|
|
286
|
-
await request;
|
|
287
|
-
return terminal;
|
|
288
|
-
};
|
|
289
|
-
const terminal = await deadline(run("conformance-turn-1"), options.timeoutMs ?? 3000);
|
|
290
|
-
requireCondition(events.filter((event) => event.type === "turn/started").length === 1, "Turn must start once");
|
|
291
|
-
requireCondition(events.filter((event) => event.type === "turn/completed").length === 1, "Turn must end once");
|
|
292
|
-
requireCondition(events.every((event, index) => event.seq === index + 1 && event.sessionId === sessionId && event.streamId === events[0].streamId), "Source identity/sequence must be contiguous");
|
|
293
|
-
if (["approval", "userInput", "elicitation", "dynamicTool"].includes(scenario))
|
|
294
|
-
requireCondition(interactionSeen && events.some((event) => event.type === "interaction/resolved"), "Declared interaction was not exercised");
|
|
295
|
-
if (scenario === "disconnect" || scenario === "backpressure")
|
|
296
|
-
requireCondition(terminal.payload.status === "failed", "Lost/overflowed source must fail explicitly");
|
|
297
|
-
else if (scenario === "interrupt" || scenario === "steer")
|
|
298
|
-
requireCondition(terminal.payload.status === "interrupted", "Interrupt must produce interrupted status");
|
|
299
|
-
else
|
|
300
|
-
requireCondition(terminal.payload.status === "completed", "Unexpected failed turn");
|
|
301
|
-
if (scenario === "lifecycle" || scenario === "immediate")
|
|
302
|
-
requireCondition(events.some((event) => event.type === "item/completed" && event.payload.item.type === "agentMessage" && event.payload.item.text === (fixture.expectedText ?? "conformance")), "Missing authoritative completed text");
|
|
303
|
-
if (scenario === "unknown-item")
|
|
304
|
-
requireCondition(events.some((event) => event.type === "item/completed" && event.payload.item.type === "toolCall" && event.payload.item.kind === "other" && !!event.payload.item.nativeType), "Unknown native item was discarded");
|
|
305
|
-
if (scenario === "identity") {
|
|
306
|
-
await deadline(run("conformance-turn-2"), options.timeoutMs ?? 3000);
|
|
307
|
-
const items = events.filter((event) => event.type === "item/completed");
|
|
308
|
-
requireCondition(items.some((event) => event.turnId === "conformance-turn-1") && items.some((event) => event.turnId === "conformance-turn-2"), "Turn-scoped item identities were lost");
|
|
309
|
-
}
|
|
310
|
-
if (scenario === "history") {
|
|
311
|
-
const history = await session.readHistory({ requestId: "history", sessionId });
|
|
312
|
-
requireCondition(history.items.length > 0 && history.items.every((entry) => !!entry.turnId && !!entry.item.id), "Declared history did not return the completed turn");
|
|
313
|
-
}
|
|
314
|
-
if (scenario === "fork") {
|
|
315
|
-
for (const relation of initialized.capabilities.session.fork) {
|
|
316
|
-
const fork = await runtime.openSession({ ...fixture.openParams, requestId: `fork:${relation}`, mode: "fork", binding: { ...session.binding, sessionId: `${sessionId}:fork:${relation}` }, fork: { sourceSessionId: sessionId, nativeRef: session.binding.nativeRef, boundary: { turnId: "conformance-turn-1", relation } } });
|
|
317
|
-
requireCondition(fork.binding.sessionId !== sessionId && fork.binding.adapterId === session.binding.adapterId, "Fork must create an independent same-adapter session");
|
|
318
|
-
if (initialized.capabilities.session?.history) {
|
|
319
|
-
const history = await fork.readHistory({ requestId: "fork-history", sessionId: fork.binding.sessionId });
|
|
320
|
-
requireCondition(relation === "before" ? !history.items.some((entry) => entry.turnId === "conformance-turn-1") : history.items.some((entry) => entry.turnId === "conformance-turn-1"), "Fork boundary was not preserved");
|
|
321
|
-
}
|
|
322
|
-
await fork.close();
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
if (scenario === "revert") {
|
|
326
|
-
const scope = initialized.capabilities.session.revert[0];
|
|
327
|
-
const result = await session.revert({ requestId: "revert", sessionId, scope, boundary: { turnId: "conformance-turn-1", relation: "before" } });
|
|
328
|
-
requireCondition(result.appliedScope === scope && result.boundary.turnId === "conformance-turn-1", "Revert did not report the actual requested scope/boundary");
|
|
329
|
-
if (scope !== "files" && initialized.capabilities.session?.history) {
|
|
330
|
-
const history = await session.readHistory({ requestId: "reverted-history", sessionId });
|
|
331
|
-
requireCondition(!history.items.some((entry) => entry.turnId === "conformance-turn-1"), "Conversation boundary was not reverted");
|
|
332
|
-
}
|
|
333
|
-
}
|
|
334
|
-
cases.push({ scenario, status: "passed" });
|
|
335
|
-
} catch (cause) {
|
|
336
|
-
cases.push({ scenario, status: "failed", message: cause instanceof Error ? cause.message : "Conformance failure" });
|
|
337
|
-
} finally {
|
|
338
|
-
await reader?.return?.();
|
|
339
|
-
await runtime?.dispose().catch(() => {});
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
return { passed: cases.every((entry) => entry.status !== "failed"), cases };
|
|
343
|
-
}
|
|
344
|
-
// packages/conformance/src/fake-claude.ts
|
|
345
|
-
import { BoundedEventQueue } from "@anureo/sdk";
|
|
346
|
-
|
|
347
|
-
class ScriptedClaudeQuery {
|
|
348
|
-
script;
|
|
349
|
-
calls = [];
|
|
350
|
-
queue = new BoundedEventQueue;
|
|
351
|
-
closed = false;
|
|
352
|
-
constructor(script) {
|
|
353
|
-
this.script = script;
|
|
354
|
-
for (const message of script.messages)
|
|
355
|
-
this.queue.push(structuredClone(message));
|
|
356
|
-
}
|
|
357
|
-
[Symbol.asyncIterator]() {
|
|
358
|
-
return this.queue[Symbol.asyncIterator]();
|
|
359
|
-
}
|
|
360
|
-
async streamInput(stream) {
|
|
361
|
-
for await (const input of stream) {
|
|
362
|
-
this.calls.push({ method: "input", value: input });
|
|
363
|
-
await this.script.onInput?.(input, (message) => this.queue.write(message));
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
async interrupt() {
|
|
367
|
-
this.calls.push({ method: "interrupt" });
|
|
368
|
-
}
|
|
369
|
-
async setModel(model) {
|
|
370
|
-
this.calls.push({ method: "setModel", ...model ? { value: model } : {} });
|
|
371
|
-
}
|
|
372
|
-
async setPermissionMode(mode) {
|
|
373
|
-
this.calls.push({ method: "setPermissionMode", value: mode });
|
|
374
|
-
}
|
|
375
|
-
async setMaxThinkingTokens(tokens) {
|
|
376
|
-
this.calls.push({ method: "setMaxThinkingTokens", value: tokens });
|
|
377
|
-
}
|
|
378
|
-
async rewindFiles(id) {
|
|
379
|
-
this.calls.push({ method: "rewindFiles", value: id });
|
|
380
|
-
return this.script.canRewind ? { canRewind: true } : { canRewind: false, error: "No checkpoint" };
|
|
381
|
-
}
|
|
382
|
-
async supportedModels() {
|
|
383
|
-
return this.script.models ?? [{ value: "fake-model", displayName: "Fake", description: "Fixture" }];
|
|
384
|
-
}
|
|
385
|
-
finish() {
|
|
386
|
-
this.queue.end();
|
|
387
|
-
}
|
|
388
|
-
close() {
|
|
389
|
-
if (!this.closed) {
|
|
390
|
-
this.closed = true;
|
|
391
|
-
this.calls.push({ method: "close" });
|
|
392
|
-
this.queue.end();
|
|
393
|
-
}
|
|
394
|
-
}
|
|
395
|
-
}
|
|
396
|
-
function createFakeClaudeQuery(script) {
|
|
397
|
-
return new ScriptedClaudeQuery(script);
|
|
398
|
-
}
|
|
399
|
-
export {
|
|
400
|
-
CONFORMANCE_SCENARIOS,
|
|
401
|
-
FAKE_CAPABILITIES,
|
|
402
|
-
ScriptedBackendSession,
|
|
403
|
-
ScriptedClaudeQuery,
|
|
404
|
-
ScriptedHarnessAdapter,
|
|
405
|
-
createFakeClaudeQuery,
|
|
406
|
-
createScriptedAdapter,
|
|
407
|
-
createScriptedConformanceFactory,
|
|
408
|
-
runConformance
|
|
409
|
-
};
|
package/dist/suite.d.ts
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import { type HarnessAdapter, type HarnessSession, type RuntimeOptions } from "@anureo/sdk";
|
|
2
|
-
import { type InteractionRequested, type InteractionResponse, type SessionOpenParams } from "@anureo/protocol";
|
|
3
|
-
export declare const CONFORMANCE_SCENARIOS: readonly ["lifecycle", "immediate", "identity", "approval", "userInput", "elicitation", "dynamicTool", "interrupt", "steer", "fork", "revert", "history", "unknown-item", "disconnect", "backpressure"];
|
|
4
|
-
export type ConformanceScenario = typeof CONFORMANCE_SCENARIOS[number];
|
|
5
|
-
export interface ConformanceDriver {
|
|
6
|
-
adapter: HarnessAdapter;
|
|
7
|
-
openParams: SessionOpenParams;
|
|
8
|
-
runtimeOptions?: RuntimeOptions;
|
|
9
|
-
/** Deterministic fake-source trigger, called after a turn has been accepted. */
|
|
10
|
-
afterStart?: (session: HarnessSession) => Promise<void>;
|
|
11
|
-
expectedText?: string;
|
|
12
|
-
answer?: (request: InteractionRequested) => InteractionResponse;
|
|
13
|
-
}
|
|
14
|
-
export type ConformanceFactory = (scenario: ConformanceScenario) => Promise<ConformanceDriver> | ConformanceDriver;
|
|
15
|
-
export interface ConformanceCaseResult {
|
|
16
|
-
scenario: ConformanceScenario;
|
|
17
|
-
status: "passed" | "failed" | "not-applicable";
|
|
18
|
-
message?: string;
|
|
19
|
-
}
|
|
20
|
-
export interface ConformanceReport {
|
|
21
|
-
passed: boolean;
|
|
22
|
-
cases: ConformanceCaseResult[];
|
|
23
|
-
}
|
|
24
|
-
/** Every declared optional capability must pass its scenario; missing fixtures are failures. */
|
|
25
|
-
export declare function runConformance(factory: ConformanceFactory, options?: {
|
|
26
|
-
timeoutMs?: number;
|
|
27
|
-
scenarios?: readonly ConformanceScenario[];
|
|
28
|
-
}): Promise<ConformanceReport>;
|
package/examples/standalone.mjs
DELETED
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
import { validateRequest } from '@anureo/protocol'
|
|
2
|
-
import { createHarnessRuntime } from '@anureo/sdk'
|
|
3
|
-
import { createScriptedConformanceFactory, runConformance } from '@anureo/conformance'
|
|
4
|
-
import { fileURLToPath } from 'node:url'
|
|
5
|
-
if (!validateRequest('turn/start', { requestId: 'start', sessionId: 's', turnId: 't', input: [{ type: 'text', text: 'standalone' }] })) throw new Error('Schema failed')
|
|
6
|
-
const report = await runConformance(createScriptedConformanceFactory(), { scenarios: ['lifecycle','approval','interrupt','history'] })
|
|
7
|
-
if (!report.passed) throw new Error(JSON.stringify(report))
|
|
8
|
-
const adapters = process.argv.slice(2)
|
|
9
|
-
const loaders = { codex: () => import('@anureo/adapter-codex'), claude: () => import('@anureo/adapter-claude'), acp: () => import('@anureo/adapter-acp'), ahp: () => import('@anureo/adapter-ahp') }
|
|
10
|
-
for (const name of adapters) {
|
|
11
|
-
const module = await loaders[name]()
|
|
12
|
-
if (typeof module['create' + name[0].toUpperCase() + name.slice(1) + 'Adapter'] !== 'function') throw new Error('Missing adapter export ' + name)
|
|
13
|
-
}
|
|
14
|
-
const nativeExamples = []
|
|
15
|
-
async function exercise(name, create, executable, text, interrupt) {
|
|
16
|
-
const runtime = createHarnessRuntime(create({ command: process.execPath, args: [fileURLToPath(new URL(executable, import.meta.url))] }))
|
|
17
|
-
let timeout
|
|
18
|
-
try {
|
|
19
|
-
await runtime.initialize({ protocolVersions: [1], clientInfo: { name: 'clean-install-example', version: '1.0.0' }, clientCapabilities: { interactions: ['approval','userInput','elicitation','dynamicTool'] } })
|
|
20
|
-
const session = await runtime.openSession({ requestId: 'open', mode: 'new', cwd: process.cwd(), binding: { sessionId: 'example-' + name, adapterId: name, runtimeConfigId: 'clean' } })
|
|
21
|
-
let sawApproval = false
|
|
22
|
-
const observing = (async () => {
|
|
23
|
-
for await (const event of session.events()) {
|
|
24
|
-
if (event.type === 'interaction/requested') {
|
|
25
|
-
sawApproval = true
|
|
26
|
-
if (event.payload.kind !== 'approval') throw new Error('Unexpected interaction')
|
|
27
|
-
const option = event.payload.options.find(option => option.semantic === 'deny') ?? event.payload.options.find(option => option.id === 'deny')
|
|
28
|
-
if (!option) throw new Error('No non-authorizing option')
|
|
29
|
-
await session.respond({ requestId: 'respond', sessionId: session.binding.sessionId, interactionId: event.payload.interactionId, response: { kind: 'approval', optionId: option.id } })
|
|
30
|
-
}
|
|
31
|
-
if (event.type === 'turn/completed') return event.payload
|
|
32
|
-
}
|
|
33
|
-
throw new Error('No terminal event')
|
|
34
|
-
})()
|
|
35
|
-
void observing.catch(() => {})
|
|
36
|
-
await session.startTurn({ requestId: 'start', sessionId: session.binding.sessionId, turnId: 'example-turn', input: [{ type: 'text', text }] })
|
|
37
|
-
if (interrupt) await session.interrupt({ requestId: 'interrupt', sessionId: session.binding.sessionId, turnId: 'example-turn' })
|
|
38
|
-
const terminal = await Promise.race([observing, new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error('Standalone example timed out')), 5000) })])
|
|
39
|
-
if (terminal.status !== (interrupt ? 'interrupted' : 'completed')) throw new Error('Wrong terminal status: ' + terminal.status)
|
|
40
|
-
if (text === '/approval' && !sawApproval) throw new Error('Approval not exercised')
|
|
41
|
-
await session.close()
|
|
42
|
-
return { adapter: name, input: text, status: terminal.status, approval: sawApproval }
|
|
43
|
-
} finally { clearTimeout(timeout); await runtime.dispose() }
|
|
44
|
-
}
|
|
45
|
-
if (adapters.includes('acp')) {
|
|
46
|
-
const { createAcpAdapter } = await import('@anureo/adapter-acp')
|
|
47
|
-
for (const text of ['hello','/approval','/wait']) nativeExamples.push(await exercise('acp', createAcpAdapter, './acp-agent.mjs', text, text === '/wait'))
|
|
48
|
-
}
|
|
49
|
-
if (adapters.includes('ahp')) {
|
|
50
|
-
const { createAhpAdapter } = await import('@anureo/adapter-ahp')
|
|
51
|
-
for (const text of ['hello','/approval','/wait']) nativeExamples.push(await exercise('ahp', createAhpAdapter, './ahp-reference.mjs', text, text === '/wait'))
|
|
52
|
-
}
|
|
53
|
-
console.log(JSON.stringify({ status: 'passed', schema: true, conformance: report.cases, adapters, nativeExamples }))
|
|
File without changes
|