@parcha/agentrun-kernel 0.0.0-stage → 0.1.0-beta.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +135 -0
- package/LICENSE +202 -0
- package/NOTICE +6 -0
- package/README.md +52 -2
- package/dist/delegate.d.ts +87 -0
- package/dist/delegate.js +143 -0
- package/dist/errors.d.ts +36 -0
- package/dist/errors.js +37 -0
- package/dist/host/alive.d.ts +2 -0
- package/dist/host/alive.js +28 -0
- package/dist/host/box-slot.d.ts +26 -0
- package/dist/host/box-slot.js +112 -0
- package/dist/host/box.d.ts +104 -0
- package/dist/host/box.js +16 -0
- package/dist/host/egress-proxy.d.ts +45 -0
- package/dist/host/egress-proxy.js +416 -0
- package/dist/host/egress-table.d.ts +24 -0
- package/dist/host/egress-table.js +165 -0
- package/dist/host/index.d.ts +16 -0
- package/dist/host/index.js +12 -0
- package/dist/host/overhead.d.ts +53 -0
- package/dist/host/overhead.js +75 -0
- package/dist/host/parent-death.d.ts +15 -0
- package/dist/host/parent-death.js +42 -0
- package/dist/host/parent-watch.d.ts +1 -0
- package/dist/host/parent-watch.js +38 -0
- package/dist/host/pinned-binary.d.ts +6 -0
- package/dist/host/pinned-binary.js +48 -0
- package/dist/host/proxy.d.ts +179 -0
- package/dist/host/proxy.js +629 -0
- package/dist/host/serial.d.ts +3 -0
- package/dist/host/serial.js +10 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +6 -0
- package/dist/kernel/blobs.d.ts +16 -0
- package/dist/kernel/blobs.js +59 -0
- package/dist/kernel/drive.d.ts +35 -0
- package/dist/kernel/drive.js +202 -0
- package/dist/kernel/errors.d.ts +27 -0
- package/dist/kernel/errors.js +42 -0
- package/dist/kernel/follow-up.d.ts +67 -0
- package/dist/kernel/follow-up.js +146 -0
- package/dist/kernel/halt.d.ts +11 -0
- package/dist/kernel/halt.js +17 -0
- package/dist/kernel/index.d.ts +8 -0
- package/dist/kernel/index.js +5 -0
- package/dist/kernel/types.d.ts +120 -0
- package/dist/kernel/types.js +1 -0
- package/dist/process/builtins.d.ts +142 -0
- package/dist/process/builtins.js +207 -0
- package/dist/process/inbox.d.ts +29 -0
- package/dist/process/inbox.js +52 -0
- package/dist/process/index.d.ts +9 -0
- package/dist/process/index.js +11 -0
- package/dist/process/journal.d.ts +111 -0
- package/dist/process/journal.js +232 -0
- package/dist/process/records.d.ts +143 -0
- package/dist/process/records.js +142 -0
- package/dist/process/recovery.d.ts +76 -0
- package/dist/process/recovery.js +139 -0
- package/dist/process/settle.d.ts +11 -0
- package/dist/process/settle.js +20 -0
- package/dist/process/tail.d.ts +20 -0
- package/dist/process/tail.js +53 -0
- package/dist/process/tools.d.ts +105 -0
- package/dist/process/tools.js +120 -0
- package/dist/process/zod.d.ts +32 -0
- package/dist/process/zod.js +109 -0
- package/dist/runner.d.ts +154 -0
- package/dist/runner.js +547 -0
- package/dist/schema-check-cases.d.ts +7 -0
- package/dist/schema-check-cases.js +73 -0
- package/dist/schema-check.d.ts +3 -0
- package/dist/schema-check.js +88 -0
- package/dist/skill-files.d.ts +23 -0
- package/dist/skill-files.js +136 -0
- package/dist/testing.d.ts +91 -0
- package/dist/testing.js +196 -0
- package/dist/types.d.ts +237 -0
- package/dist/types.js +10 -0
- package/package.json +74 -4
- package/vendor/durable-harness/UPSTREAM.json +67 -0
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// What every harness adapter refuses in a tool's input schema before a session starts. Written for AgentRun.
|
|
2
|
+
//
|
|
3
|
+
// Three checks, the same on every harness so their columns agree by construction: the root is `{ "type": "object" }`
|
|
4
|
+
// (a tool's input is an object of arguments; on the pinned Claude CLI one tool with any other root loses every tool of
|
|
5
|
+
// the session, the submit tool too); the schema is valid JSON Schema 2020-12 (what Anthropic's API checks, and an
|
|
6
|
+
// invalid schema means nothing on any harness); and no `$ref` leaves the schema (a remote one is fetched by no harness,
|
|
7
|
+
// so the model never sees what it names). A local `$ref` into `$defs` is fine. Each adapter's `check()` calls this
|
|
8
|
+
// first, then adds only its own harness's measured refusals.
|
|
9
|
+
import { Ajv2020 } from "ajv/dist/2020.js";
|
|
10
|
+
let metaschema;
|
|
11
|
+
const isPlain = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
|
|
12
|
+
const pointer = (path) => path || "/";
|
|
13
|
+
const escape = (key) => key.replace(/~/g, "~0").replace(/\//g, "~1");
|
|
14
|
+
/** The keywords whose value is a schema, a list of schemas, or a map of names to schemas (JSON Schema 2020-12, with the
|
|
15
|
+
* older spellings a generator still writes). Every other keyword holds values (default, examples, const, enum, ...). */
|
|
16
|
+
const SCHEMA_ONE = new Set(["items", "additionalItems", "additionalProperties", "contains", "propertyNames", "unevaluatedItems", "unevaluatedProperties", "not", "if", "then", "else", "contentSchema"]);
|
|
17
|
+
const SCHEMA_LISTS = new Set(["allOf", "anyOf", "oneOf", "prefixItems", "items"]);
|
|
18
|
+
const SCHEMA_MAPS = new Set(["properties", "patternProperties", "$defs", "definitions", "dependentSchemas", "dependencies"]);
|
|
19
|
+
/** The `$schema` dialects Anthropic's API takes (measured; with or without a trailing "#"). Any other value, or one that
|
|
20
|
+
* is not a string, it rejects. A schema is checked as 2020-12 whichever of these it names, as the API does. */
|
|
21
|
+
const DIALECTS = new Set([
|
|
22
|
+
"https://json-schema.org/draft/2020-12/schema", "https://json-schema.org/draft/2019-09/schema",
|
|
23
|
+
"http://json-schema.org/draft-07/schema", "http://json-schema.org/draft-06/schema", "http://json-schema.org/draft-04/schema",
|
|
24
|
+
]);
|
|
25
|
+
/** Where `schema`, the input schema of the tool `tool`, is refused on every harness, as `<tool>: <JSON pointer>: <why>`;
|
|
26
|
+
* empty when it may be used. */
|
|
27
|
+
export function toolSchemaProblems(tool, schema) {
|
|
28
|
+
if (!isPlain(schema) || schema.type !== "object") {
|
|
29
|
+
return [`${tool}: /: the root must be {"type": "object"} (a tool's input is an object of arguments), not ${String(JSON.stringify(isPlain(schema) ? schema.type ?? null : schema) ?? schema).slice(0, 60)}`];
|
|
30
|
+
}
|
|
31
|
+
// A schema is sent as JSON: one that is not JSON (a cycle in its objects, a BigInt) is refused before anything walks it.
|
|
32
|
+
try {
|
|
33
|
+
JSON.stringify(schema);
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
return [`${tool}: /: the schema is not JSON (${String(error?.message ?? error).split("\n")[0].slice(0, 80)}); a tool's input schema is sent as JSON`];
|
|
37
|
+
}
|
|
38
|
+
const problems = [];
|
|
39
|
+
const { $schema: dialect, ...body } = schema;
|
|
40
|
+
if (dialect !== undefined && !(typeof dialect === "string" && DIALECTS.has(dialect.replace(/#$/, "")))) {
|
|
41
|
+
problems.push(`${tool}: /$schema: ${String(JSON.stringify(dialect)).slice(0, 80)} names no JSON Schema dialect (2020-12, 2019-09, draft-07, -06 or -04)`);
|
|
42
|
+
}
|
|
43
|
+
metaschema ??= new Ajv2020({ strict: false, allErrors: true });
|
|
44
|
+
let valid;
|
|
45
|
+
try {
|
|
46
|
+
valid = metaschema.validateSchema(body);
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
problems.push(`${tool}: /: ${String(error?.message ?? error).slice(0, 120)} (not valid JSON Schema 2020-12)`);
|
|
50
|
+
valid = true;
|
|
51
|
+
}
|
|
52
|
+
if (!valid) {
|
|
53
|
+
// One problem per place: the metaschema's own anyOf wrappers repeat a place's failure.
|
|
54
|
+
const seen = new Set();
|
|
55
|
+
for (const error of metaschema.errors ?? []) {
|
|
56
|
+
if (error.keyword === "anyOf" || seen.has(error.instancePath))
|
|
57
|
+
continue;
|
|
58
|
+
seen.add(error.instancePath);
|
|
59
|
+
problems.push(`${tool}: ${pointer(error.instancePath)}: ${error.message ?? "invalid"} (not valid JSON Schema 2020-12)`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
// Only where a schema sits: a "$ref" inside default, examples, const or enum is a value, and a property's name is a name.
|
|
63
|
+
// A loop with its own stack, not recursion, so no depth of schema can overflow it.
|
|
64
|
+
const pending = [[schema, ""]];
|
|
65
|
+
while (pending.length) {
|
|
66
|
+
const [node, at] = pending.pop();
|
|
67
|
+
if (!isPlain(node))
|
|
68
|
+
continue;
|
|
69
|
+
const ref = node.$ref;
|
|
70
|
+
if (typeof ref === "string" && !ref.startsWith("#"))
|
|
71
|
+
problems.push(`${tool}: ${at}/$ref: a remote $ref (${JSON.stringify(ref).slice(0, 80)}) is refused; only a local one ("#/$defs/...") is followed`);
|
|
72
|
+
const next = [];
|
|
73
|
+
for (const [key, value] of Object.entries(node)) {
|
|
74
|
+
const here = `${at}/${escape(key)}`;
|
|
75
|
+
if (SCHEMA_MAPS.has(key) && isPlain(value))
|
|
76
|
+
for (const [name, sub] of Object.entries(value))
|
|
77
|
+
next.push([sub, `${here}/${escape(name)}`]);
|
|
78
|
+
else if (SCHEMA_LISTS.has(key) && Array.isArray(value))
|
|
79
|
+
value.forEach((sub, i) => next.push([sub, `${here}/${i}`]));
|
|
80
|
+
else if (SCHEMA_ONE.has(key))
|
|
81
|
+
next.push([value, here]);
|
|
82
|
+
}
|
|
83
|
+
// Children in reverse onto the stack, so they're visited in the schema's order.
|
|
84
|
+
for (let i = next.length - 1; i >= 0; i -= 1)
|
|
85
|
+
pending.push(next[i]);
|
|
86
|
+
}
|
|
87
|
+
return problems;
|
|
88
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { HostTool } from "./types.js";
|
|
2
|
+
/** A skill a host gives a harness, which the model may load: its name, and its files by path, `SKILL.md` among them. */
|
|
3
|
+
export interface HostSkill {
|
|
4
|
+
/** Lowercase letters, digits and dashes; `SKILL.md`'s front matter names it the same. */
|
|
5
|
+
name: string;
|
|
6
|
+
/** Each file's text by its path inside the skill (`SKILL.md`, `reference/sources.md`). */
|
|
7
|
+
files: Readonly<Record<string, string>>;
|
|
8
|
+
}
|
|
9
|
+
/** A skill, as the tool knows it: its name, and the paths of its files. */
|
|
10
|
+
export interface SkillFileList {
|
|
11
|
+
name: string;
|
|
12
|
+
files: readonly string[];
|
|
13
|
+
}
|
|
14
|
+
/** The tool's name, as the model calls it. */
|
|
15
|
+
export declare const READ_SKILL_FILE = "read_skill_file";
|
|
16
|
+
/** The most bytes a host's skills hold in all, every file of every skill (`skillProblems`): over it the set is refused. */
|
|
17
|
+
export declare const SKILL_BYTES: number;
|
|
18
|
+
/** The skills, checked, as every adapter that takes a host's skills checks them when it is made: each a name, a
|
|
19
|
+
* `SKILL.md` whose front matter has that name and a description, and files at plain relative paths, 1 MiB in all; or the
|
|
20
|
+
* problems. What one harness would refuse or rename (the Claude Agent CLI, Codex) is refused here for all of them. */
|
|
21
|
+
export declare function skillProblems(skills: readonly HostSkill[]): string[];
|
|
22
|
+
/** The host tool that reads the skills' files under `root` (`<root>/<skill>/<path>`): replay-safe, read-only. */
|
|
23
|
+
export declare function readSkillFileTool(root: string, skills: readonly SkillFileList[]): HostTool;
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
// A host's skills, as every adapter that takes them checks them (`skillProblems`), and a skill's own files, read by the
|
|
2
|
+
// model through a host tool. Written for AgentRun.
|
|
3
|
+
//
|
|
4
|
+
// A host that gives a harness skills writes each skill's files under one root (`<root>/<skill>/<path>`). The model reads
|
|
5
|
+
// what SKILL.md points it at through `read_skill_file`, a read-only host tool, and nothing else of the disk: the skill
|
|
6
|
+
// and its directory come from the host's own list, never from the request; the path must be one of that skill's listed
|
|
7
|
+
// files, exactly, and is refused before any filesystem call when it is absolute or climbs (`..`); no link is followed
|
|
8
|
+
// anywhere in it (each part is checked, the file is opened without following one, and the opened file is checked to
|
|
9
|
+
// be the expected one after the open, so a link swapped in between is caught).
|
|
10
|
+
import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, readlinkSync, realpathSync } from "node:fs";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
/** The tool's name, as the model calls it. */
|
|
13
|
+
export const READ_SKILL_FILE = "read_skill_file";
|
|
14
|
+
const MAX_BYTES = 1024 * 1024;
|
|
15
|
+
const SKILL_NAME = /^[a-z0-9][a-z0-9-]{0,63}$/;
|
|
16
|
+
const SKILL_PATH = /^(?:[A-Za-z0-9_][A-Za-z0-9._-]{0,99}\/){0,4}[A-Za-z0-9_][A-Za-z0-9._-]{0,99}$/;
|
|
17
|
+
/** The most bytes a host's skills hold in all, every file of every skill (`skillProblems`): over it the set is refused. */
|
|
18
|
+
export const SKILL_BYTES = 1024 * 1024;
|
|
19
|
+
/** The skills, checked, as every adapter that takes a host's skills checks them when it is made: each a name, a
|
|
20
|
+
* `SKILL.md` whose front matter has that name and a description, and files at plain relative paths, 1 MiB in all; or the
|
|
21
|
+
* problems. What one harness would refuse or rename (the Claude Agent CLI, Codex) is refused here for all of them. */
|
|
22
|
+
export function skillProblems(skills) {
|
|
23
|
+
const problems = [];
|
|
24
|
+
const seen = new Set();
|
|
25
|
+
let bytes = 0;
|
|
26
|
+
for (const skill of skills) {
|
|
27
|
+
const name = String(skill?.name ?? "");
|
|
28
|
+
if (!SKILL_NAME.test(name)) {
|
|
29
|
+
problems.push(`the skill name ${JSON.stringify(name.slice(0, 80))} is not lowercase letters, digits and dashes`);
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
if (seen.has(name))
|
|
33
|
+
problems.push(`the skill ${name} is given twice`);
|
|
34
|
+
seen.add(name);
|
|
35
|
+
const files = skill.files ?? {};
|
|
36
|
+
const md = files["SKILL.md"];
|
|
37
|
+
const front = typeof md === "string" ? /^---\r?\n([\s\S]*?)\r?\n---\r?\n/.exec(md)?.[1] : undefined;
|
|
38
|
+
if (front === undefined)
|
|
39
|
+
problems.push(`the skill ${name} has no SKILL.md with front matter`);
|
|
40
|
+
else {
|
|
41
|
+
if (!new RegExp(`^name:\\s*(['"]?)${name}\\1\\s*$`, "m").test(front))
|
|
42
|
+
problems.push(`the skill ${name}'s SKILL.md names it otherwise`);
|
|
43
|
+
if (!/^description:\s*\S/m.test(front))
|
|
44
|
+
problems.push(`the skill ${name}'s SKILL.md has no description`);
|
|
45
|
+
}
|
|
46
|
+
// A path that is another's folder (reference, and reference/sources.md) can't be written as both.
|
|
47
|
+
const paths = new Set(Object.keys(files));
|
|
48
|
+
for (const folder of new Set([...paths].flatMap((path) => path.split("/").slice(0, -1).map((_, i, parts) => parts.slice(0, i + 1).join("/"))))) {
|
|
49
|
+
if (paths.has(folder))
|
|
50
|
+
problems.push(`the skill ${name}'s ${folder} is both a file and a folder`);
|
|
51
|
+
}
|
|
52
|
+
for (const [path, text] of Object.entries(files)) {
|
|
53
|
+
if (!SKILL_PATH.test(path) || path.split("/").some((part) => part === "." || part === ".."))
|
|
54
|
+
problems.push(`the skill ${name}'s file ${JSON.stringify(path.slice(0, 80))} is not a plain relative path`);
|
|
55
|
+
if (typeof text !== "string")
|
|
56
|
+
problems.push(`the skill ${name}'s file ${path} is not text`);
|
|
57
|
+
else
|
|
58
|
+
bytes += Buffer.byteLength(text);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (bytes > SKILL_BYTES)
|
|
62
|
+
problems.push(`the skills hold ${bytes} bytes, more than ${SKILL_BYTES}`);
|
|
63
|
+
return problems;
|
|
64
|
+
}
|
|
65
|
+
/** Why `path` of `skill` is not readable, or undefined: the checks made before any filesystem call. */
|
|
66
|
+
function refusedBeforeDisk(skills, skill, path) {
|
|
67
|
+
if (typeof skill !== "string" || typeof path !== "string")
|
|
68
|
+
return "give a skill's name and a file's path, both as text";
|
|
69
|
+
if (path.startsWith("/") || path.includes("\\") || path.split("/").some((part) => part === ".." || part === "." || part === ""))
|
|
70
|
+
return `the path ${JSON.stringify(path.slice(0, 120))} is absolute or climbs (..): give a path inside the skill, as its SKILL.md names it`;
|
|
71
|
+
const files = skills.get(skill);
|
|
72
|
+
if (!files)
|
|
73
|
+
return `${JSON.stringify(skill.slice(0, 80))} is not one of your skills`;
|
|
74
|
+
if (!files.has(path))
|
|
75
|
+
return `${JSON.stringify(path.slice(0, 120))} is not one of the skill ${skill}'s files`;
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
/** The file's text, read without following a link anywhere in its path; throws a reason a model can read. */
|
|
79
|
+
function readNoFollow(root, skill, path) {
|
|
80
|
+
// The root is resolved (its own path may pass through a link, as /tmp does on some machines); the skill's directory under
|
|
81
|
+
// it is not: a link there would make its target the trusted base.
|
|
82
|
+
const dir = join(realpathSync(root), skill);
|
|
83
|
+
if (lstatSync(dir).isSymbolicLink())
|
|
84
|
+
throw new Error(`the skill ${skill}'s directory is a link, which is not followed`);
|
|
85
|
+
let at = dir;
|
|
86
|
+
for (const part of path.split("/")) {
|
|
87
|
+
at = join(at, part);
|
|
88
|
+
if (lstatSync(at).isSymbolicLink())
|
|
89
|
+
throw new Error(`the skill ${skill}'s ${path} goes through a link, which is not followed`);
|
|
90
|
+
}
|
|
91
|
+
const fd = openSync(at, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
92
|
+
try {
|
|
93
|
+
// After the open: the file opened is the one checked, not a link swapped in between.
|
|
94
|
+
if (process.platform === "linux") {
|
|
95
|
+
if (readlinkSync(`/proc/self/fd/${fd}`) !== at)
|
|
96
|
+
throw new Error(`the skill ${skill}'s ${path} is not the file it was`);
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
const opened = fstatSync(fd);
|
|
100
|
+
const expected = lstatSync(at);
|
|
101
|
+
if (opened.dev !== expected.dev || opened.ino !== expected.ino)
|
|
102
|
+
throw new Error(`the skill ${skill}'s ${path} is not the file it was`);
|
|
103
|
+
}
|
|
104
|
+
if (!fstatSync(fd).isFile())
|
|
105
|
+
throw new Error(`the skill ${skill}'s ${path} is not a file`);
|
|
106
|
+
if (fstatSync(fd).size > MAX_BYTES)
|
|
107
|
+
throw new Error(`the skill ${skill}'s ${path} is larger than ${MAX_BYTES} bytes`);
|
|
108
|
+
return readFileSync(fd, "utf8");
|
|
109
|
+
}
|
|
110
|
+
finally {
|
|
111
|
+
closeSync(fd);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
/** The host tool that reads the skills' files under `root` (`<root>/<skill>/<path>`): replay-safe, read-only. */
|
|
115
|
+
export function readSkillFileTool(root, skills) {
|
|
116
|
+
const known = new Map(skills.map((skill) => [skill.name, new Set(skill.files)]));
|
|
117
|
+
return {
|
|
118
|
+
name: READ_SKILL_FILE,
|
|
119
|
+
description: "Read a file of one of your skills: the skill's name and the file's path inside it, as its SKILL.md names it.",
|
|
120
|
+
inputSchema: { type: "object", properties: { skill: { type: "string", enum: [...known.keys()] }, path: { type: "string" } }, required: ["skill", "path"], additionalProperties: false },
|
|
121
|
+
replay: "safe",
|
|
122
|
+
async call(args) {
|
|
123
|
+
const { skill, path } = (args ?? {});
|
|
124
|
+
const refused = refusedBeforeDisk(known, skill, path);
|
|
125
|
+
if (refused)
|
|
126
|
+
return { text: `Not read: ${refused}.`, isError: true };
|
|
127
|
+
try {
|
|
128
|
+
return { text: readNoFollow(root, skill, path) };
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
const code = error?.code;
|
|
132
|
+
return { text: `Not read: ${code === "ELOOP" ? `the skill ${String(skill)}'s ${String(path)} is a link, which is not followed` : error.message}.`, isError: true };
|
|
133
|
+
}
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import type { AgentAdapter, SessionSpec, SessionState, SessionStore, TurnStatus } from "./types.js";
|
|
2
|
+
export { SCHEMA_CHECK_CASES, type SchemaCheckCase } from "./schema-check-cases.js";
|
|
3
|
+
/** A session store in memory: what one run's store holds, across the processes a test plays. Each `open` of a key
|
|
4
|
+
* counts one more holder at once, and a writer appends only while its count is the newest. */
|
|
5
|
+
export declare function memorySessionStore(): SessionStore & {
|
|
6
|
+
keys(): string[];
|
|
7
|
+
};
|
|
8
|
+
/** One model request of the faux harness, and what its reply does. */
|
|
9
|
+
export type FauxAction = {
|
|
10
|
+
say: string;
|
|
11
|
+
usd?: number | null;
|
|
12
|
+
} | {
|
|
13
|
+
call: string;
|
|
14
|
+
args?: unknown;
|
|
15
|
+
usd?: number | null;
|
|
16
|
+
}
|
|
17
|
+
/** One answer that asks for several tools, run one after another, as Claude Code and Codex run a response's calls.
|
|
18
|
+
* `bookAfter`: the calls run before the answer's cost is booked, as with Claude Code, whose cost comes in after its
|
|
19
|
+
* response, so a call can end before the request that asked for it is paid. */
|
|
20
|
+
| {
|
|
21
|
+
calls: {
|
|
22
|
+
call: string;
|
|
23
|
+
args?: unknown;
|
|
24
|
+
}[];
|
|
25
|
+
usd?: number | null;
|
|
26
|
+
bookAfter?: boolean;
|
|
27
|
+
} | {
|
|
28
|
+
submit: unknown;
|
|
29
|
+
usd?: number | null;
|
|
30
|
+
}
|
|
31
|
+
/** The reply ends the turn without a submission: `completed`, or the status `turn` names (a harness that ends its own
|
|
32
|
+
* turn interrupted, with no stop asked for, ends it so). */
|
|
33
|
+
| {
|
|
34
|
+
end: true;
|
|
35
|
+
usd?: number | null;
|
|
36
|
+
turn?: TurnStatus;
|
|
37
|
+
}
|
|
38
|
+
/** The request fails: the turn ends failed. */
|
|
39
|
+
| {
|
|
40
|
+
fail: string;
|
|
41
|
+
}
|
|
42
|
+
/** The process stops here: nothing resolves until the session's signal aborts. */
|
|
43
|
+
| {
|
|
44
|
+
hang: true;
|
|
45
|
+
}
|
|
46
|
+
/** The harness halts the drive with this reason (a spend cap, a mirror it cannot keep): the turn ends failed, naming it. */
|
|
47
|
+
| {
|
|
48
|
+
halt: string;
|
|
49
|
+
}
|
|
50
|
+
/** A paid request leaves (its beforeRequest passes) and the process dies before it answers: nothing more resolves. */
|
|
51
|
+
| {
|
|
52
|
+
cut: true;
|
|
53
|
+
}
|
|
54
|
+
/** A paid request leaves and holds unanswered until the turn is stopped: the stop cuts it, and its cost (what had
|
|
55
|
+
* streamed) is still booked, as a provider bills a response cut short. */
|
|
56
|
+
| {
|
|
57
|
+
cutBooked: true;
|
|
58
|
+
usd?: number | null;
|
|
59
|
+
};
|
|
60
|
+
export declare const FAUX_RECORDS: {
|
|
61
|
+
readonly request: "faux.request";
|
|
62
|
+
readonly user: "faux.user";
|
|
63
|
+
readonly turn: "faux.turn";
|
|
64
|
+
readonly turnEnd: "faux.turn_end";
|
|
65
|
+
readonly call: "faux.call";
|
|
66
|
+
readonly result: "faux.result";
|
|
67
|
+
};
|
|
68
|
+
export interface FauxOptions {
|
|
69
|
+
/** The session's model replies, in order, by its spec: one action per paid request. */
|
|
70
|
+
script: (spec: SessionSpec) => FauxAction[];
|
|
71
|
+
/** Whether the faux harness can fork (default false). */
|
|
72
|
+
fork?: boolean;
|
|
73
|
+
/** Problems `check` reports for a spec. */
|
|
74
|
+
refuse?: (spec: SessionSpec) => string[];
|
|
75
|
+
/** How long `open` takes, ignoring the session's signal, as a slow harness start does (default 0). */
|
|
76
|
+
openDelayMs?: number;
|
|
77
|
+
}
|
|
78
|
+
export interface FauxAdapter extends AgentAdapter {
|
|
79
|
+
/** Paid requests made, across every session and process of this adapter. */
|
|
80
|
+
readonly requests: number;
|
|
81
|
+
/** Tool calls run, by name. */
|
|
82
|
+
readonly calls: string[];
|
|
83
|
+
/** The sessions opened, by key, with the state each was found in. */
|
|
84
|
+
readonly opened: {
|
|
85
|
+
key: string;
|
|
86
|
+
state: SessionState;
|
|
87
|
+
}[];
|
|
88
|
+
/** Sessions closed. */
|
|
89
|
+
readonly closed: string[];
|
|
90
|
+
}
|
|
91
|
+
export declare function fauxAdapter(options: FauxOptions): FauxAdapter;
|
package/dist/testing.js
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
// Test kit: a session store in memory, and a scripted harness with no model and no network. The faux harness keeps its
|
|
2
|
+
// own records in the store as a real adapter does (the requests it made, each tool call's intent and outcome, its turns),
|
|
3
|
+
// so a later process resumes from them: a request already made is not made again, and a call a crash cut off is
|
|
4
|
+
// answered interrupted, never run again.
|
|
5
|
+
import { SessionFenced } from "./errors.js";
|
|
6
|
+
import { abortedOutcome, SessionHalt } from "./kernel/halt.js";
|
|
7
|
+
export { SCHEMA_CHECK_CASES } from "./schema-check-cases.js";
|
|
8
|
+
/** A session store in memory: what one run's store holds, across the processes a test plays. Each `open` of a key
|
|
9
|
+
* counts one more holder at once, and a writer appends only while its count is the newest. */
|
|
10
|
+
export function memorySessionStore() {
|
|
11
|
+
const sessions = new Map();
|
|
12
|
+
const holders = new Map();
|
|
13
|
+
const read = async (key) => structuredClone(sessions.get(key) ?? []);
|
|
14
|
+
return {
|
|
15
|
+
read,
|
|
16
|
+
async open(key) {
|
|
17
|
+
const holder = (holders.get(key) ?? 0) + 1;
|
|
18
|
+
holders.set(key, holder);
|
|
19
|
+
return {
|
|
20
|
+
key,
|
|
21
|
+
read: () => read(key),
|
|
22
|
+
async append(records) {
|
|
23
|
+
if (holders.get(key) !== holder)
|
|
24
|
+
throw new SessionFenced(key);
|
|
25
|
+
sessions.set(key, [...(sessions.get(key) ?? []), ...structuredClone(records)]);
|
|
26
|
+
},
|
|
27
|
+
};
|
|
28
|
+
},
|
|
29
|
+
keys: () => [...sessions.keys()],
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
export const FAUX_RECORDS = { request: "faux.request", user: "faux.user", turn: "faux.turn", turnEnd: "faux.turn_end", call: "faux.call", result: "faux.result" };
|
|
33
|
+
export function fauxAdapter(options) {
|
|
34
|
+
const stats = { requests: 0, calls: [], opened: [], closed: [] };
|
|
35
|
+
const adapter = {
|
|
36
|
+
id: "faux",
|
|
37
|
+
version: "0.0.0",
|
|
38
|
+
get requests() { return stats.requests; },
|
|
39
|
+
get calls() { return stats.calls; },
|
|
40
|
+
get opened() { return stats.opened; },
|
|
41
|
+
get closed() { return stats.closed; },
|
|
42
|
+
capabilities: () => ({ fork: options.fork ?? false }),
|
|
43
|
+
check: (spec) => [
|
|
44
|
+
...(spec.context === "fork" && !options.fork ? ["the faux harness cannot fork a session"] : []),
|
|
45
|
+
...(options.refuse?.(spec) ?? []),
|
|
46
|
+
],
|
|
47
|
+
async open(spec, io) {
|
|
48
|
+
if (options.openDelayMs)
|
|
49
|
+
await new Promise((resolve) => setTimeout(resolve, options.openDelayMs));
|
|
50
|
+
const script = options.script(spec);
|
|
51
|
+
const records = await io.writer.read();
|
|
52
|
+
const append = (kind, data) => io.writer.append([{ kind, data }]);
|
|
53
|
+
let cursor = records.filter((r) => r.kind === FAUX_RECORDS.request).length;
|
|
54
|
+
const turnsStarted = records.filter((r) => r.kind === FAUX_RECORDS.turn).length;
|
|
55
|
+
const turnsEnded = records.filter((r) => r.kind === FAUX_RECORDS.turnEnd).length;
|
|
56
|
+
let state = turnsStarted === 0 ? "new" : turnsStarted > turnsEnded ? "open_turn" : "idle";
|
|
57
|
+
stats.opened.push({ key: spec.key, state });
|
|
58
|
+
// A call whose intent is recorded and whose outcome is not: a crash cut it off. It is answered interrupted, once.
|
|
59
|
+
const results = new Set(records.filter((r) => r.kind === FAUX_RECORDS.result).map((r) => r.data.callId));
|
|
60
|
+
for (const r of records.filter((r) => r.kind === FAUX_RECORDS.call)) {
|
|
61
|
+
const { callId } = r.data;
|
|
62
|
+
if (!results.has(callId))
|
|
63
|
+
await append(FAUX_RECORDS.result, { callId, interrupted: true });
|
|
64
|
+
}
|
|
65
|
+
const tools = new Map(spec.tools.map((tool) => [tool.name, tool]));
|
|
66
|
+
/** What the session has been sent, as a model would be prompted with it. */
|
|
67
|
+
const prompt = records.filter((r) => r.kind === FAUX_RECORDS.user).map((r) => r.data.text);
|
|
68
|
+
let ending;
|
|
69
|
+
let turnSignal;
|
|
70
|
+
const end = async (status, error) => {
|
|
71
|
+
// A turn whose end cannot be written (its writer was fenced) still ends: failed, saying why.
|
|
72
|
+
try {
|
|
73
|
+
await append(FAUX_RECORDS.turnEnd, { status });
|
|
74
|
+
}
|
|
75
|
+
catch (cause) {
|
|
76
|
+
status = "failed";
|
|
77
|
+
error = cause.message;
|
|
78
|
+
}
|
|
79
|
+
io.onEvent({ type: "turn_end", status });
|
|
80
|
+
ending?.resolve({ status, ...(error === undefined ? {} : { error }) });
|
|
81
|
+
};
|
|
82
|
+
const hang = (signal) => new Promise((_, reject) => {
|
|
83
|
+
if (signal.aborted)
|
|
84
|
+
reject(signal.reason);
|
|
85
|
+
else
|
|
86
|
+
signal.addEventListener("abort", () => reject(signal.reason), { once: true });
|
|
87
|
+
});
|
|
88
|
+
/** The turn's model loop: one scripted action per paid request, until one ends the turn. */
|
|
89
|
+
const run = async () => {
|
|
90
|
+
const signal = AbortSignal.any([io.signal, (turnSignal = new AbortController()).signal]);
|
|
91
|
+
try {
|
|
92
|
+
for (;;) {
|
|
93
|
+
const action = script[cursor];
|
|
94
|
+
if (!action)
|
|
95
|
+
return await end("completed");
|
|
96
|
+
if ("hang" in action)
|
|
97
|
+
return await hang(signal);
|
|
98
|
+
if ("halt" in action) {
|
|
99
|
+
turnSignal.abort(new Error(action.halt));
|
|
100
|
+
return await hang(signal);
|
|
101
|
+
}
|
|
102
|
+
// Measured as a real adapter's proxy measures its body: the prompt's UTF-8 bytes, with nothing a provider adds.
|
|
103
|
+
const ticket = await io.models.beforeRequest({ model: "faux", inputTokensMax: Buffer.byteLength(JSON.stringify({ system: spec.system, user: prompt }), "utf8") });
|
|
104
|
+
if ("cut" in action) {
|
|
105
|
+
stats.requests += 1;
|
|
106
|
+
return await hang(signal);
|
|
107
|
+
}
|
|
108
|
+
if ("cutBooked" in action) {
|
|
109
|
+
stats.requests += 1;
|
|
110
|
+
await append(FAUX_RECORDS.request, { index: cursor++ });
|
|
111
|
+
try {
|
|
112
|
+
return await hang(signal);
|
|
113
|
+
}
|
|
114
|
+
finally {
|
|
115
|
+
await io.models.afterRequest({ input: 10, output: 5, usd: action.usd === undefined ? 0.001 : action.usd });
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
stats.requests += 1;
|
|
119
|
+
const index = cursor++;
|
|
120
|
+
// The output limit it was admitted with (what is left of a spend cap can buy), as a real adapter's proxy would
|
|
121
|
+
// write it into the body: filed, for a test to see.
|
|
122
|
+
const limit = ticket ? ticket.outputTokensMax : undefined;
|
|
123
|
+
await append(FAUX_RECORDS.request, { index, ...(limit === undefined ? {} : { outputTokensMax: limit }) });
|
|
124
|
+
if ("fail" in action)
|
|
125
|
+
return await end("failed", action.fail);
|
|
126
|
+
const book = () => io.models.afterRequest({ input: 10, output: 5, usd: action.usd === undefined ? 0.001 : action.usd });
|
|
127
|
+
const bookAfter = "calls" in action && action.bookAfter === true;
|
|
128
|
+
if (!bookAfter)
|
|
129
|
+
await book();
|
|
130
|
+
if ("say" in action)
|
|
131
|
+
io.onEvent({ type: "text", delta: action.say });
|
|
132
|
+
else if ("end" in action)
|
|
133
|
+
return await end(action.turn ?? "completed");
|
|
134
|
+
else {
|
|
135
|
+
const asked = "calls" in action ? action.calls.map((c) => ({ name: c.call, args: c.args ?? {} }))
|
|
136
|
+
: [{ name: "call" in action ? action.call : "submit", args: "call" in action ? action.args ?? {} : { value: action.submit } }];
|
|
137
|
+
for (const [n, { name, args }] of asked.entries()) {
|
|
138
|
+
const callId = "calls" in action ? `${spec.key}:${index}.${n}` : `${spec.key}:${index}`;
|
|
139
|
+
const tool = tools.get(name);
|
|
140
|
+
await append(FAUX_RECORDS.call, { callId, name });
|
|
141
|
+
io.onEvent({ type: "tool_start", callId, name, args });
|
|
142
|
+
const result = tool ? await tool.call(args, { callId, signal }).catch((error) => ({ text: error.message, isError: true })) : { text: `unknown tool ${name}`, isError: true };
|
|
143
|
+
if (tool)
|
|
144
|
+
stats.calls.push(name);
|
|
145
|
+
await append(FAUX_RECORDS.result, { callId, isError: result.isError === true });
|
|
146
|
+
io.onEvent({ type: "tool_end", callId, name, isError: result.isError === true });
|
|
147
|
+
}
|
|
148
|
+
if (bookAfter)
|
|
149
|
+
await book();
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
// Stopped by the session or its host: interrupted. Halted by anything else: failed, naming why.
|
|
155
|
+
if (signal.aborted) {
|
|
156
|
+
const outcome = abortedOutcome(signal, io.signal);
|
|
157
|
+
if (outcome.status === "interrupted")
|
|
158
|
+
return void ending?.resolve(outcome);
|
|
159
|
+
return await end("failed", outcome.error);
|
|
160
|
+
}
|
|
161
|
+
return await end("failed", error.message);
|
|
162
|
+
}
|
|
163
|
+
};
|
|
164
|
+
const begin = () => {
|
|
165
|
+
let resolve;
|
|
166
|
+
const promise = new Promise((r) => { resolve = r; });
|
|
167
|
+
ending = { promise, resolve };
|
|
168
|
+
void run();
|
|
169
|
+
};
|
|
170
|
+
const session = {
|
|
171
|
+
get state() { return state; },
|
|
172
|
+
async send(text) {
|
|
173
|
+
state = "idle";
|
|
174
|
+
await append(FAUX_RECORDS.user, { text });
|
|
175
|
+
prompt.push(text);
|
|
176
|
+
await append(FAUX_RECORDS.turn, { n: turnsStarted + 1 });
|
|
177
|
+
begin();
|
|
178
|
+
},
|
|
179
|
+
async recover() { state = "idle"; begin(); },
|
|
180
|
+
// The open turn ends in the session's records, as a turn's end is written (end()), and nothing runs. Its end is the
|
|
181
|
+
// kernel's for the session's own stop (abortedOutcome of its SessionHalt), as an interrupted turn's is here.
|
|
182
|
+
async settleOpenTurn() {
|
|
183
|
+
if (state !== "open_turn")
|
|
184
|
+
throw new Error("faux: there is no turn to settle");
|
|
185
|
+
await append(FAUX_RECORDS.turnEnd, { status: abortedOutcome(AbortSignal.abort(new SessionHalt("interrupted"))).status });
|
|
186
|
+
state = "idle";
|
|
187
|
+
},
|
|
188
|
+
turnEnd: () => ending ? ending.promise : Promise.resolve({ status: "completed" }),
|
|
189
|
+
async interrupt() { turnSignal?.abort(new SessionHalt("interrupted")); },
|
|
190
|
+
async close() { turnSignal?.abort(new SessionHalt("closed")); stats.closed.push(spec.key); },
|
|
191
|
+
};
|
|
192
|
+
return session;
|
|
193
|
+
},
|
|
194
|
+
};
|
|
195
|
+
return adapter;
|
|
196
|
+
}
|