@alisio/core 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/changelog/data.js +16 -0
- package/dist/eval/canonical.d.ts +14 -0
- package/dist/eval/canonical.js +40 -0
- package/dist/eval/cassette.d.ts +180 -0
- package/dist/eval/cassette.js +424 -0
- package/dist/eval/compare.d.ts +147 -0
- package/dist/eval/compare.js +436 -0
- package/dist/eval/engine.d.ts +144 -0
- package/dist/eval/engine.js +728 -0
- package/dist/eval/export.d.ts +40 -0
- package/dist/eval/export.js +90 -0
- package/dist/eval/fingerprint.d.ts +88 -0
- package/dist/eval/fingerprint.js +186 -0
- package/dist/eval/normalize.d.ts +68 -0
- package/dist/eval/normalize.js +138 -0
- package/dist/eval/paths.d.ts +11 -0
- package/dist/eval/paths.js +50 -0
- package/dist/eval/plan.d.ts +26 -0
- package/dist/eval/plan.js +44 -0
- package/dist/eval/report.d.ts +53 -0
- package/dist/eval/report.js +258 -0
- package/dist/eval/run-suite.d.ts +68 -0
- package/dist/eval/run-suite.js +94 -0
- package/dist/eval/secrets.d.ts +12 -0
- package/dist/eval/secrets.js +45 -0
- package/dist/eval/stats.d.ts +17 -0
- package/dist/eval/stats.js +50 -0
- package/dist/eval/suite.d.ts +20 -0
- package/dist/eval/suite.js +165 -0
- package/dist/eval/variants.d.ts +35 -0
- package/dist/eval/variants.js +137 -0
- package/dist/eval/verifiers.d.ts +49 -0
- package/dist/eval/verifiers.js +264 -0
- package/dist/forge/javascript/hosts/worker-host.d.ts +9 -0
- package/dist/forge/javascript/hosts/worker-host.js +16 -1
- package/dist/forge/kernel.js +3 -2
- package/dist/index.d.ts +16 -1
- package/dist/index.js +16 -1
- package/dist/plan/exit-plan.js +2 -1
- package/dist/runtime/ids.d.ts +21 -0
- package/dist/runtime/ids.js +41 -1
- package/dist/runtime/store.js +3 -2
- package/package.json +2 -2
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic, local verifiers of the reliability lab (specs/010 D4). A verifier answers
|
|
3
|
+
* `pass`, `fail` or `error`; `error` means it could not run (bad path, missing input, command that
|
|
4
|
+
* cannot start) and is never folded into `fail`. Details never echo file contents.
|
|
5
|
+
*
|
|
6
|
+
* `command` runs a subprocess with a clean environment and a timeout. A subprocess is NOT a
|
|
7
|
+
* sandbox: suites are trusted repository code and the command runs with the user's privileges.
|
|
8
|
+
*/
|
|
9
|
+
import { spawn } from "node:child_process";
|
|
10
|
+
import { createHash } from "node:crypto";
|
|
11
|
+
import { existsSync, lstatSync, readdirSync, readFileSync, readlinkSync, statSync } from "node:fs";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
import { EvalPathError, resolveInside, unsafeRelativeReason } from "./paths.js";
|
|
14
|
+
const IGNORED_DIRS = new Set([".git", "node_modules"]);
|
|
15
|
+
const MAX_COMMAND_OUTPUT = 1_000_000;
|
|
16
|
+
const sha256 = (data) => createHash("sha256").update(data).digest("hex");
|
|
17
|
+
/**
|
|
18
|
+
* Hashes every file under `dir` (symlinks by their target text, never followed). `.git` and
|
|
19
|
+
* `node_modules` are skipped at any depth: they are tooling state, not task output.
|
|
20
|
+
*/
|
|
21
|
+
export function snapshotTree(dir) {
|
|
22
|
+
const files = {};
|
|
23
|
+
const walk = (current, prefix) => {
|
|
24
|
+
for (const entry of readdirSync(current, { withFileTypes: true })) {
|
|
25
|
+
const rel = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
26
|
+
const full = join(current, entry.name);
|
|
27
|
+
if (entry.isSymbolicLink())
|
|
28
|
+
files[rel] = sha256(`symlink:${readlinkSync(full)}`);
|
|
29
|
+
else if (entry.isDirectory()) {
|
|
30
|
+
if (!IGNORED_DIRS.has(entry.name))
|
|
31
|
+
walk(full, rel);
|
|
32
|
+
}
|
|
33
|
+
else if (entry.isFile())
|
|
34
|
+
files[rel] = sha256(readFileSync(full));
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
walk(dir, "");
|
|
38
|
+
const hash = sha256(Object.keys(files)
|
|
39
|
+
.sort()
|
|
40
|
+
.map((path) => `${path}\0${files[path]}`)
|
|
41
|
+
.join("\n"));
|
|
42
|
+
return { files, hash };
|
|
43
|
+
}
|
|
44
|
+
const result = (v, status, detail) => ({
|
|
45
|
+
id: v.id,
|
|
46
|
+
status,
|
|
47
|
+
...(detail === undefined ? {} : { detail }),
|
|
48
|
+
});
|
|
49
|
+
function isMissing(error) {
|
|
50
|
+
return error?.code === "ENOENT";
|
|
51
|
+
}
|
|
52
|
+
function runCommand(v, cwd, tmpDir) {
|
|
53
|
+
return new Promise((resolve) => {
|
|
54
|
+
const [bin, ...args] = v.command;
|
|
55
|
+
if (bin === undefined)
|
|
56
|
+
return resolve(result(v, "error", "command is empty."));
|
|
57
|
+
if (!existsSync(cwd) || !statSync(cwd).isDirectory())
|
|
58
|
+
return resolve(result(v, "error", "working directory does not exist."));
|
|
59
|
+
const env = { PATH: process.env.PATH ?? "", NODE_NO_WARNINGS: "1" };
|
|
60
|
+
if (process.env.SystemRoot)
|
|
61
|
+
env.SystemRoot = process.env.SystemRoot;
|
|
62
|
+
if (tmpDir) {
|
|
63
|
+
env.HOME = tmpDir;
|
|
64
|
+
env.TMPDIR = tmpDir;
|
|
65
|
+
}
|
|
66
|
+
let stdout = "";
|
|
67
|
+
let timedOut = false;
|
|
68
|
+
let settled = false;
|
|
69
|
+
const done = (value) => {
|
|
70
|
+
if (settled)
|
|
71
|
+
return;
|
|
72
|
+
settled = true;
|
|
73
|
+
clearTimeout(timer);
|
|
74
|
+
resolve(value);
|
|
75
|
+
};
|
|
76
|
+
// The child leads its own process group (POSIX) so a timeout can stop grandchildren too.
|
|
77
|
+
const detached = process.platform !== "win32";
|
|
78
|
+
const child = spawn(bin, args, {
|
|
79
|
+
cwd,
|
|
80
|
+
env,
|
|
81
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
82
|
+
shell: false,
|
|
83
|
+
detached,
|
|
84
|
+
});
|
|
85
|
+
const timer = setTimeout(() => {
|
|
86
|
+
timedOut = true;
|
|
87
|
+
try {
|
|
88
|
+
if (detached && child.pid !== undefined)
|
|
89
|
+
process.kill(-child.pid, "SIGKILL");
|
|
90
|
+
else
|
|
91
|
+
child.kill("SIGKILL");
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
child.kill("SIGKILL");
|
|
95
|
+
}
|
|
96
|
+
}, v.timeoutMs);
|
|
97
|
+
child.stdout.on("data", (chunk) => {
|
|
98
|
+
if (stdout.length < MAX_COMMAND_OUTPUT)
|
|
99
|
+
stdout += chunk.toString("utf8");
|
|
100
|
+
});
|
|
101
|
+
child.on("error", (error) => done(result(v, "error", `command could not start: ${error.message}`)));
|
|
102
|
+
child.on("close", (code, signal) => {
|
|
103
|
+
// The command is the task's own check running agent-written code: a hang is the agent's
|
|
104
|
+
// failure, not an infrastructure error (a command that cannot start stays `error`).
|
|
105
|
+
if (timedOut)
|
|
106
|
+
return done(result(v, "fail", `command timed out after ${v.timeoutMs} ms.`));
|
|
107
|
+
const expected = v.expectExit ?? 0;
|
|
108
|
+
if (code !== expected)
|
|
109
|
+
return done(result(v, "fail", signal ? `terminated by ${signal}.` : `exit ${code}, expected exit ${expected}.`));
|
|
110
|
+
if (v.expectStdoutIncludes !== undefined && !stdout.includes(v.expectStdoutIncludes))
|
|
111
|
+
return done(result(v, "fail", "stdout does not include the expected text."));
|
|
112
|
+
done(result(v, "pass"));
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
function readTarget(path) {
|
|
117
|
+
try {
|
|
118
|
+
return { data: readFileSync(path) };
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
if (isMissing(error))
|
|
122
|
+
return { missing: true };
|
|
123
|
+
throw error;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
function diffTree(baseline, current) {
|
|
127
|
+
const added = [];
|
|
128
|
+
const deleted = [];
|
|
129
|
+
const modified = [];
|
|
130
|
+
for (const [path, hash] of Object.entries(current.files)) {
|
|
131
|
+
const before = baseline.files[path];
|
|
132
|
+
if (before === undefined)
|
|
133
|
+
added.push(path);
|
|
134
|
+
else if (before !== hash)
|
|
135
|
+
modified.push(path);
|
|
136
|
+
}
|
|
137
|
+
for (const path of Object.keys(baseline.files))
|
|
138
|
+
if (!(path in current.files))
|
|
139
|
+
deleted.push(path);
|
|
140
|
+
return { added, deleted, modified };
|
|
141
|
+
}
|
|
142
|
+
/** `a/b.ts` allows exactly that file; `a/` allows everything below the directory `a`. */
|
|
143
|
+
const isAllowed = (path, allowed) => allowed.some((entry) => (entry.endsWith("/") ? path.startsWith(entry) : path === entry));
|
|
144
|
+
function checkSequence(rule, events) {
|
|
145
|
+
switch (rule.type) {
|
|
146
|
+
case "before": {
|
|
147
|
+
const firstAt = events.findIndex((e) => e.tool === rule.first);
|
|
148
|
+
const thenAt = events.findIndex((e) => e.tool === rule.then);
|
|
149
|
+
if (thenAt === -1)
|
|
150
|
+
return { ok: true };
|
|
151
|
+
return firstAt !== -1 && firstAt < thenAt
|
|
152
|
+
? { ok: true }
|
|
153
|
+
: { ok: false, detail: `"${rule.then}" was called without a prior "${rule.first}".` };
|
|
154
|
+
}
|
|
155
|
+
case "never":
|
|
156
|
+
return events.some((e) => e.tool === rule.tool)
|
|
157
|
+
? { ok: false, detail: `"${rule.tool}" must never be called.` }
|
|
158
|
+
: { ok: true };
|
|
159
|
+
case "max-calls": {
|
|
160
|
+
const calls = events.filter((e) => e.tool === rule.tool).length;
|
|
161
|
+
return calls > rule.max
|
|
162
|
+
? { ok: false, detail: `"${rule.tool}" was called ${calls} times, maximum ${rule.max}.` }
|
|
163
|
+
: { ok: true };
|
|
164
|
+
}
|
|
165
|
+
case "no-repeat-failure": {
|
|
166
|
+
const failed = new Set();
|
|
167
|
+
for (const e of events) {
|
|
168
|
+
if (e.tool !== rule.tool)
|
|
169
|
+
continue;
|
|
170
|
+
const key = e.argsKey ?? "";
|
|
171
|
+
if (failed.has(key))
|
|
172
|
+
return {
|
|
173
|
+
ok: false,
|
|
174
|
+
detail: `"${rule.tool}" repeated an identical call that had failed.`,
|
|
175
|
+
};
|
|
176
|
+
if (!e.ok)
|
|
177
|
+
failed.add(key);
|
|
178
|
+
}
|
|
179
|
+
return { ok: true };
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
async function evaluate(v, ctx) {
|
|
184
|
+
switch (v.kind) {
|
|
185
|
+
case "command":
|
|
186
|
+
return runCommand(v, ctx.cwd, ctx.tmpDir);
|
|
187
|
+
case "file-equals": {
|
|
188
|
+
const target = readTarget(resolveInside(ctx.cwd, v.path, "path"));
|
|
189
|
+
if ("missing" in target)
|
|
190
|
+
return result(v, "fail", `${v.path} does not exist.`);
|
|
191
|
+
return sha256(target.data) === v.sha256.toLowerCase()
|
|
192
|
+
? result(v, "pass")
|
|
193
|
+
: result(v, "fail", `${v.path} sha256 differs from the expected value.`);
|
|
194
|
+
}
|
|
195
|
+
case "file-contains": {
|
|
196
|
+
const target = readTarget(resolveInside(ctx.cwd, v.path, "path"));
|
|
197
|
+
if ("missing" in target)
|
|
198
|
+
return result(v, "fail", `${v.path} does not exist.`);
|
|
199
|
+
return target.data.toString("utf8").includes(v.text)
|
|
200
|
+
? result(v, "pass")
|
|
201
|
+
: result(v, "fail", `${v.path} does not contain the expected text.`);
|
|
202
|
+
}
|
|
203
|
+
case "file-absent": {
|
|
204
|
+
const absolute = resolveInside(ctx.cwd, v.path, "path");
|
|
205
|
+
try {
|
|
206
|
+
lstatSync(absolute);
|
|
207
|
+
}
|
|
208
|
+
catch (error) {
|
|
209
|
+
if (isMissing(error))
|
|
210
|
+
return result(v, "pass");
|
|
211
|
+
throw error;
|
|
212
|
+
}
|
|
213
|
+
return result(v, "fail", `${v.path} exists.`);
|
|
214
|
+
}
|
|
215
|
+
case "tree-diff-within": {
|
|
216
|
+
if (!ctx.baseline)
|
|
217
|
+
return result(v, "error", "no fixture snapshot was provided to compare against.");
|
|
218
|
+
for (const entry of v.allowed) {
|
|
219
|
+
const reason = unsafeRelativeReason(entry);
|
|
220
|
+
if (reason)
|
|
221
|
+
return result(v, "error", `allowed entry "${entry}" ${reason}.`);
|
|
222
|
+
}
|
|
223
|
+
const { added, deleted, modified } = diffTree(ctx.baseline, snapshotTree(ctx.cwd));
|
|
224
|
+
const outside = (paths) => paths.filter((p) => !isAllowed(p, v.allowed)).sort();
|
|
225
|
+
const parts = [
|
|
226
|
+
["modified", outside(modified)],
|
|
227
|
+
["added", outside(added)],
|
|
228
|
+
["deleted", outside(deleted)],
|
|
229
|
+
]
|
|
230
|
+
.filter(([, paths]) => paths.length > 0)
|
|
231
|
+
.map(([label, paths]) => `${label}: ${paths.join(", ")}`);
|
|
232
|
+
return parts.length === 0
|
|
233
|
+
? result(v, "pass")
|
|
234
|
+
: result(v, "fail", `changes outside the allowed list (${parts.join("; ")}).`);
|
|
235
|
+
}
|
|
236
|
+
case "tool-sequence": {
|
|
237
|
+
if (!ctx.toolEvents)
|
|
238
|
+
return result(v, "error", "no tool events were recorded for this run.");
|
|
239
|
+
const checked = checkSequence(v.rule, ctx.toolEvents);
|
|
240
|
+
return checked.ok ? result(v, "pass") : result(v, "fail", checked.detail);
|
|
241
|
+
}
|
|
242
|
+
default:
|
|
243
|
+
return result(v, "error", `unknown verifier kind "${v.kind}".`);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/** Runs one verifier; anything that prevents a verdict (including exceptions) is `error`. */
|
|
247
|
+
export async function runVerifier(v, ctx) {
|
|
248
|
+
try {
|
|
249
|
+
return await evaluate(v, ctx);
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
const detail = error instanceof EvalPathError
|
|
253
|
+
? error.message
|
|
254
|
+
: `verifier could not run: ${error.code ?? error.message}`;
|
|
255
|
+
return result(v, "error", detail);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
/** Runs the verifiers in order (sequentially: `command` verifiers may share the working directory). */
|
|
259
|
+
export async function runVerifiers(verifiers, ctx) {
|
|
260
|
+
const results = [];
|
|
261
|
+
for (const v of verifiers)
|
|
262
|
+
results.push(await runVerifier(v, ctx));
|
|
263
|
+
return results;
|
|
264
|
+
}
|
|
@@ -2,6 +2,15 @@ import type { IsolateHost, IsolateSession } from "../../ports.ts";
|
|
|
2
2
|
export declare const WORKER_TERMINATE_AFTER_MS = 50;
|
|
3
3
|
/** The worker entry next to this file: `.ts` when running from sources, `.js` from `dist`. */
|
|
4
4
|
export declare function workerEntryUrl(base?: string): URL;
|
|
5
|
+
/**
|
|
6
|
+
* Node flags for the worker. A `.ts` entry (running from sources) needs type stripping, which Node
|
|
7
|
+
* enables by default only from 22.18; on 22.16-22.17 the worker would fail to load its entry. Bun
|
|
8
|
+
* runs TypeScript natively.
|
|
9
|
+
*/
|
|
10
|
+
export declare function workerExecArgv(entry: URL, runtime?: {
|
|
11
|
+
bun?: string;
|
|
12
|
+
typescript?: unknown;
|
|
13
|
+
}): string[];
|
|
5
14
|
export interface WorkerHostOptions {
|
|
6
15
|
module: WebAssembly.Module;
|
|
7
16
|
/** Overrides the entry (tests of failure paths). */
|
|
@@ -17,6 +17,19 @@ export function workerEntryUrl(base = import.meta.url) {
|
|
|
17
17
|
const extension = base.endsWith(".ts") ? "ts" : "js";
|
|
18
18
|
return new URL(`./worker-entry.${extension}`, base);
|
|
19
19
|
}
|
|
20
|
+
/**
|
|
21
|
+
* Node flags for the worker. A `.ts` entry (running from sources) needs type stripping, which Node
|
|
22
|
+
* enables by default only from 22.18; on 22.16-22.17 the worker would fail to load its entry. Bun
|
|
23
|
+
* runs TypeScript natively.
|
|
24
|
+
*/
|
|
25
|
+
export function workerExecArgv(entry, runtime = {
|
|
26
|
+
bun: process.versions.bun,
|
|
27
|
+
typescript: process.features.typescript,
|
|
28
|
+
}) {
|
|
29
|
+
if (!entry.pathname.endsWith(".ts") || runtime.bun || runtime.typescript)
|
|
30
|
+
return [];
|
|
31
|
+
return ["--experimental-strip-types", "--disable-warning=ExperimentalWarning"];
|
|
32
|
+
}
|
|
20
33
|
export class WorkerIsolateHost {
|
|
21
34
|
kind = "worker";
|
|
22
35
|
#options;
|
|
@@ -31,8 +44,10 @@ export class WorkerIsolateHost {
|
|
|
31
44
|
module: this.#options.module,
|
|
32
45
|
flag: flagBuffer,
|
|
33
46
|
};
|
|
34
|
-
const
|
|
47
|
+
const entry = this.#options.entry ?? workerEntryUrl();
|
|
48
|
+
const worker = new Worker(entry, {
|
|
35
49
|
workerData: data,
|
|
50
|
+
execArgv: workerExecArgv(entry),
|
|
36
51
|
env: {},
|
|
37
52
|
stdout: true,
|
|
38
53
|
stderr: true,
|
package/dist/forge/kernel.js
CHANGED
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
* deadline and cancellation with the host's OWN clock, the output bound and the single terminal
|
|
10
10
|
* state. The engine only knows how to run code in an isolate.
|
|
11
11
|
*/
|
|
12
|
-
import { createHash
|
|
12
|
+
import { createHash } from "node:crypto";
|
|
13
|
+
import { newUuid } from "../runtime/ids.js";
|
|
13
14
|
import { CapabilityBroker } from "./broker.js";
|
|
14
15
|
import { BudgetLedger } from "./budget.js";
|
|
15
16
|
import { ForgeError, isForgeError } from "./errors.js";
|
|
@@ -42,7 +43,7 @@ export class ExecutionKernel {
|
|
|
42
43
|
const now = deps.now ?? Date.now;
|
|
43
44
|
const call = invocation.call;
|
|
44
45
|
const startedAt = now();
|
|
45
|
-
const id = (deps.newId ?? (() => `fx_${
|
|
46
|
+
const id = (deps.newId ?? (() => `fx_${newUuid()}`))();
|
|
46
47
|
const config = deps.config();
|
|
47
48
|
const descriptors = deps.engines.descriptors();
|
|
48
49
|
const language = invocation.language ?? "javascript";
|
package/dist/index.d.ts
CHANGED
|
@@ -41,6 +41,21 @@ export { ToolRegistry } from "./core/registry.ts";
|
|
|
41
41
|
export { RunClock } from "./core/run-clock.ts";
|
|
42
42
|
export { AgentRunner, type CompactionResult, defaultTokenBudget, type RunnerOptions, type RunnerSettingsPatch, type RunOptions, } from "./core/runner.ts";
|
|
43
43
|
export { CircuitBreaker, type CircuitBreakerOptions, type CircuitState, DECISION_LIMITS, DEFAULT_DECISIONS_CONFIG, type DecisionAttempt, DecisionError, type DecisionMetricEntry, DecisionMetrics, type DecisionMetricsSnapshot, type DecisionOutcome, DecisionRegistry, DecisionService, type DecisionServiceDeps, type DecisionStatus, type DecisionsConfig, formatDecisionStatsSection, formatDecisionsReport, HEALTH_TIMEOUT_MS, type RegisteredDecisionProvider, splitSupported, validateAnswers, validateRequest, } from "./decisions/index.ts";
|
|
44
|
+
export { canonicalJson, hashCanonical } from "./eval/canonical.ts";
|
|
45
|
+
export { CASSETTE_SUFFIX, type Cassette, type CassetteCall, CassetteFormatError, type CassetteHeader, type CassetteMeta, CassetteRecorder, CassetteWriteError, MAX_CASSETTE_BYTES, MAX_CASSETTE_DIR_BYTES, parseCassette, RecordingProvider, ReplayDivergenceError, ReplayExhausted, ReplayProvider, ReplayUnconsumedError, readCassette, serializeCassette, type WriteOptions, writeCassette, } from "./eval/cassette.ts";
|
|
46
|
+
export { type Conditions, compareReports, compareVariants, EvalCompareError, type EvalComparison, type Interval, type MetricDelta, parseReport, renderComparisonMarkdown, renderVariantComparisonMarkdown, type TaskComparison, type TokenComparison, type VariantComparison, type Verdict, } from "./eval/compare.ts";
|
|
47
|
+
export { CONFIG_HASH_COVERAGE, type ConfigHashExtras, checkTaskPolicy, computeConfigHash, type EvalApplication, EvalBudgetExceeded, type EvalRunMode, EvalRunRefusedError, type InvokerPermissions, type RunTaskOptions, runTask, } from "./eval/engine.ts";
|
|
48
|
+
export { buildExportBundle, type EvalExportBundle, EvalExportRefusedError, type ExportContext, } from "./eval/export.ts";
|
|
49
|
+
export { computeFingerprint, type FingerprintDiff, type FingerprintDiffEntry, } from "./eval/fingerprint.ts";
|
|
50
|
+
export { EvalPathError, resolveInside } from "./eval/paths.ts";
|
|
51
|
+
export { interleaveVariants, mulberry32, type PlannedRun, planRuns, type VariantStep, } from "./eval/plan.ts";
|
|
52
|
+
export { type BuildReportOptions, buildReport, EvalReportError, estimateRunCost, lowPowerTasks, MIN_POWER_K, renderReportMarkdown, } from "./eval/report.ts";
|
|
53
|
+
export { type RunEvaluationOptions, type RunEvaluationResult, type RunVariantsOptions, runEvaluation, runVariantEvaluations, } from "./eval/run-suite.ts";
|
|
54
|
+
export { findSecrets, redactSecrets } from "./eval/secrets.ts";
|
|
55
|
+
export { median, passAtK, passPowK, sampleVariance, wilsonInterval, } from "./eval/stats.ts";
|
|
56
|
+
export { EvalSuiteError, type LoadedSuite, loadSuite, MAX_VERIFIER_TIMEOUT_MS, } from "./eval/suite.ts";
|
|
57
|
+
export { applyConfigOverrides, EvalVariantError, loadVariantsFile, resolveVariants, type VariantsFile, } from "./eval/variants.ts";
|
|
58
|
+
export { runVerifier, runVerifiers, snapshotTree, type ToolEvent, type TreeSnapshot, type VerifierContext, type VerifierResult, type VerifierStatus, } from "./eval/verifiers.ts";
|
|
44
59
|
export { type ExtensionConflict, ExtensionRegistry } from "./extensions/registry.ts";
|
|
45
60
|
export { ForgeError } from "./forge/errors.ts";
|
|
46
61
|
export { downloadSize as pyodideDownloadSize, inspectPyodide, installPyodide, PyodideIntegrityError, pyodideDir, } from "./forge/python/pyodide-install.ts";
|
|
@@ -69,7 +84,7 @@ export { type ExternalDirectoryDecision, type ExternalDirectoryHandler, type Ext
|
|
|
69
84
|
export { BlobStore, type BlobStoreOptions, BlobTooLarge } from "./runtime/blobs.ts";
|
|
70
85
|
export { clipLines, unifiedPatch } from "./runtime/diff.ts";
|
|
71
86
|
export { exists, fileSize, readHead, readJson, readText, which } from "./runtime/fs.ts";
|
|
72
|
-
export { newId, ulid } from "./runtime/ids.ts";
|
|
87
|
+
export { type IdSource, newId, newUuid, seededIdSource, setIdSource, ulid, } from "./runtime/ids.ts";
|
|
73
88
|
export { defaultGlobalRoots, isPathSpec, PLUGIN_KEYWORD, resolvePluginSpec, } from "./runtime/modules.ts";
|
|
74
89
|
export { findWorkspace, inside, outsideRootsMessage, safePath, workspaceKey, } from "./runtime/paths.ts";
|
|
75
90
|
export { isMissingCommand, type ProcessResult, RIPGREP_INSTALL_HINT, runProcess, } from "./runtime/process.ts";
|
package/dist/index.js
CHANGED
|
@@ -40,6 +40,21 @@ export { ToolRegistry } from "./core/registry.js";
|
|
|
40
40
|
export { RunClock } from "./core/run-clock.js";
|
|
41
41
|
export { AgentRunner, defaultTokenBudget, } from "./core/runner.js";
|
|
42
42
|
export { CircuitBreaker, DECISION_LIMITS, DEFAULT_DECISIONS_CONFIG, DecisionError, DecisionMetrics, DecisionRegistry, DecisionService, formatDecisionStatsSection, formatDecisionsReport, HEALTH_TIMEOUT_MS, splitSupported, validateAnswers, validateRequest, } from "./decisions/index.js";
|
|
43
|
+
export { canonicalJson, hashCanonical } from "./eval/canonical.js";
|
|
44
|
+
export { CASSETTE_SUFFIX, CassetteFormatError, CassetteRecorder, CassetteWriteError, MAX_CASSETTE_BYTES, MAX_CASSETTE_DIR_BYTES, parseCassette, RecordingProvider, ReplayDivergenceError, ReplayExhausted, ReplayProvider, ReplayUnconsumedError, readCassette, serializeCassette, writeCassette, } from "./eval/cassette.js";
|
|
45
|
+
export { compareReports, compareVariants, EvalCompareError, parseReport, renderComparisonMarkdown, renderVariantComparisonMarkdown, } from "./eval/compare.js";
|
|
46
|
+
export { CONFIG_HASH_COVERAGE, checkTaskPolicy, computeConfigHash, EvalBudgetExceeded, EvalRunRefusedError, runTask, } from "./eval/engine.js";
|
|
47
|
+
export { buildExportBundle, EvalExportRefusedError, } from "./eval/export.js";
|
|
48
|
+
export { computeFingerprint, } from "./eval/fingerprint.js";
|
|
49
|
+
export { EvalPathError, resolveInside } from "./eval/paths.js";
|
|
50
|
+
export { interleaveVariants, mulberry32, planRuns, } from "./eval/plan.js";
|
|
51
|
+
export { buildReport, EvalReportError, estimateRunCost, lowPowerTasks, MIN_POWER_K, renderReportMarkdown, } from "./eval/report.js";
|
|
52
|
+
export { runEvaluation, runVariantEvaluations, } from "./eval/run-suite.js";
|
|
53
|
+
export { findSecrets, redactSecrets } from "./eval/secrets.js";
|
|
54
|
+
export { median, passAtK, passPowK, sampleVariance, wilsonInterval, } from "./eval/stats.js";
|
|
55
|
+
export { EvalSuiteError, loadSuite, MAX_VERIFIER_TIMEOUT_MS, } from "./eval/suite.js";
|
|
56
|
+
export { applyConfigOverrides, EvalVariantError, loadVariantsFile, resolveVariants, } from "./eval/variants.js";
|
|
57
|
+
export { runVerifier, runVerifiers, snapshotTree, } from "./eval/verifiers.js";
|
|
43
58
|
export { ExtensionRegistry } from "./extensions/registry.js";
|
|
44
59
|
export { ForgeError } from "./forge/errors.js";
|
|
45
60
|
export { downloadSize as pyodideDownloadSize, inspectPyodide, installPyodide, PyodideIntegrityError, pyodideDir, } from "./forge/python/pyodide-install.js";
|
|
@@ -68,7 +83,7 @@ export { PathAccess, } from "./runtime/access.js";
|
|
|
68
83
|
export { BlobStore, BlobTooLarge } from "./runtime/blobs.js";
|
|
69
84
|
export { clipLines, unifiedPatch } from "./runtime/diff.js";
|
|
70
85
|
export { exists, fileSize, readHead, readJson, readText, which } from "./runtime/fs.js";
|
|
71
|
-
export { newId, ulid } from "./runtime/ids.js";
|
|
86
|
+
export { newId, newUuid, seededIdSource, setIdSource, ulid, } from "./runtime/ids.js";
|
|
72
87
|
export { defaultGlobalRoots, isPathSpec, PLUGIN_KEYWORD, resolvePluginSpec, } from "./runtime/modules.js";
|
|
73
88
|
export { findWorkspace, inside, outsideRootsMessage, safePath, workspaceKey, } from "./runtime/paths.js";
|
|
74
89
|
export { isMissingCommand, RIPGREP_INSTALL_HINT, runProcess, } from "./runtime/process.js";
|
package/dist/plan/exit-plan.js
CHANGED
|
@@ -15,6 +15,7 @@ import { tmpdir } from "node:os";
|
|
|
15
15
|
import { join } from "node:path";
|
|
16
16
|
import { textResult } from "@alisio/sdk";
|
|
17
17
|
import { EXIT_PLAN_TOOL } from "../core/opt-in.js";
|
|
18
|
+
import { newUuid } from "../runtime/ids.js";
|
|
18
19
|
import { objectSchema } from "../tools/standard.js";
|
|
19
20
|
import { DIAGRAM_MAX_BYTES, DIAGRAM_MAX_NODES, validateDiagrams, } from "./diagrams.js";
|
|
20
21
|
import { buildPlanManifest } from "./manifest.js";
|
|
@@ -118,7 +119,7 @@ export function registerExitPlan(registry, deps) {
|
|
|
118
119
|
throw new Error("exit_plan needs the whole plan in the `plan` field");
|
|
119
120
|
const sessionId = context.session;
|
|
120
121
|
const title = planTitle(plan, input.title);
|
|
121
|
-
const planId =
|
|
122
|
+
const planId = newUuid();
|
|
122
123
|
const reviewable = !!sessionId && deps.ui.interactive() && !deps.readOnly;
|
|
123
124
|
const revision = sessionId ? nextPlanRevision(deps.store, sessionId) : 1;
|
|
124
125
|
const previous = sessionId
|
package/dist/runtime/ids.d.ts
CHANGED
|
@@ -6,3 +6,24 @@
|
|
|
6
6
|
export declare function ulid(now?: number): string;
|
|
7
7
|
/** A prefixed id such as `art_01J…` or `exec_01J…`. */
|
|
8
8
|
export declare const newId: (prefix: string, now?: number) => string;
|
|
9
|
+
/**
|
|
10
|
+
* A replacement for the random identifiers the runtime generates (session ids, plan ids, forge
|
|
11
|
+
* execution ids, `ulid`-based ids). The reliability lab installs a seeded one while it records or
|
|
12
|
+
* replays a cassette so the requests a run sends do not depend on fresh randomness. Nothing
|
|
13
|
+
* installs one in normal use, and then every function here behaves exactly as before.
|
|
14
|
+
*/
|
|
15
|
+
export interface IdSource {
|
|
16
|
+
/** A UUID (the shape of `crypto.randomUUID()`). */
|
|
17
|
+
uuid(): string;
|
|
18
|
+
/** The 26-character body of a `ulid` (the argument is the caller's clock, which a source may ignore). */
|
|
19
|
+
ulid(now?: number): string;
|
|
20
|
+
}
|
|
21
|
+
/** Installs (or, with `undefined`, removes) the process-wide id source. */
|
|
22
|
+
export declare function setIdSource(source?: IdSource): void;
|
|
23
|
+
/** A random v4 UUID, or the next one of the installed id source. */
|
|
24
|
+
export declare const newUuid: () => string;
|
|
25
|
+
/**
|
|
26
|
+
* A deterministic source: the n-th UUID (and the n-th ULID) depends only on `seed`. UUIDs are
|
|
27
|
+
* v4-shaped; ULIDs carry a counter in their time part, so they stay sortable by creation order.
|
|
28
|
+
*/
|
|
29
|
+
export declare function seededIdSource(seed: string): IdSource;
|
package/dist/runtime/ids.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { randomBytes } from "node:crypto";
|
|
1
|
+
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
2
2
|
/** Crockford base32 (no I, L, O, U), the ULID alphabet. */
|
|
3
3
|
const ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
|
|
4
4
|
/**
|
|
@@ -7,6 +7,8 @@ const ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
|
|
|
7
7
|
* millisecond sort arbitrarily, which is fine for row ids and folder names.
|
|
8
8
|
*/
|
|
9
9
|
export function ulid(now = Date.now()) {
|
|
10
|
+
if (installed)
|
|
11
|
+
return installed.ulid(now);
|
|
10
12
|
let time = "";
|
|
11
13
|
let t = Math.max(0, Math.floor(now));
|
|
12
14
|
for (let i = 0; i < 10; i++) {
|
|
@@ -30,3 +32,41 @@ export function ulid(now = Date.now()) {
|
|
|
30
32
|
}
|
|
31
33
|
/** A prefixed id such as `art_01J…` or `exec_01J…`. */
|
|
32
34
|
export const newId = (prefix, now) => `${prefix}_${ulid(now)}`;
|
|
35
|
+
let installed;
|
|
36
|
+
/** Installs (or, with `undefined`, removes) the process-wide id source. */
|
|
37
|
+
export function setIdSource(source) {
|
|
38
|
+
installed = source;
|
|
39
|
+
}
|
|
40
|
+
/** A random v4 UUID, or the next one of the installed id source. */
|
|
41
|
+
export const newUuid = () => (installed ? installed.uuid() : randomUUID());
|
|
42
|
+
/**
|
|
43
|
+
* A deterministic source: the n-th UUID (and the n-th ULID) depends only on `seed`. UUIDs are
|
|
44
|
+
* v4-shaped; ULIDs carry a counter in their time part, so they stay sortable by creation order.
|
|
45
|
+
*/
|
|
46
|
+
export function seededIdSource(seed) {
|
|
47
|
+
let uuids = 0;
|
|
48
|
+
let ulids = 0;
|
|
49
|
+
const digest = (kind, n) => createHash("sha256").update(`${seed}\0${kind}\0${n}`).digest();
|
|
50
|
+
return {
|
|
51
|
+
uuid() {
|
|
52
|
+
const bytes = digest("uuid", uuids++).subarray(0, 16);
|
|
53
|
+
bytes[6] = ((bytes[6] ?? 0) & 0x0f) | 0x40;
|
|
54
|
+
bytes[8] = ((bytes[8] ?? 0) & 0x3f) | 0x80;
|
|
55
|
+
const hex = bytes.toString("hex");
|
|
56
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
57
|
+
},
|
|
58
|
+
ulid() {
|
|
59
|
+
let time = "";
|
|
60
|
+
let t = ulids;
|
|
61
|
+
for (let i = 0; i < 10; i++) {
|
|
62
|
+
time = (ALPHABET[t % 32] ?? "0") + time;
|
|
63
|
+
t = Math.floor(t / 32);
|
|
64
|
+
}
|
|
65
|
+
const bytes = digest("ulid", ulids++);
|
|
66
|
+
let random = "";
|
|
67
|
+
for (let i = 0; i < 16; i++)
|
|
68
|
+
random += ALPHABET[(bytes[i] ?? 0) & 31];
|
|
69
|
+
return time + random;
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
}
|
package/dist/runtime/store.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { textResult, } from "@alisio/sdk";
|
|
2
|
+
import { newUuid } from "./ids.js";
|
|
2
3
|
import { isProcessAlive } from "./process.js";
|
|
3
4
|
import { openDatabase } from "./sqlite.js";
|
|
4
5
|
const alive = isProcessAlive;
|
|
@@ -327,7 +328,7 @@ export class SQLiteStore {
|
|
|
327
328
|
return current;
|
|
328
329
|
}
|
|
329
330
|
create(workspace, provider, model) {
|
|
330
|
-
const id =
|
|
331
|
+
const id = newUuid();
|
|
331
332
|
const now = Date.now();
|
|
332
333
|
this.db
|
|
333
334
|
.prepare("INSERT INTO sessions(id,workspace,provider,model,created_at,updated_at) VALUES(?,?,?,?,?,?)")
|
|
@@ -367,7 +368,7 @@ export class SQLiteStore {
|
|
|
367
368
|
.map((r) => this.row(r));
|
|
368
369
|
}
|
|
369
370
|
createChild(record) {
|
|
370
|
-
const id = record.id ??
|
|
371
|
+
const id = record.id ?? newUuid();
|
|
371
372
|
const now = Date.now();
|
|
372
373
|
this.db
|
|
373
374
|
.prepare(`INSERT INTO sessions(id,workspace,provider,model,parent_id,depth,agent,status,title,usage,options,created_at,updated_at)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@alisio/core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "Embeddable provider-agnostic Alisio agent core: tool loop, context compaction, local tools, MCP, plugin host and SQLite session store. Runs on Node >=22.16 and Bun.",
|
|
5
5
|
"author": "Gustavo Gutiérrez",
|
|
6
6
|
"license": "MIT",
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"node": ">=22.16"
|
|
46
46
|
},
|
|
47
47
|
"dependencies": {
|
|
48
|
-
"@alisio/sdk": "0.
|
|
48
|
+
"@alisio/sdk": "0.7.0",
|
|
49
49
|
"@mixmark-io/domino": "2.2.0",
|
|
50
50
|
"@modelcontextprotocol/client": "2.1.0",
|
|
51
51
|
"ajv": "8.20.0",
|