@alisio/core 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/changelog/data.js +16 -0
  2. package/dist/eval/canonical.d.ts +14 -0
  3. package/dist/eval/canonical.js +40 -0
  4. package/dist/eval/cassette.d.ts +180 -0
  5. package/dist/eval/cassette.js +424 -0
  6. package/dist/eval/compare.d.ts +147 -0
  7. package/dist/eval/compare.js +436 -0
  8. package/dist/eval/engine.d.ts +144 -0
  9. package/dist/eval/engine.js +728 -0
  10. package/dist/eval/export.d.ts +40 -0
  11. package/dist/eval/export.js +90 -0
  12. package/dist/eval/fingerprint.d.ts +88 -0
  13. package/dist/eval/fingerprint.js +186 -0
  14. package/dist/eval/normalize.d.ts +68 -0
  15. package/dist/eval/normalize.js +138 -0
  16. package/dist/eval/paths.d.ts +11 -0
  17. package/dist/eval/paths.js +50 -0
  18. package/dist/eval/plan.d.ts +26 -0
  19. package/dist/eval/plan.js +44 -0
  20. package/dist/eval/report.d.ts +53 -0
  21. package/dist/eval/report.js +258 -0
  22. package/dist/eval/run-suite.d.ts +68 -0
  23. package/dist/eval/run-suite.js +94 -0
  24. package/dist/eval/secrets.d.ts +12 -0
  25. package/dist/eval/secrets.js +45 -0
  26. package/dist/eval/stats.d.ts +17 -0
  27. package/dist/eval/stats.js +50 -0
  28. package/dist/eval/suite.d.ts +20 -0
  29. package/dist/eval/suite.js +165 -0
  30. package/dist/eval/variants.d.ts +35 -0
  31. package/dist/eval/variants.js +137 -0
  32. package/dist/eval/verifiers.d.ts +49 -0
  33. package/dist/eval/verifiers.js +264 -0
  34. package/dist/forge/javascript/hosts/worker-host.d.ts +9 -0
  35. package/dist/forge/javascript/hosts/worker-host.js +16 -1
  36. package/dist/forge/kernel.js +3 -2
  37. package/dist/index.d.ts +16 -1
  38. package/dist/index.js +16 -1
  39. package/dist/plan/exit-plan.js +2 -1
  40. package/dist/runtime/ids.d.ts +21 -0
  41. package/dist/runtime/ids.js +41 -1
  42. package/dist/runtime/store.js +3 -2
  43. package/package.json +2 -2
@@ -0,0 +1,264 @@
1
+ /**
2
+ * Deterministic, local verifiers of the reliability lab (specs/010 D4). A verifier answers
3
+ * `pass`, `fail` or `error`; `error` means it could not run (bad path, missing input, command that
4
+ * cannot start) and is never folded into `fail`. Details never echo file contents.
5
+ *
6
+ * `command` runs a subprocess with a clean environment and a timeout. A subprocess is NOT a
7
+ * sandbox: suites are trusted repository code and the command runs with the user's privileges.
8
+ */
9
+ import { spawn } from "node:child_process";
10
+ import { createHash } from "node:crypto";
11
+ import { existsSync, lstatSync, readdirSync, readFileSync, readlinkSync, statSync } from "node:fs";
12
+ import { join } from "node:path";
13
+ import { EvalPathError, resolveInside, unsafeRelativeReason } from "./paths.js";
14
+ const IGNORED_DIRS = new Set([".git", "node_modules"]);
15
+ const MAX_COMMAND_OUTPUT = 1_000_000;
16
+ const sha256 = (data) => createHash("sha256").update(data).digest("hex");
17
+ /**
18
+ * Hashes every file under `dir` (symlinks by their target text, never followed). `.git` and
19
+ * `node_modules` are skipped at any depth: they are tooling state, not task output.
20
+ */
21
+ export function snapshotTree(dir) {
22
+ const files = {};
23
+ const walk = (current, prefix) => {
24
+ for (const entry of readdirSync(current, { withFileTypes: true })) {
25
+ const rel = prefix ? `${prefix}/${entry.name}` : entry.name;
26
+ const full = join(current, entry.name);
27
+ if (entry.isSymbolicLink())
28
+ files[rel] = sha256(`symlink:${readlinkSync(full)}`);
29
+ else if (entry.isDirectory()) {
30
+ if (!IGNORED_DIRS.has(entry.name))
31
+ walk(full, rel);
32
+ }
33
+ else if (entry.isFile())
34
+ files[rel] = sha256(readFileSync(full));
35
+ }
36
+ };
37
+ walk(dir, "");
38
+ const hash = sha256(Object.keys(files)
39
+ .sort()
40
+ .map((path) => `${path}\0${files[path]}`)
41
+ .join("\n"));
42
+ return { files, hash };
43
+ }
44
+ const result = (v, status, detail) => ({
45
+ id: v.id,
46
+ status,
47
+ ...(detail === undefined ? {} : { detail }),
48
+ });
49
+ function isMissing(error) {
50
+ return error?.code === "ENOENT";
51
+ }
52
+ function runCommand(v, cwd, tmpDir) {
53
+ return new Promise((resolve) => {
54
+ const [bin, ...args] = v.command;
55
+ if (bin === undefined)
56
+ return resolve(result(v, "error", "command is empty."));
57
+ if (!existsSync(cwd) || !statSync(cwd).isDirectory())
58
+ return resolve(result(v, "error", "working directory does not exist."));
59
+ const env = { PATH: process.env.PATH ?? "", NODE_NO_WARNINGS: "1" };
60
+ if (process.env.SystemRoot)
61
+ env.SystemRoot = process.env.SystemRoot;
62
+ if (tmpDir) {
63
+ env.HOME = tmpDir;
64
+ env.TMPDIR = tmpDir;
65
+ }
66
+ let stdout = "";
67
+ let timedOut = false;
68
+ let settled = false;
69
+ const done = (value) => {
70
+ if (settled)
71
+ return;
72
+ settled = true;
73
+ clearTimeout(timer);
74
+ resolve(value);
75
+ };
76
+ // The child leads its own process group (POSIX) so a timeout can stop grandchildren too.
77
+ const detached = process.platform !== "win32";
78
+ const child = spawn(bin, args, {
79
+ cwd,
80
+ env,
81
+ stdio: ["ignore", "pipe", "ignore"],
82
+ shell: false,
83
+ detached,
84
+ });
85
+ const timer = setTimeout(() => {
86
+ timedOut = true;
87
+ try {
88
+ if (detached && child.pid !== undefined)
89
+ process.kill(-child.pid, "SIGKILL");
90
+ else
91
+ child.kill("SIGKILL");
92
+ }
93
+ catch {
94
+ child.kill("SIGKILL");
95
+ }
96
+ }, v.timeoutMs);
97
+ child.stdout.on("data", (chunk) => {
98
+ if (stdout.length < MAX_COMMAND_OUTPUT)
99
+ stdout += chunk.toString("utf8");
100
+ });
101
+ child.on("error", (error) => done(result(v, "error", `command could not start: ${error.message}`)));
102
+ child.on("close", (code, signal) => {
103
+ // The command is the task's own check running agent-written code: a hang is the agent's
104
+ // failure, not an infrastructure error (a command that cannot start stays `error`).
105
+ if (timedOut)
106
+ return done(result(v, "fail", `command timed out after ${v.timeoutMs} ms.`));
107
+ const expected = v.expectExit ?? 0;
108
+ if (code !== expected)
109
+ return done(result(v, "fail", signal ? `terminated by ${signal}.` : `exit ${code}, expected exit ${expected}.`));
110
+ if (v.expectStdoutIncludes !== undefined && !stdout.includes(v.expectStdoutIncludes))
111
+ return done(result(v, "fail", "stdout does not include the expected text."));
112
+ done(result(v, "pass"));
113
+ });
114
+ });
115
+ }
116
+ function readTarget(path) {
117
+ try {
118
+ return { data: readFileSync(path) };
119
+ }
120
+ catch (error) {
121
+ if (isMissing(error))
122
+ return { missing: true };
123
+ throw error;
124
+ }
125
+ }
126
+ function diffTree(baseline, current) {
127
+ const added = [];
128
+ const deleted = [];
129
+ const modified = [];
130
+ for (const [path, hash] of Object.entries(current.files)) {
131
+ const before = baseline.files[path];
132
+ if (before === undefined)
133
+ added.push(path);
134
+ else if (before !== hash)
135
+ modified.push(path);
136
+ }
137
+ for (const path of Object.keys(baseline.files))
138
+ if (!(path in current.files))
139
+ deleted.push(path);
140
+ return { added, deleted, modified };
141
+ }
142
+ /** `a/b.ts` allows exactly that file; `a/` allows everything below the directory `a`. */
143
+ const isAllowed = (path, allowed) => allowed.some((entry) => (entry.endsWith("/") ? path.startsWith(entry) : path === entry));
144
+ function checkSequence(rule, events) {
145
+ switch (rule.type) {
146
+ case "before": {
147
+ const firstAt = events.findIndex((e) => e.tool === rule.first);
148
+ const thenAt = events.findIndex((e) => e.tool === rule.then);
149
+ if (thenAt === -1)
150
+ return { ok: true };
151
+ return firstAt !== -1 && firstAt < thenAt
152
+ ? { ok: true }
153
+ : { ok: false, detail: `"${rule.then}" was called without a prior "${rule.first}".` };
154
+ }
155
+ case "never":
156
+ return events.some((e) => e.tool === rule.tool)
157
+ ? { ok: false, detail: `"${rule.tool}" must never be called.` }
158
+ : { ok: true };
159
+ case "max-calls": {
160
+ const calls = events.filter((e) => e.tool === rule.tool).length;
161
+ return calls > rule.max
162
+ ? { ok: false, detail: `"${rule.tool}" was called ${calls} times, maximum ${rule.max}.` }
163
+ : { ok: true };
164
+ }
165
+ case "no-repeat-failure": {
166
+ const failed = new Set();
167
+ for (const e of events) {
168
+ if (e.tool !== rule.tool)
169
+ continue;
170
+ const key = e.argsKey ?? "";
171
+ if (failed.has(key))
172
+ return {
173
+ ok: false,
174
+ detail: `"${rule.tool}" repeated an identical call that had failed.`,
175
+ };
176
+ if (!e.ok)
177
+ failed.add(key);
178
+ }
179
+ return { ok: true };
180
+ }
181
+ }
182
+ }
183
+ async function evaluate(v, ctx) {
184
+ switch (v.kind) {
185
+ case "command":
186
+ return runCommand(v, ctx.cwd, ctx.tmpDir);
187
+ case "file-equals": {
188
+ const target = readTarget(resolveInside(ctx.cwd, v.path, "path"));
189
+ if ("missing" in target)
190
+ return result(v, "fail", `${v.path} does not exist.`);
191
+ return sha256(target.data) === v.sha256.toLowerCase()
192
+ ? result(v, "pass")
193
+ : result(v, "fail", `${v.path} sha256 differs from the expected value.`);
194
+ }
195
+ case "file-contains": {
196
+ const target = readTarget(resolveInside(ctx.cwd, v.path, "path"));
197
+ if ("missing" in target)
198
+ return result(v, "fail", `${v.path} does not exist.`);
199
+ return target.data.toString("utf8").includes(v.text)
200
+ ? result(v, "pass")
201
+ : result(v, "fail", `${v.path} does not contain the expected text.`);
202
+ }
203
+ case "file-absent": {
204
+ const absolute = resolveInside(ctx.cwd, v.path, "path");
205
+ try {
206
+ lstatSync(absolute);
207
+ }
208
+ catch (error) {
209
+ if (isMissing(error))
210
+ return result(v, "pass");
211
+ throw error;
212
+ }
213
+ return result(v, "fail", `${v.path} exists.`);
214
+ }
215
+ case "tree-diff-within": {
216
+ if (!ctx.baseline)
217
+ return result(v, "error", "no fixture snapshot was provided to compare against.");
218
+ for (const entry of v.allowed) {
219
+ const reason = unsafeRelativeReason(entry);
220
+ if (reason)
221
+ return result(v, "error", `allowed entry "${entry}" ${reason}.`);
222
+ }
223
+ const { added, deleted, modified } = diffTree(ctx.baseline, snapshotTree(ctx.cwd));
224
+ const outside = (paths) => paths.filter((p) => !isAllowed(p, v.allowed)).sort();
225
+ const parts = [
226
+ ["modified", outside(modified)],
227
+ ["added", outside(added)],
228
+ ["deleted", outside(deleted)],
229
+ ]
230
+ .filter(([, paths]) => paths.length > 0)
231
+ .map(([label, paths]) => `${label}: ${paths.join(", ")}`);
232
+ return parts.length === 0
233
+ ? result(v, "pass")
234
+ : result(v, "fail", `changes outside the allowed list (${parts.join("; ")}).`);
235
+ }
236
+ case "tool-sequence": {
237
+ if (!ctx.toolEvents)
238
+ return result(v, "error", "no tool events were recorded for this run.");
239
+ const checked = checkSequence(v.rule, ctx.toolEvents);
240
+ return checked.ok ? result(v, "pass") : result(v, "fail", checked.detail);
241
+ }
242
+ default:
243
+ return result(v, "error", `unknown verifier kind "${v.kind}".`);
244
+ }
245
+ }
246
+ /** Runs one verifier; anything that prevents a verdict (including exceptions) is `error`. */
247
+ export async function runVerifier(v, ctx) {
248
+ try {
249
+ return await evaluate(v, ctx);
250
+ }
251
+ catch (error) {
252
+ const detail = error instanceof EvalPathError
253
+ ? error.message
254
+ : `verifier could not run: ${error.code ?? error.message}`;
255
+ return result(v, "error", detail);
256
+ }
257
+ }
258
+ /** Runs the verifiers in order (sequentially: `command` verifiers may share the working directory). */
259
+ export async function runVerifiers(verifiers, ctx) {
260
+ const results = [];
261
+ for (const v of verifiers)
262
+ results.push(await runVerifier(v, ctx));
263
+ return results;
264
+ }
@@ -2,6 +2,15 @@ import type { IsolateHost, IsolateSession } from "../../ports.ts";
2
2
  export declare const WORKER_TERMINATE_AFTER_MS = 50;
3
3
  /** The worker entry next to this file: `.ts` when running from sources, `.js` from `dist`. */
4
4
  export declare function workerEntryUrl(base?: string): URL;
5
+ /**
6
+ * Node flags for the worker. A `.ts` entry (running from sources) needs type stripping, which Node
7
+ * enables by default only from 22.18; on 22.16-22.17 the worker would fail to load its entry. Bun
8
+ * runs TypeScript natively.
9
+ */
10
+ export declare function workerExecArgv(entry: URL, runtime?: {
11
+ bun?: string;
12
+ typescript?: unknown;
13
+ }): string[];
5
14
  export interface WorkerHostOptions {
6
15
  module: WebAssembly.Module;
7
16
  /** Overrides the entry (tests of failure paths). */
@@ -17,6 +17,19 @@ export function workerEntryUrl(base = import.meta.url) {
17
17
  const extension = base.endsWith(".ts") ? "ts" : "js";
18
18
  return new URL(`./worker-entry.${extension}`, base);
19
19
  }
20
+ /**
21
+ * Node flags for the worker. A `.ts` entry (running from sources) needs type stripping, which Node
22
+ * enables by default only from 22.18; on 22.16-22.17 the worker would fail to load its entry. Bun
23
+ * runs TypeScript natively.
24
+ */
25
+ export function workerExecArgv(entry, runtime = {
26
+ bun: process.versions.bun,
27
+ typescript: process.features.typescript,
28
+ }) {
29
+ if (!entry.pathname.endsWith(".ts") || runtime.bun || runtime.typescript)
30
+ return [];
31
+ return ["--experimental-strip-types", "--disable-warning=ExperimentalWarning"];
32
+ }
20
33
  export class WorkerIsolateHost {
21
34
  kind = "worker";
22
35
  #options;
@@ -31,8 +44,10 @@ export class WorkerIsolateHost {
31
44
  module: this.#options.module,
32
45
  flag: flagBuffer,
33
46
  };
34
- const worker = new Worker(this.#options.entry ?? workerEntryUrl(), {
47
+ const entry = this.#options.entry ?? workerEntryUrl();
48
+ const worker = new Worker(entry, {
35
49
  workerData: data,
50
+ execArgv: workerExecArgv(entry),
36
51
  env: {},
37
52
  stdout: true,
38
53
  stderr: true,
@@ -9,7 +9,8 @@
9
9
  * deadline and cancellation with the host's OWN clock, the output bound and the single terminal
10
10
  * state. The engine only knows how to run code in an isolate.
11
11
  */
12
- import { createHash, randomUUID } from "node:crypto";
12
+ import { createHash } from "node:crypto";
13
+ import { newUuid } from "../runtime/ids.js";
13
14
  import { CapabilityBroker } from "./broker.js";
14
15
  import { BudgetLedger } from "./budget.js";
15
16
  import { ForgeError, isForgeError } from "./errors.js";
@@ -42,7 +43,7 @@ export class ExecutionKernel {
42
43
  const now = deps.now ?? Date.now;
43
44
  const call = invocation.call;
44
45
  const startedAt = now();
45
- const id = (deps.newId ?? (() => `fx_${randomUUID()}`))();
46
+ const id = (deps.newId ?? (() => `fx_${newUuid()}`))();
46
47
  const config = deps.config();
47
48
  const descriptors = deps.engines.descriptors();
48
49
  const language = invocation.language ?? "javascript";
package/dist/index.d.ts CHANGED
@@ -41,6 +41,21 @@ export { ToolRegistry } from "./core/registry.ts";
41
41
  export { RunClock } from "./core/run-clock.ts";
42
42
  export { AgentRunner, type CompactionResult, defaultTokenBudget, type RunnerOptions, type RunnerSettingsPatch, type RunOptions, } from "./core/runner.ts";
43
43
  export { CircuitBreaker, type CircuitBreakerOptions, type CircuitState, DECISION_LIMITS, DEFAULT_DECISIONS_CONFIG, type DecisionAttempt, DecisionError, type DecisionMetricEntry, DecisionMetrics, type DecisionMetricsSnapshot, type DecisionOutcome, DecisionRegistry, DecisionService, type DecisionServiceDeps, type DecisionStatus, type DecisionsConfig, formatDecisionStatsSection, formatDecisionsReport, HEALTH_TIMEOUT_MS, type RegisteredDecisionProvider, splitSupported, validateAnswers, validateRequest, } from "./decisions/index.ts";
44
+ export { canonicalJson, hashCanonical } from "./eval/canonical.ts";
45
+ export { CASSETTE_SUFFIX, type Cassette, type CassetteCall, CassetteFormatError, type CassetteHeader, type CassetteMeta, CassetteRecorder, CassetteWriteError, MAX_CASSETTE_BYTES, MAX_CASSETTE_DIR_BYTES, parseCassette, RecordingProvider, ReplayDivergenceError, ReplayExhausted, ReplayProvider, ReplayUnconsumedError, readCassette, serializeCassette, type WriteOptions, writeCassette, } from "./eval/cassette.ts";
46
+ export { type Conditions, compareReports, compareVariants, EvalCompareError, type EvalComparison, type Interval, type MetricDelta, parseReport, renderComparisonMarkdown, renderVariantComparisonMarkdown, type TaskComparison, type TokenComparison, type VariantComparison, type Verdict, } from "./eval/compare.ts";
47
+ export { CONFIG_HASH_COVERAGE, type ConfigHashExtras, checkTaskPolicy, computeConfigHash, type EvalApplication, EvalBudgetExceeded, type EvalRunMode, EvalRunRefusedError, type InvokerPermissions, type RunTaskOptions, runTask, } from "./eval/engine.ts";
48
+ export { buildExportBundle, type EvalExportBundle, EvalExportRefusedError, type ExportContext, } from "./eval/export.ts";
49
+ export { computeFingerprint, type FingerprintDiff, type FingerprintDiffEntry, } from "./eval/fingerprint.ts";
50
+ export { EvalPathError, resolveInside } from "./eval/paths.ts";
51
+ export { interleaveVariants, mulberry32, type PlannedRun, planRuns, type VariantStep, } from "./eval/plan.ts";
52
+ export { type BuildReportOptions, buildReport, EvalReportError, estimateRunCost, lowPowerTasks, MIN_POWER_K, renderReportMarkdown, } from "./eval/report.ts";
53
+ export { type RunEvaluationOptions, type RunEvaluationResult, type RunVariantsOptions, runEvaluation, runVariantEvaluations, } from "./eval/run-suite.ts";
54
+ export { findSecrets, redactSecrets } from "./eval/secrets.ts";
55
+ export { median, passAtK, passPowK, sampleVariance, wilsonInterval, } from "./eval/stats.ts";
56
+ export { EvalSuiteError, type LoadedSuite, loadSuite, MAX_VERIFIER_TIMEOUT_MS, } from "./eval/suite.ts";
57
+ export { applyConfigOverrides, EvalVariantError, loadVariantsFile, resolveVariants, type VariantsFile, } from "./eval/variants.ts";
58
+ export { runVerifier, runVerifiers, snapshotTree, type ToolEvent, type TreeSnapshot, type VerifierContext, type VerifierResult, type VerifierStatus, } from "./eval/verifiers.ts";
44
59
  export { type ExtensionConflict, ExtensionRegistry } from "./extensions/registry.ts";
45
60
  export { ForgeError } from "./forge/errors.ts";
46
61
  export { downloadSize as pyodideDownloadSize, inspectPyodide, installPyodide, PyodideIntegrityError, pyodideDir, } from "./forge/python/pyodide-install.ts";
@@ -69,7 +84,7 @@ export { type ExternalDirectoryDecision, type ExternalDirectoryHandler, type Ext
69
84
  export { BlobStore, type BlobStoreOptions, BlobTooLarge } from "./runtime/blobs.ts";
70
85
  export { clipLines, unifiedPatch } from "./runtime/diff.ts";
71
86
  export { exists, fileSize, readHead, readJson, readText, which } from "./runtime/fs.ts";
72
- export { newId, ulid } from "./runtime/ids.ts";
87
+ export { type IdSource, newId, newUuid, seededIdSource, setIdSource, ulid, } from "./runtime/ids.ts";
73
88
  export { defaultGlobalRoots, isPathSpec, PLUGIN_KEYWORD, resolvePluginSpec, } from "./runtime/modules.ts";
74
89
  export { findWorkspace, inside, outsideRootsMessage, safePath, workspaceKey, } from "./runtime/paths.ts";
75
90
  export { isMissingCommand, type ProcessResult, RIPGREP_INSTALL_HINT, runProcess, } from "./runtime/process.ts";
package/dist/index.js CHANGED
@@ -40,6 +40,21 @@ export { ToolRegistry } from "./core/registry.js";
40
40
  export { RunClock } from "./core/run-clock.js";
41
41
  export { AgentRunner, defaultTokenBudget, } from "./core/runner.js";
42
42
  export { CircuitBreaker, DECISION_LIMITS, DEFAULT_DECISIONS_CONFIG, DecisionError, DecisionMetrics, DecisionRegistry, DecisionService, formatDecisionStatsSection, formatDecisionsReport, HEALTH_TIMEOUT_MS, splitSupported, validateAnswers, validateRequest, } from "./decisions/index.js";
43
+ export { canonicalJson, hashCanonical } from "./eval/canonical.js";
44
+ export { CASSETTE_SUFFIX, CassetteFormatError, CassetteRecorder, CassetteWriteError, MAX_CASSETTE_BYTES, MAX_CASSETTE_DIR_BYTES, parseCassette, RecordingProvider, ReplayDivergenceError, ReplayExhausted, ReplayProvider, ReplayUnconsumedError, readCassette, serializeCassette, writeCassette, } from "./eval/cassette.js";
45
+ export { compareReports, compareVariants, EvalCompareError, parseReport, renderComparisonMarkdown, renderVariantComparisonMarkdown, } from "./eval/compare.js";
46
+ export { CONFIG_HASH_COVERAGE, checkTaskPolicy, computeConfigHash, EvalBudgetExceeded, EvalRunRefusedError, runTask, } from "./eval/engine.js";
47
+ export { buildExportBundle, EvalExportRefusedError, } from "./eval/export.js";
48
+ export { computeFingerprint, } from "./eval/fingerprint.js";
49
+ export { EvalPathError, resolveInside } from "./eval/paths.js";
50
+ export { interleaveVariants, mulberry32, planRuns, } from "./eval/plan.js";
51
+ export { buildReport, EvalReportError, estimateRunCost, lowPowerTasks, MIN_POWER_K, renderReportMarkdown, } from "./eval/report.js";
52
+ export { runEvaluation, runVariantEvaluations, } from "./eval/run-suite.js";
53
+ export { findSecrets, redactSecrets } from "./eval/secrets.js";
54
+ export { median, passAtK, passPowK, sampleVariance, wilsonInterval, } from "./eval/stats.js";
55
+ export { EvalSuiteError, loadSuite, MAX_VERIFIER_TIMEOUT_MS, } from "./eval/suite.js";
56
+ export { applyConfigOverrides, EvalVariantError, loadVariantsFile, resolveVariants, } from "./eval/variants.js";
57
+ export { runVerifier, runVerifiers, snapshotTree, } from "./eval/verifiers.js";
43
58
  export { ExtensionRegistry } from "./extensions/registry.js";
44
59
  export { ForgeError } from "./forge/errors.js";
45
60
  export { downloadSize as pyodideDownloadSize, inspectPyodide, installPyodide, PyodideIntegrityError, pyodideDir, } from "./forge/python/pyodide-install.js";
@@ -68,7 +83,7 @@ export { PathAccess, } from "./runtime/access.js";
68
83
  export { BlobStore, BlobTooLarge } from "./runtime/blobs.js";
69
84
  export { clipLines, unifiedPatch } from "./runtime/diff.js";
70
85
  export { exists, fileSize, readHead, readJson, readText, which } from "./runtime/fs.js";
71
- export { newId, ulid } from "./runtime/ids.js";
86
+ export { newId, newUuid, seededIdSource, setIdSource, ulid, } from "./runtime/ids.js";
72
87
  export { defaultGlobalRoots, isPathSpec, PLUGIN_KEYWORD, resolvePluginSpec, } from "./runtime/modules.js";
73
88
  export { findWorkspace, inside, outsideRootsMessage, safePath, workspaceKey, } from "./runtime/paths.js";
74
89
  export { isMissingCommand, RIPGREP_INSTALL_HINT, runProcess, } from "./runtime/process.js";
@@ -15,6 +15,7 @@ import { tmpdir } from "node:os";
15
15
  import { join } from "node:path";
16
16
  import { textResult } from "@alisio/sdk";
17
17
  import { EXIT_PLAN_TOOL } from "../core/opt-in.js";
18
+ import { newUuid } from "../runtime/ids.js";
18
19
  import { objectSchema } from "../tools/standard.js";
19
20
  import { DIAGRAM_MAX_BYTES, DIAGRAM_MAX_NODES, validateDiagrams, } from "./diagrams.js";
20
21
  import { buildPlanManifest } from "./manifest.js";
@@ -118,7 +119,7 @@ export function registerExitPlan(registry, deps) {
118
119
  throw new Error("exit_plan needs the whole plan in the `plan` field");
119
120
  const sessionId = context.session;
120
121
  const title = planTitle(plan, input.title);
121
- const planId = crypto.randomUUID();
122
+ const planId = newUuid();
122
123
  const reviewable = !!sessionId && deps.ui.interactive() && !deps.readOnly;
123
124
  const revision = sessionId ? nextPlanRevision(deps.store, sessionId) : 1;
124
125
  const previous = sessionId
@@ -6,3 +6,24 @@
6
6
  export declare function ulid(now?: number): string;
7
7
  /** A prefixed id such as `art_01J…` or `exec_01J…`. */
8
8
  export declare const newId: (prefix: string, now?: number) => string;
9
+ /**
10
+ * A replacement for the random identifiers the runtime generates (session ids, plan ids, forge
11
+ * execution ids, `ulid`-based ids). The reliability lab installs a seeded one while it records or
12
+ * replays a cassette so the requests a run sends do not depend on fresh randomness. Nothing
13
+ * installs one in normal use, and then every function here behaves exactly as before.
14
+ */
15
+ export interface IdSource {
16
+ /** A UUID (the shape of `crypto.randomUUID()`). */
17
+ uuid(): string;
18
+ /** The 26-character body of a `ulid` (the argument is the caller's clock, which a source may ignore). */
19
+ ulid(now?: number): string;
20
+ }
21
+ /** Installs (or, with `undefined`, removes) the process-wide id source. */
22
+ export declare function setIdSource(source?: IdSource): void;
23
+ /** A random v4 UUID, or the next one of the installed id source. */
24
+ export declare const newUuid: () => string;
25
+ /**
26
+ * A deterministic source: the n-th UUID (and the n-th ULID) depends only on `seed`. UUIDs are
27
+ * v4-shaped; ULIDs carry a counter in their time part, so they stay sortable by creation order.
28
+ */
29
+ export declare function seededIdSource(seed: string): IdSource;
@@ -1,4 +1,4 @@
1
- import { randomBytes } from "node:crypto";
1
+ import { createHash, randomBytes, randomUUID } from "node:crypto";
2
2
  /** Crockford base32 (no I, L, O, U), the ULID alphabet. */
3
3
  const ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
4
4
  /**
@@ -7,6 +7,8 @@ const ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
7
7
  * millisecond sort arbitrarily, which is fine for row ids and folder names.
8
8
  */
9
9
  export function ulid(now = Date.now()) {
10
+ if (installed)
11
+ return installed.ulid(now);
10
12
  let time = "";
11
13
  let t = Math.max(0, Math.floor(now));
12
14
  for (let i = 0; i < 10; i++) {
@@ -30,3 +32,41 @@ export function ulid(now = Date.now()) {
30
32
  }
31
33
  /** A prefixed id such as `art_01J…` or `exec_01J…`. */
32
34
  export const newId = (prefix, now) => `${prefix}_${ulid(now)}`;
35
+ let installed;
36
+ /** Installs (or, with `undefined`, removes) the process-wide id source. */
37
+ export function setIdSource(source) {
38
+ installed = source;
39
+ }
40
+ /** A random v4 UUID, or the next one of the installed id source. */
41
+ export const newUuid = () => (installed ? installed.uuid() : randomUUID());
42
+ /**
43
+ * A deterministic source: the n-th UUID (and the n-th ULID) depends only on `seed`. UUIDs are
44
+ * v4-shaped; ULIDs carry a counter in their time part, so they stay sortable by creation order.
45
+ */
46
+ export function seededIdSource(seed) {
47
+ let uuids = 0;
48
+ let ulids = 0;
49
+ const digest = (kind, n) => createHash("sha256").update(`${seed}\0${kind}\0${n}`).digest();
50
+ return {
51
+ uuid() {
52
+ const bytes = digest("uuid", uuids++).subarray(0, 16);
53
+ bytes[6] = ((bytes[6] ?? 0) & 0x0f) | 0x40;
54
+ bytes[8] = ((bytes[8] ?? 0) & 0x3f) | 0x80;
55
+ const hex = bytes.toString("hex");
56
+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
57
+ },
58
+ ulid() {
59
+ let time = "";
60
+ let t = ulids;
61
+ for (let i = 0; i < 10; i++) {
62
+ time = (ALPHABET[t % 32] ?? "0") + time;
63
+ t = Math.floor(t / 32);
64
+ }
65
+ const bytes = digest("ulid", ulids++);
66
+ let random = "";
67
+ for (let i = 0; i < 16; i++)
68
+ random += ALPHABET[(bytes[i] ?? 0) & 31];
69
+ return time + random;
70
+ },
71
+ };
72
+ }
@@ -1,4 +1,5 @@
1
1
  import { textResult, } from "@alisio/sdk";
2
+ import { newUuid } from "./ids.js";
2
3
  import { isProcessAlive } from "./process.js";
3
4
  import { openDatabase } from "./sqlite.js";
4
5
  const alive = isProcessAlive;
@@ -327,7 +328,7 @@ export class SQLiteStore {
327
328
  return current;
328
329
  }
329
330
  create(workspace, provider, model) {
330
- const id = crypto.randomUUID();
331
+ const id = newUuid();
331
332
  const now = Date.now();
332
333
  this.db
333
334
  .prepare("INSERT INTO sessions(id,workspace,provider,model,created_at,updated_at) VALUES(?,?,?,?,?,?)")
@@ -367,7 +368,7 @@ export class SQLiteStore {
367
368
  .map((r) => this.row(r));
368
369
  }
369
370
  createChild(record) {
370
- const id = record.id ?? crypto.randomUUID();
371
+ const id = record.id ?? newUuid();
371
372
  const now = Date.now();
372
373
  this.db
373
374
  .prepare(`INSERT INTO sessions(id,workspace,provider,model,parent_id,depth,agent,status,title,usage,options,created_at,updated_at)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@alisio/core",
3
- "version": "0.6.0",
3
+ "version": "0.7.0",
4
4
  "description": "Embeddable provider-agnostic Alisio agent core: tool loop, context compaction, local tools, MCP, plugin host and SQLite session store. Runs on Node >=22.16 and Bun.",
5
5
  "author": "Gustavo Gutiérrez",
6
6
  "license": "MIT",
@@ -45,7 +45,7 @@
45
45
  "node": ">=22.16"
46
46
  },
47
47
  "dependencies": {
48
- "@alisio/sdk": "0.6.0",
48
+ "@alisio/sdk": "0.7.0",
49
49
  "@mixmark-io/domino": "2.2.0",
50
50
  "@modelcontextprotocol/client": "2.1.0",
51
51
  "ajv": "8.20.0",