@alisio/core 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application.d.ts +15 -0
- package/dist/application.js +38 -3
- package/dist/changelog/data.js +39 -0
- package/dist/commands/builtins.js +7 -0
- package/dist/config.d.ts +30 -0
- package/dist/config.js +32 -0
- package/dist/core/runner.d.ts +4 -0
- package/dist/core/runner.js +57 -1
- package/dist/eval/canonical.d.ts +14 -0
- package/dist/eval/canonical.js +40 -0
- package/dist/eval/cassette.d.ts +180 -0
- package/dist/eval/cassette.js +424 -0
- package/dist/eval/compare.d.ts +147 -0
- package/dist/eval/compare.js +436 -0
- package/dist/eval/engine.d.ts +145 -0
- package/dist/eval/engine.js +731 -0
- package/dist/eval/export.d.ts +40 -0
- package/dist/eval/export.js +90 -0
- package/dist/eval/fingerprint.d.ts +88 -0
- package/dist/eval/fingerprint.js +186 -0
- package/dist/eval/normalize.d.ts +68 -0
- package/dist/eval/normalize.js +138 -0
- package/dist/eval/paths.d.ts +11 -0
- package/dist/eval/paths.js +50 -0
- package/dist/eval/plan.d.ts +26 -0
- package/dist/eval/plan.js +44 -0
- package/dist/eval/report.d.ts +53 -0
- package/dist/eval/report.js +258 -0
- package/dist/eval/run-suite.d.ts +68 -0
- package/dist/eval/run-suite.js +94 -0
- package/dist/eval/secrets.d.ts +14 -0
- package/dist/eval/secrets.js +55 -0
- package/dist/eval/stats.d.ts +17 -0
- package/dist/eval/stats.js +50 -0
- package/dist/eval/suite.d.ts +20 -0
- package/dist/eval/suite.js +165 -0
- package/dist/eval/variants.d.ts +35 -0
- package/dist/eval/variants.js +137 -0
- package/dist/eval/verifiers.d.ts +49 -0
- package/dist/eval/verifiers.js +264 -0
- package/dist/forge/javascript/hosts/worker-host.d.ts +9 -0
- package/dist/forge/javascript/hosts/worker-host.js +16 -1
- package/dist/forge/kernel.js +3 -2
- package/dist/goal/command.d.ts +7 -2
- package/dist/goal/command.js +10 -2
- package/dist/goal/format.js +2 -0
- package/dist/goal/machine.d.ts +50 -3
- package/dist/goal/machine.js +80 -2
- package/dist/goal/service.d.ts +22 -2
- package/dist/goal/service.js +94 -7
- package/dist/goal/store.js +22 -3
- package/dist/index.d.ts +18 -2
- package/dist/index.js +18 -2
- package/dist/plan/criteria.d.ts +30 -0
- package/dist/plan/criteria.js +256 -0
- package/dist/plan/exit-plan.d.ts +8 -0
- package/dist/plan/exit-plan.js +91 -9
- package/dist/plan/manifest.d.ts +4 -2
- package/dist/plan/manifest.js +6 -1
- package/dist/plan/state.d.ts +13 -2
- package/dist/plan/state.js +7 -1
- package/dist/plugins/host.d.ts +7 -0
- package/dist/plugins/host.js +15 -0
- package/dist/proof/command.d.ts +24 -0
- package/dist/proof/command.js +109 -0
- package/dist/proof/coverage.d.ts +40 -0
- package/dist/proof/coverage.js +102 -0
- package/dist/proof/divergence.d.ts +24 -0
- package/dist/proof/divergence.js +120 -0
- package/dist/proof/export.d.ts +65 -0
- package/dist/proof/export.js +144 -0
- package/dist/proof/index.d.ts +12 -0
- package/dist/proof/index.js +12 -0
- package/dist/proof/ledger.d.ts +38 -0
- package/dist/proof/ledger.js +57 -0
- package/dist/proof/plugin-verifiers.d.ts +23 -0
- package/dist/proof/plugin-verifiers.js +39 -0
- package/dist/proof/recorder.d.ts +69 -0
- package/dist/proof/recorder.js +347 -0
- package/dist/proof/scope.d.ts +12 -0
- package/dist/proof/scope.js +121 -0
- package/dist/proof/store.d.ts +52 -0
- package/dist/proof/store.js +192 -0
- package/dist/proof/tree.d.ts +6 -0
- package/dist/proof/tree.js +78 -0
- package/dist/proof/types.d.ts +21 -0
- package/dist/proof/types.js +1 -0
- package/dist/proof/verifiers.d.ts +46 -0
- package/dist/proof/verifiers.js +176 -0
- package/dist/proof/verify.d.ts +36 -0
- package/dist/proof/verify.js +171 -0
- package/dist/runtime/ids.d.ts +21 -0
- package/dist/runtime/ids.js +41 -1
- package/dist/runtime/store.js +63 -2
- package/dist/tools/goal.d.ts +5 -1
- package/dist/tools/goal.js +35 -4
- package/package.json +2 -2
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic, local verifiers of the reliability lab (specs/010 D4). A verifier answers
|
|
3
|
+
* `pass`, `fail` or `error`; `error` means it could not run (bad path, missing input, command that
|
|
4
|
+
* cannot start) and is never folded into `fail`. Details never echo file contents.
|
|
5
|
+
*
|
|
6
|
+
* `command` runs a subprocess with a clean environment and a timeout. A subprocess is NOT a
|
|
7
|
+
* sandbox: suites are trusted repository code and the command runs with the user's privileges.
|
|
8
|
+
*/
|
|
9
|
+
import { spawn } from "node:child_process";
|
|
10
|
+
import { createHash } from "node:crypto";
|
|
11
|
+
import { existsSync, lstatSync, readdirSync, readFileSync, readlinkSync, statSync } from "node:fs";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
import { EvalPathError, resolveInside, unsafeRelativeReason } from "./paths.js";
|
|
14
|
+
const IGNORED_DIRS = new Set([".git", "node_modules"]);
|
|
15
|
+
const MAX_COMMAND_OUTPUT = 1_000_000;
|
|
16
|
+
const sha256 = (data) => createHash("sha256").update(data).digest("hex");
|
|
17
|
+
/**
|
|
18
|
+
* Hashes every file under `dir` (symlinks by their target text, never followed). `.git` and
|
|
19
|
+
* `node_modules` are skipped at any depth: they are tooling state, not task output.
|
|
20
|
+
*/
|
|
21
|
+
export function snapshotTree(dir) {
|
|
22
|
+
const files = {};
|
|
23
|
+
const walk = (current, prefix) => {
|
|
24
|
+
for (const entry of readdirSync(current, { withFileTypes: true })) {
|
|
25
|
+
const rel = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
26
|
+
const full = join(current, entry.name);
|
|
27
|
+
if (entry.isSymbolicLink())
|
|
28
|
+
files[rel] = sha256(`symlink:${readlinkSync(full)}`);
|
|
29
|
+
else if (entry.isDirectory()) {
|
|
30
|
+
if (!IGNORED_DIRS.has(entry.name))
|
|
31
|
+
walk(full, rel);
|
|
32
|
+
}
|
|
33
|
+
else if (entry.isFile())
|
|
34
|
+
files[rel] = sha256(readFileSync(full));
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
walk(dir, "");
|
|
38
|
+
const hash = sha256(Object.keys(files)
|
|
39
|
+
.sort()
|
|
40
|
+
.map((path) => `${path}\0${files[path]}`)
|
|
41
|
+
.join("\n"));
|
|
42
|
+
return { files, hash };
|
|
43
|
+
}
|
|
44
|
+
const result = (v, status, detail) => ({
|
|
45
|
+
id: v.id,
|
|
46
|
+
status,
|
|
47
|
+
...(detail === undefined ? {} : { detail }),
|
|
48
|
+
});
|
|
49
|
+
function isMissing(error) {
|
|
50
|
+
return error?.code === "ENOENT";
|
|
51
|
+
}
|
|
52
|
+
function runCommand(v, cwd, tmpDir) {
|
|
53
|
+
return new Promise((resolve) => {
|
|
54
|
+
const [bin, ...args] = v.command;
|
|
55
|
+
if (bin === undefined)
|
|
56
|
+
return resolve(result(v, "error", "command is empty."));
|
|
57
|
+
if (!existsSync(cwd) || !statSync(cwd).isDirectory())
|
|
58
|
+
return resolve(result(v, "error", "working directory does not exist."));
|
|
59
|
+
const env = { PATH: process.env.PATH ?? "", NODE_NO_WARNINGS: "1" };
|
|
60
|
+
if (process.env.SystemRoot)
|
|
61
|
+
env.SystemRoot = process.env.SystemRoot;
|
|
62
|
+
if (tmpDir) {
|
|
63
|
+
env.HOME = tmpDir;
|
|
64
|
+
env.TMPDIR = tmpDir;
|
|
65
|
+
}
|
|
66
|
+
let stdout = "";
|
|
67
|
+
let timedOut = false;
|
|
68
|
+
let settled = false;
|
|
69
|
+
const done = (value) => {
|
|
70
|
+
if (settled)
|
|
71
|
+
return;
|
|
72
|
+
settled = true;
|
|
73
|
+
clearTimeout(timer);
|
|
74
|
+
resolve(value);
|
|
75
|
+
};
|
|
76
|
+
// The child leads its own process group (POSIX) so a timeout can stop grandchildren too.
|
|
77
|
+
const detached = process.platform !== "win32";
|
|
78
|
+
const child = spawn(bin, args, {
|
|
79
|
+
cwd,
|
|
80
|
+
env,
|
|
81
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
82
|
+
shell: false,
|
|
83
|
+
detached,
|
|
84
|
+
});
|
|
85
|
+
const timer = setTimeout(() => {
|
|
86
|
+
timedOut = true;
|
|
87
|
+
try {
|
|
88
|
+
if (detached && child.pid !== undefined)
|
|
89
|
+
process.kill(-child.pid, "SIGKILL");
|
|
90
|
+
else
|
|
91
|
+
child.kill("SIGKILL");
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
child.kill("SIGKILL");
|
|
95
|
+
}
|
|
96
|
+
}, v.timeoutMs);
|
|
97
|
+
child.stdout.on("data", (chunk) => {
|
|
98
|
+
if (stdout.length < MAX_COMMAND_OUTPUT)
|
|
99
|
+
stdout += chunk.toString("utf8");
|
|
100
|
+
});
|
|
101
|
+
child.on("error", (error) => done(result(v, "error", `command could not start: ${error.message}`)));
|
|
102
|
+
child.on("close", (code, signal) => {
|
|
103
|
+
// The command is the task's own check running agent-written code: a hang is the agent's
|
|
104
|
+
// failure, not an infrastructure error (a command that cannot start stays `error`).
|
|
105
|
+
if (timedOut)
|
|
106
|
+
return done(result(v, "fail", `command timed out after ${v.timeoutMs} ms.`));
|
|
107
|
+
const expected = v.expectExit ?? 0;
|
|
108
|
+
if (code !== expected)
|
|
109
|
+
return done(result(v, "fail", signal ? `terminated by ${signal}.` : `exit ${code}, expected exit ${expected}.`));
|
|
110
|
+
if (v.expectStdoutIncludes !== undefined && !stdout.includes(v.expectStdoutIncludes))
|
|
111
|
+
return done(result(v, "fail", "stdout does not include the expected text."));
|
|
112
|
+
done(result(v, "pass"));
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
function readTarget(path) {
|
|
117
|
+
try {
|
|
118
|
+
return { data: readFileSync(path) };
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
if (isMissing(error))
|
|
122
|
+
return { missing: true };
|
|
123
|
+
throw error;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
function diffTree(baseline, current) {
|
|
127
|
+
const added = [];
|
|
128
|
+
const deleted = [];
|
|
129
|
+
const modified = [];
|
|
130
|
+
for (const [path, hash] of Object.entries(current.files)) {
|
|
131
|
+
const before = baseline.files[path];
|
|
132
|
+
if (before === undefined)
|
|
133
|
+
added.push(path);
|
|
134
|
+
else if (before !== hash)
|
|
135
|
+
modified.push(path);
|
|
136
|
+
}
|
|
137
|
+
for (const path of Object.keys(baseline.files))
|
|
138
|
+
if (!(path in current.files))
|
|
139
|
+
deleted.push(path);
|
|
140
|
+
return { added, deleted, modified };
|
|
141
|
+
}
|
|
142
|
+
/** `a/b.ts` allows exactly that file; `a/` allows everything below the directory `a`. */
|
|
143
|
+
const isAllowed = (path, allowed) => allowed.some((entry) => (entry.endsWith("/") ? path.startsWith(entry) : path === entry));
|
|
144
|
+
function checkSequence(rule, events) {
|
|
145
|
+
switch (rule.type) {
|
|
146
|
+
case "before": {
|
|
147
|
+
const firstAt = events.findIndex((e) => e.tool === rule.first);
|
|
148
|
+
const thenAt = events.findIndex((e) => e.tool === rule.then);
|
|
149
|
+
if (thenAt === -1)
|
|
150
|
+
return { ok: true };
|
|
151
|
+
return firstAt !== -1 && firstAt < thenAt
|
|
152
|
+
? { ok: true }
|
|
153
|
+
: { ok: false, detail: `"${rule.then}" was called without a prior "${rule.first}".` };
|
|
154
|
+
}
|
|
155
|
+
case "never":
|
|
156
|
+
return events.some((e) => e.tool === rule.tool)
|
|
157
|
+
? { ok: false, detail: `"${rule.tool}" must never be called.` }
|
|
158
|
+
: { ok: true };
|
|
159
|
+
case "max-calls": {
|
|
160
|
+
const calls = events.filter((e) => e.tool === rule.tool).length;
|
|
161
|
+
return calls > rule.max
|
|
162
|
+
? { ok: false, detail: `"${rule.tool}" was called ${calls} times, maximum ${rule.max}.` }
|
|
163
|
+
: { ok: true };
|
|
164
|
+
}
|
|
165
|
+
case "no-repeat-failure": {
|
|
166
|
+
const failed = new Set();
|
|
167
|
+
for (const e of events) {
|
|
168
|
+
if (e.tool !== rule.tool)
|
|
169
|
+
continue;
|
|
170
|
+
const key = e.argsKey ?? "";
|
|
171
|
+
if (failed.has(key))
|
|
172
|
+
return {
|
|
173
|
+
ok: false,
|
|
174
|
+
detail: `"${rule.tool}" repeated an identical call that had failed.`,
|
|
175
|
+
};
|
|
176
|
+
if (!e.ok)
|
|
177
|
+
failed.add(key);
|
|
178
|
+
}
|
|
179
|
+
return { ok: true };
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
async function evaluate(v, ctx) {
|
|
184
|
+
switch (v.kind) {
|
|
185
|
+
case "command":
|
|
186
|
+
return runCommand(v, ctx.cwd, ctx.tmpDir);
|
|
187
|
+
case "file-equals": {
|
|
188
|
+
const target = readTarget(resolveInside(ctx.cwd, v.path, "path"));
|
|
189
|
+
if ("missing" in target)
|
|
190
|
+
return result(v, "fail", `${v.path} does not exist.`);
|
|
191
|
+
return sha256(target.data) === v.sha256.toLowerCase()
|
|
192
|
+
? result(v, "pass")
|
|
193
|
+
: result(v, "fail", `${v.path} sha256 differs from the expected value.`);
|
|
194
|
+
}
|
|
195
|
+
case "file-contains": {
|
|
196
|
+
const target = readTarget(resolveInside(ctx.cwd, v.path, "path"));
|
|
197
|
+
if ("missing" in target)
|
|
198
|
+
return result(v, "fail", `${v.path} does not exist.`);
|
|
199
|
+
return target.data.toString("utf8").includes(v.text)
|
|
200
|
+
? result(v, "pass")
|
|
201
|
+
: result(v, "fail", `${v.path} does not contain the expected text.`);
|
|
202
|
+
}
|
|
203
|
+
case "file-absent": {
|
|
204
|
+
const absolute = resolveInside(ctx.cwd, v.path, "path");
|
|
205
|
+
try {
|
|
206
|
+
lstatSync(absolute);
|
|
207
|
+
}
|
|
208
|
+
catch (error) {
|
|
209
|
+
if (isMissing(error))
|
|
210
|
+
return result(v, "pass");
|
|
211
|
+
throw error;
|
|
212
|
+
}
|
|
213
|
+
return result(v, "fail", `${v.path} exists.`);
|
|
214
|
+
}
|
|
215
|
+
case "tree-diff-within": {
|
|
216
|
+
if (!ctx.baseline)
|
|
217
|
+
return result(v, "error", "no fixture snapshot was provided to compare against.");
|
|
218
|
+
for (const entry of v.allowed) {
|
|
219
|
+
const reason = unsafeRelativeReason(entry);
|
|
220
|
+
if (reason)
|
|
221
|
+
return result(v, "error", `allowed entry "${entry}" ${reason}.`);
|
|
222
|
+
}
|
|
223
|
+
const { added, deleted, modified } = diffTree(ctx.baseline, snapshotTree(ctx.cwd));
|
|
224
|
+
const outside = (paths) => paths.filter((p) => !isAllowed(p, v.allowed)).sort();
|
|
225
|
+
const parts = [
|
|
226
|
+
["modified", outside(modified)],
|
|
227
|
+
["added", outside(added)],
|
|
228
|
+
["deleted", outside(deleted)],
|
|
229
|
+
]
|
|
230
|
+
.filter(([, paths]) => paths.length > 0)
|
|
231
|
+
.map(([label, paths]) => `${label}: ${paths.join(", ")}`);
|
|
232
|
+
return parts.length === 0
|
|
233
|
+
? result(v, "pass")
|
|
234
|
+
: result(v, "fail", `changes outside the allowed list (${parts.join("; ")}).`);
|
|
235
|
+
}
|
|
236
|
+
case "tool-sequence": {
|
|
237
|
+
if (!ctx.toolEvents)
|
|
238
|
+
return result(v, "error", "no tool events were recorded for this run.");
|
|
239
|
+
const checked = checkSequence(v.rule, ctx.toolEvents);
|
|
240
|
+
return checked.ok ? result(v, "pass") : result(v, "fail", checked.detail);
|
|
241
|
+
}
|
|
242
|
+
default:
|
|
243
|
+
return result(v, "error", `unknown verifier kind "${v.kind}".`);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/** Runs one verifier; anything that prevents a verdict (including exceptions) is `error`. */
|
|
247
|
+
export async function runVerifier(v, ctx) {
|
|
248
|
+
try {
|
|
249
|
+
return await evaluate(v, ctx);
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
const detail = error instanceof EvalPathError
|
|
253
|
+
? error.message
|
|
254
|
+
: `verifier could not run: ${error.code ?? error.message}`;
|
|
255
|
+
return result(v, "error", detail);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
/** Runs the verifiers in order (sequentially: `command` verifiers may share the working directory). */
|
|
259
|
+
export async function runVerifiers(verifiers, ctx) {
|
|
260
|
+
const results = [];
|
|
261
|
+
for (const v of verifiers)
|
|
262
|
+
results.push(await runVerifier(v, ctx));
|
|
263
|
+
return results;
|
|
264
|
+
}
|
|
@@ -2,6 +2,15 @@ import type { IsolateHost, IsolateSession } from "../../ports.ts";
|
|
|
2
2
|
export declare const WORKER_TERMINATE_AFTER_MS = 50;
|
|
3
3
|
/** The worker entry next to this file: `.ts` when running from sources, `.js` from `dist`. */
|
|
4
4
|
export declare function workerEntryUrl(base?: string): URL;
|
|
5
|
+
/**
|
|
6
|
+
* Node flags for the worker. A `.ts` entry (running from sources) needs type stripping, which Node
|
|
7
|
+
* enables by default only from 22.18; on 22.16-22.17 the worker would fail to load its entry. Bun
|
|
8
|
+
* runs TypeScript natively.
|
|
9
|
+
*/
|
|
10
|
+
export declare function workerExecArgv(entry: URL, runtime?: {
|
|
11
|
+
bun?: string;
|
|
12
|
+
typescript?: unknown;
|
|
13
|
+
}): string[];
|
|
5
14
|
export interface WorkerHostOptions {
|
|
6
15
|
module: WebAssembly.Module;
|
|
7
16
|
/** Overrides the entry (tests of failure paths). */
|
|
@@ -17,6 +17,19 @@ export function workerEntryUrl(base = import.meta.url) {
|
|
|
17
17
|
const extension = base.endsWith(".ts") ? "ts" : "js";
|
|
18
18
|
return new URL(`./worker-entry.${extension}`, base);
|
|
19
19
|
}
|
|
20
|
+
/**
|
|
21
|
+
* Node flags for the worker. A `.ts` entry (running from sources) needs type stripping, which Node
|
|
22
|
+
* enables by default only from 22.18; on 22.16-22.17 the worker would fail to load its entry. Bun
|
|
23
|
+
* runs TypeScript natively.
|
|
24
|
+
*/
|
|
25
|
+
export function workerExecArgv(entry, runtime = {
|
|
26
|
+
bun: process.versions.bun,
|
|
27
|
+
typescript: process.features.typescript,
|
|
28
|
+
}) {
|
|
29
|
+
if (!entry.pathname.endsWith(".ts") || runtime.bun || runtime.typescript)
|
|
30
|
+
return [];
|
|
31
|
+
return ["--experimental-strip-types", "--disable-warning=ExperimentalWarning"];
|
|
32
|
+
}
|
|
20
33
|
export class WorkerIsolateHost {
|
|
21
34
|
kind = "worker";
|
|
22
35
|
#options;
|
|
@@ -31,8 +44,10 @@ export class WorkerIsolateHost {
|
|
|
31
44
|
module: this.#options.module,
|
|
32
45
|
flag: flagBuffer,
|
|
33
46
|
};
|
|
34
|
-
const
|
|
47
|
+
const entry = this.#options.entry ?? workerEntryUrl();
|
|
48
|
+
const worker = new Worker(entry, {
|
|
35
49
|
workerData: data,
|
|
50
|
+
execArgv: workerExecArgv(entry),
|
|
36
51
|
env: {},
|
|
37
52
|
stdout: true,
|
|
38
53
|
stderr: true,
|
package/dist/forge/kernel.js
CHANGED
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
* deadline and cancellation with the host's OWN clock, the output bound and the single terminal
|
|
10
10
|
* state. The engine only knows how to run code in an isolate.
|
|
11
11
|
*/
|
|
12
|
-
import { createHash
|
|
12
|
+
import { createHash } from "node:crypto";
|
|
13
|
+
import { newUuid } from "../runtime/ids.js";
|
|
13
14
|
import { CapabilityBroker } from "./broker.js";
|
|
14
15
|
import { BudgetLedger } from "./budget.js";
|
|
15
16
|
import { ForgeError, isForgeError } from "./errors.js";
|
|
@@ -42,7 +43,7 @@ export class ExecutionKernel {
|
|
|
42
43
|
const now = deps.now ?? Date.now;
|
|
43
44
|
const call = invocation.call;
|
|
44
45
|
const startedAt = now();
|
|
45
|
-
const id = (deps.newId ?? (() => `fx_${
|
|
46
|
+
const id = (deps.newId ?? (() => `fx_${newUuid()}`))();
|
|
46
47
|
const config = deps.config();
|
|
47
48
|
const descriptors = deps.engines.descriptors();
|
|
48
49
|
const language = invocation.language ?? "javascript";
|
package/dist/goal/command.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** The longest objective accepted, in characters (it is sent to the model on the kickoff). */
|
|
2
2
|
export declare const GOAL_OBJECTIVE_MAX = 4000;
|
|
3
|
-
export declare const GOAL_SUBCOMMANDS: readonly ["pause", "resume", "edit", "clear", "help"];
|
|
3
|
+
export declare const GOAL_SUBCOMMANDS: readonly ["pause", "resume", "edit", "clear", "help", "override"];
|
|
4
4
|
export type GoalCommand = {
|
|
5
5
|
type: "show";
|
|
6
6
|
} | {
|
|
@@ -12,6 +12,11 @@ export type GoalCommand = {
|
|
|
12
12
|
} | {
|
|
13
13
|
type: "clear";
|
|
14
14
|
}
|
|
15
|
+
/** The user completes the goal over unmet required Proof criteria (a reason is required). */
|
|
16
|
+
| {
|
|
17
|
+
type: "override";
|
|
18
|
+
reason: string;
|
|
19
|
+
}
|
|
15
20
|
/** `objective` is absent when the UI should ask for the new text. */
|
|
16
21
|
| {
|
|
17
22
|
type: "edit";
|
|
@@ -27,7 +32,7 @@ export type GoalCommand = {
|
|
|
27
32
|
type: "error";
|
|
28
33
|
message: string;
|
|
29
34
|
};
|
|
30
|
-
export declare const GOAL_USAGE = "Usage: /goal [<objective> [budget=50k]] | pause | resume | edit [<objective>] | clear | budget=<n|clear> | help";
|
|
35
|
+
export declare const GOAL_USAGE = "Usage: /goal [<objective> [budget=50k]] | pause | resume | edit [<objective>] | clear | override <reason> | budget=<n|clear> | help";
|
|
31
36
|
/** Markdown help shown by `/goal help` (identical on every surface). */
|
|
32
37
|
export declare const GOAL_HELP: string;
|
|
33
38
|
/** Words of `args` split on whitespace, keeping the original spacing of the objective. */
|
package/dist/goal/command.js
CHANGED
|
@@ -5,14 +5,15 @@
|
|
|
5
5
|
* /goal <objective> [budget=N] create (or replace) the goal and start working on it
|
|
6
6
|
* /goal pause | resume | clear | help
|
|
7
7
|
* /goal edit [<objective>] change the objective of the current goal
|
|
8
|
+
* /goal override <reason> complete the goal over unmet required Proof criteria (you only)
|
|
8
9
|
* /goal budget=50k set the token budget of the current goal (budget=clear removes it)
|
|
9
10
|
*/
|
|
10
11
|
import { parseTokenBudget } from "./budget.js";
|
|
11
12
|
/** The longest objective accepted, in characters (it is sent to the model on the kickoff). */
|
|
12
13
|
export const GOAL_OBJECTIVE_MAX = 4000;
|
|
13
|
-
export const GOAL_SUBCOMMANDS = ["pause", "resume", "edit", "clear", "help"];
|
|
14
|
+
export const GOAL_SUBCOMMANDS = ["pause", "resume", "edit", "clear", "help", "override"];
|
|
14
15
|
const BUDGET_WORD = /^budget=(.*)$/i;
|
|
15
|
-
export const GOAL_USAGE = "Usage: /goal [<objective> [budget=50k]] | pause | resume | edit [<objective>] | clear | budget=<n|clear> | help";
|
|
16
|
+
export const GOAL_USAGE = "Usage: /goal [<objective> [budget=50k]] | pause | resume | edit [<objective>] | clear | override <reason> | budget=<n|clear> | help";
|
|
16
17
|
/** Markdown help shown by `/goal help` (identical on every surface). */
|
|
17
18
|
export const GOAL_HELP = [
|
|
18
19
|
"**/goal** keeps the agent working on one objective until it is done, blocked, paused or out of budget.",
|
|
@@ -22,6 +23,7 @@ export const GOAL_HELP = [
|
|
|
22
23
|
"- `/goal <objective> budget=50k` also sets a token budget (`50k`, `1.5M`, `250000`).",
|
|
23
24
|
"- `/goal pause` stops the automatic continuation; `/goal resume` continues a paused or blocked goal.",
|
|
24
25
|
"- `/goal edit <objective>` changes the objective; `/goal clear` removes the goal.",
|
|
26
|
+
"- `/goal override <reason>` completes the goal over acceptance criteria that are not verified (Proof); only you can, and the reason is kept.",
|
|
25
27
|
"- `/goal budget=50k` sets the token budget; `/goal budget=clear` (also `none`, `off`, `0`) removes it.",
|
|
26
28
|
"",
|
|
27
29
|
`The objective can be up to ${GOAL_OBJECTIVE_MAX} characters. Only you pause, resume, edit or clear a goal or change its budget; the agent marks it complete or blocked, with evidence.`,
|
|
@@ -62,6 +64,12 @@ export function parseGoalCommand(args) {
|
|
|
62
64
|
return { type: "edit" };
|
|
63
65
|
return validateObjective(objective, (value) => ({ type: "edit", objective: value }));
|
|
64
66
|
}
|
|
67
|
+
if (subcommand === "override") {
|
|
68
|
+
const reason = text.slice(text.search(/\s/) + 1).trim();
|
|
69
|
+
if (rest.length === 1)
|
|
70
|
+
return { type: "error", message: "/goal override needs a reason." };
|
|
71
|
+
return { type: "override", reason };
|
|
72
|
+
}
|
|
65
73
|
if (rest.length > 1)
|
|
66
74
|
return {
|
|
67
75
|
type: "error",
|
package/dist/goal/format.js
CHANGED
|
@@ -28,10 +28,12 @@ const REASON_TEXT = {
|
|
|
28
28
|
max_wall: "The time limit was reached",
|
|
29
29
|
no_progress: "No progress",
|
|
30
30
|
restart: "Alisio restarted while the goal was running",
|
|
31
|
+
user_override: "You completed the goal over unmet acceptance criteria",
|
|
31
32
|
};
|
|
32
33
|
const DETAIL_TEXT = {
|
|
33
34
|
repeated_reply: "the same reply repeated",
|
|
34
35
|
no_tool_turns: "several turns without tool calls",
|
|
36
|
+
completion_refused: "completion was refused while required criteria were not verified",
|
|
35
37
|
};
|
|
36
38
|
/** Why the goal is in its state, as one short sentence (empty when there is nothing to say). */
|
|
37
39
|
export function goalReasonText(goal) {
|
package/dist/goal/machine.d.ts
CHANGED
|
@@ -9,12 +9,14 @@
|
|
|
9
9
|
* └──── budget raised / cleared (token_budget only) ── budget_limited
|
|
10
10
|
*
|
|
11
11
|
* Who may do what:
|
|
12
|
-
* - user: create, pause, resume (paused|blocked), edit, clear, change the budget
|
|
13
|
-
*
|
|
12
|
+
* - user: create, pause, resume (paused|blocked), edit, clear, change the budget, and the Proof
|
|
13
|
+
* override (complete over unmet required criteria; never reachable from the model).
|
|
14
|
+
* - model: `update_goal` complete / blocked, only while the goal is `active`. `complete` is
|
|
15
|
+
* refused while a required criterion of the Proof ledger is failed or not checked.
|
|
14
16
|
* - system: the caps and breakers (pause / budget_limited / blocked on error), user interrupt,
|
|
15
17
|
* restart. The system never resumes a goal.
|
|
16
18
|
*/
|
|
17
|
-
import type { GoalAction, GoalEvidence, GoalInfo, GoalReason, GoalStatus } from "@alisio/sdk";
|
|
19
|
+
import type { GoalAction, GoalEvidence, GoalInfo, GoalProofInfo, GoalReason, GoalStatus, ProofCriterion, ProofLedger } from "@alisio/sdk";
|
|
18
20
|
export type GoalActor = "user" | "model" | "system";
|
|
19
21
|
export declare const GOAL_STATUSES: readonly GoalStatus[];
|
|
20
22
|
/** The longest `detail` kept (an error text is truncated). */
|
|
@@ -40,6 +42,8 @@ export interface GoalRecord {
|
|
|
40
42
|
detail?: string;
|
|
41
43
|
summary?: string;
|
|
42
44
|
evidence?: GoalEvidence[];
|
|
45
|
+
/** The Proof snapshot taken when the goal completed with criteria (absent without criteria). */
|
|
46
|
+
proof?: GoalProofInfo;
|
|
43
47
|
epoch: number;
|
|
44
48
|
tokenBudget?: number;
|
|
45
49
|
tokensUsed: number;
|
|
@@ -104,6 +108,12 @@ export declare function userActions(status: GoalStatus): GoalAction[];
|
|
|
104
108
|
export declare const isFinalGoalStatus: (status: GoalStatus) => boolean;
|
|
105
109
|
export declare const clip: (text: string, max?: number) => string;
|
|
106
110
|
export type GoalRefusal = "not_allowed" | "not_active";
|
|
111
|
+
/** A required criterion that blocks completion (`failed` or `not_checked`). */
|
|
112
|
+
export interface GoalUnmet {
|
|
113
|
+
criterionId: string;
|
|
114
|
+
text: string;
|
|
115
|
+
status: "failed" | "not_checked";
|
|
116
|
+
}
|
|
107
117
|
export type GoalResult<T = GoalRecord> = {
|
|
108
118
|
ok: true;
|
|
109
119
|
goal: T;
|
|
@@ -111,7 +121,23 @@ export type GoalResult<T = GoalRecord> = {
|
|
|
111
121
|
ok: false;
|
|
112
122
|
code: GoalRefusal;
|
|
113
123
|
message: string;
|
|
124
|
+
unmet?: GoalUnmet[];
|
|
114
125
|
};
|
|
126
|
+
/**
|
|
127
|
+
* What the Proof ledger says about one goal (specs/011 3.6): the criteria that apply to it and
|
|
128
|
+
* the evidence ids the model declared that resolved to captured evidence.
|
|
129
|
+
*/
|
|
130
|
+
export interface GoalProofInput {
|
|
131
|
+
criteria: ProofCriterion[];
|
|
132
|
+
linkedEvidence: string[];
|
|
133
|
+
}
|
|
134
|
+
/** The criteria that apply to a goal: its own and the session-wide ones (no `goalId`). */
|
|
135
|
+
export declare const criteriaForGoal: (ledger: ProofLedger, goalId: string) => ProofCriterion[];
|
|
136
|
+
/**
|
|
137
|
+
* Required criteria that are `failed` or `not_checked`. `manual_review` and `not_applicable` do
|
|
138
|
+
* not block (O6 names only the first two; a manual review is the user's own call).
|
|
139
|
+
*/
|
|
140
|
+
export declare function unmetRequired(criteria: readonly ProofCriterion[]): GoalUnmet[];
|
|
115
141
|
/** A new goal (the user's `/goal <objective>`). The caller supplies the id and the limits. */
|
|
116
142
|
export declare function newGoal(input: {
|
|
117
143
|
sessionId: string;
|
|
@@ -145,6 +171,27 @@ export declare function userBudget(goal: GoalRecord, now: number, tokens: number
|
|
|
145
171
|
export declare function modelComplete(goal: GoalRecord, now: number, report: {
|
|
146
172
|
summary: string;
|
|
147
173
|
evidence: GoalEvidence[];
|
|
174
|
+
}, proof?: GoalProofInput): GoalResult;
|
|
175
|
+
/** Different runs that may be refused a completion over the ledger before the goal pauses. */
|
|
176
|
+
export declare const COMPLETION_REFUSAL_LIMIT = 3;
|
|
177
|
+
/**
|
|
178
|
+
* A model that keeps reporting `complete` over unmet required criteria, run after run, is not
|
|
179
|
+
* making progress: the goal pauses as `no_progress` (the user resumes or overrides it). The caller
|
|
180
|
+
* counts the refusals; the in-flight claim stays so the running turn still settles.
|
|
181
|
+
*/
|
|
182
|
+
export declare function pauseForRefusedCompletions(goal: GoalRecord, now: number): GoalRecord;
|
|
183
|
+
/** The longest override reason kept. */
|
|
184
|
+
export declare const GOAL_OVERRIDE_REASON_MAX = 500;
|
|
185
|
+
/**
|
|
186
|
+
* The USER completes a goal over unmet required criteria (O6). It exists only to bypass that
|
|
187
|
+
* gate: with nothing unmet there is nothing to override. Only a goal that can still be worked
|
|
188
|
+
* on (`active`, `paused`, `blocked`) can be overridden; `budget_limited` and `complete` are final
|
|
189
|
+
* (raise the budget first). The in-flight claim stays, as for a model completion: the running
|
|
190
|
+
* turn still settles. The caller (the service) has already checked that a person asked for it.
|
|
191
|
+
*/
|
|
192
|
+
export declare function userOverrideComplete(goal: GoalRecord, now: number, input: {
|
|
193
|
+
reason: string;
|
|
194
|
+
proof: GoalProofInput;
|
|
148
195
|
}): GoalResult;
|
|
149
196
|
export interface BlockedReport {
|
|
150
197
|
summary: string;
|
package/dist/goal/machine.js
CHANGED
|
@@ -44,11 +44,46 @@ function moveTo(goal, now, status, reason, extra = {}) {
|
|
|
44
44
|
...extra,
|
|
45
45
|
}, true);
|
|
46
46
|
}
|
|
47
|
-
const refuse = (code, message) => ({
|
|
47
|
+
const refuse = (code, message, unmet) => ({
|
|
48
48
|
ok: false,
|
|
49
49
|
code,
|
|
50
50
|
message,
|
|
51
|
+
...(unmet ? { unmet } : {}),
|
|
51
52
|
});
|
|
53
|
+
/** The criteria that apply to a goal: its own and the session-wide ones (no `goalId`). */
|
|
54
|
+
export const criteriaForGoal = (ledger, goalId) => ledger.criteria.filter((c) => c.goalId === undefined || c.goalId === goalId);
|
|
55
|
+
/**
|
|
56
|
+
* Required criteria that are `failed` or `not_checked`. `manual_review` and `not_applicable` do
|
|
57
|
+
* not block (O6 names only the first two; a manual review is the user's own call).
|
|
58
|
+
*/
|
|
59
|
+
export function unmetRequired(criteria) {
|
|
60
|
+
const unmet = [];
|
|
61
|
+
for (const c of criteria)
|
|
62
|
+
if (c.required && (c.status === "failed" || c.status === "not_checked"))
|
|
63
|
+
unmet.push({ criterionId: c.criterionId, text: c.text, status: c.status });
|
|
64
|
+
return unmet;
|
|
65
|
+
}
|
|
66
|
+
const proofInfo = (proof, unmet, override) => ({
|
|
67
|
+
criteria: proof.criteria.map((c) => ({
|
|
68
|
+
criterionId: c.criterionId,
|
|
69
|
+
required: c.required,
|
|
70
|
+
status: c.status,
|
|
71
|
+
})),
|
|
72
|
+
unmet: unmet.map((u) => u.criterionId),
|
|
73
|
+
linkedEvidence: [...new Set(proof.linkedEvidence)],
|
|
74
|
+
...(override ? { override } : {}),
|
|
75
|
+
});
|
|
76
|
+
const UNMET_LISTED = 5;
|
|
77
|
+
const unmetMessage = (unmet) => {
|
|
78
|
+
const listed = unmet
|
|
79
|
+
.slice(0, UNMET_LISTED)
|
|
80
|
+
.map((u) => `"${clip(u.text, 80)}" (${u.status})`)
|
|
81
|
+
.join(", ");
|
|
82
|
+
const more = unmet.length > UNMET_LISTED ? ` and ${unmet.length - UNMET_LISTED} more` : "";
|
|
83
|
+
return (`Completion refused: required acceptance criteria are not verified: ${listed}${more}. ` +
|
|
84
|
+
"Run their verifiers and report again, or report the goal blocked. Only the user can " +
|
|
85
|
+
"override this.");
|
|
86
|
+
};
|
|
52
87
|
/** A new goal (the user's `/goal <objective>`). The caller supplies the id and the limits. */
|
|
53
88
|
export function newGoal(input) {
|
|
54
89
|
return {
|
|
@@ -146,17 +181,59 @@ export function userBudget(goal, now, tokens, ownerPid) {
|
|
|
146
181
|
* The model reports the goal complete. Evidence is mandatory, and only an `active` goal can be
|
|
147
182
|
* completed (a goal the user paused stays paused).
|
|
148
183
|
*/
|
|
149
|
-
export function modelComplete(goal, now, report) {
|
|
184
|
+
export function modelComplete(goal, now, report, proof) {
|
|
150
185
|
if (goal.status !== "active")
|
|
151
186
|
return refuse("not_active", `The goal is ${goal.status}; only an active goal can be completed.`);
|
|
152
187
|
if (!report.evidence.length)
|
|
153
188
|
return refuse("not_allowed", "Completion needs evidence (files, tests, logs or commands).");
|
|
189
|
+
// O6: with criteria the ledger decides, not the model's declaration (free text backs nothing).
|
|
190
|
+
const unmet = proof ? unmetRequired(proof.criteria) : [];
|
|
191
|
+
if (unmet.length)
|
|
192
|
+
return refuse("not_allowed", unmetMessage(unmet), unmet);
|
|
154
193
|
return {
|
|
155
194
|
ok: true,
|
|
156
195
|
goal: moveTo(goal, now, "complete", "model_complete", {
|
|
157
196
|
summary: report.summary,
|
|
158
197
|
evidence: report.evidence,
|
|
159
198
|
inflight: goal.inflight,
|
|
199
|
+
...(proof?.criteria.length ? { proof: proofInfo(proof, unmet) } : {}),
|
|
200
|
+
}),
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
/** Different runs that may be refused a completion over the ledger before the goal pauses. */
|
|
204
|
+
export const COMPLETION_REFUSAL_LIMIT = 3;
|
|
205
|
+
/**
|
|
206
|
+
* A model that keeps reporting `complete` over unmet required criteria, run after run, is not
|
|
207
|
+
* making progress: the goal pauses as `no_progress` (the user resumes or overrides it). The caller
|
|
208
|
+
* counts the refusals; the in-flight claim stays so the running turn still settles.
|
|
209
|
+
*/
|
|
210
|
+
export function pauseForRefusedCompletions(goal, now) {
|
|
211
|
+
return moveTo(goal, now, "paused", "no_progress", {
|
|
212
|
+
detail: "completion_refused",
|
|
213
|
+
inflight: goal.inflight,
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
/** The longest override reason kept. */
|
|
217
|
+
export const GOAL_OVERRIDE_REASON_MAX = 500;
|
|
218
|
+
/**
|
|
219
|
+
* The USER completes a goal over unmet required criteria (O6). It exists only to bypass that
|
|
220
|
+
* gate: with nothing unmet there is nothing to override. Only a goal that can still be worked
|
|
221
|
+
* on (`active`, `paused`, `blocked`) can be overridden; `budget_limited` and `complete` are final
|
|
222
|
+
* (raise the budget first). The in-flight claim stays, as for a model completion: the running
|
|
223
|
+
* turn still settles. The caller (the service) has already checked that a person asked for it.
|
|
224
|
+
*/
|
|
225
|
+
export function userOverrideComplete(goal, now, input) {
|
|
226
|
+
if (goal.status !== "active" && goal.status !== "paused" && goal.status !== "blocked")
|
|
227
|
+
return refuse("not_allowed", `The goal is ${goal.status}; only an active, paused or blocked goal can be overridden.`);
|
|
228
|
+
const unmet = unmetRequired(input.proof.criteria);
|
|
229
|
+
if (!unmet.length)
|
|
230
|
+
return refuse("not_allowed", "Nothing to override: no required criterion is failed or not checked.");
|
|
231
|
+
const reason = clip(input.reason.trim(), GOAL_OVERRIDE_REASON_MAX);
|
|
232
|
+
return {
|
|
233
|
+
ok: true,
|
|
234
|
+
goal: moveTo(goal, now, "complete", "user_override", {
|
|
235
|
+
proof: proofInfo(input.proof, unmet, { reason, at: now }),
|
|
236
|
+
inflight: goal.inflight,
|
|
160
237
|
}),
|
|
161
238
|
};
|
|
162
239
|
}
|
|
@@ -336,6 +413,7 @@ export function toGoalInfo(goal) {
|
|
|
336
413
|
maxWallMs: goal.maxWallMs,
|
|
337
414
|
...(goal.summary ? { summary: goal.summary } : {}),
|
|
338
415
|
...(goal.evidence?.length ? { evidence: goal.evidence } : {}),
|
|
416
|
+
...(goal.proof ? { proof: goal.proof } : {}),
|
|
339
417
|
actions: userActions(goal.status),
|
|
340
418
|
createdAt: goal.createdAt,
|
|
341
419
|
updatedAt: goal.updatedAt,
|