@alisio/core 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application.d.ts +15 -0
- package/dist/application.js +38 -3
- package/dist/changelog/data.js +39 -0
- package/dist/commands/builtins.js +7 -0
- package/dist/config.d.ts +30 -0
- package/dist/config.js +32 -0
- package/dist/core/runner.d.ts +4 -0
- package/dist/core/runner.js +57 -1
- package/dist/eval/canonical.d.ts +14 -0
- package/dist/eval/canonical.js +40 -0
- package/dist/eval/cassette.d.ts +180 -0
- package/dist/eval/cassette.js +424 -0
- package/dist/eval/compare.d.ts +147 -0
- package/dist/eval/compare.js +436 -0
- package/dist/eval/engine.d.ts +145 -0
- package/dist/eval/engine.js +731 -0
- package/dist/eval/export.d.ts +40 -0
- package/dist/eval/export.js +90 -0
- package/dist/eval/fingerprint.d.ts +88 -0
- package/dist/eval/fingerprint.js +186 -0
- package/dist/eval/normalize.d.ts +68 -0
- package/dist/eval/normalize.js +138 -0
- package/dist/eval/paths.d.ts +11 -0
- package/dist/eval/paths.js +50 -0
- package/dist/eval/plan.d.ts +26 -0
- package/dist/eval/plan.js +44 -0
- package/dist/eval/report.d.ts +53 -0
- package/dist/eval/report.js +258 -0
- package/dist/eval/run-suite.d.ts +68 -0
- package/dist/eval/run-suite.js +94 -0
- package/dist/eval/secrets.d.ts +14 -0
- package/dist/eval/secrets.js +55 -0
- package/dist/eval/stats.d.ts +17 -0
- package/dist/eval/stats.js +50 -0
- package/dist/eval/suite.d.ts +20 -0
- package/dist/eval/suite.js +165 -0
- package/dist/eval/variants.d.ts +35 -0
- package/dist/eval/variants.js +137 -0
- package/dist/eval/verifiers.d.ts +49 -0
- package/dist/eval/verifiers.js +264 -0
- package/dist/forge/javascript/hosts/worker-host.d.ts +9 -0
- package/dist/forge/javascript/hosts/worker-host.js +16 -1
- package/dist/forge/kernel.js +3 -2
- package/dist/goal/command.d.ts +7 -2
- package/dist/goal/command.js +10 -2
- package/dist/goal/format.js +2 -0
- package/dist/goal/machine.d.ts +50 -3
- package/dist/goal/machine.js +80 -2
- package/dist/goal/service.d.ts +22 -2
- package/dist/goal/service.js +94 -7
- package/dist/goal/store.js +22 -3
- package/dist/index.d.ts +18 -2
- package/dist/index.js +18 -2
- package/dist/plan/criteria.d.ts +30 -0
- package/dist/plan/criteria.js +256 -0
- package/dist/plan/exit-plan.d.ts +8 -0
- package/dist/plan/exit-plan.js +91 -9
- package/dist/plan/manifest.d.ts +4 -2
- package/dist/plan/manifest.js +6 -1
- package/dist/plan/state.d.ts +13 -2
- package/dist/plan/state.js +7 -1
- package/dist/plugins/host.d.ts +7 -0
- package/dist/plugins/host.js +15 -0
- package/dist/proof/command.d.ts +24 -0
- package/dist/proof/command.js +109 -0
- package/dist/proof/coverage.d.ts +40 -0
- package/dist/proof/coverage.js +102 -0
- package/dist/proof/divergence.d.ts +24 -0
- package/dist/proof/divergence.js +120 -0
- package/dist/proof/export.d.ts +65 -0
- package/dist/proof/export.js +144 -0
- package/dist/proof/index.d.ts +12 -0
- package/dist/proof/index.js +12 -0
- package/dist/proof/ledger.d.ts +38 -0
- package/dist/proof/ledger.js +57 -0
- package/dist/proof/plugin-verifiers.d.ts +23 -0
- package/dist/proof/plugin-verifiers.js +39 -0
- package/dist/proof/recorder.d.ts +69 -0
- package/dist/proof/recorder.js +347 -0
- package/dist/proof/scope.d.ts +12 -0
- package/dist/proof/scope.js +121 -0
- package/dist/proof/store.d.ts +52 -0
- package/dist/proof/store.js +192 -0
- package/dist/proof/tree.d.ts +6 -0
- package/dist/proof/tree.js +78 -0
- package/dist/proof/types.d.ts +21 -0
- package/dist/proof/types.js +1 -0
- package/dist/proof/verifiers.d.ts +46 -0
- package/dist/proof/verifiers.js +176 -0
- package/dist/proof/verify.d.ts +36 -0
- package/dist/proof/verify.js +171 -0
- package/dist/runtime/ids.d.ts +21 -0
- package/dist/runtime/ids.js +41 -1
- package/dist/runtime/store.js +63 -2
- package/dist/tools/goal.d.ts +5 -1
- package/dist/tools/goal.js +35 -4
- package/package.json +2 -2
package/dist/application.d.ts
CHANGED
|
@@ -21,6 +21,8 @@ import { GoalService } from "./goal/service.ts";
|
|
|
21
21
|
import { HerdrBridge } from "./integrations/herdr.ts";
|
|
22
22
|
import { McpConnector } from "./mcp/connector.ts";
|
|
23
23
|
import { PluginHost } from "./plugins/host.ts";
|
|
24
|
+
import { ProofVerifierRegistry } from "./proof/plugin-verifiers.ts";
|
|
25
|
+
import { ProofStore } from "./proof/store.ts";
|
|
24
26
|
import { ActiveProvider, ProviderRegistry } from "./providers/registry.ts";
|
|
25
27
|
import { ProviderSettingsStore } from "./providers/settings.ts";
|
|
26
28
|
import { ProjectContext } from "./resources/context.ts";
|
|
@@ -44,6 +46,8 @@ export interface AppOptions {
|
|
|
44
46
|
/** Extra directories the tools may touch outside the workspace (per-run `--add-dir`). */
|
|
45
47
|
addDirs?: string[];
|
|
46
48
|
noHerdr?: boolean;
|
|
49
|
+
/** Home directory for user-level resources (skills, agents); defaults to `os.homedir()`. */
|
|
50
|
+
home?: string;
|
|
47
51
|
model?: string;
|
|
48
52
|
baseURL?: string;
|
|
49
53
|
apiMode?: string;
|
|
@@ -183,6 +187,17 @@ export declare function createApplication(options?: AppOptions): Promise<{
|
|
|
183
187
|
tasks: BackgroundTasks;
|
|
184
188
|
/** Session goals (`/goal`): state, user actions, the continuation controller. */
|
|
185
189
|
goals: GoalService;
|
|
190
|
+
/** The Proof ledger (specs/011): criteria and evidence of every session; user surfaces only. */
|
|
191
|
+
proof: ProofStore;
|
|
192
|
+
/** Verifiers plugins registered with `api.proof.registerVerifier`, keyed `<plugin>:<id>`. */
|
|
193
|
+
proofVerifiers: ProofVerifierRegistry;
|
|
194
|
+
/**
|
|
195
|
+
* Runs the verifier of one criterion on demand (`file`, `data-check`, `coverage`, `plugin`): a USER
|
|
196
|
+
* action, never reachable by the model. Core tool calls go through the normal gate.
|
|
197
|
+
*/
|
|
198
|
+
verifyCriterion: (sessionId: string, criterionId: string, verifyOptions?: {
|
|
199
|
+
signal?: AbortSignal;
|
|
200
|
+
}) => Promise<import("./index.ts").VerifyOutcome>;
|
|
186
201
|
/** Python discovery and job folders of `python_run`; `enabled` is false under --read-only. */
|
|
187
202
|
analysis: {
|
|
188
203
|
enabled: boolean;
|
package/dist/application.js
CHANGED
|
@@ -32,6 +32,10 @@ import { GoalStore } from "./goal/store.js";
|
|
|
32
32
|
import { HerdrBridge } from "./integrations/herdr.js";
|
|
33
33
|
import { McpConnector } from "./mcp/connector.js";
|
|
34
34
|
import { discoverPlugins, PluginHost } from "./plugins/host.js";
|
|
35
|
+
import { ProofVerifierRegistry } from "./proof/plugin-verifiers.js";
|
|
36
|
+
import { ProofRecorder } from "./proof/recorder.js";
|
|
37
|
+
import { ProofStore } from "./proof/store.js";
|
|
38
|
+
import { verifyCriterion } from "./proof/verify.js";
|
|
35
39
|
import { ActiveProvider, ProviderRegistry, UnconfiguredProvider } from "./providers/registry.js";
|
|
36
40
|
import { availableProviderModels, resolveProviderModel as resolveConfiguredModel, } from "./providers/routing.js";
|
|
37
41
|
import { ProviderSettingsStore } from "./providers/settings.js";
|
|
@@ -197,6 +201,9 @@ export async function createApplication(options = {}) {
|
|
|
197
201
|
},
|
|
198
202
|
},
|
|
199
203
|
});
|
|
204
|
+
// Proof verifiers contributed by plugins (`api.proof`); bound before any plugin loads.
|
|
205
|
+
const proofVerifiers = new ProofVerifierRegistry();
|
|
206
|
+
plugins.setProofVerifiers(proofVerifiers);
|
|
200
207
|
const stateRoot = options.db && options.db !== ":memory:" ? dirname(options.db) : stateHome();
|
|
201
208
|
plugins.setPluginEnvironment({
|
|
202
209
|
stateRoot,
|
|
@@ -216,7 +223,7 @@ export async function createApplication(options = {}) {
|
|
|
216
223
|
cwd,
|
|
217
224
|
stateHome: stateHome(),
|
|
218
225
|
configHome: configHome(),
|
|
219
|
-
home: homedir(),
|
|
226
|
+
home: options.home ?? homedir(),
|
|
220
227
|
trusted: !!options.trustProject || !!options.config,
|
|
221
228
|
configDir: dirname(configFile(workspace, options.config ? { file: options.config } : { trustProject: options.trustProject })),
|
|
222
229
|
};
|
|
@@ -380,7 +387,7 @@ export async function createApplication(options = {}) {
|
|
|
380
387
|
await skills.discover(skillRoots({
|
|
381
388
|
workspace,
|
|
382
389
|
cwd,
|
|
383
|
-
home: homedir(),
|
|
390
|
+
home: options.home ?? homedir(),
|
|
384
391
|
configHome: configHome(),
|
|
385
392
|
trusted: !!options.trustProject || !!options.config,
|
|
386
393
|
configSkills: config.skills,
|
|
@@ -543,6 +550,8 @@ export async function createApplication(options = {}) {
|
|
|
543
550
|
// tools are opt-in: a run only sees them while its session has an active goal.
|
|
544
551
|
const goals = new GoalService({
|
|
545
552
|
store: new GoalStore(store.db),
|
|
553
|
+
// Proof (specs/011): `update_goal complete` is judged against the session's criteria.
|
|
554
|
+
proof: new ProofStore(store.db),
|
|
546
555
|
settings: () => config.goal,
|
|
547
556
|
...(options.goals?.onChange ? { onChange: options.goals.onChange } : {}),
|
|
548
557
|
...(options.goals?.onLog ? { onLog: options.goals.onLog } : {}),
|
|
@@ -642,6 +651,8 @@ export async function createApplication(options = {}) {
|
|
|
642
651
|
store,
|
|
643
652
|
ui: plugins.ui,
|
|
644
653
|
readOnly: !!options.readOnly,
|
|
654
|
+
// Criteria the model proposes become active only when the user approves the plan.
|
|
655
|
+
proof: new ProofStore(store.db),
|
|
645
656
|
diagrams: () => ({
|
|
646
657
|
enabled: config.plan.diagrams,
|
|
647
658
|
max: config.plan.diagrams ? config.plan.maxDiagrams : 0,
|
|
@@ -982,6 +993,7 @@ export async function createApplication(options = {}) {
|
|
|
982
993
|
return known;
|
|
983
994
|
},
|
|
984
995
|
...(options.approve && !options.readOnly ? { approve: options.approve } : {}),
|
|
996
|
+
proof: new ProofRecorder(store.db, { coverage: () => config.proof.coverage.enabled }),
|
|
985
997
|
pathAccess,
|
|
986
998
|
...(analysisEnabled
|
|
987
999
|
? {
|
|
@@ -1118,6 +1130,29 @@ export async function createApplication(options = {}) {
|
|
|
1118
1130
|
tasks,
|
|
1119
1131
|
/** Session goals (`/goal`): state, user actions, the continuation controller. */
|
|
1120
1132
|
goals,
|
|
1133
|
+
/** The Proof ledger (specs/011): criteria and evidence of every session; user surfaces only. */
|
|
1134
|
+
proof: new ProofStore(store.db),
|
|
1135
|
+
/** Verifiers plugins registered with `api.proof.registerVerifier`, keyed `<plugin>:<id>`. */
|
|
1136
|
+
proofVerifiers,
|
|
1137
|
+
/**
|
|
1138
|
+
* Runs the verifier of one criterion on demand (`file`, `data-check`, `coverage`, `plugin`): a USER
|
|
1139
|
+
* action, never reachable by the model. Core tool calls go through the normal gate.
|
|
1140
|
+
*/
|
|
1141
|
+
verifyCriterion: (sessionId, criterionId, verifyOptions = {}) => verifyCriterion({
|
|
1142
|
+
store: new ProofStore(store.db),
|
|
1143
|
+
registry: proofVerifiers,
|
|
1144
|
+
db: store.db,
|
|
1145
|
+
coverage: () => config.proof.coverage.enabled,
|
|
1146
|
+
callTool: (id, name, input, signal) => runner.runToolCall(id, name, input, {
|
|
1147
|
+
signal,
|
|
1148
|
+
workspace: sessionWorkspace(id),
|
|
1149
|
+
}),
|
|
1150
|
+
}, {
|
|
1151
|
+
sessionId,
|
|
1152
|
+
criterionId,
|
|
1153
|
+
workspace: sessionWorkspace(sessionId),
|
|
1154
|
+
...(verifyOptions.signal ? { signal: verifyOptions.signal } : {}),
|
|
1155
|
+
}),
|
|
1121
1156
|
/** Python discovery and job folders of `python_run`; `enabled` is false under --read-only. */
|
|
1122
1157
|
analysis: {
|
|
1123
1158
|
enabled: analysisEnabled,
|
|
@@ -1522,7 +1557,7 @@ export async function createApplication(options = {}) {
|
|
|
1522
1557
|
*/
|
|
1523
1558
|
agentDefinitions: new AgentDefinitionService({
|
|
1524
1559
|
workspace,
|
|
1525
|
-
home: homedir(),
|
|
1560
|
+
home: options.home ?? homedir(),
|
|
1526
1561
|
registry: {
|
|
1527
1562
|
async reload() {
|
|
1528
1563
|
const handler = plugins.commands.get("agents");
|
package/dist/changelog/data.js
CHANGED
|
@@ -2,6 +2,45 @@
|
|
|
2
2
|
export const CHANGELOG_DATA = {
|
|
3
3
|
schema: 1,
|
|
4
4
|
entries: [
|
|
5
|
+
{
|
|
6
|
+
version: "0.8.0",
|
|
7
|
+
date: "2026-10-14",
|
|
8
|
+
sections: [
|
|
9
|
+
{
|
|
10
|
+
title: "Added",
|
|
11
|
+
items: [
|
|
12
|
+
"Proof ledger: acceptance criteria whose status (`verified`, `failed`, `not_checked`, `manual_review`, `not_applicable`) is computed from evidence captured from the commands, file reads and data queries the agent really ran. A criterion without a verifier stays `not_checked`.",
|
|
13
|
+
"Verifiers `command`, `file`, `data-check`, opt-in `coverage` (`proof.coverage.enabled`) and plugin verifiers (`api.proof.registerVerifier`). An edit after the evidence expires it.",
|
|
14
|
+
"The Proof card and tab in the web UI, `/proof` and `/goal override <reason>` in the terminal, and per-criterion review of the criteria a plan proposes (all-or-nothing in the terminal).",
|
|
15
|
+
"`alisio run --proof`, `--criterion`, `--verify` and `--proof-file`: the run prints the ledger, ends with a `proof_ledger` JSONL line under `--json`, exits `0`, `10` or `11` (`proof.exitCodes`, `proof.ci`) and publishes `proof.md` and `proof.json`.",
|
|
16
|
+
"A warning when the final answer claims tests, types or lint passed without matching evidence.",
|
|
17
|
+
"Export in Markdown and versioned JSON (`proof/v1`) and the routes `GET /api/sessions/:sid/proof`, `POST` `…/proof/criteria`, `…/not-applicable` and `…/goal/override`.",
|
|
18
|
+
],
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
title: "Improved",
|
|
22
|
+
items: [
|
|
23
|
+
"`update_goal` with `complete` is refused while a required criterion is `failed` or `not_checked`; only you can complete the goal over it. Without criteria, goal or plan nothing changes.",
|
|
24
|
+
],
|
|
25
|
+
},
|
|
26
|
+
],
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
version: "0.7.0",
|
|
30
|
+
date: "2026-10-14",
|
|
31
|
+
sections: [
|
|
32
|
+
{
|
|
33
|
+
title: "Added",
|
|
34
|
+
items: [
|
|
35
|
+
"`alisio eval list|run|record|compare|variants|export`: run evaluation suites against any configured provider, compare variants and export the results.",
|
|
36
|
+
"Reliability Lab suites with deterministic verifiers (files, commands, tool sequences), so a pass or fail never depends on a model judging itself.",
|
|
37
|
+
"`pass@k` and `pass^k` metrics with Wilson confidence intervals, so a comparison tells you whether a difference is real or noise.",
|
|
38
|
+
"Cassettes: `alisio eval record` captures provider responses and later runs replay them offline, without network or cost.",
|
|
39
|
+
"Bundled suites `smoke-coding`, `tool-sequences` and `long-context`, available from a source checkout.",
|
|
40
|
+
],
|
|
41
|
+
},
|
|
42
|
+
],
|
|
43
|
+
},
|
|
5
44
|
{
|
|
6
45
|
version: "0.6.0",
|
|
7
46
|
date: "2026-10-12",
|
|
@@ -151,6 +151,13 @@ export const BUILTIN_COMMANDS = [
|
|
|
151
151
|
surfaces: INTERACTIVE,
|
|
152
152
|
execution: "surface",
|
|
153
153
|
},
|
|
154
|
+
{
|
|
155
|
+
name: "proof",
|
|
156
|
+
description: "Show the acceptance criteria of this session and the evidence behind each one",
|
|
157
|
+
argumentHint: "[add | edit | required | optional | na | export | help]",
|
|
158
|
+
surfaces: INTERACTIVE,
|
|
159
|
+
execution: "surface",
|
|
160
|
+
},
|
|
154
161
|
{
|
|
155
162
|
name: "decisions",
|
|
156
163
|
description: "Show Decision Intelligence status, provider health and session metrics",
|
package/dist/config.d.ts
CHANGED
|
@@ -255,6 +255,16 @@ export declare const configSchema: z.ZodPipe<z.ZodObject<{
|
|
|
255
255
|
noToolTurnsLimit: z.ZodDefault<z.ZodNumber>;
|
|
256
256
|
blockedRepeats: z.ZodDefault<z.ZodNumber>;
|
|
257
257
|
}, z.core.$strict>>;
|
|
258
|
+
proof: z.ZodDefault<z.ZodObject<{
|
|
259
|
+
ci: z.ZodDefault<z.ZodBoolean>;
|
|
260
|
+
exitCodes: z.ZodDefault<z.ZodObject<{
|
|
261
|
+
failed: z.ZodDefault<z.ZodNumber>;
|
|
262
|
+
unverified: z.ZodDefault<z.ZodNumber>;
|
|
263
|
+
}, z.core.$strict>>;
|
|
264
|
+
coverage: z.ZodDefault<z.ZodObject<{
|
|
265
|
+
enabled: z.ZodDefault<z.ZodBoolean>;
|
|
266
|
+
}, z.core.$strict>>;
|
|
267
|
+
}, z.core.$strict>>;
|
|
258
268
|
plan: z.ZodDefault<z.ZodObject<{
|
|
259
269
|
diagrams: z.ZodDefault<z.ZodBoolean>;
|
|
260
270
|
maxDiagrams: z.ZodDefault<z.ZodNumber>;
|
|
@@ -415,6 +425,16 @@ export declare const configSchema: z.ZodPipe<z.ZodObject<{
|
|
|
415
425
|
noToolTurnsLimit: number;
|
|
416
426
|
blockedRepeats: number;
|
|
417
427
|
};
|
|
428
|
+
proof: {
|
|
429
|
+
ci: boolean;
|
|
430
|
+
exitCodes: {
|
|
431
|
+
failed: number;
|
|
432
|
+
unverified: number;
|
|
433
|
+
};
|
|
434
|
+
coverage: {
|
|
435
|
+
enabled: boolean;
|
|
436
|
+
};
|
|
437
|
+
};
|
|
418
438
|
plan: {
|
|
419
439
|
diagrams: boolean;
|
|
420
440
|
maxDiagrams: number;
|
|
@@ -654,6 +674,16 @@ export declare const configSchema: z.ZodPipe<z.ZodObject<{
|
|
|
654
674
|
noToolTurnsLimit: number;
|
|
655
675
|
blockedRepeats: number;
|
|
656
676
|
};
|
|
677
|
+
proof: {
|
|
678
|
+
ci: boolean;
|
|
679
|
+
exitCodes: {
|
|
680
|
+
failed: number;
|
|
681
|
+
unverified: number;
|
|
682
|
+
};
|
|
683
|
+
coverage: {
|
|
684
|
+
enabled: boolean;
|
|
685
|
+
};
|
|
686
|
+
};
|
|
657
687
|
plan: {
|
|
658
688
|
diagrams: boolean;
|
|
659
689
|
maxDiagrams: number;
|
package/dist/config.js
CHANGED
|
@@ -528,6 +528,38 @@ const configObjectSchema = z
|
|
|
528
528
|
noToolTurnsLimit: 3,
|
|
529
529
|
blockedRepeats: 2,
|
|
530
530
|
})),
|
|
531
|
+
/**
|
|
532
|
+
* Proof (specs/011): acceptance criteria with evidence. `ci` makes `alisio run` behave as
|
|
533
|
+
* with `--proof` (the ledger as the final event and the exit codes below). `exitCodes` are
|
|
534
|
+
* what `--proof` exits with: `failed` when a required criterion failed, `unverified` when a
|
|
535
|
+
* required one is not checked or needs a manual review (0 when every required one is verified).
|
|
536
|
+
*/
|
|
537
|
+
proof: z
|
|
538
|
+
.object({
|
|
539
|
+
ci: z.boolean().default(false),
|
|
540
|
+
exitCodes: z
|
|
541
|
+
.object({
|
|
542
|
+
failed: z.number().int().min(1).max(125).default(10),
|
|
543
|
+
unverified: z.number().int().min(1).max(125).default(11),
|
|
544
|
+
})
|
|
545
|
+
.strict()
|
|
546
|
+
.default(() => ({ failed: 10, unverified: 11 })),
|
|
547
|
+
/**
|
|
548
|
+
* The `coverage` verifier (opt-in). Off: a `coverage` criterion stays `not_checked` and
|
|
549
|
+
* Proof records nothing for its command. On: the percentage a coverage command printed
|
|
550
|
+
* (through the normal permission gate) is compared with the criterion's `minPercent`.
|
|
551
|
+
*/
|
|
552
|
+
coverage: z
|
|
553
|
+
.object({ enabled: z.boolean().default(false) })
|
|
554
|
+
.strict()
|
|
555
|
+
.default(() => ({ enabled: false })),
|
|
556
|
+
})
|
|
557
|
+
.strict()
|
|
558
|
+
.default(() => ({
|
|
559
|
+
ci: false,
|
|
560
|
+
exitCodes: { failed: 10, unverified: 11 },
|
|
561
|
+
coverage: { enabled: false },
|
|
562
|
+
})),
|
|
531
563
|
/**
|
|
532
564
|
* Plan diagrams (`exit_plan`): the plan agent may add Mermaid diagrams to a plan. `diagrams`
|
|
533
565
|
* off removes them from the tool and from the agent's instructions; `maxDiagrams` caps how
|
package/dist/core/runner.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type Attachment, type DatasetRef, type Message, type ModelProvider, type RunEvent, type ToolDefinition, type ToolResult } from "@alisio/sdk";
|
|
2
|
+
import type { ProofRecorder } from "../proof/recorder.ts";
|
|
2
3
|
import type { PathAccess } from "../runtime/access.ts";
|
|
3
4
|
import type { ApprovalHandler, ArtifactPublisherFactory, CapabilityGate, ContextSource, DecisionsFactory, Policy, RunnerExtensions, SessionStore } from "./contracts.ts";
|
|
4
5
|
import { HumanWaits } from "./human-wait.ts";
|
|
@@ -83,6 +84,8 @@ export interface RunnerOptions {
|
|
|
83
84
|
artifacts?: ArtifactPublisherFactory;
|
|
84
85
|
/** Decision Intelligence bound to each tool call (`ToolContext.decisions`). */
|
|
85
86
|
decisions?: DecisionsFactory;
|
|
87
|
+
/** Captures Proof evidence from executed calls (specs/011). Absent: the runner is unchanged. */
|
|
88
|
+
proof?: ProofRecorder;
|
|
86
89
|
/**
|
|
87
90
|
* Provider-native tool definitions (for example a hosted `web_search` tool) appended to every
|
|
88
91
|
* request's `tools` alongside the registry's function tools. Opt-in, set once for the whole
|
|
@@ -285,6 +288,7 @@ export declare class AgentRunner {
|
|
|
285
288
|
signal?: AbortSignal;
|
|
286
289
|
}): Promise<{
|
|
287
290
|
runId: string;
|
|
291
|
+
callId: string;
|
|
288
292
|
status: "completed";
|
|
289
293
|
result: ToolResult;
|
|
290
294
|
}>;
|
package/dist/core/runner.js
CHANGED
|
@@ -414,6 +414,8 @@ export class AgentRunner {
|
|
|
414
414
|
/** Set by the call itself (see `CallInternals`): it may have left effects. */
|
|
415
415
|
let effectful = false;
|
|
416
416
|
let dispatcher;
|
|
417
|
+
/** Proof ledger changes caused by this call, announced after `tool_completed`. */
|
|
418
|
+
let proofUpdates = [];
|
|
417
419
|
/** Held from the scratchpad quota check until the write is done (L1). */
|
|
418
420
|
let releaseScratch;
|
|
419
421
|
// `effect` rides along so the TUI can group consecutive calls by capability kind
|
|
@@ -692,6 +694,25 @@ export class AgentRunner {
|
|
|
692
694
|
})),
|
|
693
695
|
});
|
|
694
696
|
result = await p.tool.execute(p.input, toolContext);
|
|
697
|
+
// Proof reads the RAW result of a call that really ran, before the model-facing bound below.
|
|
698
|
+
// It never throws into the call: a failed capture only means no evidence.
|
|
699
|
+
if (o.proof) {
|
|
700
|
+
try {
|
|
701
|
+
proofUpdates = o.proof.capture({
|
|
702
|
+
sessionId,
|
|
703
|
+
runId,
|
|
704
|
+
callId: call.id,
|
|
705
|
+
name: call.name,
|
|
706
|
+
input: p.input,
|
|
707
|
+
result,
|
|
708
|
+
workspace,
|
|
709
|
+
durationMs: Date.now() - started,
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
catch {
|
|
713
|
+
proofUpdates = [];
|
|
714
|
+
}
|
|
715
|
+
}
|
|
695
716
|
// The bound applies to what the model sees (the text projection): display-only
|
|
696
717
|
// ui/image parts are bounded by their producers and must not push text over it.
|
|
697
718
|
const projected = textProjection(result);
|
|
@@ -702,6 +723,25 @@ export class AgentRunner {
|
|
|
702
723
|
}
|
|
703
724
|
catch (error) {
|
|
704
725
|
result = textResult(error instanceof Error ? error.message : String(error), true);
|
|
726
|
+
// A `read_file` of a missing file is evidence for a `file` criterion (specs/011): the call
|
|
727
|
+
// threw, so it never reached the capture above. Capture never throws into the call.
|
|
728
|
+
if (o.proof && call.name === "read_file") {
|
|
729
|
+
try {
|
|
730
|
+
proofUpdates = proofUpdates.concat(o.proof.captureFailure({
|
|
731
|
+
sessionId,
|
|
732
|
+
runId,
|
|
733
|
+
callId: call.id,
|
|
734
|
+
name: call.name,
|
|
735
|
+
input: p.input ?? {},
|
|
736
|
+
result,
|
|
737
|
+
workspace,
|
|
738
|
+
durationMs: Date.now() - started,
|
|
739
|
+
}));
|
|
740
|
+
}
|
|
741
|
+
catch {
|
|
742
|
+
/* A failed capture only means no evidence. */
|
|
743
|
+
}
|
|
744
|
+
}
|
|
705
745
|
// A cancellation after execution may have left effects, so the journal stays pending for
|
|
706
746
|
// the user to inspect. A `read` call cannot have left any (and `exit_plan`, which is one,
|
|
707
747
|
// is exactly what a stop or the run limit interrupts while a person decides): it is closed
|
|
@@ -725,6 +765,8 @@ export class AgentRunner {
|
|
|
725
765
|
.join("\n")
|
|
726
766
|
.slice(0, 2_000),
|
|
727
767
|
});
|
|
768
|
+
for (const update of proofUpdates)
|
|
769
|
+
emit("proof_updated", update);
|
|
728
770
|
return result;
|
|
729
771
|
}
|
|
730
772
|
/**
|
|
@@ -813,7 +855,7 @@ export class AgentRunner {
|
|
|
813
855
|
o.store.append(sessionId, { role: "assistant", text, calls: [] });
|
|
814
856
|
emit("run_completed", { tokens: 0, text });
|
|
815
857
|
finish("completed");
|
|
816
|
-
return { runId, status: "completed", result };
|
|
858
|
+
return { runId, callId: call.id, status: "completed", result };
|
|
817
859
|
}
|
|
818
860
|
catch (error) {
|
|
819
861
|
if (combined.aborted && isTimeoutReason(combined.reason)) {
|
|
@@ -1270,6 +1312,20 @@ export class AgentRunner {
|
|
|
1270
1312
|
});
|
|
1271
1313
|
final.truncated = true;
|
|
1272
1314
|
}
|
|
1315
|
+
if (o.proof) {
|
|
1316
|
+
// Display only (D8): the claims of the answer that the Proof ledger does not back.
|
|
1317
|
+
try {
|
|
1318
|
+
for (const d of o.proof.divergences(sessionId, lastText))
|
|
1319
|
+
emit("proof_divergence", {
|
|
1320
|
+
claim: d.claim,
|
|
1321
|
+
...(d.criterionId ? { criterionId: d.criterionId } : {}),
|
|
1322
|
+
actual: d.actualState,
|
|
1323
|
+
});
|
|
1324
|
+
}
|
|
1325
|
+
catch {
|
|
1326
|
+
// a failed check only means no warning
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1273
1329
|
emit("run_completed", final);
|
|
1274
1330
|
finish("completed");
|
|
1275
1331
|
return {
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical JSON of the reliability lab (specs/010 D10): equal values give equal text, so the same
|
|
3
|
+
* text can be hashed for the configuration hash, argument keys and cassette fingerprints.
|
|
4
|
+
*
|
|
5
|
+
* Rules: object keys sorted by UTF-16 code unit (the default string order), `undefined` members
|
|
6
|
+
* dropped (in arrays they become `null`, as `JSON.stringify` does), `null` kept, `-0` written as
|
|
7
|
+
* `0`, non-finite numbers rejected (JSON cannot carry them), string VALUES normalized to NFC
|
|
8
|
+
* (keys are left as written).
|
|
9
|
+
*/
|
|
10
|
+
/** JSON text with the canonical rules above. */
|
|
11
|
+
export declare function canonicalJson(value: unknown): string;
|
|
12
|
+
export declare const sha256Hex: (data: string | Buffer) => string;
|
|
13
|
+
/** `sha256:<64 hex>` of the canonical JSON of `value`. */
|
|
14
|
+
export declare function hashCanonical(value: unknown): string;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical JSON of the reliability lab (specs/010 D10): equal values give equal text, so the same
|
|
3
|
+
* text can be hashed for the configuration hash, argument keys and cassette fingerprints.
|
|
4
|
+
*
|
|
5
|
+
* Rules: object keys sorted by UTF-16 code unit (the default string order), `undefined` members
|
|
6
|
+
* dropped (in arrays they become `null`, as `JSON.stringify` does), `null` kept, `-0` written as
|
|
7
|
+
* `0`, non-finite numbers rejected (JSON cannot carry them), string VALUES normalized to NFC
|
|
8
|
+
* (keys are left as written).
|
|
9
|
+
*/
|
|
10
|
+
import { createHash } from "node:crypto";
|
|
11
|
+
function normalize(value) {
|
|
12
|
+
if (typeof value === "string")
|
|
13
|
+
return value.normalize("NFC");
|
|
14
|
+
if (typeof value === "number") {
|
|
15
|
+
if (!Number.isFinite(value))
|
|
16
|
+
throw new TypeError(`Cannot canonicalize a non-finite number (${value}).`);
|
|
17
|
+
return value === 0 ? 0 : value;
|
|
18
|
+
}
|
|
19
|
+
if (Array.isArray(value))
|
|
20
|
+
return value.map((item) => (item === undefined ? null : normalize(item)));
|
|
21
|
+
if (value && typeof value === "object") {
|
|
22
|
+
const out = {};
|
|
23
|
+
for (const key of Object.keys(value).sort()) {
|
|
24
|
+
const member = value[key];
|
|
25
|
+
if (member !== undefined)
|
|
26
|
+
out[key] = normalize(member);
|
|
27
|
+
}
|
|
28
|
+
return out;
|
|
29
|
+
}
|
|
30
|
+
return value;
|
|
31
|
+
}
|
|
32
|
+
/** JSON text with the canonical rules above. */
|
|
33
|
+
export function canonicalJson(value) {
|
|
34
|
+
return JSON.stringify(normalize(value));
|
|
35
|
+
}
|
|
36
|
+
export const sha256Hex = (data) => createHash("sha256").update(data).digest("hex");
|
|
37
|
+
/** `sha256:<64 hex>` of the canonical JSON of `value`. */
|
|
38
|
+
export function hashCanonical(value) {
|
|
39
|
+
return `sha256:${sha256Hex(canonicalJson(value))}`;
|
|
40
|
+
}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cassettes of the reliability lab (specs/010 5.3, D10, D11): `cassette/v1` JSONL, the provider
|
|
3
|
+
* that records a live run and the provider that serves it back.
|
|
4
|
+
*
|
|
5
|
+
* line 1 header: {v:1, cassetteVersion, suiteId, taskId, harnessVersion, recordedAt, configHash,
|
|
6
|
+
* provider, model, models?, texts, toolSets}
|
|
7
|
+
* line n one call to `ModelProvider.stream`: {call, lane, fingerprint, parts, messages?, events,
|
|
8
|
+
* error?}
|
|
9
|
+
*
|
|
10
|
+
* `completed.message` is stored VERBATIM (tool-call ids, `providerData`, `truncated`) together with
|
|
11
|
+
* its usage; `reasoning_delta` is dropped (the SDK says it is never replayed). A provider error
|
|
12
|
+
* (for example the truncation signal) is stored as `error: {message, code}` so recovery replays.
|
|
13
|
+
*
|
|
14
|
+
* Replay serves the recorded events in strict order PER LANE. A lane is a session (the registered
|
|
15
|
+
* main session by its `<SESSION#n>` placeholder); a session the run did not register (a child /
|
|
16
|
+
* subagent) is bound to the lane whose first request has the same fingerprint, so concurrent
|
|
17
|
+
* children may interleave differently. v1 limitation: two children with an identical first request
|
|
18
|
+
* are interchangeable, and a child whose first request changes is reported as a divergence.
|
|
19
|
+
*
|
|
20
|
+
* This module (and everything it imports) must not touch the network: a test checks that
|
|
21
|
+
* statically, and replay needs no credentials. Replay promises no timing.
|
|
22
|
+
*/
|
|
23
|
+
import type { ModelInfo, ModelProvider, ProviderEvent, Usage } from "@alisio/sdk";
|
|
24
|
+
import { type CassetteTexts, type FingerprintDiff, type FingerprintParts, type MessageView, type StreamRequest } from "./fingerprint.ts";
|
|
25
|
+
import type { Normalizer } from "./normalize.ts";
|
|
26
|
+
export declare const MAX_CASSETTE_BYTES: number;
|
|
27
|
+
export declare const MAX_CASSETTE_DIR_BYTES: number;
|
|
28
|
+
export declare const CASSETTE_SUFFIX = ".cassette.jsonl";
|
|
29
|
+
export type RecordedEvent = {
|
|
30
|
+
type: "text_delta";
|
|
31
|
+
delta: string;
|
|
32
|
+
} | {
|
|
33
|
+
type: "completed";
|
|
34
|
+
message: Extract<ProviderEvent, {
|
|
35
|
+
type: "completed";
|
|
36
|
+
}>["message"];
|
|
37
|
+
usage?: Usage;
|
|
38
|
+
};
|
|
39
|
+
export interface CassetteHeader {
|
|
40
|
+
v: 1;
|
|
41
|
+
cassetteVersion: number;
|
|
42
|
+
suiteId: string;
|
|
43
|
+
taskId: string;
|
|
44
|
+
harnessVersion: string;
|
|
45
|
+
recordedAt: number;
|
|
46
|
+
configHash: string;
|
|
47
|
+
provider: string;
|
|
48
|
+
model: string;
|
|
49
|
+
models?: ModelInfo[];
|
|
50
|
+
texts: CassetteTexts["texts"];
|
|
51
|
+
toolSets: CassetteTexts["toolSets"];
|
|
52
|
+
}
|
|
53
|
+
export interface CassetteCall {
|
|
54
|
+
call: number;
|
|
55
|
+
lane: string;
|
|
56
|
+
fingerprint: string;
|
|
57
|
+
parts: FingerprintParts;
|
|
58
|
+
/** Views of the messages that first appear in this call (the earlier ones are in earlier lines). */
|
|
59
|
+
messages?: Record<string, MessageView>;
|
|
60
|
+
events: RecordedEvent[];
|
|
61
|
+
error?: {
|
|
62
|
+
message: string;
|
|
63
|
+
code?: string;
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
export interface Cassette {
|
|
67
|
+
header: CassetteHeader;
|
|
68
|
+
calls: CassetteCall[];
|
|
69
|
+
}
|
|
70
|
+
export declare class CassetteFormatError extends Error {
|
|
71
|
+
constructor(message: string);
|
|
72
|
+
}
|
|
73
|
+
export declare class CassetteWriteError extends Error {
|
|
74
|
+
readonly reason: "secret" | "size" | "total";
|
|
75
|
+
constructor(message: string, reason: "secret" | "size" | "total");
|
|
76
|
+
}
|
|
77
|
+
export declare function serializeCassette(cassette: Cassette): string;
|
|
78
|
+
export declare function parseCassette(text: string): Cassette;
|
|
79
|
+
export declare function readCassette(path: string): Cassette;
|
|
80
|
+
export interface WriteOptions {
|
|
81
|
+
/** Exact credential values of the run (provider profile, matching env vars). */
|
|
82
|
+
secrets?: readonly string[];
|
|
83
|
+
maxBytes?: number;
|
|
84
|
+
maxDirBytes?: number;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Writes `cassette` to `path` unless it holds a secret (REFUSES: it never strips, which would make
|
|
88
|
+
* the cassette lie about what the model saw) or exceeds the size limits (256 KB each, 2 MB for the
|
|
89
|
+
* directory). Nothing is written on refusal.
|
|
90
|
+
*/
|
|
91
|
+
export declare function writeCassette(path: string, cassette: Cassette, options?: WriteOptions): void;
|
|
92
|
+
export interface CassetteMeta {
|
|
93
|
+
suiteId: string;
|
|
94
|
+
taskId: string;
|
|
95
|
+
harnessVersion: string;
|
|
96
|
+
configHash: string;
|
|
97
|
+
recordedAt: number;
|
|
98
|
+
provider: string;
|
|
99
|
+
model: string;
|
|
100
|
+
cassetteVersion?: number;
|
|
101
|
+
}
|
|
102
|
+
/** Collects the calls a `RecordingProvider` sees; `toCassette` turns them into the file content. */
|
|
103
|
+
export declare class CassetteRecorder {
|
|
104
|
+
readonly calls: CassetteCall[];
|
|
105
|
+
/** Why the recording cannot be trusted (set, never cleared); `toCassette` refuses then. */
|
|
106
|
+
unstable: string | undefined;
|
|
107
|
+
models: ModelInfo[] | undefined;
|
|
108
|
+
private normalizer;
|
|
109
|
+
private readonly store;
|
|
110
|
+
private readonly childLanes;
|
|
111
|
+
private readonly laneCounts;
|
|
112
|
+
bind(normalizer: Normalizer): void;
|
|
113
|
+
/** Opens the record of the next call. */
|
|
114
|
+
begin(request: StreamRequest): CassetteCall;
|
|
115
|
+
private laneFor;
|
|
116
|
+
markUnstable(reason: string): void;
|
|
117
|
+
toCassette(meta: CassetteMeta): Cassette;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Wraps a real provider and tees what it emits into `sink`. The caller still receives every event
|
|
121
|
+
* (reasoning included). A call aborted before any event (a first-token retry, a timeout) or one the
|
|
122
|
+
* consumer abandons makes the recording unstable: replaying it would not reproduce the run.
|
|
123
|
+
*/
|
|
124
|
+
export declare class RecordingProvider implements ModelProvider {
|
|
125
|
+
private readonly inner;
|
|
126
|
+
private readonly sink;
|
|
127
|
+
readonly id: string;
|
|
128
|
+
readonly model: string;
|
|
129
|
+
readonly listModels?: ModelProvider["listModels"];
|
|
130
|
+
readonly dispose?: ModelProvider["dispose"];
|
|
131
|
+
constructor(inner: ModelProvider, sink: CassetteRecorder);
|
|
132
|
+
stream(request: StreamRequest): AsyncGenerator<ProviderEvent>;
|
|
133
|
+
}
|
|
134
|
+
/** The request of a call differs from the recorded one. Carries the structured diff. */
|
|
135
|
+
export declare class ReplayDivergenceError extends Error {
|
|
136
|
+
readonly call: number;
|
|
137
|
+
readonly lane: string;
|
|
138
|
+
readonly diff: FingerprintDiff;
|
|
139
|
+
constructor(call: number, lane: string, diff: FingerprintDiff);
|
|
140
|
+
}
|
|
141
|
+
/** The run asked for a call the cassette has no (more) recording for. */
|
|
142
|
+
export declare class ReplayExhausted extends Error {
|
|
143
|
+
readonly call: number;
|
|
144
|
+
readonly lane: string;
|
|
145
|
+
constructor(call: number, lane: string, detail: string);
|
|
146
|
+
}
|
|
147
|
+
/** Recorded calls the run never asked for (`ReplayProvider.assertConsumed`). */
|
|
148
|
+
export declare class ReplayUnconsumedError extends Error {
|
|
149
|
+
readonly leftover: Array<{
|
|
150
|
+
call: number;
|
|
151
|
+
lane: string;
|
|
152
|
+
}>;
|
|
153
|
+
constructor(leftover: Array<{
|
|
154
|
+
call: number;
|
|
155
|
+
lane: string;
|
|
156
|
+
}>);
|
|
157
|
+
}
|
|
158
|
+
export declare class ReplayProvider implements ModelProvider {
|
|
159
|
+
readonly id: string;
|
|
160
|
+
readonly model: string;
|
|
161
|
+
readonly listModels?: ModelProvider["listModels"];
|
|
162
|
+
private readonly store;
|
|
163
|
+
private readonly lanes;
|
|
164
|
+
private readonly position;
|
|
165
|
+
private readonly bindings;
|
|
166
|
+
private readonly claimed;
|
|
167
|
+
private requests;
|
|
168
|
+
private readonly normalizer;
|
|
169
|
+
constructor(cassette: Cassette, options: {
|
|
170
|
+
normalizer: Normalizer;
|
|
171
|
+
});
|
|
172
|
+
/** The first divergence or exhaustion, kept even if the runner wraps the error it received. */
|
|
173
|
+
failure: ReplayDivergenceError | ReplayExhausted | undefined;
|
|
174
|
+
stream(request: StreamRequest): AsyncGenerator<ProviderEvent>;
|
|
175
|
+
/** The recorded call this request must be, consumed; throws when it is not. */
|
|
176
|
+
private match;
|
|
177
|
+
private laneFor;
|
|
178
|
+
/** Throws `ReplayUnconsumedError` when recorded calls were never requested (extra calls). */
|
|
179
|
+
assertConsumed(): void;
|
|
180
|
+
}
|