@alisio/core 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application.d.ts +15 -0
- package/dist/application.js +38 -3
- package/dist/changelog/data.js +23 -0
- package/dist/commands/builtins.js +7 -0
- package/dist/config.d.ts +30 -0
- package/dist/config.js +32 -0
- package/dist/core/runner.d.ts +4 -0
- package/dist/core/runner.js +57 -1
- package/dist/eval/engine.d.ts +1 -0
- package/dist/eval/engine.js +3 -0
- package/dist/eval/secrets.d.ts +2 -0
- package/dist/eval/secrets.js +10 -0
- package/dist/goal/command.d.ts +7 -2
- package/dist/goal/command.js +10 -2
- package/dist/goal/format.js +2 -0
- package/dist/goal/machine.d.ts +50 -3
- package/dist/goal/machine.js +80 -2
- package/dist/goal/service.d.ts +22 -2
- package/dist/goal/service.js +94 -7
- package/dist/goal/store.js +22 -3
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/plan/criteria.d.ts +30 -0
- package/dist/plan/criteria.js +256 -0
- package/dist/plan/exit-plan.d.ts +8 -0
- package/dist/plan/exit-plan.js +89 -8
- package/dist/plan/manifest.d.ts +4 -2
- package/dist/plan/manifest.js +6 -1
- package/dist/plan/state.d.ts +13 -2
- package/dist/plan/state.js +7 -1
- package/dist/plugins/host.d.ts +7 -0
- package/dist/plugins/host.js +15 -0
- package/dist/proof/command.d.ts +24 -0
- package/dist/proof/command.js +109 -0
- package/dist/proof/coverage.d.ts +40 -0
- package/dist/proof/coverage.js +102 -0
- package/dist/proof/divergence.d.ts +24 -0
- package/dist/proof/divergence.js +120 -0
- package/dist/proof/export.d.ts +65 -0
- package/dist/proof/export.js +144 -0
- package/dist/proof/index.d.ts +12 -0
- package/dist/proof/index.js +12 -0
- package/dist/proof/ledger.d.ts +38 -0
- package/dist/proof/ledger.js +57 -0
- package/dist/proof/plugin-verifiers.d.ts +23 -0
- package/dist/proof/plugin-verifiers.js +39 -0
- package/dist/proof/recorder.d.ts +69 -0
- package/dist/proof/recorder.js +347 -0
- package/dist/proof/scope.d.ts +12 -0
- package/dist/proof/scope.js +121 -0
- package/dist/proof/store.d.ts +52 -0
- package/dist/proof/store.js +192 -0
- package/dist/proof/tree.d.ts +6 -0
- package/dist/proof/tree.js +78 -0
- package/dist/proof/types.d.ts +21 -0
- package/dist/proof/types.js +1 -0
- package/dist/proof/verifiers.d.ts +46 -0
- package/dist/proof/verifiers.js +176 -0
- package/dist/proof/verify.d.ts +36 -0
- package/dist/proof/verify.js +171 -0
- package/dist/runtime/store.js +60 -0
- package/dist/tools/goal.d.ts +5 -1
- package/dist/tools/goal.js +35 -4
- package/package.json +2 -2
package/dist/application.d.ts
CHANGED
|
@@ -21,6 +21,8 @@ import { GoalService } from "./goal/service.ts";
|
|
|
21
21
|
import { HerdrBridge } from "./integrations/herdr.ts";
|
|
22
22
|
import { McpConnector } from "./mcp/connector.ts";
|
|
23
23
|
import { PluginHost } from "./plugins/host.ts";
|
|
24
|
+
import { ProofVerifierRegistry } from "./proof/plugin-verifiers.ts";
|
|
25
|
+
import { ProofStore } from "./proof/store.ts";
|
|
24
26
|
import { ActiveProvider, ProviderRegistry } from "./providers/registry.ts";
|
|
25
27
|
import { ProviderSettingsStore } from "./providers/settings.ts";
|
|
26
28
|
import { ProjectContext } from "./resources/context.ts";
|
|
@@ -44,6 +46,8 @@ export interface AppOptions {
|
|
|
44
46
|
/** Extra directories the tools may touch outside the workspace (per-run `--add-dir`). */
|
|
45
47
|
addDirs?: string[];
|
|
46
48
|
noHerdr?: boolean;
|
|
49
|
+
/** Home directory for user-level resources (skills, agents); defaults to `os.homedir()`. */
|
|
50
|
+
home?: string;
|
|
47
51
|
model?: string;
|
|
48
52
|
baseURL?: string;
|
|
49
53
|
apiMode?: string;
|
|
@@ -183,6 +187,17 @@ export declare function createApplication(options?: AppOptions): Promise<{
|
|
|
183
187
|
tasks: BackgroundTasks;
|
|
184
188
|
/** Session goals (`/goal`): state, user actions, the continuation controller. */
|
|
185
189
|
goals: GoalService;
|
|
190
|
+
/** The Proof ledger (specs/011): criteria and evidence of every session; user surfaces only. */
|
|
191
|
+
proof: ProofStore;
|
|
192
|
+
/** Verifiers plugins registered with `api.proof.registerVerifier`, keyed `<plugin>:<id>`. */
|
|
193
|
+
proofVerifiers: ProofVerifierRegistry;
|
|
194
|
+
/**
|
|
195
|
+
* Runs the verifier of one criterion on demand (`file`, `data-check`, `coverage`, `plugin`): a USER
|
|
196
|
+
* action, never reachable by the model. Core tool calls go through the normal gate.
|
|
197
|
+
*/
|
|
198
|
+
verifyCriterion: (sessionId: string, criterionId: string, verifyOptions?: {
|
|
199
|
+
signal?: AbortSignal;
|
|
200
|
+
}) => Promise<import("./index.ts").VerifyOutcome>;
|
|
186
201
|
/** Python discovery and job folders of `python_run`; `enabled` is false under --read-only. */
|
|
187
202
|
analysis: {
|
|
188
203
|
enabled: boolean;
|
package/dist/application.js
CHANGED
|
@@ -32,6 +32,10 @@ import { GoalStore } from "./goal/store.js";
|
|
|
32
32
|
import { HerdrBridge } from "./integrations/herdr.js";
|
|
33
33
|
import { McpConnector } from "./mcp/connector.js";
|
|
34
34
|
import { discoverPlugins, PluginHost } from "./plugins/host.js";
|
|
35
|
+
import { ProofVerifierRegistry } from "./proof/plugin-verifiers.js";
|
|
36
|
+
import { ProofRecorder } from "./proof/recorder.js";
|
|
37
|
+
import { ProofStore } from "./proof/store.js";
|
|
38
|
+
import { verifyCriterion } from "./proof/verify.js";
|
|
35
39
|
import { ActiveProvider, ProviderRegistry, UnconfiguredProvider } from "./providers/registry.js";
|
|
36
40
|
import { availableProviderModels, resolveProviderModel as resolveConfiguredModel, } from "./providers/routing.js";
|
|
37
41
|
import { ProviderSettingsStore } from "./providers/settings.js";
|
|
@@ -197,6 +201,9 @@ export async function createApplication(options = {}) {
|
|
|
197
201
|
},
|
|
198
202
|
},
|
|
199
203
|
});
|
|
204
|
+
// Proof verifiers contributed by plugins (`api.proof`); bound before any plugin loads.
|
|
205
|
+
const proofVerifiers = new ProofVerifierRegistry();
|
|
206
|
+
plugins.setProofVerifiers(proofVerifiers);
|
|
200
207
|
const stateRoot = options.db && options.db !== ":memory:" ? dirname(options.db) : stateHome();
|
|
201
208
|
plugins.setPluginEnvironment({
|
|
202
209
|
stateRoot,
|
|
@@ -216,7 +223,7 @@ export async function createApplication(options = {}) {
|
|
|
216
223
|
cwd,
|
|
217
224
|
stateHome: stateHome(),
|
|
218
225
|
configHome: configHome(),
|
|
219
|
-
home: homedir(),
|
|
226
|
+
home: options.home ?? homedir(),
|
|
220
227
|
trusted: !!options.trustProject || !!options.config,
|
|
221
228
|
configDir: dirname(configFile(workspace, options.config ? { file: options.config } : { trustProject: options.trustProject })),
|
|
222
229
|
};
|
|
@@ -380,7 +387,7 @@ export async function createApplication(options = {}) {
|
|
|
380
387
|
await skills.discover(skillRoots({
|
|
381
388
|
workspace,
|
|
382
389
|
cwd,
|
|
383
|
-
home: homedir(),
|
|
390
|
+
home: options.home ?? homedir(),
|
|
384
391
|
configHome: configHome(),
|
|
385
392
|
trusted: !!options.trustProject || !!options.config,
|
|
386
393
|
configSkills: config.skills,
|
|
@@ -543,6 +550,8 @@ export async function createApplication(options = {}) {
|
|
|
543
550
|
// tools are opt-in: a run only sees them while its session has an active goal.
|
|
544
551
|
const goals = new GoalService({
|
|
545
552
|
store: new GoalStore(store.db),
|
|
553
|
+
// Proof (specs/011): `update_goal complete` is judged against the session's criteria.
|
|
554
|
+
proof: new ProofStore(store.db),
|
|
546
555
|
settings: () => config.goal,
|
|
547
556
|
...(options.goals?.onChange ? { onChange: options.goals.onChange } : {}),
|
|
548
557
|
...(options.goals?.onLog ? { onLog: options.goals.onLog } : {}),
|
|
@@ -642,6 +651,8 @@ export async function createApplication(options = {}) {
|
|
|
642
651
|
store,
|
|
643
652
|
ui: plugins.ui,
|
|
644
653
|
readOnly: !!options.readOnly,
|
|
654
|
+
// Criteria the model proposes become active only when the user approves the plan.
|
|
655
|
+
proof: new ProofStore(store.db),
|
|
645
656
|
diagrams: () => ({
|
|
646
657
|
enabled: config.plan.diagrams,
|
|
647
658
|
max: config.plan.diagrams ? config.plan.maxDiagrams : 0,
|
|
@@ -982,6 +993,7 @@ export async function createApplication(options = {}) {
|
|
|
982
993
|
return known;
|
|
983
994
|
},
|
|
984
995
|
...(options.approve && !options.readOnly ? { approve: options.approve } : {}),
|
|
996
|
+
proof: new ProofRecorder(store.db, { coverage: () => config.proof.coverage.enabled }),
|
|
985
997
|
pathAccess,
|
|
986
998
|
...(analysisEnabled
|
|
987
999
|
? {
|
|
@@ -1118,6 +1130,29 @@ export async function createApplication(options = {}) {
|
|
|
1118
1130
|
tasks,
|
|
1119
1131
|
/** Session goals (`/goal`): state, user actions, the continuation controller. */
|
|
1120
1132
|
goals,
|
|
1133
|
+
/** The Proof ledger (specs/011): criteria and evidence of every session; user surfaces only. */
|
|
1134
|
+
proof: new ProofStore(store.db),
|
|
1135
|
+
/** Verifiers plugins registered with `api.proof.registerVerifier`, keyed `<plugin>:<id>`. */
|
|
1136
|
+
proofVerifiers,
|
|
1137
|
+
/**
|
|
1138
|
+
* Runs the verifier of one criterion on demand (`file`, `data-check`, `coverage`, `plugin`): a USER
|
|
1139
|
+
* action, never reachable by the model. Core tool calls go through the normal gate.
|
|
1140
|
+
*/
|
|
1141
|
+
verifyCriterion: (sessionId, criterionId, verifyOptions = {}) => verifyCriterion({
|
|
1142
|
+
store: new ProofStore(store.db),
|
|
1143
|
+
registry: proofVerifiers,
|
|
1144
|
+
db: store.db,
|
|
1145
|
+
coverage: () => config.proof.coverage.enabled,
|
|
1146
|
+
callTool: (id, name, input, signal) => runner.runToolCall(id, name, input, {
|
|
1147
|
+
signal,
|
|
1148
|
+
workspace: sessionWorkspace(id),
|
|
1149
|
+
}),
|
|
1150
|
+
}, {
|
|
1151
|
+
sessionId,
|
|
1152
|
+
criterionId,
|
|
1153
|
+
workspace: sessionWorkspace(sessionId),
|
|
1154
|
+
...(verifyOptions.signal ? { signal: verifyOptions.signal } : {}),
|
|
1155
|
+
}),
|
|
1121
1156
|
/** Python discovery and job folders of `python_run`; `enabled` is false under --read-only. */
|
|
1122
1157
|
analysis: {
|
|
1123
1158
|
enabled: analysisEnabled,
|
|
@@ -1522,7 +1557,7 @@ export async function createApplication(options = {}) {
|
|
|
1522
1557
|
*/
|
|
1523
1558
|
agentDefinitions: new AgentDefinitionService({
|
|
1524
1559
|
workspace,
|
|
1525
|
-
home: homedir(),
|
|
1560
|
+
home: options.home ?? homedir(),
|
|
1526
1561
|
registry: {
|
|
1527
1562
|
async reload() {
|
|
1528
1563
|
const handler = plugins.commands.get("agents");
|
package/dist/changelog/data.js
CHANGED
|
@@ -2,6 +2,29 @@
|
|
|
2
2
|
export const CHANGELOG_DATA = {
|
|
3
3
|
schema: 1,
|
|
4
4
|
entries: [
|
|
5
|
+
{
|
|
6
|
+
version: "0.8.0",
|
|
7
|
+
date: "2026-10-14",
|
|
8
|
+
sections: [
|
|
9
|
+
{
|
|
10
|
+
title: "Added",
|
|
11
|
+
items: [
|
|
12
|
+
"Proof ledger: acceptance criteria whose status (`verified`, `failed`, `not_checked`, `manual_review`, `not_applicable`) is computed from evidence captured from the commands, file reads and data queries the agent really ran. A criterion without a verifier stays `not_checked`.",
|
|
13
|
+
"Verifiers `command`, `file`, `data-check`, opt-in `coverage` (`proof.coverage.enabled`) and plugin verifiers (`api.proof.registerVerifier`). An edit after the evidence expires it.",
|
|
14
|
+
"The Proof card and tab in the web UI, `/proof` and `/goal override <reason>` in the terminal, and per-criterion review of the criteria a plan proposes (all-or-nothing in the terminal).",
|
|
15
|
+
"`alisio run --proof`, `--criterion`, `--verify` and `--proof-file`: the run prints the ledger, ends with a `proof_ledger` JSONL line under `--json`, exits `0`, `10` or `11` (`proof.exitCodes`, `proof.ci`) and publishes `proof.md` and `proof.json`.",
|
|
16
|
+
"A warning when the final answer claims tests, types or lint passed without matching evidence.",
|
|
17
|
+
"Export in Markdown and versioned JSON (`proof/v1`) and the routes `GET /api/sessions/:sid/proof`, `POST` `…/proof/criteria`, `…/not-applicable` and `…/goal/override`.",
|
|
18
|
+
],
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
title: "Improved",
|
|
22
|
+
items: [
|
|
23
|
+
"`update_goal` with `complete` is refused while a required criterion is `failed` or `not_checked`; only you can complete the goal over it. Without criteria, goal or plan nothing changes.",
|
|
24
|
+
],
|
|
25
|
+
},
|
|
26
|
+
],
|
|
27
|
+
},
|
|
5
28
|
{
|
|
6
29
|
version: "0.7.0",
|
|
7
30
|
date: "2026-10-14",
|
|
@@ -151,6 +151,13 @@ export const BUILTIN_COMMANDS = [
|
|
|
151
151
|
surfaces: INTERACTIVE,
|
|
152
152
|
execution: "surface",
|
|
153
153
|
},
|
|
154
|
+
{
|
|
155
|
+
name: "proof",
|
|
156
|
+
description: "Show the acceptance criteria of this session and the evidence behind each one",
|
|
157
|
+
argumentHint: "[add | edit | required | optional | na | export | help]",
|
|
158
|
+
surfaces: INTERACTIVE,
|
|
159
|
+
execution: "surface",
|
|
160
|
+
},
|
|
154
161
|
{
|
|
155
162
|
name: "decisions",
|
|
156
163
|
description: "Show Decision Intelligence status, provider health and session metrics",
|
package/dist/config.d.ts
CHANGED
|
@@ -255,6 +255,16 @@ export declare const configSchema: z.ZodPipe<z.ZodObject<{
|
|
|
255
255
|
noToolTurnsLimit: z.ZodDefault<z.ZodNumber>;
|
|
256
256
|
blockedRepeats: z.ZodDefault<z.ZodNumber>;
|
|
257
257
|
}, z.core.$strict>>;
|
|
258
|
+
proof: z.ZodDefault<z.ZodObject<{
|
|
259
|
+
ci: z.ZodDefault<z.ZodBoolean>;
|
|
260
|
+
exitCodes: z.ZodDefault<z.ZodObject<{
|
|
261
|
+
failed: z.ZodDefault<z.ZodNumber>;
|
|
262
|
+
unverified: z.ZodDefault<z.ZodNumber>;
|
|
263
|
+
}, z.core.$strict>>;
|
|
264
|
+
coverage: z.ZodDefault<z.ZodObject<{
|
|
265
|
+
enabled: z.ZodDefault<z.ZodBoolean>;
|
|
266
|
+
}, z.core.$strict>>;
|
|
267
|
+
}, z.core.$strict>>;
|
|
258
268
|
plan: z.ZodDefault<z.ZodObject<{
|
|
259
269
|
diagrams: z.ZodDefault<z.ZodBoolean>;
|
|
260
270
|
maxDiagrams: z.ZodDefault<z.ZodNumber>;
|
|
@@ -415,6 +425,16 @@ export declare const configSchema: z.ZodPipe<z.ZodObject<{
|
|
|
415
425
|
noToolTurnsLimit: number;
|
|
416
426
|
blockedRepeats: number;
|
|
417
427
|
};
|
|
428
|
+
proof: {
|
|
429
|
+
ci: boolean;
|
|
430
|
+
exitCodes: {
|
|
431
|
+
failed: number;
|
|
432
|
+
unverified: number;
|
|
433
|
+
};
|
|
434
|
+
coverage: {
|
|
435
|
+
enabled: boolean;
|
|
436
|
+
};
|
|
437
|
+
};
|
|
418
438
|
plan: {
|
|
419
439
|
diagrams: boolean;
|
|
420
440
|
maxDiagrams: number;
|
|
@@ -654,6 +674,16 @@ export declare const configSchema: z.ZodPipe<z.ZodObject<{
|
|
|
654
674
|
noToolTurnsLimit: number;
|
|
655
675
|
blockedRepeats: number;
|
|
656
676
|
};
|
|
677
|
+
proof: {
|
|
678
|
+
ci: boolean;
|
|
679
|
+
exitCodes: {
|
|
680
|
+
failed: number;
|
|
681
|
+
unverified: number;
|
|
682
|
+
};
|
|
683
|
+
coverage: {
|
|
684
|
+
enabled: boolean;
|
|
685
|
+
};
|
|
686
|
+
};
|
|
657
687
|
plan: {
|
|
658
688
|
diagrams: boolean;
|
|
659
689
|
maxDiagrams: number;
|
package/dist/config.js
CHANGED
|
@@ -528,6 +528,38 @@ const configObjectSchema = z
|
|
|
528
528
|
noToolTurnsLimit: 3,
|
|
529
529
|
blockedRepeats: 2,
|
|
530
530
|
})),
|
|
531
|
+
/**
|
|
532
|
+
* Proof (specs/011): acceptance criteria with evidence. `ci` makes `alisio run` behave as
|
|
533
|
+
* with `--proof` (the ledger as the final event and the exit codes below). `exitCodes` are
|
|
534
|
+
* what `--proof` exits with: `failed` when a required criterion failed, `unverified` when a
|
|
535
|
+
* required one is not checked or needs a manual review (0 when every required one is verified).
|
|
536
|
+
*/
|
|
537
|
+
proof: z
|
|
538
|
+
.object({
|
|
539
|
+
ci: z.boolean().default(false),
|
|
540
|
+
exitCodes: z
|
|
541
|
+
.object({
|
|
542
|
+
failed: z.number().int().min(1).max(125).default(10),
|
|
543
|
+
unverified: z.number().int().min(1).max(125).default(11),
|
|
544
|
+
})
|
|
545
|
+
.strict()
|
|
546
|
+
.default(() => ({ failed: 10, unverified: 11 })),
|
|
547
|
+
/**
|
|
548
|
+
* The `coverage` verifier (opt-in). Off: a `coverage` criterion stays `not_checked` and
|
|
549
|
+
* Proof records nothing for its command. On: the percentage a coverage command printed
|
|
550
|
+
* (through the normal permission gate) is compared with the criterion's `minPercent`.
|
|
551
|
+
*/
|
|
552
|
+
coverage: z
|
|
553
|
+
.object({ enabled: z.boolean().default(false) })
|
|
554
|
+
.strict()
|
|
555
|
+
.default(() => ({ enabled: false })),
|
|
556
|
+
})
|
|
557
|
+
.strict()
|
|
558
|
+
.default(() => ({
|
|
559
|
+
ci: false,
|
|
560
|
+
exitCodes: { failed: 10, unverified: 11 },
|
|
561
|
+
coverage: { enabled: false },
|
|
562
|
+
})),
|
|
531
563
|
/**
|
|
532
564
|
* Plan diagrams (`exit_plan`): the plan agent may add Mermaid diagrams to a plan. `diagrams`
|
|
533
565
|
* off removes them from the tool and from the agent's instructions; `maxDiagrams` caps how
|
package/dist/core/runner.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type Attachment, type DatasetRef, type Message, type ModelProvider, type RunEvent, type ToolDefinition, type ToolResult } from "@alisio/sdk";
|
|
2
|
+
import type { ProofRecorder } from "../proof/recorder.ts";
|
|
2
3
|
import type { PathAccess } from "../runtime/access.ts";
|
|
3
4
|
import type { ApprovalHandler, ArtifactPublisherFactory, CapabilityGate, ContextSource, DecisionsFactory, Policy, RunnerExtensions, SessionStore } from "./contracts.ts";
|
|
4
5
|
import { HumanWaits } from "./human-wait.ts";
|
|
@@ -83,6 +84,8 @@ export interface RunnerOptions {
|
|
|
83
84
|
artifacts?: ArtifactPublisherFactory;
|
|
84
85
|
/** Decision Intelligence bound to each tool call (`ToolContext.decisions`). */
|
|
85
86
|
decisions?: DecisionsFactory;
|
|
87
|
+
/** Captures Proof evidence from executed calls (specs/011). Absent: the runner is unchanged. */
|
|
88
|
+
proof?: ProofRecorder;
|
|
86
89
|
/**
|
|
87
90
|
* Provider-native tool definitions (for example a hosted `web_search` tool) appended to every
|
|
88
91
|
* request's `tools` alongside the registry's function tools. Opt-in, set once for the whole
|
|
@@ -285,6 +288,7 @@ export declare class AgentRunner {
|
|
|
285
288
|
signal?: AbortSignal;
|
|
286
289
|
}): Promise<{
|
|
287
290
|
runId: string;
|
|
291
|
+
callId: string;
|
|
288
292
|
status: "completed";
|
|
289
293
|
result: ToolResult;
|
|
290
294
|
}>;
|
package/dist/core/runner.js
CHANGED
|
@@ -414,6 +414,8 @@ export class AgentRunner {
|
|
|
414
414
|
/** Set by the call itself (see `CallInternals`): it may have left effects. */
|
|
415
415
|
let effectful = false;
|
|
416
416
|
let dispatcher;
|
|
417
|
+
/** Proof ledger changes caused by this call, announced after `tool_completed`. */
|
|
418
|
+
let proofUpdates = [];
|
|
417
419
|
/** Held from the scratchpad quota check until the write is done (L1). */
|
|
418
420
|
let releaseScratch;
|
|
419
421
|
// `effect` rides along so the TUI can group consecutive calls by capability kind
|
|
@@ -692,6 +694,25 @@ export class AgentRunner {
|
|
|
692
694
|
})),
|
|
693
695
|
});
|
|
694
696
|
result = await p.tool.execute(p.input, toolContext);
|
|
697
|
+
// Proof reads the RAW result of a call that really ran, before the model-facing bound below.
|
|
698
|
+
// It never throws into the call: a failed capture only means no evidence.
|
|
699
|
+
if (o.proof) {
|
|
700
|
+
try {
|
|
701
|
+
proofUpdates = o.proof.capture({
|
|
702
|
+
sessionId,
|
|
703
|
+
runId,
|
|
704
|
+
callId: call.id,
|
|
705
|
+
name: call.name,
|
|
706
|
+
input: p.input,
|
|
707
|
+
result,
|
|
708
|
+
workspace,
|
|
709
|
+
durationMs: Date.now() - started,
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
catch {
|
|
713
|
+
proofUpdates = [];
|
|
714
|
+
}
|
|
715
|
+
}
|
|
695
716
|
// The bound applies to what the model sees (the text projection): display-only
|
|
696
717
|
// ui/image parts are bounded by their producers and must not push text over it.
|
|
697
718
|
const projected = textProjection(result);
|
|
@@ -702,6 +723,25 @@ export class AgentRunner {
|
|
|
702
723
|
}
|
|
703
724
|
catch (error) {
|
|
704
725
|
result = textResult(error instanceof Error ? error.message : String(error), true);
|
|
726
|
+
// A `read_file` of a missing file is evidence for a `file` criterion (specs/011): the call
|
|
727
|
+
// threw, so it never reached the capture above. Capture never throws into the call.
|
|
728
|
+
if (o.proof && call.name === "read_file") {
|
|
729
|
+
try {
|
|
730
|
+
proofUpdates = proofUpdates.concat(o.proof.captureFailure({
|
|
731
|
+
sessionId,
|
|
732
|
+
runId,
|
|
733
|
+
callId: call.id,
|
|
734
|
+
name: call.name,
|
|
735
|
+
input: p.input ?? {},
|
|
736
|
+
result,
|
|
737
|
+
workspace,
|
|
738
|
+
durationMs: Date.now() - started,
|
|
739
|
+
}));
|
|
740
|
+
}
|
|
741
|
+
catch {
|
|
742
|
+
/* A failed capture only means no evidence. */
|
|
743
|
+
}
|
|
744
|
+
}
|
|
705
745
|
// A cancellation after execution may have left effects, so the journal stays pending for
|
|
706
746
|
// the user to inspect. A `read` call cannot have left any (and `exit_plan`, which is one,
|
|
707
747
|
// is exactly what a stop or the run limit interrupts while a person decides): it is closed
|
|
@@ -725,6 +765,8 @@ export class AgentRunner {
|
|
|
725
765
|
.join("\n")
|
|
726
766
|
.slice(0, 2_000),
|
|
727
767
|
});
|
|
768
|
+
for (const update of proofUpdates)
|
|
769
|
+
emit("proof_updated", update);
|
|
728
770
|
return result;
|
|
729
771
|
}
|
|
730
772
|
/**
|
|
@@ -813,7 +855,7 @@ export class AgentRunner {
|
|
|
813
855
|
o.store.append(sessionId, { role: "assistant", text, calls: [] });
|
|
814
856
|
emit("run_completed", { tokens: 0, text });
|
|
815
857
|
finish("completed");
|
|
816
|
-
return { runId, status: "completed", result };
|
|
858
|
+
return { runId, callId: call.id, status: "completed", result };
|
|
817
859
|
}
|
|
818
860
|
catch (error) {
|
|
819
861
|
if (combined.aborted && isTimeoutReason(combined.reason)) {
|
|
@@ -1270,6 +1312,20 @@ export class AgentRunner {
|
|
|
1270
1312
|
});
|
|
1271
1313
|
final.truncated = true;
|
|
1272
1314
|
}
|
|
1315
|
+
if (o.proof) {
|
|
1316
|
+
// Display only (D8): the claims of the answer that the Proof ledger does not back.
|
|
1317
|
+
try {
|
|
1318
|
+
for (const d of o.proof.divergences(sessionId, lastText))
|
|
1319
|
+
emit("proof_divergence", {
|
|
1320
|
+
claim: d.claim,
|
|
1321
|
+
...(d.criterionId ? { criterionId: d.criterionId } : {}),
|
|
1322
|
+
actual: d.actualState,
|
|
1323
|
+
});
|
|
1324
|
+
}
|
|
1325
|
+
catch {
|
|
1326
|
+
// a failed check only means no warning
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1273
1329
|
emit("run_completed", final);
|
|
1274
1330
|
finish("completed");
|
|
1275
1331
|
return {
|
package/dist/eval/engine.d.ts
CHANGED
|
@@ -104,6 +104,7 @@ export declare const CONFIG_HASH_COVERAGE: {
|
|
|
104
104
|
readonly tasks: "excluded";
|
|
105
105
|
readonly goal: "excluded";
|
|
106
106
|
readonly plan: "excluded";
|
|
107
|
+
readonly proof: "excluded";
|
|
107
108
|
readonly tui: "excluded";
|
|
108
109
|
readonly agents: "excluded";
|
|
109
110
|
readonly web: "excluded";
|
package/dist/eval/engine.js
CHANGED
|
@@ -78,6 +78,7 @@ export const CONFIG_HASH_COVERAGE = {
|
|
|
78
78
|
tasks: "excluded",
|
|
79
79
|
goal: "excluded",
|
|
80
80
|
plan: "excluded",
|
|
81
|
+
proof: "excluded",
|
|
81
82
|
tui: "excluded",
|
|
82
83
|
agents: "excluded",
|
|
83
84
|
web: "excluded",
|
|
@@ -540,6 +541,8 @@ async function execute(task, opts, plan, fixtureDir) {
|
|
|
540
541
|
db: join(dirs.state, "sessions.sqlite"),
|
|
541
542
|
provider: effectiveProvider,
|
|
542
543
|
noHerdr: true,
|
|
544
|
+
// Hermetic: user-level skills and agents never come from the developer's real home.
|
|
545
|
+
home: dirs.home,
|
|
543
546
|
trustProject: false,
|
|
544
547
|
allowWrite: plan.policy.write,
|
|
545
548
|
allowProcess: plan.policy.process,
|
package/dist/eval/secrets.d.ts
CHANGED
|
@@ -10,3 +10,5 @@
|
|
|
10
10
|
export declare function findSecrets(text: string, exact?: readonly string[]): string[];
|
|
11
11
|
/** `text` with every match replaced by `[redacted]`. */
|
|
12
12
|
export declare function redactSecrets(text: string, exact?: readonly string[]): string;
|
|
13
|
+
/** `redactSecrets` plus generic credential assignments, for excerpts of tool output (Proof). */
|
|
14
|
+
export declare function redactOutput(text: string, exact?: readonly string[]): string;
|
package/dist/eval/secrets.js
CHANGED
|
@@ -43,3 +43,13 @@ export function redactSecrets(text, exact = []) {
|
|
|
43
43
|
out = out.replace(new RegExp(escape(value), "g"), "[redacted]");
|
|
44
44
|
return out;
|
|
45
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* Generic `password=`/`TOKEN=` style assignments (also `"api_token": "..."`). Kept apart from
|
|
48
|
+
* `RULES` on purpose: a hit there REFUSES a cassette, and widening that net would change the
|
|
49
|
+
* eval harness. Meant for text that is only displayed or stored as an excerpt.
|
|
50
|
+
*/
|
|
51
|
+
const GENERIC_ASSIGNMENT = /\b[\w.-]*(?:password|passwd|secret|token)[\w.-]*["']?\s*[=:]\s*["']?[^\s"',;&]{3,}/gi;
|
|
52
|
+
/** `redactSecrets` plus generic credential assignments, for excerpts of tool output (Proof). */
|
|
53
|
+
export function redactOutput(text, exact = []) {
|
|
54
|
+
return redactSecrets(text, exact).replace(GENERIC_ASSIGNMENT, "[redacted]");
|
|
55
|
+
}
|
package/dist/goal/command.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** The longest objective accepted, in characters (it is sent to the model on the kickoff). */
|
|
2
2
|
export declare const GOAL_OBJECTIVE_MAX = 4000;
|
|
3
|
-
export declare const GOAL_SUBCOMMANDS: readonly ["pause", "resume", "edit", "clear", "help"];
|
|
3
|
+
export declare const GOAL_SUBCOMMANDS: readonly ["pause", "resume", "edit", "clear", "help", "override"];
|
|
4
4
|
export type GoalCommand = {
|
|
5
5
|
type: "show";
|
|
6
6
|
} | {
|
|
@@ -12,6 +12,11 @@ export type GoalCommand = {
|
|
|
12
12
|
} | {
|
|
13
13
|
type: "clear";
|
|
14
14
|
}
|
|
15
|
+
/** The user completes the goal over unmet required Proof criteria (a reason is required). */
|
|
16
|
+
| {
|
|
17
|
+
type: "override";
|
|
18
|
+
reason: string;
|
|
19
|
+
}
|
|
15
20
|
/** `objective` is absent when the UI should ask for the new text. */
|
|
16
21
|
| {
|
|
17
22
|
type: "edit";
|
|
@@ -27,7 +32,7 @@ export type GoalCommand = {
|
|
|
27
32
|
type: "error";
|
|
28
33
|
message: string;
|
|
29
34
|
};
|
|
30
|
-
export declare const GOAL_USAGE = "Usage: /goal [<objective> [budget=50k]] | pause | resume | edit [<objective>] | clear | budget=<n|clear> | help";
|
|
35
|
+
export declare const GOAL_USAGE = "Usage: /goal [<objective> [budget=50k]] | pause | resume | edit [<objective>] | clear | override <reason> | budget=<n|clear> | help";
|
|
31
36
|
/** Markdown help shown by `/goal help` (identical on every surface). */
|
|
32
37
|
export declare const GOAL_HELP: string;
|
|
33
38
|
/** Words of `args` split on whitespace, keeping the original spacing of the objective. */
|
package/dist/goal/command.js
CHANGED
|
@@ -5,14 +5,15 @@
|
|
|
5
5
|
* /goal <objective> [budget=N] create (or replace) the goal and start working on it
|
|
6
6
|
* /goal pause | resume | clear | help
|
|
7
7
|
* /goal edit [<objective>] change the objective of the current goal
|
|
8
|
+
* /goal override <reason> complete the goal over unmet required Proof criteria (you only)
|
|
8
9
|
* /goal budget=50k set the token budget of the current goal (budget=clear removes it)
|
|
9
10
|
*/
|
|
10
11
|
import { parseTokenBudget } from "./budget.js";
|
|
11
12
|
/** The longest objective accepted, in characters (it is sent to the model on the kickoff). */
|
|
12
13
|
export const GOAL_OBJECTIVE_MAX = 4000;
|
|
13
|
-
export const GOAL_SUBCOMMANDS = ["pause", "resume", "edit", "clear", "help"];
|
|
14
|
+
export const GOAL_SUBCOMMANDS = ["pause", "resume", "edit", "clear", "help", "override"];
|
|
14
15
|
const BUDGET_WORD = /^budget=(.*)$/i;
|
|
15
|
-
export const GOAL_USAGE = "Usage: /goal [<objective> [budget=50k]] | pause | resume | edit [<objective>] | clear | budget=<n|clear> | help";
|
|
16
|
+
export const GOAL_USAGE = "Usage: /goal [<objective> [budget=50k]] | pause | resume | edit [<objective>] | clear | override <reason> | budget=<n|clear> | help";
|
|
16
17
|
/** Markdown help shown by `/goal help` (identical on every surface). */
|
|
17
18
|
export const GOAL_HELP = [
|
|
18
19
|
"**/goal** keeps the agent working on one objective until it is done, blocked, paused or out of budget.",
|
|
@@ -22,6 +23,7 @@ export const GOAL_HELP = [
|
|
|
22
23
|
"- `/goal <objective> budget=50k` also sets a token budget (`50k`, `1.5M`, `250000`).",
|
|
23
24
|
"- `/goal pause` stops the automatic continuation; `/goal resume` continues a paused or blocked goal.",
|
|
24
25
|
"- `/goal edit <objective>` changes the objective; `/goal clear` removes the goal.",
|
|
26
|
+
"- `/goal override <reason>` completes the goal over acceptance criteria that are not verified (Proof); only you can, and the reason is kept.",
|
|
25
27
|
"- `/goal budget=50k` sets the token budget; `/goal budget=clear` (also `none`, `off`, `0`) removes it.",
|
|
26
28
|
"",
|
|
27
29
|
`The objective can be up to ${GOAL_OBJECTIVE_MAX} characters. Only you pause, resume, edit or clear a goal or change its budget; the agent marks it complete or blocked, with evidence.`,
|
|
@@ -62,6 +64,12 @@ export function parseGoalCommand(args) {
|
|
|
62
64
|
return { type: "edit" };
|
|
63
65
|
return validateObjective(objective, (value) => ({ type: "edit", objective: value }));
|
|
64
66
|
}
|
|
67
|
+
if (subcommand === "override") {
|
|
68
|
+
const reason = text.slice(text.search(/\s/) + 1).trim();
|
|
69
|
+
if (rest.length === 1)
|
|
70
|
+
return { type: "error", message: "/goal override needs a reason." };
|
|
71
|
+
return { type: "override", reason };
|
|
72
|
+
}
|
|
65
73
|
if (rest.length > 1)
|
|
66
74
|
return {
|
|
67
75
|
type: "error",
|
package/dist/goal/format.js
CHANGED
|
@@ -28,10 +28,12 @@ const REASON_TEXT = {
|
|
|
28
28
|
max_wall: "The time limit was reached",
|
|
29
29
|
no_progress: "No progress",
|
|
30
30
|
restart: "Alisio restarted while the goal was running",
|
|
31
|
+
user_override: "You completed the goal over unmet acceptance criteria",
|
|
31
32
|
};
|
|
32
33
|
const DETAIL_TEXT = {
|
|
33
34
|
repeated_reply: "the same reply repeated",
|
|
34
35
|
no_tool_turns: "several turns without tool calls",
|
|
36
|
+
completion_refused: "completion was refused while required criteria were not verified",
|
|
35
37
|
};
|
|
36
38
|
/** Why the goal is in its state, as one short sentence (empty when there is nothing to say). */
|
|
37
39
|
export function goalReasonText(goal) {
|
package/dist/goal/machine.d.ts
CHANGED
|
@@ -9,12 +9,14 @@
|
|
|
9
9
|
* └──── budget raised / cleared (token_budget only) ── budget_limited
|
|
10
10
|
*
|
|
11
11
|
* Who may do what:
|
|
12
|
-
* - user: create, pause, resume (paused|blocked), edit, clear, change the budget
|
|
13
|
-
*
|
|
12
|
+
* - user: create, pause, resume (paused|blocked), edit, clear, change the budget, and the Proof
|
|
13
|
+
* override (complete over unmet required criteria; never reachable from the model).
|
|
14
|
+
* - model: `update_goal` complete / blocked, only while the goal is `active`. `complete` is
|
|
15
|
+
* refused while a required criterion of the Proof ledger is failed or not checked.
|
|
14
16
|
* - system: the caps and breakers (pause / budget_limited / blocked on error), user interrupt,
|
|
15
17
|
* restart. The system never resumes a goal.
|
|
16
18
|
*/
|
|
17
|
-
import type { GoalAction, GoalEvidence, GoalInfo, GoalReason, GoalStatus } from "@alisio/sdk";
|
|
19
|
+
import type { GoalAction, GoalEvidence, GoalInfo, GoalProofInfo, GoalReason, GoalStatus, ProofCriterion, ProofLedger } from "@alisio/sdk";
|
|
18
20
|
export type GoalActor = "user" | "model" | "system";
|
|
19
21
|
export declare const GOAL_STATUSES: readonly GoalStatus[];
|
|
20
22
|
/** The longest `detail` kept (an error text is truncated). */
|
|
@@ -40,6 +42,8 @@ export interface GoalRecord {
|
|
|
40
42
|
detail?: string;
|
|
41
43
|
summary?: string;
|
|
42
44
|
evidence?: GoalEvidence[];
|
|
45
|
+
/** The Proof snapshot taken when the goal completed with criteria (absent without criteria). */
|
|
46
|
+
proof?: GoalProofInfo;
|
|
43
47
|
epoch: number;
|
|
44
48
|
tokenBudget?: number;
|
|
45
49
|
tokensUsed: number;
|
|
@@ -104,6 +108,12 @@ export declare function userActions(status: GoalStatus): GoalAction[];
|
|
|
104
108
|
export declare const isFinalGoalStatus: (status: GoalStatus) => boolean;
|
|
105
109
|
export declare const clip: (text: string, max?: number) => string;
|
|
106
110
|
export type GoalRefusal = "not_allowed" | "not_active";
|
|
111
|
+
/** A required criterion that blocks completion (`failed` or `not_checked`). */
|
|
112
|
+
export interface GoalUnmet {
|
|
113
|
+
criterionId: string;
|
|
114
|
+
text: string;
|
|
115
|
+
status: "failed" | "not_checked";
|
|
116
|
+
}
|
|
107
117
|
export type GoalResult<T = GoalRecord> = {
|
|
108
118
|
ok: true;
|
|
109
119
|
goal: T;
|
|
@@ -111,7 +121,23 @@ export type GoalResult<T = GoalRecord> = {
|
|
|
111
121
|
ok: false;
|
|
112
122
|
code: GoalRefusal;
|
|
113
123
|
message: string;
|
|
124
|
+
unmet?: GoalUnmet[];
|
|
114
125
|
};
|
|
126
|
+
/**
|
|
127
|
+
* What the Proof ledger says about one goal (specs/011 3.6): the criteria that apply to it and
|
|
128
|
+
* the evidence ids the model declared that resolved to captured evidence.
|
|
129
|
+
*/
|
|
130
|
+
export interface GoalProofInput {
|
|
131
|
+
criteria: ProofCriterion[];
|
|
132
|
+
linkedEvidence: string[];
|
|
133
|
+
}
|
|
134
|
+
/** The criteria that apply to a goal: its own and the session-wide ones (no `goalId`). */
|
|
135
|
+
export declare const criteriaForGoal: (ledger: ProofLedger, goalId: string) => ProofCriterion[];
|
|
136
|
+
/**
|
|
137
|
+
* Required criteria that are `failed` or `not_checked`. `manual_review` and `not_applicable` do
|
|
138
|
+
* not block (O6 names only the first two; a manual review is the user's own call).
|
|
139
|
+
*/
|
|
140
|
+
export declare function unmetRequired(criteria: readonly ProofCriterion[]): GoalUnmet[];
|
|
115
141
|
/** A new goal (the user's `/goal <objective>`). The caller supplies the id and the limits. */
|
|
116
142
|
export declare function newGoal(input: {
|
|
117
143
|
sessionId: string;
|
|
@@ -145,6 +171,27 @@ export declare function userBudget(goal: GoalRecord, now: number, tokens: number
|
|
|
145
171
|
export declare function modelComplete(goal: GoalRecord, now: number, report: {
|
|
146
172
|
summary: string;
|
|
147
173
|
evidence: GoalEvidence[];
|
|
174
|
+
}, proof?: GoalProofInput): GoalResult;
|
|
175
|
+
/** Different runs that may be refused a completion over the ledger before the goal pauses. */
|
|
176
|
+
export declare const COMPLETION_REFUSAL_LIMIT = 3;
|
|
177
|
+
/**
|
|
178
|
+
* A model that keeps reporting `complete` over unmet required criteria, run after run, is not
|
|
179
|
+
* making progress: the goal pauses as `no_progress` (the user resumes or overrides it). The caller
|
|
180
|
+
* counts the refusals; the in-flight claim stays so the running turn still settles.
|
|
181
|
+
*/
|
|
182
|
+
export declare function pauseForRefusedCompletions(goal: GoalRecord, now: number): GoalRecord;
|
|
183
|
+
/** The longest override reason kept. */
|
|
184
|
+
export declare const GOAL_OVERRIDE_REASON_MAX = 500;
|
|
185
|
+
/**
|
|
186
|
+
* The USER completes a goal over unmet required criteria (O6). It exists only to bypass that
|
|
187
|
+
* gate: with nothing unmet there is nothing to override. Only a goal that can still be worked
|
|
188
|
+
* on (`active`, `paused`, `blocked`) can be overridden; `budget_limited` and `complete` are final
|
|
189
|
+
* (raise the budget first). The in-flight claim stays, as for a model completion: the running
|
|
190
|
+
* turn still settles. The caller (the service) has already checked that a person asked for it.
|
|
191
|
+
*/
|
|
192
|
+
export declare function userOverrideComplete(goal: GoalRecord, now: number, input: {
|
|
193
|
+
reason: string;
|
|
194
|
+
proof: GoalProofInput;
|
|
148
195
|
}): GoalResult;
|
|
149
196
|
export interface BlockedReport {
|
|
150
197
|
summary: string;
|