@alisio/core 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analysis/data/engine-client.d.ts +2 -0
- package/dist/analysis/data/engine-client.js +11 -4
- package/dist/application.d.ts +11 -1
- package/dist/application.js +46 -0
- package/dist/changelog/data.js +42 -0
- package/dist/commands/builtins.js +6 -0
- package/dist/commands/catalog.d.ts +4 -2
- package/dist/commands/catalog.js +23 -1
- package/dist/config.d.ts +31 -0
- package/dist/config.js +31 -1
- package/dist/context/elide.d.ts +50 -0
- package/dist/context/elide.js +222 -0
- package/dist/context/format.d.ts +21 -0
- package/dist/context/format.js +81 -0
- package/dist/context/governance.d.ts +58 -0
- package/dist/context/governance.js +96 -0
- package/dist/context/invariants.d.ts +88 -0
- package/dist/context/invariants.js +141 -0
- package/dist/context/inventory.d.ts +22 -0
- package/dist/context/inventory.js +178 -0
- package/dist/context/report.d.ts +35 -0
- package/dist/context/report.js +120 -0
- package/dist/context/sources.d.ts +20 -0
- package/dist/context/sources.js +41 -0
- package/dist/context/types.d.ts +43 -0
- package/dist/context/types.js +5 -0
- package/dist/context/usage.d.ts +96 -0
- package/dist/context/usage.js +264 -0
- package/dist/context/util.d.ts +22 -0
- package/dist/context/util.js +36 -0
- package/dist/core/compaction.d.ts +7 -2
- package/dist/core/compaction.js +9 -3
- package/dist/core/contracts.d.ts +8 -0
- package/dist/core/runner.d.ts +67 -0
- package/dist/core/runner.js +272 -10
- package/dist/eval/compare.d.ts +1 -1
- package/dist/eval/compare.js +18 -1
- package/dist/eval/engine.js +31 -1
- package/dist/eval/report.js +44 -1
- package/dist/forge/javascript/hosts/process-host.d.ts +1 -1
- package/dist/forge/javascript/hosts/process-host.js +4 -1
- package/dist/forge/python/wasm-node.d.ts +1 -1
- package/dist/forge/python/wasm-node.js +4 -1
- package/dist/index.d.ts +8 -2
- package/dist/index.js +8 -2
- package/dist/plugins/host.d.ts +5 -0
- package/dist/plugins/host.js +15 -1
- package/dist/plugins/install.d.ts +50 -2
- package/dist/plugins/install.js +102 -12
- package/dist/plugins/local-install.d.ts +33 -0
- package/dist/plugins/local-install.js +121 -0
- package/dist/plugins/npm-runner.d.ts +30 -0
- package/dist/plugins/npm-runner.js +66 -0
- package/dist/plugins/tarball.d.ts +39 -0
- package/dist/plugins/tarball.js +277 -0
- package/dist/plugins/version-compat.d.ts +20 -0
- package/dist/plugins/version-compat.js +154 -0
- package/package.json +2 -2
package/dist/core/runner.js
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { isEphemeralRunEventType, textProjection, textResult, } from "@alisio/sdk";
|
|
2
|
+
import { projectContext } from "../context/elide.js";
|
|
3
|
+
import { governanceSnapshot, governedEventData, } from "../context/governance.js";
|
|
4
|
+
import { collectInvariants, promoteInstructions, readInvariantState, renderInvariants, } from "../context/invariants.js";
|
|
5
|
+
import { UsageLedger } from "../context/usage.js";
|
|
2
6
|
import { createNestedDispatcher } from "../forge/nested.js";
|
|
3
7
|
import { hasScratchEstimator, scratchIncomingBytes } from "../forge/scratch-quota.js";
|
|
4
8
|
import { decideEffect, effectAskable } from "../permissions/decide.js";
|
|
5
9
|
import { bindCallInternals } from "./call-internals.js";
|
|
6
|
-
import { checkpointInstructions, estimateTokens, MAX_TRUSTED_WINDOW, parseCheckpointOutput, planCompaction, reduceMessagesToBudget, shouldCompactContext, summarize, summaryMessage, } from "./compaction.js";
|
|
10
|
+
import { checkpointInstructions, estimateTokens, INVARIANTS_FIELD, MAX_TRUSTED_WINDOW, parseCheckpointOutput, planCompaction, reduceMessagesToBudget, shouldCompactContext, summarize, summaryMessage, } from "./compaction.js";
|
|
7
11
|
import { HumanWaits } from "./human-wait.js";
|
|
8
12
|
import { EXIT_PLAN_TOOL, optInAllows } from "./opt-in.js";
|
|
9
13
|
import { resolveMaxOutputTokens } from "./output-limit.js";
|
|
@@ -52,8 +56,12 @@ const MIN_MESSAGE_BUDGET_FLOOR = 4_000;
|
|
|
52
56
|
* would pay for a summarizer call. The hard-limit last-chance compaction ignores it.
|
|
53
57
|
*/
|
|
54
58
|
const RECOMPACT_GROWTH_MARGIN = 0.1;
|
|
59
|
+
/** Sessions whose usage ledger one runner keeps in memory (least recently used go first). */
|
|
60
|
+
const USAGE_LEDGER_SESSIONS = 64;
|
|
55
61
|
/** First 40 lines of a script input, for capability approvals. */
|
|
56
62
|
const scriptPreview = (input) => typeof input.code === "string" ? input.code.split(/\r?\n/).slice(0, 40).join("\n") : undefined;
|
|
63
|
+
/** `{ [key]: value }`, or nothing when there is no value (optional properties stay absent). */
|
|
64
|
+
const optional = (key, value) => (value === undefined ? {} : { [key]: value });
|
|
57
65
|
/** A short id for a tool call the user started from a UI (alphanumeric, 9 characters). */
|
|
58
66
|
const uiCallId = () => {
|
|
59
67
|
const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
|
@@ -70,6 +78,16 @@ export class AgentRunner {
|
|
|
70
78
|
reported = new Map();
|
|
71
79
|
/** Conversation size in characters right after the last automatic compaction, per session. */
|
|
72
80
|
compactedChars = new Map();
|
|
81
|
+
/** What the Context Governor decided for the latest request of each session (shadow and on). */
|
|
82
|
+
governed = new Map();
|
|
83
|
+
/** Usage ledger per session (shadow and on; process-local, LRU of `USAGE_LEDGER_SESSIONS`). */
|
|
84
|
+
usage = new Map();
|
|
85
|
+
/** Effects the user approved for the whole session, per session (`on` mode; process-local). */
|
|
86
|
+
sessionEffects = new Map();
|
|
87
|
+
/** Instruction quotes of a session whose store cannot keep session options (process-local). */
|
|
88
|
+
invariantMemory = new Map();
|
|
89
|
+
/** Quotes promoted and refused by the compactions of this process, per session. */
|
|
90
|
+
promotions = new Map();
|
|
73
91
|
/** Where human waits are opened; the clocks of the active runs listen to it. */
|
|
74
92
|
humanWaits;
|
|
75
93
|
constructor(options) {
|
|
@@ -267,6 +285,173 @@ export class AgentRunner {
|
|
|
267
285
|
fallback: o.fallbackMaxOutputTokens ?? 4096,
|
|
268
286
|
});
|
|
269
287
|
}
|
|
288
|
+
/** Latest Context Governor decision for a session (core-internal; counts and ids only). */
|
|
289
|
+
governance(sessionId) {
|
|
290
|
+
return this.governed.get(sessionId);
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* Heuristic usage of the tool results this process showed the model in a session (shadow and on
|
|
294
|
+
* only; undefined otherwise). Lost on restart; advisory, it never drives elision.
|
|
295
|
+
*/
|
|
296
|
+
contextUsage(sessionId) {
|
|
297
|
+
return this.usage.get(sessionId)?.summary();
|
|
298
|
+
}
|
|
299
|
+
usageLedger(sessionId, workspace) {
|
|
300
|
+
let ledger = this.usage.get(sessionId);
|
|
301
|
+
if (ledger)
|
|
302
|
+
this.usage.delete(sessionId);
|
|
303
|
+
else {
|
|
304
|
+
const registry = this.options.registry;
|
|
305
|
+
ledger = new UsageLedger({
|
|
306
|
+
workspace,
|
|
307
|
+
pathsOf: (tool, args) => {
|
|
308
|
+
try {
|
|
309
|
+
return registry.get(tool).paths?.(args);
|
|
310
|
+
}
|
|
311
|
+
catch {
|
|
312
|
+
return undefined;
|
|
313
|
+
}
|
|
314
|
+
},
|
|
315
|
+
isRead: (tool) => {
|
|
316
|
+
try {
|
|
317
|
+
return registry.get(tool).effect === "read";
|
|
318
|
+
}
|
|
319
|
+
catch {
|
|
320
|
+
return false;
|
|
321
|
+
}
|
|
322
|
+
},
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
this.usage.set(sessionId, ledger);
|
|
326
|
+
while (this.usage.size > USAGE_LEDGER_SESSIONS) {
|
|
327
|
+
const oldest = this.usage.keys().next().value;
|
|
328
|
+
if (oldest === undefined)
|
|
329
|
+
break;
|
|
330
|
+
this.usage.delete(oldest);
|
|
331
|
+
}
|
|
332
|
+
return ledger;
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Forgets the user instructions kept for a session: the stored `contextInvariants` and every
|
|
336
|
+
* in-memory entry. Core-internal; there is no per-item removal before phase 5.
|
|
337
|
+
*/
|
|
338
|
+
clearInvariants(sessionId) {
|
|
339
|
+
this.invariantMemory.delete(sessionId);
|
|
340
|
+
this.usage.delete(sessionId);
|
|
341
|
+
this.sessionEffects.delete(sessionId);
|
|
342
|
+
this.promotions.delete(sessionId);
|
|
343
|
+
this.options.store.mutateSessionOptions?.(sessionId, (current) => ({
|
|
344
|
+
...("contextInvariants" in current ? { patch: { contextInvariants: undefined } } : {}),
|
|
345
|
+
value: undefined,
|
|
346
|
+
}));
|
|
347
|
+
}
|
|
348
|
+
/** The stored instruction list of a session, or undefined for old or never-compacted sessions. */
|
|
349
|
+
invariantState(sessionId) {
|
|
350
|
+
const store = this.options.store;
|
|
351
|
+
return store.mutateSessionOptions
|
|
352
|
+
? readInvariantState(store.get(sessionId).options)
|
|
353
|
+
: this.invariantMemory.get(sessionId);
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* The `<invariants>` block for the next request of a session: empty before the first compaction
|
|
357
|
+
* (a history that starts with a checkpoint, or a stored compaction count). Built fresh from the
|
|
358
|
+
* stored instructions and the live sources, never saved.
|
|
359
|
+
*/
|
|
360
|
+
invariantBlock(sessionId, messages, maxChars) {
|
|
361
|
+
const list = this.invariantList(sessionId, messages);
|
|
362
|
+
return list ? renderInvariants(list, maxChars) : { text: "", included: 0, omitted: 0 };
|
|
363
|
+
}
|
|
364
|
+
/** The collected invariants, or undefined before the first compaction (see `invariantBlock`). */
|
|
365
|
+
invariantList(sessionId, messages) {
|
|
366
|
+
const state = this.invariantState(sessionId);
|
|
367
|
+
const first = messages[0];
|
|
368
|
+
const compacted = (state?.compactions ?? 0) > 0 ||
|
|
369
|
+
(first?.role === "user" && !!first.summary && first.text.startsWith("[Checkpoint of"));
|
|
370
|
+
if (!compacted)
|
|
371
|
+
return undefined;
|
|
372
|
+
const sources = this.options.invariantSources;
|
|
373
|
+
// A broken source never stops a request: its section is simply left out.
|
|
374
|
+
const read = (source) => {
|
|
375
|
+
try {
|
|
376
|
+
return source?.();
|
|
377
|
+
}
|
|
378
|
+
catch {
|
|
379
|
+
return undefined;
|
|
380
|
+
}
|
|
381
|
+
};
|
|
382
|
+
const grants = read(sources?.grants?.bind(sources, sessionId));
|
|
383
|
+
const effects = [...(this.sessionEffects.get(sessionId) ?? [])];
|
|
384
|
+
return collectInvariants({
|
|
385
|
+
instructions: state?.instructions ?? [],
|
|
386
|
+
...optional("goal", read(sources?.goal?.bind(sources, sessionId))),
|
|
387
|
+
...optional("plan", read(sources?.plan?.bind(sources, sessionId))),
|
|
388
|
+
...optional("criteria", read(sources?.criteria?.bind(sources, sessionId))),
|
|
389
|
+
...(grants || effects.length
|
|
390
|
+
? {
|
|
391
|
+
grants: {
|
|
392
|
+
effects,
|
|
393
|
+
capabilities: grants?.capabilities ?? [],
|
|
394
|
+
externalDirectories: grants?.externalDirectories ?? 0,
|
|
395
|
+
},
|
|
396
|
+
}
|
|
397
|
+
: {}),
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* What the `ContextReport` of a session is built from (spec 012 phase 5): the governor settings,
|
|
402
|
+
* the active history, the invariants of the next request and the in-memory usage ledger. Read
|
|
403
|
+
* only. `off` and runners without a governor return no history, because `off` measures nothing.
|
|
404
|
+
*/
|
|
405
|
+
contextInputs(sessionId) {
|
|
406
|
+
const governor = this.options.contextGovernor;
|
|
407
|
+
if (!governor)
|
|
408
|
+
return undefined;
|
|
409
|
+
if (governor.mode === "off")
|
|
410
|
+
return { governor, messages: [], invariants: { items: [], omitted: 0 } };
|
|
411
|
+
const messages = this.options.store.messages(sessionId);
|
|
412
|
+
const list = this.invariantList(sessionId, messages) ?? [];
|
|
413
|
+
const omitted = list.length
|
|
414
|
+
? renderInvariants(list, governor.invariantMaxChars ?? 4000).omitted
|
|
415
|
+
: 0;
|
|
416
|
+
const ledger = this.usage.get(sessionId);
|
|
417
|
+
const governance = this.governed.get(sessionId);
|
|
418
|
+
return {
|
|
419
|
+
governor,
|
|
420
|
+
messages,
|
|
421
|
+
invariants: { items: list.map((i) => ({ kind: i.kind, text: i.text })), omitted },
|
|
422
|
+
...(ledger ? { ledger } : {}),
|
|
423
|
+
...(governance ? { governance } : {}),
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
/** Keeps the user's verbatim quotes the summarizer proposed (see `promoteInstructions`). */
|
|
427
|
+
promoteInvariants(sessionId, proposed, summarized) {
|
|
428
|
+
const store = this.options.store;
|
|
429
|
+
const apply = (state) => {
|
|
430
|
+
const out = promoteInstructions(state?.instructions ?? [], proposed, summarized);
|
|
431
|
+
const next = {
|
|
432
|
+
version: 1,
|
|
433
|
+
instructions: out.instructions,
|
|
434
|
+
compactions: (state?.compactions ?? 0) + 1,
|
|
435
|
+
updatedAt: Date.now(),
|
|
436
|
+
};
|
|
437
|
+
return { next, out };
|
|
438
|
+
};
|
|
439
|
+
const { out } = store.mutateSessionOptions
|
|
440
|
+
? store.mutateSessionOptions(sessionId, (current) => {
|
|
441
|
+
const result = apply(readInvariantState(current));
|
|
442
|
+
return { patch: { contextInvariants: result.next }, value: result };
|
|
443
|
+
})
|
|
444
|
+
: (() => {
|
|
445
|
+
const result = apply(this.invariantMemory.get(sessionId));
|
|
446
|
+
this.invariantMemory.set(sessionId, result.next);
|
|
447
|
+
return result;
|
|
448
|
+
})();
|
|
449
|
+
const total = this.promotions.get(sessionId) ?? { promoted: 0, rejected: 0 };
|
|
450
|
+
this.promotions.set(sessionId, {
|
|
451
|
+
promoted: total.promoted + out.accepted,
|
|
452
|
+
rejected: total.rejected + out.rejected,
|
|
453
|
+
});
|
|
454
|
+
}
|
|
270
455
|
/** Change the model used by subsequent turns of a session; recorded in the store. */
|
|
271
456
|
setModel(sessionId, model) {
|
|
272
457
|
const id = model.trim();
|
|
@@ -277,6 +462,7 @@ export class AgentRunner {
|
|
|
277
462
|
return;
|
|
278
463
|
this.options.store.setModel(sessionId, id);
|
|
279
464
|
this.reported.delete(sessionId);
|
|
465
|
+
this.governed.delete(sessionId);
|
|
280
466
|
this.emitter(sessionId)("model_changed", { model: id, previous });
|
|
281
467
|
}
|
|
282
468
|
/** Manually compact a session outside a run. */
|
|
@@ -337,10 +523,14 @@ export class AgentRunner {
|
|
|
337
523
|
});
|
|
338
524
|
}
|
|
339
525
|
reportFailures(contribution.failures);
|
|
340
|
-
|
|
526
|
+
// Only `on` asks the summarizer for verbatim quotes; a plugin field of the same name is ignored.
|
|
527
|
+
const govern = o.contextGovernor?.mode === "on";
|
|
528
|
+
const fields = Object.fromEntries(Object.entries(contribution.fields)
|
|
529
|
+
.filter(([name]) => !(govern && name === INVARIANTS_FIELD))
|
|
530
|
+
.map(([name, f]) => [name, f.description]));
|
|
341
531
|
const provider = await (o.providerFor?.(o.store.get(sessionId)) ?? o.provider);
|
|
342
532
|
const { text: raw, truncated: summaryTruncated } = await summarize(provider, {
|
|
343
|
-
instructions: checkpointInstructions(contribution.instructions, fields),
|
|
533
|
+
instructions: checkpointInstructions(contribution.instructions, fields, govern),
|
|
344
534
|
messages: plan.summarized,
|
|
345
535
|
focus,
|
|
346
536
|
model,
|
|
@@ -348,7 +538,9 @@ export class AgentRunner {
|
|
|
348
538
|
signal,
|
|
349
539
|
sessionId,
|
|
350
540
|
});
|
|
351
|
-
const
|
|
541
|
+
const parsed = parseCheckpointOutput(raw, govern ? [...Object.keys(fields), INVARIANTS_FIELD] : Object.keys(fields));
|
|
542
|
+
const { [INVARIANTS_FIELD]: proposed, ...extracted } = parsed.extracted;
|
|
543
|
+
const output = govern ? { ...parsed, extracted } : parsed;
|
|
352
544
|
let after = {
|
|
353
545
|
inject: [],
|
|
354
546
|
reports: {},
|
|
@@ -374,8 +566,12 @@ export class AgentRunner {
|
|
|
374
566
|
.filter((x) => x.trim())
|
|
375
567
|
.join("\n\n");
|
|
376
568
|
const summary = summaryMessage(body, plan.cut);
|
|
569
|
+
// Before the history is replaced: if the process dies in between, the next compaction dedupes.
|
|
570
|
+
if (govern)
|
|
571
|
+
this.promoteInvariants(sessionId, proposed, plan.summarized);
|
|
377
572
|
o.store.compact(sessionId, plan.cut, summary);
|
|
378
573
|
this.reported.delete(sessionId);
|
|
574
|
+
this.governed.delete(sessionId);
|
|
379
575
|
const afterTokens = await this.estimateContext(sessionId);
|
|
380
576
|
emit("compaction_completed", {
|
|
381
577
|
reason,
|
|
@@ -568,8 +764,14 @@ export class AgentRunner {
|
|
|
568
764
|
if (decision === "deny")
|
|
569
765
|
throw new Error(`Capability ${capability ?? effect} denied by the user for this call`);
|
|
570
766
|
// A capability grant never widens the effect: only effect approvals do.
|
|
571
|
-
if (decision === "session" && !capability)
|
|
767
|
+
if (decision === "session" && !capability) {
|
|
572
768
|
policy[effect] = true;
|
|
769
|
+
if (o.contextGovernor?.mode === "on" &&
|
|
770
|
+
(effect === "write" || effect === "process" || effect === "external")) {
|
|
771
|
+
const effects = this.sessionEffects.get(sessionId) ?? new Set();
|
|
772
|
+
this.sessionEffects.set(sessionId, effects.add(effect));
|
|
773
|
+
}
|
|
774
|
+
}
|
|
573
775
|
}
|
|
574
776
|
o.store.beginCall(sessionId, call, { runId, effect });
|
|
575
777
|
const publisher = o.artifacts?.({ sessionId, runId, callId: call.id }, (published) => emit("artifact_published", {
|
|
@@ -1009,7 +1211,20 @@ export class AgentRunner {
|
|
|
1009
1211
|
let messages = o.store.messages(sessionId);
|
|
1010
1212
|
const tools = this.availableTools(options, sessionId);
|
|
1011
1213
|
const toolsText = this.toolsText(tools);
|
|
1012
|
-
|
|
1214
|
+
// The `<invariants>` block (`on` only) is part of every request after the first compaction,
|
|
1215
|
+
// so it counts like instructions. Built once per history; shadow measures it at the seam.
|
|
1216
|
+
const maxInvariantChars = o.contextGovernor?.invariantMaxChars ?? 4000;
|
|
1217
|
+
let invariantMemo;
|
|
1218
|
+
const invariants = () => {
|
|
1219
|
+
if (invariantMemo?.for !== messages)
|
|
1220
|
+
invariantMemo = {
|
|
1221
|
+
for: messages,
|
|
1222
|
+
value: this.invariantBlock(sessionId, messages, maxInvariantChars),
|
|
1223
|
+
};
|
|
1224
|
+
return invariantMemo.value;
|
|
1225
|
+
};
|
|
1226
|
+
const blockChars = () => (o.contextGovernor?.mode === "on" ? invariants().text.length : 0);
|
|
1227
|
+
const chars = () => instructions.length + blockChars() + JSON.stringify(messages).length + toolsText.length;
|
|
1013
1228
|
// REDUCIBLE content only (instructions + transcript). The fixed tool catalog
|
|
1014
1229
|
// (`toolsText`) is a deployment reality — a huge MCP catalog with dozens of tools is
|
|
1015
1230
|
// part of every request regardless of history, so it is deliberately excluded from the
|
|
@@ -1017,7 +1232,7 @@ export class AgentRunner {
|
|
|
1017
1232
|
// and still fake-fail. Oversized catalogs are a `/plugins` decision, not session growth;
|
|
1018
1233
|
// auto-compaction above still measures the FULL request (`chars()`), because window
|
|
1019
1234
|
// thresholds protect what the model really sees, tools included.
|
|
1020
|
-
const charsConversation = () => instructions.length + JSON.stringify(messages).length;
|
|
1235
|
+
const charsConversation = () => instructions.length + blockChars() + JSON.stringify(messages).length;
|
|
1021
1236
|
const used = () => {
|
|
1022
1237
|
const last = this.reported.get(sessionId);
|
|
1023
1238
|
return last && last.count <= messages.length
|
|
@@ -1060,12 +1275,17 @@ export class AgentRunner {
|
|
|
1060
1275
|
// limit) the run fails with an actionable error.
|
|
1061
1276
|
if (charsConversation() > limit) {
|
|
1062
1277
|
const reduceInPlace = () => {
|
|
1063
|
-
const targetChars = Math.max(MIN_MESSAGE_BUDGET_FLOOR, limit - instructions.length);
|
|
1278
|
+
const targetChars = Math.max(MIN_MESSAGE_BUDGET_FLOOR, limit - instructions.length - blockChars());
|
|
1064
1279
|
const reduction = reduceMessagesToBudget(messages, targetChars);
|
|
1065
1280
|
if (reduction.truncated > 0) {
|
|
1066
1281
|
o.store.overwrite(sessionId, reduction.messages);
|
|
1282
|
+
const tokensSaved = Math.max(0, estimateTokens(messages) - estimateTokens(reduction.messages));
|
|
1067
1283
|
messages = reduction.messages;
|
|
1068
|
-
emit("context_reduced", {
|
|
1284
|
+
emit("context_reduced", {
|
|
1285
|
+
messages: reduction.truncated,
|
|
1286
|
+
reason: "budget",
|
|
1287
|
+
tokensSaved,
|
|
1288
|
+
});
|
|
1069
1289
|
}
|
|
1070
1290
|
};
|
|
1071
1291
|
reduceInPlace();
|
|
@@ -1105,7 +1325,49 @@ export class AgentRunner {
|
|
|
1105
1325
|
// Providers only ever see the text projection of tool results: ui/image parts are a
|
|
1106
1326
|
// display-only extension of the persisted transcript and never reach a model prompt
|
|
1107
1327
|
// (raw image bytes included). The store keeps the rich parts for TUI replay.
|
|
1108
|
-
|
|
1328
|
+
// Context Governor: `off` costs one property read. `shadow` only measures; `on` sends the
|
|
1329
|
+
// projected history. The stored transcript, the compaction triggers and the limits above
|
|
1330
|
+
// always use the raw `messages`.
|
|
1331
|
+
let sent = messages;
|
|
1332
|
+
let invariantsText = "";
|
|
1333
|
+
const governor = o.contextGovernor;
|
|
1334
|
+
if (governor && governor.mode !== "off") {
|
|
1335
|
+
const projection = projectContext(messages, governor);
|
|
1336
|
+
const snapshot = governanceSnapshot(projection, governor, requests + 1, this.governed.get(sessionId));
|
|
1337
|
+
const block = invariants();
|
|
1338
|
+
const done = this.promotions.get(sessionId);
|
|
1339
|
+
snapshot.invariants = {
|
|
1340
|
+
count: block.included,
|
|
1341
|
+
chars: block.text.length,
|
|
1342
|
+
omitted: block.omitted,
|
|
1343
|
+
promoted: done?.promoted ?? 0,
|
|
1344
|
+
rejected: done?.rejected ?? 0,
|
|
1345
|
+
};
|
|
1346
|
+
this.governed.set(sessionId, snapshot);
|
|
1347
|
+
// The ledger only measures: a fault in it never fails the request or changes `sent`.
|
|
1348
|
+
let usageSummary;
|
|
1349
|
+
try {
|
|
1350
|
+
const ledger = this.usageLedger(sessionId, workspace);
|
|
1351
|
+
const stubbed = governor.mode === "on"
|
|
1352
|
+
? new Set(projection.elisions.map((e) => e.callId))
|
|
1353
|
+
: new Set();
|
|
1354
|
+
ledger.record(ledger.seams + 1, messages, stubbed);
|
|
1355
|
+
usageSummary = ledger.summary();
|
|
1356
|
+
}
|
|
1357
|
+
catch {
|
|
1358
|
+
/* Advisory numbers only. */
|
|
1359
|
+
}
|
|
1360
|
+
emit("context_governed", governedEventData(snapshot, usageSummary));
|
|
1361
|
+
if (governor.mode === "on") {
|
|
1362
|
+
sent = projection.messages;
|
|
1363
|
+
invariantsText = block.text;
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
const providerMessages = sent.map((m) => m.role === "tool" ? { ...m, result: textProjection(m.result) } : m);
|
|
1367
|
+
// The invariants block is the last message: built fresh, never saved, never an elision
|
|
1368
|
+
// candidate (it joins the request after the projection).
|
|
1369
|
+
if (invariantsText)
|
|
1370
|
+
providerMessages.push({ role: "user", text: invariantsText });
|
|
1109
1371
|
// After a cut-off response the request carries a transient continuation notice (never
|
|
1110
1372
|
// persisted: the stored transcript only ever holds what the model really answered).
|
|
1111
1373
|
if (recoveryNotice)
|
package/dist/eval/compare.d.ts
CHANGED
|
@@ -46,7 +46,7 @@ export interface Interval {
|
|
|
46
46
|
wilsonHigh?: number;
|
|
47
47
|
}
|
|
48
48
|
declare const CHANGED_FIELDS: readonly ["configHash", "model", "harnessVersion", "mode", "provider"];
|
|
49
|
-
declare const MEASURED_KEYS: readonly ["medianWallMs", "varianceWallMs", "varianceTokens", "tokensPerSuccess", "wallMsPerSuccess"];
|
|
49
|
+
declare const MEASURED_KEYS: readonly ["medianWallMs", "varianceWallMs", "varianceTokens", "tokensPerSuccess", "wallMsPerSuccess", "cachedInputRatio"];
|
|
50
50
|
type MeasuredKey = (typeof MEASURED_KEYS)[number];
|
|
51
51
|
export interface TaskComparison {
|
|
52
52
|
taskId: string;
|
package/dist/eval/compare.js
CHANGED
|
@@ -41,13 +41,18 @@ const MEASURED_KEYS = [
|
|
|
41
41
|
"varianceTokens",
|
|
42
42
|
"tokensPerSuccess",
|
|
43
43
|
"wallMsPerSuccess",
|
|
44
|
+
"cachedInputRatio",
|
|
44
45
|
];
|
|
45
46
|
const delta = (baseline, candidate) => ({
|
|
46
47
|
baseline,
|
|
47
48
|
candidate,
|
|
48
49
|
delta: candidate - baseline,
|
|
49
50
|
});
|
|
50
|
-
const TOKEN_KEYS = [
|
|
51
|
+
const TOKEN_KEYS = [
|
|
52
|
+
"varianceTokens",
|
|
53
|
+
"tokensPerSuccess",
|
|
54
|
+
"cachedInputRatio",
|
|
55
|
+
];
|
|
51
56
|
function compareTask(a, b, ctx) {
|
|
52
57
|
const lowPower = a.k < MIN_POWER_K || b.k < MIN_POWER_K;
|
|
53
58
|
const difference = b.successRate.value - a.successRate.value;
|
|
@@ -263,6 +268,8 @@ const reportSchema = z.object({
|
|
|
263
268
|
varianceTokens: z.number().optional(),
|
|
264
269
|
tokensPerSuccess: z.number().optional(),
|
|
265
270
|
wallMsPerSuccess: z.number().optional(),
|
|
271
|
+
cachedInputRatio: z.number().optional(),
|
|
272
|
+
cachedInputRuns: z.number().optional(),
|
|
266
273
|
}),
|
|
267
274
|
estimated: z.object({ costUsdPerSuccess: z.number().optional() }).optional(),
|
|
268
275
|
})),
|
|
@@ -274,6 +281,15 @@ const reportSchema = z.object({
|
|
|
274
281
|
budgetExceeded: z.number(),
|
|
275
282
|
runs: z.number().optional(),
|
|
276
283
|
failed: z.number().optional(),
|
|
284
|
+
cachedInput: z
|
|
285
|
+
.object({
|
|
286
|
+
tokens: z.number(),
|
|
287
|
+
inputTokens: z.number(),
|
|
288
|
+
ratio: z.number(),
|
|
289
|
+
runsReporting: z.number(),
|
|
290
|
+
runsJudged: z.number(),
|
|
291
|
+
})
|
|
292
|
+
.optional(),
|
|
277
293
|
}),
|
|
278
294
|
rates: z.object({ inputPerMTokUsd: z.number(), outputPerMTokUsd: z.number() }).optional(),
|
|
279
295
|
});
|
|
@@ -315,6 +331,7 @@ const METRIC_LABEL = {
|
|
|
315
331
|
varianceTokens: "Token variance (main-loop tokens)",
|
|
316
332
|
tokensPerSuccess: "Tokens per success (main-loop tokens, excludes compaction calls)",
|
|
317
333
|
wallMsPerSuccess: "Wall per success (ms)",
|
|
334
|
+
cachedInputRatio: "Cached input ratio (provider-reported cached / input tokens)",
|
|
318
335
|
};
|
|
319
336
|
const number = (value) => String(Math.round(value * 1000) / 1000);
|
|
320
337
|
function table(headers, rows) {
|
package/dist/eval/engine.js
CHANGED
|
@@ -83,6 +83,19 @@ export const CONFIG_HASH_COVERAGE = {
|
|
|
83
83
|
agents: "excluded",
|
|
84
84
|
web: "excluded",
|
|
85
85
|
};
|
|
86
|
+
/**
|
|
87
|
+
* The Context Governor keys exist only since spec 012. A switched-off governor with default
|
|
88
|
+
* settings leaves them out, so the hashes of runs recorded before they existed stay valid.
|
|
89
|
+
*/
|
|
90
|
+
function hashedContext(context) {
|
|
91
|
+
const { governor, elide, policyVersion, invariants, ...rest } = context;
|
|
92
|
+
const untouched = governor === "off" &&
|
|
93
|
+
policyVersion === undefined &&
|
|
94
|
+
elide.minAgeTurns === 2 &&
|
|
95
|
+
elide.epochMessages === 32 &&
|
|
96
|
+
invariants.maxChars === 4000;
|
|
97
|
+
return untouched ? rest : context;
|
|
98
|
+
}
|
|
86
99
|
/**
|
|
87
100
|
* `sha256:` of the allowlisted effective configuration plus policy, approval mode, budget and
|
|
88
101
|
* enabled built-ins. Excludes secrets, env names, plugin options, absolute paths, endpoints, the
|
|
@@ -93,7 +106,7 @@ export function computeConfigHash(config, extras) {
|
|
|
93
106
|
const picked = {
|
|
94
107
|
limits: config.limits,
|
|
95
108
|
compaction: config.compaction,
|
|
96
|
-
context: config.context,
|
|
109
|
+
context: hashedContext(config.context),
|
|
97
110
|
forge: {
|
|
98
111
|
javascript: forge.javascript,
|
|
99
112
|
state: forge.state,
|
|
@@ -289,6 +302,10 @@ class RunObserver {
|
|
|
289
302
|
/** Cumulative tokens reported by the last `turn_completed`. */
|
|
290
303
|
tokens = 0;
|
|
291
304
|
usage;
|
|
305
|
+
/** Sum of the cached input tokens reported; undefined until a turn reports any. */
|
|
306
|
+
cachedInput;
|
|
307
|
+
/** Turns whose usage carried `cachedInput`. */
|
|
308
|
+
turnsWithCached = 0;
|
|
292
309
|
model;
|
|
293
310
|
timedOut = false;
|
|
294
311
|
turnsExceeded = false;
|
|
@@ -311,6 +328,10 @@ class RunObserver {
|
|
|
311
328
|
this.usage ??= { input: 0, output: 0 };
|
|
312
329
|
this.usage.input += usage.input;
|
|
313
330
|
this.usage.output += usage.output;
|
|
331
|
+
if (typeof usage.cachedInput === "number") {
|
|
332
|
+
this.turnsWithCached += 1;
|
|
333
|
+
this.cachedInput = (this.cachedInput ?? 0) + usage.cachedInput;
|
|
334
|
+
}
|
|
314
335
|
}
|
|
315
336
|
break;
|
|
316
337
|
}
|
|
@@ -386,6 +407,10 @@ class RunObserver {
|
|
|
386
407
|
get usagePartial() {
|
|
387
408
|
return this.turnsWithUsage > 0 && this.turnsWithUsage < this.turns;
|
|
388
409
|
}
|
|
410
|
+
/** Some turns that reported usage did not report cached tokens: the cached sum understates. */
|
|
411
|
+
get cachedPartial() {
|
|
412
|
+
return this.turnsWithCached > 0 && this.turnsWithCached < this.turnsWithUsage;
|
|
413
|
+
}
|
|
389
414
|
markAborted() {
|
|
390
415
|
this.bump("run_aborted");
|
|
391
416
|
}
|
|
@@ -718,6 +743,11 @@ async function execute(task, opts, plan, fixtureDir) {
|
|
|
718
743
|
measured: {
|
|
719
744
|
...(usage ? { inputTokens: usage.input, outputTokens: usage.output } : {}),
|
|
720
745
|
...(usage && observer.usagePartial ? { usagePartial: true } : {}),
|
|
746
|
+
// Only the observer sees `cachedInput`; absent stays absent (unknown is not 0).
|
|
747
|
+
...(observer.cachedInput !== undefined ? { cachedInputTokens: observer.cachedInput } : {}),
|
|
748
|
+
...(observer.cachedInput !== undefined && observer.cachedPartial
|
|
749
|
+
? { cachedPartial: true }
|
|
750
|
+
: {}),
|
|
721
751
|
wallMs,
|
|
722
752
|
turns: observer.turns,
|
|
723
753
|
toolCalls: observer.startedCalls,
|
package/dist/eval/report.js
CHANGED
|
@@ -56,6 +56,21 @@ function assertUniform(records, field, hint) {
|
|
|
56
56
|
if (values.length > 1)
|
|
57
57
|
throw new EvalReportError(`Cannot aggregate runs measured under different ${field}: ${values.map((v) => `"${v}"`).join(", ")}. ${hint}`);
|
|
58
58
|
}
|
|
59
|
+
/** Runs that reported cached input completely: judged, no partial usage, no partial cached sum. */
|
|
60
|
+
function cachedReporters(judged) {
|
|
61
|
+
return judged.filter((r) => r.measured.cachedInputTokens !== undefined &&
|
|
62
|
+
!r.measured.cachedPartial &&
|
|
63
|
+
r.measured.inputTokens !== undefined &&
|
|
64
|
+
!r.measured.usagePartial);
|
|
65
|
+
}
|
|
66
|
+
function cachedTotals(judged) {
|
|
67
|
+
const reporters = cachedReporters(judged);
|
|
68
|
+
if (reporters.length === 0)
|
|
69
|
+
return undefined;
|
|
70
|
+
const tokens = sum(reporters.map((r) => r.measured.cachedInputTokens));
|
|
71
|
+
const inputTokens = sum(reporters.map((r) => r.measured.inputTokens));
|
|
72
|
+
return { tokens, inputTokens, runsReporting: reporters.length };
|
|
73
|
+
}
|
|
59
74
|
function taskMetrics(taskId, runs, rates) {
|
|
60
75
|
const judged = runs.filter((r) => r.status !== "error");
|
|
61
76
|
const errored = runs.length - judged.length;
|
|
@@ -71,6 +86,7 @@ function taskMetrics(taskId, runs, rates) {
|
|
|
71
86
|
const varianceWallMs = sampleVariance(wallTimes);
|
|
72
87
|
// Same rule as tokensPerSuccess: a spread over the runs that happened to report would mislead.
|
|
73
88
|
const varianceTokens = allReportedUsage ? sampleVariance(tokenSamples) : null;
|
|
89
|
+
const cached = cachedTotals(judged);
|
|
74
90
|
const measured = {
|
|
75
91
|
...(medianWallMs === null ? {} : { medianWallMs }),
|
|
76
92
|
...(varianceWallMs === null ? {} : { varianceWallMs }),
|
|
@@ -79,6 +95,13 @@ function taskMetrics(taskId, runs, rates) {
|
|
|
79
95
|
? { tokensPerSuccess: sum(tokenSamples) / successes }
|
|
80
96
|
: {}),
|
|
81
97
|
...(successes > 0 ? { wallMsPerSuccess: sum(wallTimes) / successes } : {}),
|
|
98
|
+
// A ratio over zero input tokens means nothing: leave it out rather than divide by zero.
|
|
99
|
+
...(cached && cached.inputTokens > 0
|
|
100
|
+
? {
|
|
101
|
+
cachedInputRatio: cached.tokens / cached.inputTokens,
|
|
102
|
+
cachedInputRuns: cached.runsReporting,
|
|
103
|
+
}
|
|
104
|
+
: {}),
|
|
82
105
|
};
|
|
83
106
|
const costs = rates ? judged.map((r) => estimateRunCost(r, rates)) : [];
|
|
84
107
|
const costKnown = rates !== undefined && successes > 0 && costs.every((c) => c !== undefined);
|
|
@@ -147,6 +170,8 @@ export function buildReport(records, options = {}) {
|
|
|
147
170
|
? taskHashes[0][1]
|
|
148
171
|
: `sha256:${createHash("sha256").update(JSON.stringify(taskHashes)).digest("hex")}`;
|
|
149
172
|
const count = (status) => records.filter((r) => r.status === status).length;
|
|
173
|
+
const judgedRuns = records.filter((r) => r.status !== "error");
|
|
174
|
+
const armCached = cachedTotals(judgedRuns);
|
|
150
175
|
return {
|
|
151
176
|
schemaVersion: 1,
|
|
152
177
|
suiteId: first.suiteId,
|
|
@@ -174,6 +199,17 @@ export function buildReport(records, options = {}) {
|
|
|
174
199
|
budgetExceeded: count("budget_exceeded"),
|
|
175
200
|
runs: records.length,
|
|
176
201
|
failed: count("fail"),
|
|
202
|
+
...(armCached && armCached.inputTokens > 0
|
|
203
|
+
? {
|
|
204
|
+
cachedInput: {
|
|
205
|
+
tokens: armCached.tokens,
|
|
206
|
+
inputTokens: armCached.inputTokens,
|
|
207
|
+
ratio: armCached.tokens / armCached.inputTokens,
|
|
208
|
+
runsReporting: armCached.runsReporting,
|
|
209
|
+
runsJudged: judgedRuns.length,
|
|
210
|
+
},
|
|
211
|
+
}
|
|
212
|
+
: {}),
|
|
177
213
|
},
|
|
178
214
|
...(options.rates ? { rates: { ...options.rates } } : {}),
|
|
179
215
|
};
|
|
@@ -198,6 +234,13 @@ function table(headers, rows) {
|
|
|
198
234
|
export function lowPowerTasks(report) {
|
|
199
235
|
return report.tasks.filter((t) => t.k < MIN_POWER_K);
|
|
200
236
|
}
|
|
237
|
+
/** The arm's cached-input line: the provider's own figure, or an explicit "not reported". */
|
|
238
|
+
function cachedLine(report) {
|
|
239
|
+
const c = report.summary.cachedInput;
|
|
240
|
+
if (!c)
|
|
241
|
+
return "Cached input: not reported by the provider (no ratio is computed; absent is not zero).";
|
|
242
|
+
return `Cached input: ${percent(c.ratio)} of input tokens (${count(c.tokens)} of ${count(c.inputTokens)}), from ${c.runsReporting} of ${c.runsJudged} judged runs that reported it.`;
|
|
243
|
+
}
|
|
201
244
|
/** Markdown of a report: conditions, warnings, then Measured and Estimated as separate sections. */
|
|
202
245
|
export function renderReportMarkdown(report) {
|
|
203
246
|
const { conditions: c } = report;
|
|
@@ -219,7 +262,7 @@ export function renderReportMarkdown(report) {
|
|
|
219
262
|
if (weak.length > 0)
|
|
220
263
|
lines.push(`> **Low power:** ${weak.map((t) => `${t.taskId} (k=${t.k})`).join(", ")} ${weak.length === 1 ? "has" : "have"} fewer than ${MIN_POWER_K} judged runs. The intervals are wide; do not read an improvement or a regression from these numbers.`, "");
|
|
221
264
|
const { summary: s } = report;
|
|
222
|
-
lines.push(`Runs: ${s.runs ?? report.runs.length} (passed ${s.passed}, failed ${s.failed ?? "-"}, budget exceeded ${s.budgetExceeded}, errored ${s.errored}). Errored runs are infrastructure failures: they are excluded from every rate and spread below.`, "", "## Measured", "", "Provider-reported usage and wall time. Success is passes over judged runs, with a 95% Wilson interval; sd is the square root of the sample variance. Token figures are main-loop tokens (excludes compaction calls).", "", table([
|
|
265
|
+
lines.push(`Runs: ${s.runs ?? report.runs.length} (passed ${s.passed}, failed ${s.failed ?? "-"}, budget exceeded ${s.budgetExceeded}, errored ${s.errored}). Errored runs are infrastructure failures: they are excluded from every rate and spread below.`, "", "## Measured", "", cachedLine(report), "", "Provider-reported usage and wall time. Success is passes over judged runs, with a 95% Wilson interval; sd is the square root of the sample variance. Token figures are main-loop tokens (excludes compaction calls).", "", table([
|
|
223
266
|
"Task",
|
|
224
267
|
"k",
|
|
225
268
|
"Success [Wilson 95%]",
|
|
@@ -4,7 +4,7 @@ export declare const PROCESS_KILL_BACKSTOP_MS = 500;
|
|
|
4
4
|
* The isolate's whole environment: the marker, and on Windows `SystemRoot` and nothing else (the
|
|
5
5
|
* runtime cannot start without it). No other variable of the parent ever reaches the guest process.
|
|
6
6
|
*/
|
|
7
|
-
export declare function isolateEnv(platform?: NodeJS.Platform, parent?: NodeJS.ProcessEnv): Record<string, string>;
|
|
7
|
+
export declare function isolateEnv(platform?: NodeJS.Platform, parent?: NodeJS.ProcessEnv, electron?: boolean): Record<string, string>;
|
|
8
8
|
export interface IsolateCommand {
|
|
9
9
|
execPath: string;
|
|
10
10
|
args: string[];
|
|
@@ -23,8 +23,11 @@ const EXIT_CONFIRM_MS = 2_000;
|
|
|
23
23
|
* The isolate's whole environment: the marker, and on Windows `SystemRoot` and nothing else (the
|
|
24
24
|
* runtime cannot start without it). No other variable of the parent ever reaches the guest process.
|
|
25
25
|
*/
|
|
26
|
-
export function isolateEnv(platform = process.platform, parent = process.env) {
|
|
26
|
+
export function isolateEnv(platform = process.platform, parent = process.env, electron = !!process.versions.electron) {
|
|
27
27
|
const env = { ALISIO_FORGE_ISOLATE: "1" };
|
|
28
|
+
// The Electron binary only behaves as Node with this flag; it carries no parent data.
|
|
29
|
+
if (electron)
|
|
30
|
+
env.ELECTRON_RUN_AS_NODE = "1";
|
|
28
31
|
if (platform === "win32" && parent.SystemRoot)
|
|
29
32
|
env.SystemRoot = parent.SystemRoot;
|
|
30
33
|
return env;
|
|
@@ -38,7 +38,7 @@ export declare function permissionArgs(paths: {
|
|
|
38
38
|
write: readonly string[];
|
|
39
39
|
}): string[];
|
|
40
40
|
/** Environment of the child: nothing but the job's `ALISIO_*` variables. */
|
|
41
|
-
export declare function childEnv(vars: Record<string, string>, platform?: NodeJS.Platform): Record<string, string>;
|
|
41
|
+
export declare function childEnv(vars: Record<string, string>, platform?: NodeJS.Platform, electron?: boolean): Record<string, string>;
|
|
42
42
|
export interface NetworkProbeResult {
|
|
43
43
|
/**
|
|
44
44
|
* The Permission Model refused the connection (`ERR_ACCESS_DENIED`). `false` also when the probe
|
|
@@ -126,11 +126,14 @@ export function permissionArgs(paths) {
|
|
|
126
126
|
];
|
|
127
127
|
}
|
|
128
128
|
/** Environment of the child: nothing but the job's `ALISIO_*` variables. */
|
|
129
|
-
export function childEnv(vars, platform = process.platform) {
|
|
129
|
+
export function childEnv(vars, platform = process.platform, electron = !!process.versions.electron) {
|
|
130
130
|
const env = {};
|
|
131
131
|
for (const [key, value] of Object.entries(vars))
|
|
132
132
|
if (key.startsWith("ALISIO_"))
|
|
133
133
|
env[key] = value;
|
|
134
|
+
// The Electron binary only behaves as Node with this flag; it carries no parent data.
|
|
135
|
+
if (electron)
|
|
136
|
+
env.ELECTRON_RUN_AS_NODE = "1";
|
|
134
137
|
// Windows cannot start a process without these.
|
|
135
138
|
if (platform === "win32" && process.env.SystemRoot)
|
|
136
139
|
env.SystemRoot = process.env.SystemRoot;
|