@alisio/core 0.8.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/dist/analysis/data/engine-client.d.ts +2 -0
  2. package/dist/analysis/data/engine-client.js +11 -4
  3. package/dist/application.d.ts +11 -1
  4. package/dist/application.js +46 -0
  5. package/dist/changelog/data.js +42 -0
  6. package/dist/commands/builtins.js +6 -0
  7. package/dist/commands/catalog.d.ts +4 -2
  8. package/dist/commands/catalog.js +23 -1
  9. package/dist/config.d.ts +31 -0
  10. package/dist/config.js +31 -1
  11. package/dist/context/elide.d.ts +50 -0
  12. package/dist/context/elide.js +222 -0
  13. package/dist/context/format.d.ts +21 -0
  14. package/dist/context/format.js +81 -0
  15. package/dist/context/governance.d.ts +58 -0
  16. package/dist/context/governance.js +96 -0
  17. package/dist/context/invariants.d.ts +88 -0
  18. package/dist/context/invariants.js +141 -0
  19. package/dist/context/inventory.d.ts +22 -0
  20. package/dist/context/inventory.js +178 -0
  21. package/dist/context/report.d.ts +35 -0
  22. package/dist/context/report.js +120 -0
  23. package/dist/context/sources.d.ts +20 -0
  24. package/dist/context/sources.js +41 -0
  25. package/dist/context/types.d.ts +43 -0
  26. package/dist/context/types.js +5 -0
  27. package/dist/context/usage.d.ts +96 -0
  28. package/dist/context/usage.js +264 -0
  29. package/dist/context/util.d.ts +22 -0
  30. package/dist/context/util.js +36 -0
  31. package/dist/core/compaction.d.ts +7 -2
  32. package/dist/core/compaction.js +9 -3
  33. package/dist/core/contracts.d.ts +8 -0
  34. package/dist/core/runner.d.ts +67 -0
  35. package/dist/core/runner.js +272 -10
  36. package/dist/eval/compare.d.ts +1 -1
  37. package/dist/eval/compare.js +18 -1
  38. package/dist/eval/engine.js +31 -1
  39. package/dist/eval/report.js +44 -1
  40. package/dist/forge/javascript/hosts/process-host.d.ts +1 -1
  41. package/dist/forge/javascript/hosts/process-host.js +4 -1
  42. package/dist/forge/python/wasm-node.d.ts +1 -1
  43. package/dist/forge/python/wasm-node.js +4 -1
  44. package/dist/index.d.ts +8 -2
  45. package/dist/index.js +8 -2
  46. package/dist/plugins/host.d.ts +5 -0
  47. package/dist/plugins/host.js +15 -1
  48. package/dist/plugins/install.d.ts +50 -2
  49. package/dist/plugins/install.js +102 -12
  50. package/dist/plugins/local-install.d.ts +33 -0
  51. package/dist/plugins/local-install.js +121 -0
  52. package/dist/plugins/npm-runner.d.ts +30 -0
  53. package/dist/plugins/npm-runner.js +66 -0
  54. package/dist/plugins/tarball.d.ts +39 -0
  55. package/dist/plugins/tarball.js +277 -0
  56. package/dist/plugins/version-compat.d.ts +20 -0
  57. package/dist/plugins/version-compat.js +154 -0
  58. package/package.json +2 -2
@@ -1,9 +1,13 @@
1
1
  import { isEphemeralRunEventType, textProjection, textResult, } from "@alisio/sdk";
2
+ import { projectContext } from "../context/elide.js";
3
+ import { governanceSnapshot, governedEventData, } from "../context/governance.js";
4
+ import { collectInvariants, promoteInstructions, readInvariantState, renderInvariants, } from "../context/invariants.js";
5
+ import { UsageLedger } from "../context/usage.js";
2
6
  import { createNestedDispatcher } from "../forge/nested.js";
3
7
  import { hasScratchEstimator, scratchIncomingBytes } from "../forge/scratch-quota.js";
4
8
  import { decideEffect, effectAskable } from "../permissions/decide.js";
5
9
  import { bindCallInternals } from "./call-internals.js";
6
- import { checkpointInstructions, estimateTokens, MAX_TRUSTED_WINDOW, parseCheckpointOutput, planCompaction, reduceMessagesToBudget, shouldCompactContext, summarize, summaryMessage, } from "./compaction.js";
10
+ import { checkpointInstructions, estimateTokens, INVARIANTS_FIELD, MAX_TRUSTED_WINDOW, parseCheckpointOutput, planCompaction, reduceMessagesToBudget, shouldCompactContext, summarize, summaryMessage, } from "./compaction.js";
7
11
  import { HumanWaits } from "./human-wait.js";
8
12
  import { EXIT_PLAN_TOOL, optInAllows } from "./opt-in.js";
9
13
  import { resolveMaxOutputTokens } from "./output-limit.js";
@@ -52,8 +56,12 @@ const MIN_MESSAGE_BUDGET_FLOOR = 4_000;
52
56
  * would pay for a summarizer call. The hard-limit last-chance compaction ignores it.
53
57
  */
54
58
  const RECOMPACT_GROWTH_MARGIN = 0.1;
59
+ /** Sessions whose usage ledger one runner keeps in memory (least recently used go first). */
60
+ const USAGE_LEDGER_SESSIONS = 64;
55
61
  /** First 40 lines of a script input, for capability approvals. */
56
62
  const scriptPreview = (input) => typeof input.code === "string" ? input.code.split(/\r?\n/).slice(0, 40).join("\n") : undefined;
63
+ /** `{ [key]: value }`, or nothing when there is no value (optional properties stay absent). */
64
+ const optional = (key, value) => (value === undefined ? {} : { [key]: value });
57
65
  /** A short id for a tool call the user started from a UI (alphanumeric, 9 characters). */
58
66
  const uiCallId = () => {
59
67
  const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
@@ -70,6 +78,16 @@ export class AgentRunner {
70
78
  reported = new Map();
71
79
  /** Conversation size in characters right after the last automatic compaction, per session. */
72
80
  compactedChars = new Map();
81
+ /** What the Context Governor decided for the latest request of each session (shadow and on). */
82
+ governed = new Map();
83
+ /** Usage ledger per session (shadow and on; process-local, LRU of `USAGE_LEDGER_SESSIONS`). */
84
+ usage = new Map();
85
+ /** Effects the user approved for the whole session, per session (`on` mode; process-local). */
86
+ sessionEffects = new Map();
87
+ /** Instruction quotes of a session whose store cannot keep session options (process-local). */
88
+ invariantMemory = new Map();
89
+ /** Quotes promoted and refused by the compactions of this process, per session. */
90
+ promotions = new Map();
73
91
  /** Where human waits are opened; the clocks of the active runs listen to it. */
74
92
  humanWaits;
75
93
  constructor(options) {
@@ -267,6 +285,173 @@ export class AgentRunner {
267
285
  fallback: o.fallbackMaxOutputTokens ?? 4096,
268
286
  });
269
287
  }
288
+ /** Latest Context Governor decision for a session (core-internal; counts and ids only). */
289
+ governance(sessionId) {
290
+ return this.governed.get(sessionId);
291
+ }
292
+ /**
293
+ * Heuristic usage of the tool results this process showed the model in a session (shadow and on
294
+ * only; undefined otherwise). Lost on restart; advisory, it never drives elision.
295
+ */
296
+ contextUsage(sessionId) {
297
+ return this.usage.get(sessionId)?.summary();
298
+ }
299
+ usageLedger(sessionId, workspace) {
300
+ let ledger = this.usage.get(sessionId);
301
+ if (ledger)
302
+ this.usage.delete(sessionId);
303
+ else {
304
+ const registry = this.options.registry;
305
+ ledger = new UsageLedger({
306
+ workspace,
307
+ pathsOf: (tool, args) => {
308
+ try {
309
+ return registry.get(tool).paths?.(args);
310
+ }
311
+ catch {
312
+ return undefined;
313
+ }
314
+ },
315
+ isRead: (tool) => {
316
+ try {
317
+ return registry.get(tool).effect === "read";
318
+ }
319
+ catch {
320
+ return false;
321
+ }
322
+ },
323
+ });
324
+ }
325
+ this.usage.set(sessionId, ledger);
326
+ while (this.usage.size > USAGE_LEDGER_SESSIONS) {
327
+ const oldest = this.usage.keys().next().value;
328
+ if (oldest === undefined)
329
+ break;
330
+ this.usage.delete(oldest);
331
+ }
332
+ return ledger;
333
+ }
334
+ /**
335
+ * Forgets the user instructions kept for a session: the stored `contextInvariants` and every
336
+ * in-memory entry. Core-internal; there is no per-item removal before phase 5.
337
+ */
338
+ clearInvariants(sessionId) {
339
+ this.invariantMemory.delete(sessionId);
340
+ this.usage.delete(sessionId);
341
+ this.sessionEffects.delete(sessionId);
342
+ this.promotions.delete(sessionId);
343
+ this.options.store.mutateSessionOptions?.(sessionId, (current) => ({
344
+ ...("contextInvariants" in current ? { patch: { contextInvariants: undefined } } : {}),
345
+ value: undefined,
346
+ }));
347
+ }
348
+ /** The stored instruction list of a session, or undefined for old or never-compacted sessions. */
349
+ invariantState(sessionId) {
350
+ const store = this.options.store;
351
+ return store.mutateSessionOptions
352
+ ? readInvariantState(store.get(sessionId).options)
353
+ : this.invariantMemory.get(sessionId);
354
+ }
355
+ /**
356
+ * The `<invariants>` block for the next request of a session: empty before the first compaction
357
+ * (a history that starts with a checkpoint, or a stored compaction count). Built fresh from the
358
+ * stored instructions and the live sources, never saved.
359
+ */
360
+ invariantBlock(sessionId, messages, maxChars) {
361
+ const list = this.invariantList(sessionId, messages);
362
+ return list ? renderInvariants(list, maxChars) : { text: "", included: 0, omitted: 0 };
363
+ }
364
+ /** The collected invariants, or undefined before the first compaction (see `invariantBlock`). */
365
+ invariantList(sessionId, messages) {
366
+ const state = this.invariantState(sessionId);
367
+ const first = messages[0];
368
+ const compacted = (state?.compactions ?? 0) > 0 ||
369
+ (first?.role === "user" && !!first.summary && first.text.startsWith("[Checkpoint of"));
370
+ if (!compacted)
371
+ return undefined;
372
+ const sources = this.options.invariantSources;
373
+ // A broken source never stops a request: its section is simply left out.
374
+ const read = (source) => {
375
+ try {
376
+ return source?.();
377
+ }
378
+ catch {
379
+ return undefined;
380
+ }
381
+ };
382
+ const grants = read(sources?.grants?.bind(sources, sessionId));
383
+ const effects = [...(this.sessionEffects.get(sessionId) ?? [])];
384
+ return collectInvariants({
385
+ instructions: state?.instructions ?? [],
386
+ ...optional("goal", read(sources?.goal?.bind(sources, sessionId))),
387
+ ...optional("plan", read(sources?.plan?.bind(sources, sessionId))),
388
+ ...optional("criteria", read(sources?.criteria?.bind(sources, sessionId))),
389
+ ...(grants || effects.length
390
+ ? {
391
+ grants: {
392
+ effects,
393
+ capabilities: grants?.capabilities ?? [],
394
+ externalDirectories: grants?.externalDirectories ?? 0,
395
+ },
396
+ }
397
+ : {}),
398
+ });
399
+ }
400
+ /**
401
+ * What the `ContextReport` of a session is built from (spec 012 phase 5): the governor settings,
402
+ * the active history, the invariants of the next request and the in-memory usage ledger. Read
403
+ * only. `off` and runners without a governor return no history, because `off` measures nothing.
404
+ */
405
+ contextInputs(sessionId) {
406
+ const governor = this.options.contextGovernor;
407
+ if (!governor)
408
+ return undefined;
409
+ if (governor.mode === "off")
410
+ return { governor, messages: [], invariants: { items: [], omitted: 0 } };
411
+ const messages = this.options.store.messages(sessionId);
412
+ const list = this.invariantList(sessionId, messages) ?? [];
413
+ const omitted = list.length
414
+ ? renderInvariants(list, governor.invariantMaxChars ?? 4000).omitted
415
+ : 0;
416
+ const ledger = this.usage.get(sessionId);
417
+ const governance = this.governed.get(sessionId);
418
+ return {
419
+ governor,
420
+ messages,
421
+ invariants: { items: list.map((i) => ({ kind: i.kind, text: i.text })), omitted },
422
+ ...(ledger ? { ledger } : {}),
423
+ ...(governance ? { governance } : {}),
424
+ };
425
+ }
426
+ /** Keeps the user's verbatim quotes the summarizer proposed (see `promoteInstructions`). */
427
+ promoteInvariants(sessionId, proposed, summarized) {
428
+ const store = this.options.store;
429
+ const apply = (state) => {
430
+ const out = promoteInstructions(state?.instructions ?? [], proposed, summarized);
431
+ const next = {
432
+ version: 1,
433
+ instructions: out.instructions,
434
+ compactions: (state?.compactions ?? 0) + 1,
435
+ updatedAt: Date.now(),
436
+ };
437
+ return { next, out };
438
+ };
439
+ const { out } = store.mutateSessionOptions
440
+ ? store.mutateSessionOptions(sessionId, (current) => {
441
+ const result = apply(readInvariantState(current));
442
+ return { patch: { contextInvariants: result.next }, value: result };
443
+ })
444
+ : (() => {
445
+ const result = apply(this.invariantMemory.get(sessionId));
446
+ this.invariantMemory.set(sessionId, result.next);
447
+ return result;
448
+ })();
449
+ const total = this.promotions.get(sessionId) ?? { promoted: 0, rejected: 0 };
450
+ this.promotions.set(sessionId, {
451
+ promoted: total.promoted + out.accepted,
452
+ rejected: total.rejected + out.rejected,
453
+ });
454
+ }
270
455
  /** Change the model used by subsequent turns of a session; recorded in the store. */
271
456
  setModel(sessionId, model) {
272
457
  const id = model.trim();
@@ -277,6 +462,7 @@ export class AgentRunner {
277
462
  return;
278
463
  this.options.store.setModel(sessionId, id);
279
464
  this.reported.delete(sessionId);
465
+ this.governed.delete(sessionId);
280
466
  this.emitter(sessionId)("model_changed", { model: id, previous });
281
467
  }
282
468
  /** Manually compact a session outside a run. */
@@ -337,10 +523,14 @@ export class AgentRunner {
337
523
  });
338
524
  }
339
525
  reportFailures(contribution.failures);
340
- const fields = Object.fromEntries(Object.entries(contribution.fields).map(([name, f]) => [name, f.description]));
526
+ // Only `on` asks the summarizer for verbatim quotes; a plugin field of the same name is ignored.
527
+ const govern = o.contextGovernor?.mode === "on";
528
+ const fields = Object.fromEntries(Object.entries(contribution.fields)
529
+ .filter(([name]) => !(govern && name === INVARIANTS_FIELD))
530
+ .map(([name, f]) => [name, f.description]));
341
531
  const provider = await (o.providerFor?.(o.store.get(sessionId)) ?? o.provider);
342
532
  const { text: raw, truncated: summaryTruncated } = await summarize(provider, {
343
- instructions: checkpointInstructions(contribution.instructions, fields),
533
+ instructions: checkpointInstructions(contribution.instructions, fields, govern),
344
534
  messages: plan.summarized,
345
535
  focus,
346
536
  model,
@@ -348,7 +538,9 @@ export class AgentRunner {
348
538
  signal,
349
539
  sessionId,
350
540
  });
351
- const output = parseCheckpointOutput(raw, Object.keys(fields));
541
+ const parsed = parseCheckpointOutput(raw, govern ? [...Object.keys(fields), INVARIANTS_FIELD] : Object.keys(fields));
542
+ const { [INVARIANTS_FIELD]: proposed, ...extracted } = parsed.extracted;
543
+ const output = govern ? { ...parsed, extracted } : parsed;
352
544
  let after = {
353
545
  inject: [],
354
546
  reports: {},
@@ -374,8 +566,12 @@ export class AgentRunner {
374
566
  .filter((x) => x.trim())
375
567
  .join("\n\n");
376
568
  const summary = summaryMessage(body, plan.cut);
569
+ // Before the history is replaced: if the process dies in between, the next compaction dedupes.
570
+ if (govern)
571
+ this.promoteInvariants(sessionId, proposed, plan.summarized);
377
572
  o.store.compact(sessionId, plan.cut, summary);
378
573
  this.reported.delete(sessionId);
574
+ this.governed.delete(sessionId);
379
575
  const afterTokens = await this.estimateContext(sessionId);
380
576
  emit("compaction_completed", {
381
577
  reason,
@@ -568,8 +764,14 @@ export class AgentRunner {
568
764
  if (decision === "deny")
569
765
  throw new Error(`Capability ${capability ?? effect} denied by the user for this call`);
570
766
  // A capability grant never widens the effect: only effect approvals do.
571
- if (decision === "session" && !capability)
767
+ if (decision === "session" && !capability) {
572
768
  policy[effect] = true;
769
+ if (o.contextGovernor?.mode === "on" &&
770
+ (effect === "write" || effect === "process" || effect === "external")) {
771
+ const effects = this.sessionEffects.get(sessionId) ?? new Set();
772
+ this.sessionEffects.set(sessionId, effects.add(effect));
773
+ }
774
+ }
573
775
  }
574
776
  o.store.beginCall(sessionId, call, { runId, effect });
575
777
  const publisher = o.artifacts?.({ sessionId, runId, callId: call.id }, (published) => emit("artifact_published", {
@@ -1009,7 +1211,20 @@ export class AgentRunner {
1009
1211
  let messages = o.store.messages(sessionId);
1010
1212
  const tools = this.availableTools(options, sessionId);
1011
1213
  const toolsText = this.toolsText(tools);
1012
- const chars = () => instructions.length + JSON.stringify(messages).length + toolsText.length;
1214
+ // The `<invariants>` block (`on` only) is part of every request after the first compaction,
1215
+ // so it counts like instructions. Built once per history; shadow measures it at the seam.
1216
+ const maxInvariantChars = o.contextGovernor?.invariantMaxChars ?? 4000;
1217
+ let invariantMemo;
1218
+ const invariants = () => {
1219
+ if (invariantMemo?.for !== messages)
1220
+ invariantMemo = {
1221
+ for: messages,
1222
+ value: this.invariantBlock(sessionId, messages, maxInvariantChars),
1223
+ };
1224
+ return invariantMemo.value;
1225
+ };
1226
+ const blockChars = () => (o.contextGovernor?.mode === "on" ? invariants().text.length : 0);
1227
+ const chars = () => instructions.length + blockChars() + JSON.stringify(messages).length + toolsText.length;
1013
1228
  // REDUCIBLE content only (instructions + transcript). The fixed tool catalog
1014
1229
  // (`toolsText`) is a deployment reality — a huge MCP catalog with dozens of tools is
1015
1230
  // part of every request regardless of history, so it is deliberately excluded from the
@@ -1017,7 +1232,7 @@ export class AgentRunner {
1017
1232
  // and still fake-fail. Oversized catalogs are a `/plugins` decision, not session growth;
1018
1233
  // auto-compaction above still measures the FULL request (`chars()`), because window
1019
1234
  // thresholds protect what the model really sees, tools included.
1020
- const charsConversation = () => instructions.length + JSON.stringify(messages).length;
1235
+ const charsConversation = () => instructions.length + blockChars() + JSON.stringify(messages).length;
1021
1236
  const used = () => {
1022
1237
  const last = this.reported.get(sessionId);
1023
1238
  return last && last.count <= messages.length
@@ -1060,12 +1275,17 @@ export class AgentRunner {
1060
1275
  // limit) the run fails with an actionable error.
1061
1276
  if (charsConversation() > limit) {
1062
1277
  const reduceInPlace = () => {
1063
- const targetChars = Math.max(MIN_MESSAGE_BUDGET_FLOOR, limit - instructions.length);
1278
+ const targetChars = Math.max(MIN_MESSAGE_BUDGET_FLOOR, limit - instructions.length - blockChars());
1064
1279
  const reduction = reduceMessagesToBudget(messages, targetChars);
1065
1280
  if (reduction.truncated > 0) {
1066
1281
  o.store.overwrite(sessionId, reduction.messages);
1282
+ const tokensSaved = Math.max(0, estimateTokens(messages) - estimateTokens(reduction.messages));
1067
1283
  messages = reduction.messages;
1068
- emit("context_reduced", { messages: reduction.truncated });
1284
+ emit("context_reduced", {
1285
+ messages: reduction.truncated,
1286
+ reason: "budget",
1287
+ tokensSaved,
1288
+ });
1069
1289
  }
1070
1290
  };
1071
1291
  reduceInPlace();
@@ -1105,7 +1325,49 @@ export class AgentRunner {
1105
1325
  // Providers only ever see the text projection of tool results: ui/image parts are a
1106
1326
  // display-only extension of the persisted transcript and never reach a model prompt
1107
1327
  // (raw image bytes included). The store keeps the rich parts for TUI replay.
1108
- const providerMessages = messages.map((m) => m.role === "tool" ? { ...m, result: textProjection(m.result) } : m);
1328
+ // Context Governor: `off` costs one property read. `shadow` only measures; `on` sends the
1329
+ // projected history. The stored transcript, the compaction triggers and the limits above
1330
+ // always use the raw `messages`.
1331
+ let sent = messages;
1332
+ let invariantsText = "";
1333
+ const governor = o.contextGovernor;
1334
+ if (governor && governor.mode !== "off") {
1335
+ const projection = projectContext(messages, governor);
1336
+ const snapshot = governanceSnapshot(projection, governor, requests + 1, this.governed.get(sessionId));
1337
+ const block = invariants();
1338
+ const done = this.promotions.get(sessionId);
1339
+ snapshot.invariants = {
1340
+ count: block.included,
1341
+ chars: block.text.length,
1342
+ omitted: block.omitted,
1343
+ promoted: done?.promoted ?? 0,
1344
+ rejected: done?.rejected ?? 0,
1345
+ };
1346
+ this.governed.set(sessionId, snapshot);
1347
+ // The ledger only measures: a fault in it never fails the request or changes `sent`.
1348
+ let usageSummary;
1349
+ try {
1350
+ const ledger = this.usageLedger(sessionId, workspace);
1351
+ const stubbed = governor.mode === "on"
1352
+ ? new Set(projection.elisions.map((e) => e.callId))
1353
+ : new Set();
1354
+ ledger.record(ledger.seams + 1, messages, stubbed);
1355
+ usageSummary = ledger.summary();
1356
+ }
1357
+ catch {
1358
+ /* Advisory numbers only. */
1359
+ }
1360
+ emit("context_governed", governedEventData(snapshot, usageSummary));
1361
+ if (governor.mode === "on") {
1362
+ sent = projection.messages;
1363
+ invariantsText = block.text;
1364
+ }
1365
+ }
1366
+ const providerMessages = sent.map((m) => m.role === "tool" ? { ...m, result: textProjection(m.result) } : m);
1367
+ // The invariants block is the last message: built fresh, never saved, never an elision
1368
+ // candidate (it joins the request after the projection).
1369
+ if (invariantsText)
1370
+ providerMessages.push({ role: "user", text: invariantsText });
1109
1371
  // After a cut-off response the request carries a transient continuation notice (never
1110
1372
  // persisted: the stored transcript only ever holds what the model really answered).
1111
1373
  if (recoveryNotice)
@@ -46,7 +46,7 @@ export interface Interval {
46
46
  wilsonHigh?: number;
47
47
  }
48
48
  declare const CHANGED_FIELDS: readonly ["configHash", "model", "harnessVersion", "mode", "provider"];
49
- declare const MEASURED_KEYS: readonly ["medianWallMs", "varianceWallMs", "varianceTokens", "tokensPerSuccess", "wallMsPerSuccess"];
49
+ declare const MEASURED_KEYS: readonly ["medianWallMs", "varianceWallMs", "varianceTokens", "tokensPerSuccess", "wallMsPerSuccess", "cachedInputRatio"];
50
50
  type MeasuredKey = (typeof MEASURED_KEYS)[number];
51
51
  export interface TaskComparison {
52
52
  taskId: string;
@@ -41,13 +41,18 @@ const MEASURED_KEYS = [
41
41
  "varianceTokens",
42
42
  "tokensPerSuccess",
43
43
  "wallMsPerSuccess",
44
+ "cachedInputRatio",
44
45
  ];
45
46
  const delta = (baseline, candidate) => ({
46
47
  baseline,
47
48
  candidate,
48
49
  delta: candidate - baseline,
49
50
  });
50
- const TOKEN_KEYS = ["varianceTokens", "tokensPerSuccess"];
51
+ const TOKEN_KEYS = [
52
+ "varianceTokens",
53
+ "tokensPerSuccess",
54
+ "cachedInputRatio",
55
+ ];
51
56
  function compareTask(a, b, ctx) {
52
57
  const lowPower = a.k < MIN_POWER_K || b.k < MIN_POWER_K;
53
58
  const difference = b.successRate.value - a.successRate.value;
@@ -263,6 +268,8 @@ const reportSchema = z.object({
263
268
  varianceTokens: z.number().optional(),
264
269
  tokensPerSuccess: z.number().optional(),
265
270
  wallMsPerSuccess: z.number().optional(),
271
+ cachedInputRatio: z.number().optional(),
272
+ cachedInputRuns: z.number().optional(),
266
273
  }),
267
274
  estimated: z.object({ costUsdPerSuccess: z.number().optional() }).optional(),
268
275
  })),
@@ -274,6 +281,15 @@ const reportSchema = z.object({
274
281
  budgetExceeded: z.number(),
275
282
  runs: z.number().optional(),
276
283
  failed: z.number().optional(),
284
+ cachedInput: z
285
+ .object({
286
+ tokens: z.number(),
287
+ inputTokens: z.number(),
288
+ ratio: z.number(),
289
+ runsReporting: z.number(),
290
+ runsJudged: z.number(),
291
+ })
292
+ .optional(),
277
293
  }),
278
294
  rates: z.object({ inputPerMTokUsd: z.number(), outputPerMTokUsd: z.number() }).optional(),
279
295
  });
@@ -315,6 +331,7 @@ const METRIC_LABEL = {
315
331
  varianceTokens: "Token variance (main-loop tokens)",
316
332
  tokensPerSuccess: "Tokens per success (main-loop tokens, excludes compaction calls)",
317
333
  wallMsPerSuccess: "Wall per success (ms)",
334
+ cachedInputRatio: "Cached input ratio (provider-reported cached / input tokens)",
318
335
  };
319
336
  const number = (value) => String(Math.round(value * 1000) / 1000);
320
337
  function table(headers, rows) {
@@ -83,6 +83,19 @@ export const CONFIG_HASH_COVERAGE = {
83
83
  agents: "excluded",
84
84
  web: "excluded",
85
85
  };
86
+ /**
87
+ * The Context Governor keys exist only since spec 012. A switched-off governor with default
88
+ * settings leaves them out, so the hashes of runs recorded before they existed stay valid.
89
+ */
90
+ function hashedContext(context) {
91
+ const { governor, elide, policyVersion, invariants, ...rest } = context;
92
+ const untouched = governor === "off" &&
93
+ policyVersion === undefined &&
94
+ elide.minAgeTurns === 2 &&
95
+ elide.epochMessages === 32 &&
96
+ invariants.maxChars === 4000;
97
+ return untouched ? rest : context;
98
+ }
86
99
  /**
87
100
  * `sha256:` of the allowlisted effective configuration plus policy, approval mode, budget and
88
101
  * enabled built-ins. Excludes secrets, env names, plugin options, absolute paths, endpoints, the
@@ -93,7 +106,7 @@ export function computeConfigHash(config, extras) {
93
106
  const picked = {
94
107
  limits: config.limits,
95
108
  compaction: config.compaction,
96
- context: config.context,
109
+ context: hashedContext(config.context),
97
110
  forge: {
98
111
  javascript: forge.javascript,
99
112
  state: forge.state,
@@ -289,6 +302,10 @@ class RunObserver {
289
302
  /** Cumulative tokens reported by the last `turn_completed`. */
290
303
  tokens = 0;
291
304
  usage;
305
+ /** Sum of the cached input tokens reported; undefined until a turn reports any. */
306
+ cachedInput;
307
+ /** Turns whose usage carried `cachedInput`. */
308
+ turnsWithCached = 0;
292
309
  model;
293
310
  timedOut = false;
294
311
  turnsExceeded = false;
@@ -311,6 +328,10 @@ class RunObserver {
311
328
  this.usage ??= { input: 0, output: 0 };
312
329
  this.usage.input += usage.input;
313
330
  this.usage.output += usage.output;
331
+ if (typeof usage.cachedInput === "number") {
332
+ this.turnsWithCached += 1;
333
+ this.cachedInput = (this.cachedInput ?? 0) + usage.cachedInput;
334
+ }
314
335
  }
315
336
  break;
316
337
  }
@@ -386,6 +407,10 @@ class RunObserver {
386
407
  get usagePartial() {
387
408
  return this.turnsWithUsage > 0 && this.turnsWithUsage < this.turns;
388
409
  }
410
+ /** Some turns that reported usage did not report cached tokens: the cached sum understates. */
411
+ get cachedPartial() {
412
+ return this.turnsWithCached > 0 && this.turnsWithCached < this.turnsWithUsage;
413
+ }
389
414
  markAborted() {
390
415
  this.bump("run_aborted");
391
416
  }
@@ -718,6 +743,11 @@ async function execute(task, opts, plan, fixtureDir) {
718
743
  measured: {
719
744
  ...(usage ? { inputTokens: usage.input, outputTokens: usage.output } : {}),
720
745
  ...(usage && observer.usagePartial ? { usagePartial: true } : {}),
746
+ // Only the observer sees `cachedInput`; absent stays absent (unknown is not 0).
747
+ ...(observer.cachedInput !== undefined ? { cachedInputTokens: observer.cachedInput } : {}),
748
+ ...(observer.cachedInput !== undefined && observer.cachedPartial
749
+ ? { cachedPartial: true }
750
+ : {}),
721
751
  wallMs,
722
752
  turns: observer.turns,
723
753
  toolCalls: observer.startedCalls,
@@ -56,6 +56,21 @@ function assertUniform(records, field, hint) {
56
56
  if (values.length > 1)
57
57
  throw new EvalReportError(`Cannot aggregate runs measured under different ${field}: ${values.map((v) => `"${v}"`).join(", ")}. ${hint}`);
58
58
  }
59
+ /** Runs that reported cached input completely: judged, no partial usage, no partial cached sum. */
60
+ function cachedReporters(judged) {
61
+ return judged.filter((r) => r.measured.cachedInputTokens !== undefined &&
62
+ !r.measured.cachedPartial &&
63
+ r.measured.inputTokens !== undefined &&
64
+ !r.measured.usagePartial);
65
+ }
66
+ function cachedTotals(judged) {
67
+ const reporters = cachedReporters(judged);
68
+ if (reporters.length === 0)
69
+ return undefined;
70
+ const tokens = sum(reporters.map((r) => r.measured.cachedInputTokens));
71
+ const inputTokens = sum(reporters.map((r) => r.measured.inputTokens));
72
+ return { tokens, inputTokens, runsReporting: reporters.length };
73
+ }
59
74
  function taskMetrics(taskId, runs, rates) {
60
75
  const judged = runs.filter((r) => r.status !== "error");
61
76
  const errored = runs.length - judged.length;
@@ -71,6 +86,7 @@ function taskMetrics(taskId, runs, rates) {
71
86
  const varianceWallMs = sampleVariance(wallTimes);
72
87
  // Same rule as tokensPerSuccess: a spread over the runs that happened to report would mislead.
73
88
  const varianceTokens = allReportedUsage ? sampleVariance(tokenSamples) : null;
89
+ const cached = cachedTotals(judged);
74
90
  const measured = {
75
91
  ...(medianWallMs === null ? {} : { medianWallMs }),
76
92
  ...(varianceWallMs === null ? {} : { varianceWallMs }),
@@ -79,6 +95,13 @@ function taskMetrics(taskId, runs, rates) {
79
95
  ? { tokensPerSuccess: sum(tokenSamples) / successes }
80
96
  : {}),
81
97
  ...(successes > 0 ? { wallMsPerSuccess: sum(wallTimes) / successes } : {}),
98
+ // A ratio over zero input tokens means nothing: leave it out rather than divide by zero.
99
+ ...(cached && cached.inputTokens > 0
100
+ ? {
101
+ cachedInputRatio: cached.tokens / cached.inputTokens,
102
+ cachedInputRuns: cached.runsReporting,
103
+ }
104
+ : {}),
82
105
  };
83
106
  const costs = rates ? judged.map((r) => estimateRunCost(r, rates)) : [];
84
107
  const costKnown = rates !== undefined && successes > 0 && costs.every((c) => c !== undefined);
@@ -147,6 +170,8 @@ export function buildReport(records, options = {}) {
147
170
  ? taskHashes[0][1]
148
171
  : `sha256:${createHash("sha256").update(JSON.stringify(taskHashes)).digest("hex")}`;
149
172
  const count = (status) => records.filter((r) => r.status === status).length;
173
+ const judgedRuns = records.filter((r) => r.status !== "error");
174
+ const armCached = cachedTotals(judgedRuns);
150
175
  return {
151
176
  schemaVersion: 1,
152
177
  suiteId: first.suiteId,
@@ -174,6 +199,17 @@ export function buildReport(records, options = {}) {
174
199
  budgetExceeded: count("budget_exceeded"),
175
200
  runs: records.length,
176
201
  failed: count("fail"),
202
+ ...(armCached && armCached.inputTokens > 0
203
+ ? {
204
+ cachedInput: {
205
+ tokens: armCached.tokens,
206
+ inputTokens: armCached.inputTokens,
207
+ ratio: armCached.tokens / armCached.inputTokens,
208
+ runsReporting: armCached.runsReporting,
209
+ runsJudged: judgedRuns.length,
210
+ },
211
+ }
212
+ : {}),
177
213
  },
178
214
  ...(options.rates ? { rates: { ...options.rates } } : {}),
179
215
  };
@@ -198,6 +234,13 @@ function table(headers, rows) {
198
234
  export function lowPowerTasks(report) {
199
235
  return report.tasks.filter((t) => t.k < MIN_POWER_K);
200
236
  }
237
+ /** The arm's cached-input line: the provider's own figure, or an explicit "not reported". */
238
+ function cachedLine(report) {
239
+ const c = report.summary.cachedInput;
240
+ if (!c)
241
+ return "Cached input: not reported by the provider (no ratio is computed; absent is not zero).";
242
+ return `Cached input: ${percent(c.ratio)} of input tokens (${count(c.tokens)} of ${count(c.inputTokens)}), from ${c.runsReporting} of ${c.runsJudged} judged runs that reported it.`;
243
+ }
201
244
  /** Markdown of a report: conditions, warnings, then Measured and Estimated as separate sections. */
202
245
  export function renderReportMarkdown(report) {
203
246
  const { conditions: c } = report;
@@ -219,7 +262,7 @@ export function renderReportMarkdown(report) {
219
262
  if (weak.length > 0)
220
263
  lines.push(`> **Low power:** ${weak.map((t) => `${t.taskId} (k=${t.k})`).join(", ")} ${weak.length === 1 ? "has" : "have"} fewer than ${MIN_POWER_K} judged runs. The intervals are wide; do not read an improvement or a regression from these numbers.`, "");
221
264
  const { summary: s } = report;
222
- lines.push(`Runs: ${s.runs ?? report.runs.length} (passed ${s.passed}, failed ${s.failed ?? "-"}, budget exceeded ${s.budgetExceeded}, errored ${s.errored}). Errored runs are infrastructure failures: they are excluded from every rate and spread below.`, "", "## Measured", "", "Provider-reported usage and wall time. Success is passes over judged runs, with a 95% Wilson interval; sd is the square root of the sample variance. Token figures are main-loop tokens (excludes compaction calls).", "", table([
265
+ lines.push(`Runs: ${s.runs ?? report.runs.length} (passed ${s.passed}, failed ${s.failed ?? "-"}, budget exceeded ${s.budgetExceeded}, errored ${s.errored}). Errored runs are infrastructure failures: they are excluded from every rate and spread below.`, "", "## Measured", "", cachedLine(report), "", "Provider-reported usage and wall time. Success is passes over judged runs, with a 95% Wilson interval; sd is the square root of the sample variance. Token figures are main-loop tokens (excludes compaction calls).", "", table([
223
266
  "Task",
224
267
  "k",
225
268
  "Success [Wilson 95%]",
@@ -4,7 +4,7 @@ export declare const PROCESS_KILL_BACKSTOP_MS = 500;
4
4
  * The isolate's whole environment: the marker, and on Windows `SystemRoot` and nothing else (the
5
5
  * runtime cannot start without it). No other variable of the parent ever reaches the guest process.
6
6
  */
7
- export declare function isolateEnv(platform?: NodeJS.Platform, parent?: NodeJS.ProcessEnv): Record<string, string>;
7
+ export declare function isolateEnv(platform?: NodeJS.Platform, parent?: NodeJS.ProcessEnv, electron?: boolean): Record<string, string>;
8
8
  export interface IsolateCommand {
9
9
  execPath: string;
10
10
  args: string[];
@@ -23,8 +23,11 @@ const EXIT_CONFIRM_MS = 2_000;
23
23
  * The isolate's whole environment: the marker, and on Windows `SystemRoot` and nothing else (the
24
24
  * runtime cannot start without it). No other variable of the parent ever reaches the guest process.
25
25
  */
26
- export function isolateEnv(platform = process.platform, parent = process.env) {
26
+ export function isolateEnv(platform = process.platform, parent = process.env, electron = !!process.versions.electron) {
27
27
  const env = { ALISIO_FORGE_ISOLATE: "1" };
28
+ // The Electron binary only behaves as Node with this flag; it carries no parent data.
29
+ if (electron)
30
+ env.ELECTRON_RUN_AS_NODE = "1";
28
31
  if (platform === "win32" && parent.SystemRoot)
29
32
  env.SystemRoot = parent.SystemRoot;
30
33
  return env;
@@ -38,7 +38,7 @@ export declare function permissionArgs(paths: {
38
38
  write: readonly string[];
39
39
  }): string[];
40
40
  /** Environment of the child: nothing but the job's `ALISIO_*` variables. */
41
- export declare function childEnv(vars: Record<string, string>, platform?: NodeJS.Platform): Record<string, string>;
41
+ export declare function childEnv(vars: Record<string, string>, platform?: NodeJS.Platform, electron?: boolean): Record<string, string>;
42
42
  export interface NetworkProbeResult {
43
43
  /**
44
44
  * The Permission Model refused the connection (`ERR_ACCESS_DENIED`). `false` also when the probe
@@ -126,11 +126,14 @@ export function permissionArgs(paths) {
126
126
  ];
127
127
  }
128
128
  /** Environment of the child: nothing but the job's `ALISIO_*` variables. */
129
- export function childEnv(vars, platform = process.platform) {
129
+ export function childEnv(vars, platform = process.platform, electron = !!process.versions.electron) {
130
130
  const env = {};
131
131
  for (const [key, value] of Object.entries(vars))
132
132
  if (key.startsWith("ALISIO_"))
133
133
  env[key] = value;
134
+ // The Electron binary only behaves as Node with this flag; it carries no parent data.
135
+ if (electron)
136
+ env.ELECTRON_RUN_AS_NODE = "1";
134
137
  // Windows cannot start a process without these.
135
138
  if (platform === "win32" && process.env.SystemRoot)
136
139
  env.SystemRoot = process.env.SystemRoot;