@animalabs/agent-framework 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/framework.d.ts +31 -4
- package/dist/src/framework.d.ts.map +1 -1
- package/dist/src/framework.js +327 -49
- package/dist/src/framework.js.map +1 -1
- package/dist/src/gate/event-gate.d.ts +55 -1
- package/dist/src/gate/event-gate.d.ts.map +1 -1
- package/dist/src/gate/event-gate.js +41 -1
- package/dist/src/gate/event-gate.js.map +1 -1
- package/dist/src/gate/index.d.ts +1 -1
- package/dist/src/gate/index.d.ts.map +1 -1
- package/dist/src/gate/index.js +1 -1
- package/dist/src/gate/index.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/tool-image-ledger.d.ts +175 -0
- package/dist/src/tool-image-ledger.d.ts.map +1 -0
- package/dist/src/tool-image-ledger.js +257 -0
- package/dist/src/tool-image-ledger.js.map +1 -0
- package/dist/src/tool-result-history.d.ts +15 -2
- package/dist/src/tool-result-history.d.ts.map +1 -1
- package/dist/src/tool-result-history.js +16 -9
- package/dist/src/tool-result-history.js.map +1 -1
- package/dist/src/types/agent.d.ts +10 -0
- package/dist/src/types/agent.d.ts.map +1 -1
- package/dist/test/event-gate.test.js +84 -1
- package/dist/test/event-gate.test.js.map +1 -1
- package/dist/test/puppet-tool-call.test.js +1 -0
- package/dist/test/puppet-tool-call.test.js.map +1 -1
- package/dist/test/save-recent-image.test.d.ts +26 -0
- package/dist/test/save-recent-image.test.d.ts.map +1 -0
- package/dist/test/save-recent-image.test.js +707 -0
- package/dist/test/save-recent-image.test.js.map +1 -0
- package/dist/test/turn-trigger-provenance.test.js +17 -0
- package/dist/test/turn-trigger-provenance.test.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/src/framework.ts +389 -54
- package/src/gate/event-gate.ts +71 -3
- package/src/gate/index.ts +1 -1
- package/src/index.ts +1 -1
- package/src/tool-image-ledger.ts +334 -0
- package/src/tool-result-history.ts +30 -9
- package/src/types/agent.ts +10 -0
package/package.json
CHANGED
package/src/framework.ts
CHANGED
|
@@ -80,6 +80,15 @@ import {
|
|
|
80
80
|
truncateForHistory,
|
|
81
81
|
DEFAULT_TOOL_RESULT_INLINE_MAX_CHARS,
|
|
82
82
|
} from './tool-result-history.js';
|
|
83
|
+
import {
|
|
84
|
+
TOOL_IMAGE_REF_RE,
|
|
85
|
+
ToolImageLedger,
|
|
86
|
+
formatPreservedImageSlots,
|
|
87
|
+
parseImagePlaceholders,
|
|
88
|
+
sha256Hex,
|
|
89
|
+
splitPreservingImageSlots,
|
|
90
|
+
type ParsedImagePlaceholder,
|
|
91
|
+
} from './tool-image-ledger.js';
|
|
83
92
|
import { randomUUID } from 'node:crypto';
|
|
84
93
|
import { PyRunner, buildInjectedTools } from './code-execution/py-runner.js';
|
|
85
94
|
import {
|
|
@@ -960,6 +969,11 @@ export class AgentFramework {
|
|
|
960
969
|
* framework state like the core runtime settings and restored at create
|
|
961
970
|
* (antra + Sol, 08-06); reset clears it back to the residence default. */
|
|
962
971
|
private toolResultInlineMaxCharsOverride: Map<string, number> = new Map();
|
|
972
|
+
/** Per-agent retention of tool-result images with provenance (issue
|
|
973
|
+
* #104). In-memory and bounded by design: history keeps only the
|
|
974
|
+
* placeholder; this is what lets `save_recent_image` reach the bytes the
|
|
975
|
+
* resident actually saw — or fail at that exact index when they're gone. */
|
|
976
|
+
private toolImageLedgers: Map<string, ToolImageLedger> = new Map();
|
|
963
977
|
/** Durable residence-configured inline cap from
|
|
964
978
|
* FrameworkConfig.toolResultInlineMaxChars; null → house default. */
|
|
965
979
|
private toolResultInlineMaxCharsConfig: number | null = null;
|
|
@@ -1221,8 +1235,17 @@ export class AgentFramework {
|
|
|
1221
1235
|
privilegedUsersPath: config.gate.privilegedUsersPath,
|
|
1222
1236
|
emitTrace: (e) => framework.emitTrace(e as { type: TraceEvent['type']; [key: string]: unknown }),
|
|
1223
1237
|
addMessage: (p, c, m, forAgent) => framework.addMessage(p, c, m as MessageMetadata, forAgent ? { forAgent } : undefined),
|
|
1224
|
-
requestInference: (agentName, reason, source) => {
|
|
1225
|
-
framework.pendingRequests.push({
|
|
1238
|
+
requestInference: (agentName, reason, source, provenance) => {
|
|
1239
|
+
framework.pendingRequests.push({
|
|
1240
|
+
agentName, reason, source, timestamp: Date.now(),
|
|
1241
|
+
// gate-requested wakes carry where/who (EventGate wakeProvenance)
|
|
1242
|
+
// as TELEMETRY fields — the host's stamp reads them; the turn's
|
|
1243
|
+
// speech locus (channelId / addressed) is deliberately NOT set
|
|
1244
|
+
// here, so a batched wake routes exactly as it did before.
|
|
1245
|
+
...(provenance?.channelId ? { wakeChannelId: provenance.channelId } : {}),
|
|
1246
|
+
...(provenance?.counterparty ? { counterparty: provenance.counterparty } : {}),
|
|
1247
|
+
...(provenance?.at ? { wakeAt: provenance.at } : {}),
|
|
1248
|
+
});
|
|
1226
1249
|
},
|
|
1227
1250
|
getAgentNames: () => [...framework.agents.keys()].filter(
|
|
1228
1251
|
(n) => n !== framework.subconsciousAgentName),
|
|
@@ -1411,6 +1434,9 @@ export class AgentFramework {
|
|
|
1411
1434
|
record.runner.dispose();
|
|
1412
1435
|
}
|
|
1413
1436
|
this.backgroundScripts.clear();
|
|
1437
|
+
// Retained tool images are per-process by design; a retained stopped
|
|
1438
|
+
// framework must not keep up to the whole ledger budget referenced.
|
|
1439
|
+
this.toolImageLedgers.clear();
|
|
1414
1440
|
|
|
1415
1441
|
// A stopped host must never hang behind its own cooldown.
|
|
1416
1442
|
this.cancelProviderAdmission();
|
|
@@ -2719,6 +2745,7 @@ export class AgentFramework {
|
|
|
2719
2745
|
this.eventGate?.onInferenceEnded(agent.name);
|
|
2720
2746
|
this.agents.delete(agent.name);
|
|
2721
2747
|
}
|
|
2748
|
+
this.toolImageLedgers.delete(agent.name);
|
|
2722
2749
|
// Spawn-and-dispose bookkeeping (main, d453165/fee96a7): without this,
|
|
2723
2750
|
// ephemeral agents leave checkpoint-tree keys and diagnostics map
|
|
2724
2751
|
// entries behind for the life of the store/session.
|
|
@@ -2979,11 +3006,18 @@ export class AgentFramework {
|
|
|
2979
3006
|
name: 'save_recent_image',
|
|
2980
3007
|
description:
|
|
2981
3008
|
'Save one or more recent images from your own context to workspace files. ' +
|
|
2982
|
-
'Images are counted back from the most recent (index 0)
|
|
3009
|
+
'Images are counted back from the most recent (index 0), across BOTH ' +
|
|
3010
|
+
'attachments in messages and images returned by tools (their history ' +
|
|
3011
|
+
'placeholder shows a ref like `[image: image/png, ~691KB, ref img_k7x3q2_7]`; pass ' +
|
|
3012
|
+
'`ref` to save that exact image regardless of position). A single image is ' +
|
|
2983
3013
|
'written to `path` as given (e.g. "project/photos/cat.png"); when `count` > 1 ' +
|
|
2984
3014
|
'the range index..index+count-1 is saved with numeric suffixes ' +
|
|
2985
|
-
'("cat-0.png", "cat-1.png", …; 0 = the newest of the range).
|
|
2986
|
-
'
|
|
3015
|
+
'("cat-0.png", "cat-1.png", …; 0 = the newest of the range). If the image at ' +
|
|
3016
|
+
'the requested position is no longer retained (evicted, or from before a ' +
|
|
3017
|
+
'restart) the call fails and writes nothing — it never substitutes an older ' +
|
|
3018
|
+
'image. When other tool calls in the same batch are still running, the save ' +
|
|
3019
|
+
'waits for them so their images are counted. Receipts carry source tool call, MIME, byte size and SHA-256. Saved ' +
|
|
3020
|
+
'files are visible via workspace tools and the /files/ endpoint.',
|
|
2987
3021
|
inputSchema: {
|
|
2988
3022
|
type: 'object',
|
|
2989
3023
|
properties: {
|
|
@@ -2999,6 +3033,12 @@ export class AgentFramework {
|
|
|
2999
3033
|
type: 'number',
|
|
3000
3034
|
description: 'How many images to save, starting at `index` and going further back. Default 1.',
|
|
3001
3035
|
},
|
|
3036
|
+
ref: {
|
|
3037
|
+
type: 'string',
|
|
3038
|
+
description:
|
|
3039
|
+
'Save a specific tool-result image by its history ref (e.g. "img_k7x3q2_7", as shown in its placeholder in your context) instead of by position. ' +
|
|
3040
|
+
'Mutually exclusive with `index`/`count`.',
|
|
3041
|
+
},
|
|
3002
3042
|
},
|
|
3003
3043
|
required: ['path'],
|
|
3004
3044
|
},
|
|
@@ -4436,7 +4476,7 @@ export class AgentFramework {
|
|
|
4436
4476
|
// byte-matched (the window stores what the membrane sends;
|
|
4437
4477
|
// divergence breaks the compile prefix).
|
|
4438
4478
|
const { blocks: toolResultContent, spilled } =
|
|
4439
|
-
await this.buildStoredToolResultContent(currentState.toolResults, maxChars);
|
|
4479
|
+
await this.buildStoredToolResultContent(agent.name, currentState.toolResults, maxChars);
|
|
4440
4480
|
agent.getContextManager().addMessage('user', toolResultContent);
|
|
4441
4481
|
|
|
4442
4482
|
// Flush any messages that were deferred while this turn was in
|
|
@@ -5117,6 +5157,7 @@ export class AgentFramework {
|
|
|
5117
5157
|
const channelId = this.conversationAgentHomes.get(agentName);
|
|
5118
5158
|
const agent = this.agents.get(agentName);
|
|
5119
5159
|
this.agents.delete(agentName);
|
|
5160
|
+
this.toolImageLedgers.delete(agentName);
|
|
5120
5161
|
this.agentConfigs.delete(agentName);
|
|
5121
5162
|
this.conversationAgentHomes.delete(agentName);
|
|
5122
5163
|
this.evictTurnCheckpoints(agentName);
|
|
@@ -5600,6 +5641,17 @@ export class AgentFramework {
|
|
|
5600
5641
|
if (r.addressed) addressedReq = r;
|
|
5601
5642
|
}
|
|
5602
5643
|
const channelReq = addressedReq ?? ambientReq;
|
|
5644
|
+
// Telemetry provenance (who/where woke the agent): from the routing
|
|
5645
|
+
// winner when it carries one; else from the most recent request that
|
|
5646
|
+
// does (gate wakes name a counterparty without setting a locus),
|
|
5647
|
+
// ordered by the event's own time (wakeAt) rather than flush order.
|
|
5648
|
+
let provReq: InferenceRequest | undefined = channelReq?.counterparty || channelReq?.wakeChannelId ? channelReq : undefined;
|
|
5649
|
+
if (!provReq) {
|
|
5650
|
+
for (const r of requests) {
|
|
5651
|
+
if (!r.counterparty && !r.wakeChannelId) continue;
|
|
5652
|
+
if (!provReq || (r.wakeAt ?? r.timestamp) >= (provReq.wakeAt ?? provReq.timestamp)) provReq = r;
|
|
5653
|
+
}
|
|
5654
|
+
}
|
|
5603
5655
|
await this.startAgentStream(agent, {
|
|
5604
5656
|
...trigger,
|
|
5605
5657
|
channelId: channelReq?.channelId,
|
|
@@ -5608,7 +5660,9 @@ export class AgentFramework {
|
|
|
5608
5660
|
// the channel for routing but names no author — the restart is its
|
|
5609
5661
|
// own cause, and borrowing another request's author would be false
|
|
5610
5662
|
// provenance.
|
|
5611
|
-
counterparty: budgetRestart ? undefined :
|
|
5663
|
+
counterparty: budgetRestart ? undefined : provReq?.counterparty,
|
|
5664
|
+
wakeChannelId: budgetRestart ? undefined : provReq?.wakeChannelId,
|
|
5665
|
+
wakeAt: budgetRestart ? undefined : provReq?.wakeAt,
|
|
5612
5666
|
});
|
|
5613
5667
|
}
|
|
5614
5668
|
}
|
|
@@ -6859,6 +6913,7 @@ export class AgentFramework {
|
|
|
6859
6913
|
const readyState = agent.state as AgentState;
|
|
6860
6914
|
if (readyState.status === 'ready') {
|
|
6861
6915
|
const { blocks: toolResultContent } = await this.buildStoredToolResultContent(
|
|
6916
|
+
agent.name,
|
|
6862
6917
|
readyState.toolResults,
|
|
6863
6918
|
this.resolveToolResultInlineCap(agent).cap,
|
|
6864
6919
|
);
|
|
@@ -7793,6 +7848,7 @@ export class AgentFramework {
|
|
|
7793
7848
|
// never between tool_use and its tool_result. The two addMessage calls
|
|
7794
7849
|
// below are synchronous and adjacent — nothing can interleave.
|
|
7795
7850
|
const { blocks } = await this.buildStoredToolResultContent(
|
|
7851
|
+
agentName,
|
|
7796
7852
|
[{ id: toolUseId, name: toolName, input, result, durationMs }],
|
|
7797
7853
|
this.resolveToolResultInlineCap(agent).cap,
|
|
7798
7854
|
);
|
|
@@ -8299,6 +8355,7 @@ export class AgentFramework {
|
|
|
8299
8355
|
* live wire copy can reuse the identical strings.
|
|
8300
8356
|
*/
|
|
8301
8357
|
private async buildStoredToolResultContent(
|
|
8358
|
+
agentName: string,
|
|
8302
8359
|
toolResults: CompletedToolCall[],
|
|
8303
8360
|
maxChars: number | undefined,
|
|
8304
8361
|
): Promise<{
|
|
@@ -8307,10 +8364,22 @@ export class AgentFramework {
|
|
|
8307
8364
|
}> {
|
|
8308
8365
|
const spilled = new Map<string, { text: string; filePath: string | null }>();
|
|
8309
8366
|
const dateLabel = new Date().toISOString().slice(0, 10);
|
|
8367
|
+
const ledger = this.toolImageLedgerFor(agentName);
|
|
8310
8368
|
for (const tc of toolResults) {
|
|
8369
|
+
// Retain every image the result carries BEFORE the placeholder is
|
|
8370
|
+
// written (issue #104): the placeholder embeds the ref, so the image
|
|
8371
|
+
// stays reachable by provenance after the bytes leave the wire.
|
|
8311
8372
|
const raw = tc.result.isError
|
|
8312
8373
|
? tc.result.error ?? 'Unknown error'
|
|
8313
|
-
: toolResultDataToHistoryString(tc.result.data, undefined
|
|
8374
|
+
: toolResultDataToHistoryString(tc.result.data, undefined, {
|
|
8375
|
+
imageRef: (blockIndex, image) => ledger.retain({
|
|
8376
|
+
toolCallId: tc.id,
|
|
8377
|
+
toolName: tc.name,
|
|
8378
|
+
blockIndex,
|
|
8379
|
+
data: image.data,
|
|
8380
|
+
mediaType: image.mimeType,
|
|
8381
|
+
}).ref,
|
|
8382
|
+
});
|
|
8314
8383
|
spilled.set(tc.id, await this.spillOrTruncate(raw, maxChars, `${dateLabel}-${tc.id}`));
|
|
8315
8384
|
}
|
|
8316
8385
|
const blocks: ContentBlock[] = toolResults.map(tc => ({
|
|
@@ -8331,6 +8400,13 @@ export class AgentFramework {
|
|
|
8331
8400
|
label: string,
|
|
8332
8401
|
): Promise<{ text: string; filePath: string | null }> {
|
|
8333
8402
|
if (!cap || content.length <= cap) return { text: content, filePath: null };
|
|
8403
|
+
// The stored text is the only place save_recent_image discovers tool
|
|
8404
|
+
// images, and the live wire delivers every image regardless of text
|
|
8405
|
+
// length — so an image slot dropped by the cut is an image the resident
|
|
8406
|
+
// saw that the index would silently slide past (issue #104). Keep the
|
|
8407
|
+
// slots at or past the cut, re-appended after the notice.
|
|
8408
|
+
const { head, tail } = splitPreservingImageSlots(content, cap);
|
|
8409
|
+
const kept = formatPreservedImageSlots(tail);
|
|
8334
8410
|
|
|
8335
8411
|
const workspace = this.getWorkspaceModule();
|
|
8336
8412
|
const mountName = workspace ? this.firstWritableMountName(workspace) : null;
|
|
@@ -8342,10 +8418,11 @@ export class AgentFramework {
|
|
|
8342
8418
|
const result = await workspace.writeBinary(path, Buffer.from(content, 'utf8'), 'text/plain');
|
|
8343
8419
|
if (result.success) {
|
|
8344
8420
|
return {
|
|
8345
|
-
text:
|
|
8346
|
-
+ `\n\n[truncated — showing ${
|
|
8421
|
+
text: head
|
|
8422
|
+
+ `\n\n[truncated — showing ${head.length} of ${content.length} chars; full content: workspace file ${path}. `
|
|
8347
8423
|
+ 'Read/grep it with your file tools, or raise the inline cap via '
|
|
8348
|
-
+ 'agent_settings update tool_result_inline_max_chars.]'
|
|
8424
|
+
+ 'agent_settings update tool_result_inline_max_chars.]'
|
|
8425
|
+
+ kept,
|
|
8349
8426
|
filePath: path,
|
|
8350
8427
|
};
|
|
8351
8428
|
}
|
|
@@ -8368,15 +8445,17 @@ export class AgentFramework {
|
|
|
8368
8445
|
`[spill] workspace write failed for ${path} (${content.length} chars): ${failure}`,
|
|
8369
8446
|
);
|
|
8370
8447
|
return {
|
|
8371
|
-
text:
|
|
8372
|
-
+ `\n\n[truncated — showing ${
|
|
8373
|
-
+ ` (${failure}); content over the cap was not retained]
|
|
8448
|
+
text: head
|
|
8449
|
+
+ `\n\n[truncated — showing ${head.length} of ${content.length} chars; spill to workspace file ${path} FAILED`
|
|
8450
|
+
+ ` (${failure}); content over the cap was not retained]`
|
|
8451
|
+
+ kept,
|
|
8374
8452
|
filePath: null,
|
|
8375
8453
|
};
|
|
8376
8454
|
}
|
|
8377
8455
|
return {
|
|
8378
|
-
text:
|
|
8379
|
-
+ '\n\n[truncated — original was ' + content.length + ' chars; no writable workspace, full content not retained]'
|
|
8456
|
+
text: head
|
|
8457
|
+
+ '\n\n[truncated — original was ' + content.length + ' chars; no writable workspace, full content not retained]'
|
|
8458
|
+
+ kept,
|
|
8380
8459
|
filePath: null,
|
|
8381
8460
|
};
|
|
8382
8461
|
}
|
|
@@ -8594,6 +8673,15 @@ export class AgentFramework {
|
|
|
8594
8673
|
return hasImage ? blocks : null;
|
|
8595
8674
|
}
|
|
8596
8675
|
|
|
8676
|
+
private toolImageLedgerFor(agentName: string): ToolImageLedger {
|
|
8677
|
+
let ledger = this.toolImageLedgers.get(agentName);
|
|
8678
|
+
if (!ledger) {
|
|
8679
|
+
ledger = new ToolImageLedger();
|
|
8680
|
+
this.toolImageLedgers.set(agentName, ledger);
|
|
8681
|
+
}
|
|
8682
|
+
return ledger;
|
|
8683
|
+
}
|
|
8684
|
+
|
|
8597
8685
|
/** Strategy-derived per-message bound (maxMessageTokens * 4 chars). */
|
|
8598
8686
|
private strategyDerivedToolResultChars(agent: Agent): number | undefined {
|
|
8599
8687
|
const strategy = agent.getContextManager().getStrategy();
|
|
@@ -11690,10 +11778,53 @@ export class AgentFramework {
|
|
|
11690
11778
|
}
|
|
11691
11779
|
|
|
11692
11780
|
/**
|
|
11693
|
-
*
|
|
11694
|
-
*
|
|
11695
|
-
*
|
|
11696
|
-
*
|
|
11781
|
+
* `save_recent_image` barrier. Module tool calls in a batch are only
|
|
11782
|
+
* ENQUEUED by dispatch, while the save runs at once — so a [snap, save]
|
|
11783
|
+
* round would inventory before the snapshot exists and silently save the
|
|
11784
|
+
* newest OLDER image (issue #104, same-batch representation). Wait for
|
|
11785
|
+
* every sibling in the round to settle; if one never does, fail closed
|
|
11786
|
+
* rather than trust an index the batch may still change. Sibling saves
|
|
11787
|
+
* produce no images and are not waited on (two saves in one batch would
|
|
11788
|
+
* otherwise deadlock into a double timeout). Script-inner calls
|
|
11789
|
+
* (`pytc-…`) run sequentially inside their script host and skip the
|
|
11790
|
+
* barrier — the host call itself is the only pending sibling they'd see.
|
|
11791
|
+
*/
|
|
11792
|
+
private async awaitSiblingToolCalls(agent: Agent, call: ToolCall): Promise<void> {
|
|
11793
|
+
if (call.id.startsWith('pytc-')) return;
|
|
11794
|
+
const BARRIER_MS = 120_000;
|
|
11795
|
+
const POLL_MS = 20;
|
|
11796
|
+
const startedAt = Date.now();
|
|
11797
|
+
for (;;) {
|
|
11798
|
+
const state = agent.state;
|
|
11799
|
+
// Idle (puppet path) or the round was torn down: nothing to wait on.
|
|
11800
|
+
if (state.status !== 'waiting_for_tools') return;
|
|
11801
|
+
const open = [...state.pending.values()].filter((p) => p.id !== call.id && p.name !== 'save_recent_image');
|
|
11802
|
+
if (open.length === 0) return;
|
|
11803
|
+
if (Date.now() - startedAt > BARRIER_MS) {
|
|
11804
|
+
throw new Error(
|
|
11805
|
+
`save_recent_image: ${open.length} sibling tool call(s) in this batch still pending after ` +
|
|
11806
|
+
`${Math.round(BARRIER_MS / 1000)}s (${open.map((p) => `${p.name} (call ${p.id})`).join(', ')}) — their results ` +
|
|
11807
|
+
'may carry newer images than anything in your context, so no index can be trusted yet. Nothing written. ' +
|
|
11808
|
+
'Save again once they have finished.',
|
|
11809
|
+
);
|
|
11810
|
+
}
|
|
11811
|
+
await new Promise((r) => setTimeout(r, POLL_MS));
|
|
11812
|
+
}
|
|
11813
|
+
}
|
|
11814
|
+
|
|
11815
|
+
/**
|
|
11816
|
+
* Handle the synthesized `save_recent_image` tool (issue #104): build ONE
|
|
11817
|
+
* ordered inventory of every image the calling agent has seen — attachment
|
|
11818
|
+
* image blocks (blobs re-inlined) AND tool-result images (their history
|
|
11819
|
+
* placeholders carry a ref, resolved through the agent's ToolImageLedger)
|
|
11820
|
+
* — counted back from the most recent, then write the requested bytes to
|
|
11821
|
+
* a workspace mount via WorkspaceModule.writeBinary.
|
|
11822
|
+
*
|
|
11823
|
+
* Fail-closed: when the image at a requested position cannot be produced
|
|
11824
|
+
* (bytes evicted, minted by an earlier process, or a placeholder written
|
|
11825
|
+
* before retention existed) the call fails AT THAT INDEX and writes
|
|
11826
|
+
* nothing. It never slides to an older image — a filename must not
|
|
11827
|
+
* authenticate bytes from another surface.
|
|
11697
11828
|
*/
|
|
11698
11829
|
private dispatchSaveImageToolCall(agentName: string, call: ToolCall): void {
|
|
11699
11830
|
this.emitTrace({ type: 'tool:started', module: 'workspace', tool: call.name, callId: call.id, input: call.input });
|
|
@@ -11708,26 +11839,219 @@ export class AgentFramework {
|
|
|
11708
11839
|
});
|
|
11709
11840
|
this.pushEvent({ type: 'tool-result', callId: call.id, agentName, moduleName: 'workspace', result });
|
|
11710
11841
|
};
|
|
11842
|
+
type Candidate =
|
|
11843
|
+
| { kind: 'attachment'; data: string | null; mediaType: string; messagesBack: number }
|
|
11844
|
+
| {
|
|
11845
|
+
kind: 'tool';
|
|
11846
|
+
ref: string | null;
|
|
11847
|
+
/** The tool_result block the placeholder was found in — NOT trusted
|
|
11848
|
+
* text; a resolved ref must belong to this very call. */
|
|
11849
|
+
toolCallId: string;
|
|
11850
|
+
toolName: string;
|
|
11851
|
+
mediaType: string;
|
|
11852
|
+
messagesBack: number;
|
|
11853
|
+
}
|
|
11854
|
+
| {
|
|
11855
|
+
kind: 'reference';
|
|
11856
|
+
refId: string;
|
|
11857
|
+
mediaType: string;
|
|
11858
|
+
toolCallId: string;
|
|
11859
|
+
toolName: string;
|
|
11860
|
+
messagesBack: number;
|
|
11861
|
+
};
|
|
11862
|
+
interface Resolved {
|
|
11863
|
+
rangeOffset: number;
|
|
11864
|
+
data: string;
|
|
11865
|
+
mediaType: string;
|
|
11866
|
+
receipt: Record<string, unknown>;
|
|
11867
|
+
}
|
|
11711
11868
|
void (async (): Promise<void> => {
|
|
11712
11869
|
try {
|
|
11713
11870
|
const workspace = this.getWorkspaceModule();
|
|
11714
11871
|
if (!workspace) throw new Error('save_recent_image requires a workspace module');
|
|
11715
11872
|
const agent = this.agents.get(agentName);
|
|
11716
11873
|
if (!agent) throw new Error(`Unknown agent: ${agentName}`);
|
|
11717
|
-
const input = (call.input ?? {}) as { path?: unknown; index?: unknown; count?: unknown };
|
|
11874
|
+
const input = (call.input ?? {}) as { path?: unknown; index?: unknown; count?: unknown; ref?: unknown };
|
|
11718
11875
|
if (typeof input.path !== 'string' || input.path.length === 0) {
|
|
11719
11876
|
throw new Error('save_recent_image: `path` (mount-prefixed) is required');
|
|
11720
11877
|
}
|
|
11721
|
-
const
|
|
11722
|
-
|
|
11723
|
-
|
|
11724
|
-
|
|
11725
|
-
const
|
|
11726
|
-
|
|
11727
|
-
|
|
11728
|
-
|
|
11878
|
+
const ledger = this.toolImageLedgerFor(agentName);
|
|
11879
|
+
|
|
11880
|
+
const describeTool = (c: { toolName: string; toolCallId: string }): string =>
|
|
11881
|
+
`${c.toolName} (call ${c.toolCallId})`;
|
|
11882
|
+
const unavailable = (status: 'evicted' | 'unknown'): string =>
|
|
11883
|
+
status === 'evicted'
|
|
11884
|
+
? 'its bytes were evicted from the bounded retention budget'
|
|
11885
|
+
: 'this process has no record of that ref (refs are per-process and a restart clears them; ' +
|
|
11886
|
+
'very old refs age out; a mistyped ref looks the same)';
|
|
11887
|
+
const resolve = (candidate: Candidate, position: number): Resolved => {
|
|
11888
|
+
const rangeOffset = 0; // filled by the caller
|
|
11889
|
+
if (candidate.kind === 'attachment') {
|
|
11890
|
+
if (candidate.data === null) {
|
|
11891
|
+
throw new Error(
|
|
11892
|
+
`save_recent_image: image at index ${position} is not stored inline (base64) — cannot save it. Nothing written.`,
|
|
11893
|
+
);
|
|
11894
|
+
}
|
|
11895
|
+
const bytes = Buffer.from(candidate.data, 'base64');
|
|
11896
|
+
return {
|
|
11897
|
+
rangeOffset,
|
|
11898
|
+
data: candidate.data,
|
|
11899
|
+
mediaType: candidate.mediaType,
|
|
11900
|
+
receipt: {
|
|
11901
|
+
source: 'attachment',
|
|
11902
|
+
mediaType: candidate.mediaType,
|
|
11903
|
+
byteSize: bytes.byteLength,
|
|
11904
|
+
sha256: sha256Hex(bytes),
|
|
11905
|
+
messagesBack: candidate.messagesBack,
|
|
11906
|
+
},
|
|
11907
|
+
};
|
|
11908
|
+
}
|
|
11909
|
+
if (candidate.kind === 'reference') {
|
|
11910
|
+
throw new Error(
|
|
11911
|
+
`save_recent_image: image at index ${position} is a reference (${candidate.refId}, ${candidate.mediaType}, ` +
|
|
11912
|
+
`from ${describeTool(candidate)}) — its bytes are not held here. Nothing written. ` +
|
|
11913
|
+
'Fetch it with fetch_reference and save the materialized file instead.',
|
|
11914
|
+
);
|
|
11915
|
+
}
|
|
11916
|
+
if (candidate.ref === null) {
|
|
11917
|
+
throw new Error(
|
|
11918
|
+
`save_recent_image: image at index ${position} is a tool-result image from ${describeTool(candidate)} ` +
|
|
11919
|
+
'recorded without a retention ref (written before tool-image retention existed, or inside a ' +
|
|
11920
|
+
'script run, where tool images are not retained) — its bytes are not available. Nothing written. ' +
|
|
11921
|
+
'Re-run the tool for a fresh image rather than saving an older one.',
|
|
11922
|
+
);
|
|
11923
|
+
}
|
|
11924
|
+
const lookup = ledger.lookup(candidate.ref);
|
|
11925
|
+
// The placeholder came out of stored TEXT, which a tool result can
|
|
11926
|
+
// quote from anywhere (a fetch_history of a channel where someone
|
|
11927
|
+
// pasted their save receipt). A genuine placeholder is written by
|
|
11928
|
+
// its own call's serialization, so the ref's provenance names that
|
|
11929
|
+
// call; a quoted or forged one cannot. Refuse the mismatch.
|
|
11930
|
+
if (lookup.status !== 'unknown' && lookup.image.toolCallId !== candidate.toolCallId) {
|
|
11931
|
+
throw new Error(
|
|
11932
|
+
`save_recent_image: image at index ${position} cites ${candidate.ref}, but that image belongs to ` +
|
|
11933
|
+
`${describeTool(lookup.image)}, not to ${describeTool(candidate)} where the placeholder appears — ` +
|
|
11934
|
+
'the placeholder is quoted or forged text, not this result\'s own image. Nothing written.',
|
|
11935
|
+
);
|
|
11936
|
+
}
|
|
11937
|
+
if (lookup.status !== 'retained') {
|
|
11938
|
+
throw new Error(
|
|
11939
|
+
`save_recent_image: image at index ${position} (${candidate.ref}, from ${describeTool(candidate)}) ` +
|
|
11940
|
+
`is no longer retained — ${unavailable(lookup.status)}. Nothing written. ` +
|
|
11941
|
+
'Re-run the tool for a fresh image rather than saving an older one.',
|
|
11942
|
+
);
|
|
11943
|
+
}
|
|
11944
|
+
const image = lookup.image;
|
|
11945
|
+
return {
|
|
11946
|
+
rangeOffset,
|
|
11947
|
+
data: image.data,
|
|
11948
|
+
mediaType: image.mediaType,
|
|
11949
|
+
receipt: {
|
|
11950
|
+
source: 'tool-result',
|
|
11951
|
+
ref: image.ref,
|
|
11952
|
+
toolName: image.toolName,
|
|
11953
|
+
toolCallId: image.toolCallId,
|
|
11954
|
+
blockIndex: image.blockIndex,
|
|
11955
|
+
mediaType: image.mediaType,
|
|
11956
|
+
byteSize: image.byteSize,
|
|
11957
|
+
sha256: image.sha256,
|
|
11958
|
+
messagesBack: candidate.messagesBack,
|
|
11959
|
+
},
|
|
11960
|
+
};
|
|
11961
|
+
};
|
|
11962
|
+
|
|
11963
|
+
// Barrier: a save dispatched alongside other calls in one batch must
|
|
11964
|
+
// not inventory before their results exist — those are the newest
|
|
11965
|
+
// images. Waits (bounded) for every sibling to settle, or fails closed.
|
|
11966
|
+
await this.awaitSiblingToolCalls(agent, call);
|
|
11967
|
+
|
|
11968
|
+
const found: Resolved[] = [];
|
|
11969
|
+
let index = 0;
|
|
11970
|
+
let count = 1;
|
|
11971
|
+
/** Direct-ref mode: the one ref to find in the scannable context. */
|
|
11972
|
+
let wantedRef: string | null = null;
|
|
11973
|
+
if (input.ref !== undefined) {
|
|
11974
|
+
if (typeof input.ref !== 'string' || !TOOL_IMAGE_REF_RE.test(input.ref)) {
|
|
11975
|
+
throw new Error('save_recent_image: `ref` must look like "img_k7x3q2_7" (copy it from the image placeholder in your context)');
|
|
11976
|
+
}
|
|
11977
|
+
if (input.index !== undefined || input.count !== undefined) {
|
|
11978
|
+
throw new Error('save_recent_image: `ref` is mutually exclusive with `index`/`count`');
|
|
11979
|
+
}
|
|
11980
|
+
wantedRef = input.ref;
|
|
11981
|
+
} else {
|
|
11982
|
+
index = input.index === undefined ? 0 : Number(input.index);
|
|
11983
|
+
if (!Number.isInteger(index) || index < 0) {
|
|
11984
|
+
throw new Error('save_recent_image: `index` must be a non-negative integer');
|
|
11985
|
+
}
|
|
11986
|
+
count = input.count === undefined ? 1 : Number(input.count);
|
|
11987
|
+
const MAX_COUNT = 20;
|
|
11988
|
+
if (!Number.isInteger(count) || count < 1 || count > MAX_COUNT) {
|
|
11989
|
+
throw new Error(`save_recent_image: \`count\` must be an integer in 1..${MAX_COUNT}`);
|
|
11990
|
+
}
|
|
11729
11991
|
}
|
|
11730
11992
|
const lastWanted = index + count - 1;
|
|
11993
|
+
let seen = 0;
|
|
11994
|
+
const collecting = (): boolean => (wantedRef !== null ? found.length === 0 : found.length < count);
|
|
11995
|
+
// One walk serves both modes. By index: the requested range of the
|
|
11996
|
+
// ordered inventory. By ref: the slot whose placeholder cites that
|
|
11997
|
+
// ref — the ref is resolved THROUGH its occurrence in context, so
|
|
11998
|
+
// the same provenance cross-check applies (a ref that is not visible
|
|
11999
|
+
// in the scannable window — an undone branch, quoted text naming a
|
|
12000
|
+
// real ref — is not saveable by name either).
|
|
12001
|
+
const consider = (candidate: Candidate): void => {
|
|
12002
|
+
if (wantedRef !== null) {
|
|
12003
|
+
if (candidate.kind === 'tool' && candidate.ref === wantedRef && found.length === 0) {
|
|
12004
|
+
found.push(resolve(candidate, seen));
|
|
12005
|
+
}
|
|
12006
|
+
} else if (seen >= index && seen <= lastWanted) {
|
|
12007
|
+
const resolved = resolve(candidate, seen);
|
|
12008
|
+
resolved.rangeOffset = seen - index;
|
|
12009
|
+
found.push(resolved);
|
|
12010
|
+
}
|
|
12011
|
+
seen++;
|
|
12012
|
+
};
|
|
12013
|
+
const considerToolResultImages = (
|
|
12014
|
+
toolCallId: string,
|
|
12015
|
+
toolName: string,
|
|
12016
|
+
placeholders: ParsedImagePlaceholder[],
|
|
12017
|
+
messagesBack: number,
|
|
12018
|
+
): void => {
|
|
12019
|
+
for (let p = placeholders.length - 1; p >= 0 && collecting(); p--) {
|
|
12020
|
+
const ph = placeholders[p]!;
|
|
12021
|
+
if (ph.kind === 'inline') {
|
|
12022
|
+
consider({ kind: 'tool', ref: ph.ref, toolCallId, toolName, mediaType: ph.mediaType, messagesBack });
|
|
12023
|
+
continue;
|
|
12024
|
+
}
|
|
12025
|
+
// Reference stub: the registry's testimony (when this process
|
|
12026
|
+
// still has the record) says whether it was an image; the
|
|
12027
|
+
// sniffed stub text is the fallback. Non-image references
|
|
12028
|
+
// are not slots.
|
|
12029
|
+
const testimony = referenceRegistry.get(ph.refId)?.testimony.mimeType?.toLowerCase() ?? ph.mediaType;
|
|
12030
|
+
if (!testimony || !testimony.startsWith('image/')) continue;
|
|
12031
|
+
consider({ kind: 'reference', refId: ph.refId, mediaType: testimony, toolCallId, toolName, messagesBack });
|
|
12032
|
+
}
|
|
12033
|
+
};
|
|
12034
|
+
|
|
12035
|
+
// Same-batch results (e.g. a snapshot dispatched alongside the save)
|
|
12036
|
+
// are not in the store yet but ARE the newest images. Classify them
|
|
12037
|
+
// with the very serializer the commit path will run — contradiction
|
|
12038
|
+
// blocks withheld, uri-form blocks as reference stubs, unknown shapes
|
|
12039
|
+
// as no slot at all — so the inventory means the same thing before
|
|
12040
|
+
// and after commit. Retention is idempotent per block, so the
|
|
12041
|
+
// placeholder written at commit time carries the same ref.
|
|
12042
|
+
const state = agent.state;
|
|
12043
|
+
if (state.status === 'waiting_for_tools') {
|
|
12044
|
+
for (let i = state.completed.length - 1; i >= 0 && collecting(); i--) {
|
|
12045
|
+
const tc = state.completed[i]!;
|
|
12046
|
+
if (tc.result.isError) continue;
|
|
12047
|
+
const text = toolResultDataToHistoryString(tc.result.data, undefined, {
|
|
12048
|
+
imageRef: (blockIndex, image) => ledger.retain({
|
|
12049
|
+
toolCallId: tc.id, toolName: tc.name, blockIndex, data: image.data, mediaType: image.mimeType,
|
|
12050
|
+
}).ref,
|
|
12051
|
+
});
|
|
12052
|
+
considerToolResultImages(tc.id, tc.name, parseImagePlaceholders(text), 0);
|
|
12053
|
+
}
|
|
12054
|
+
}
|
|
11731
12055
|
|
|
11732
12056
|
// Walk the message store tail-first in bounded windows, re-inlining
|
|
11733
12057
|
// blob media, until we've collected the requested range. Scanning is
|
|
@@ -11737,45 +12061,56 @@ export class AgentFramework {
|
|
|
11737
12061
|
const WINDOW = 25;
|
|
11738
12062
|
const cm = agent.getContextManager();
|
|
11739
12063
|
const total = cm.getMessageCount();
|
|
11740
|
-
let seen = 0;
|
|
11741
12064
|
let scanned = 0;
|
|
11742
|
-
|
|
11743
|
-
rangeOffset: number;
|
|
11744
|
-
data: string;
|
|
11745
|
-
mediaType: string;
|
|
11746
|
-
messagesBack: number;
|
|
11747
|
-
}> = [];
|
|
11748
|
-
for (let end = total; end > 0 && scanned < MAX_SCAN && found.length < count; end -= WINDOW) {
|
|
12065
|
+
for (let end = total; end > 0 && scanned < MAX_SCAN && collecting(); end -= WINDOW) {
|
|
11749
12066
|
const start = Math.max(0, end - WINDOW);
|
|
11750
|
-
|
|
12067
|
+
// By ref, attachment bytes are never needed — skip blob resolution.
|
|
12068
|
+
const { messages } = cm.getMessageWindow(start, end - start, { resolveBlobs: wantedRef === null });
|
|
11751
12069
|
scanned += end - start;
|
|
11752
|
-
for (let i = messages.length - 1; i >= 0 &&
|
|
12070
|
+
for (let i = messages.length - 1; i >= 0 && collecting(); i--) {
|
|
11753
12071
|
const content = messages[i]?.content;
|
|
11754
12072
|
if (!Array.isArray(content)) continue;
|
|
11755
|
-
|
|
12073
|
+
const messagesBack = total - (start + i);
|
|
12074
|
+
for (let b = content.length - 1; b >= 0 && collecting(); b--) {
|
|
11756
12075
|
const block = content[b] as {
|
|
11757
12076
|
type?: string;
|
|
11758
12077
|
source?: { type?: string; data?: string; mediaType?: string };
|
|
12078
|
+
toolUseId?: string;
|
|
12079
|
+
toolName?: string;
|
|
12080
|
+
content?: unknown;
|
|
11759
12081
|
};
|
|
11760
|
-
if (block?.type
|
|
11761
|
-
|
|
11762
|
-
|
|
11763
|
-
|
|
11764
|
-
|
|
11765
|
-
|
|
11766
|
-
|
|
11767
|
-
found.push({
|
|
11768
|
-
rangeOffset: seen - index,
|
|
11769
|
-
data: block.source.data,
|
|
11770
|
-
mediaType: block.source.mediaType ?? 'image/png',
|
|
11771
|
-
messagesBack: total - (start + i),
|
|
12082
|
+
if (block?.type === 'image') {
|
|
12083
|
+
const inline = block.source?.type === 'base64' && typeof block.source.data === 'string';
|
|
12084
|
+
consider({
|
|
12085
|
+
kind: 'attachment',
|
|
12086
|
+
data: inline ? block.source!.data! : null,
|
|
12087
|
+
mediaType: block.source?.mediaType ?? 'image/png',
|
|
12088
|
+
messagesBack,
|
|
11772
12089
|
});
|
|
12090
|
+
} else if (block?.type === 'tool_result' && typeof block.content === 'string') {
|
|
12091
|
+
considerToolResultImages(
|
|
12092
|
+
block.toolUseId ?? 'unknown',
|
|
12093
|
+
block.toolName ?? 'unknown tool',
|
|
12094
|
+
parseImagePlaceholders(block.content),
|
|
12095
|
+
messagesBack,
|
|
12096
|
+
);
|
|
11773
12097
|
}
|
|
11774
|
-
seen++;
|
|
11775
12098
|
}
|
|
11776
12099
|
}
|
|
11777
12100
|
}
|
|
11778
12101
|
if (found.length === 0) {
|
|
12102
|
+
if (wantedRef !== null) {
|
|
12103
|
+
const lookup = ledger.lookup(wantedRef);
|
|
12104
|
+
throw new Error(
|
|
12105
|
+
lookup.status === 'unknown'
|
|
12106
|
+
? `save_recent_image: ref ${wantedRef} cannot be saved — ${unavailable('unknown')}. Nothing written.`
|
|
12107
|
+
: `save_recent_image: ref ${wantedRef} (from ${describeTool(lookup.image)}) does not appear in your recent ` +
|
|
12108
|
+
`context (scanned ${Math.min(scanned, MAX_SCAN)} messages) — a ref is saved through its placeholder, ` +
|
|
12109
|
+
'which must be visible to you' +
|
|
12110
|
+
(lookup.status === 'evicted' ? '; its bytes were also evicted from the bounded retention budget' : '') +
|
|
12111
|
+
'. Nothing written.',
|
|
12112
|
+
);
|
|
12113
|
+
}
|
|
11779
12114
|
throw new Error(
|
|
11780
12115
|
seen === 0
|
|
11781
12116
|
? `save_recent_image: no images found in the most recent ${Math.min(scanned, MAX_SCAN)} messages`
|
|
@@ -11814,7 +12149,7 @@ export class AgentFramework {
|
|
|
11814
12149
|
savedFiles.push({
|
|
11815
12150
|
...(result.data as Record<string, unknown>),
|
|
11816
12151
|
imageIndex: index + image.rangeOffset,
|
|
11817
|
-
|
|
12152
|
+
...image.receipt,
|
|
11818
12153
|
});
|
|
11819
12154
|
}
|
|
11820
12155
|
finish({
|