@inflexa-ai/harness 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.js +5 -3
- package/dist/lib/page-capture.d.ts +44 -15
- package/dist/lib/page-capture.js +78 -19
- package/dist/loop/run-agent.js +36 -19
- package/dist/memory/card-builders.d.ts +4 -6
- package/dist/memory/card-builders.js +4 -6
- package/dist/memory/conversation-display-replay.d.ts +7 -6
- package/dist/memory/conversation-display-replay.js +7 -6
- package/dist/memory/conversation-display-storage.d.ts +2 -10
- package/dist/memory/conversation-display-storage.js +79 -32
- package/dist/memory/thread-history.d.ts +31 -20
- package/dist/memory/thread-history.js +45 -45
- package/dist/prompts/report-session.d.ts +1 -1
- package/dist/prompts/report-session.js +22 -13
- package/dist/runtime/boot.d.ts +7 -8
- package/dist/runtime/boot.js +7 -19
- package/dist/tools/define-tool.d.ts +19 -8
- package/dist/tools/define-tool.js +28 -12
- package/dist/tools/report-session/examine-page.d.ts +22 -13
- package/dist/tools/report-session/examine-page.js +15 -10
- package/dist/tools/report-session/index.d.ts +1 -1
- package/dist/tools/research/generate-plan.js +3 -8
- package/dist/tools/research/inspect-data-profile.js +10 -11
- package/package.json +1 -1
- package/dist/app/data-profile-policy.d.ts +0 -62
- package/dist/app/data-profile-policy.js +0 -59
- package/dist/memory/content-to-cortex.d.ts +0 -75
- package/dist/memory/content-to-cortex.js +0 -201
- package/dist/memory/conversation-display-backfill.d.ts +0 -23
- package/dist/memory/conversation-display-backfill.js +0 -91
- package/dist/memory/reconstruct-cards.d.ts +0 -27
- package/dist/memory/reconstruct-cards.js +0 -57
package/dist/index.d.ts
CHANGED
|
@@ -77,7 +77,7 @@ export type { PrepareChatTurnDeps, PrepareChatTurnParams, PrepareChatTurnResult
|
|
|
77
77
|
export { createThreadStore } from "./memory/thread-store.js";
|
|
78
78
|
export type { ThreadStore, Thread, ThreadType, ThreadInputError, CreateThreadInput, ListThreadsInput, ThreadPage } from "./memory/thread-store.js";
|
|
79
79
|
export { createThreadHistory, conversationRecordTurn } from "./memory/thread-history.js";
|
|
80
|
-
export type { ThreadHistory, StoredMessage,
|
|
80
|
+
export type { ThreadHistory, StoredMessage, RetractOutcome, ConversationTurn } from "./memory/thread-history.js";
|
|
81
81
|
export { syntheticUserMessage, syntheticRecordMessage, isSyntheticUserMessage, isSyntheticRecordMessage } from "./memory/ai-sdk-message-storage.js";
|
|
82
82
|
export { createConversationDisplayRecorder } from "./memory/conversation-display-recorder.js";
|
|
83
83
|
export type { ConversationDisplayRecorder, ConversationDisplayRecorderOptions } from "./memory/conversation-display-recorder.js";
|
package/dist/index.js
CHANGED
|
@@ -141,9 +141,11 @@ export { createConversationDisplayRecorder } from "./memory/conversation-display
|
|
|
141
141
|
export { envelopeDisplayMessages, parseStoredDisplayEnvelope } from "./memory/conversation-display-storage.js";
|
|
142
142
|
// The transcript read. Every append persists what it displayed, so replay concatenates stored
|
|
143
143
|
// projections and consults nothing else — no tool names, no card builders, no workspace or database
|
|
144
|
-
// lookups.
|
|
145
|
-
//
|
|
146
|
-
//
|
|
144
|
+
// lookups. A row written before the projection existed carries none and is skipped; there is no
|
|
145
|
+
// reconstruction to fall back to, the migration renderer that once supplied one having been deleted
|
|
146
|
+
// along with the startup backfill that was its only caller. `parseStoredDisplayEnvelope` absorbs the
|
|
147
|
+
// one thing a read can now meet and not understand — a part whose key this vocabulary has retired —
|
|
148
|
+
// by dropping that part, so one stale row cannot fail a whole thread.
|
|
147
149
|
export { storedMessagesToCortex } from "./memory/conversation-display-replay.js";
|
|
148
150
|
export { PART_REGISTRY, isTransient, isReconciling, isSidebarPart } from "./contracts/part-registry.js";
|
|
149
151
|
// Embedder runtime surface — what a same-process host needs to run durable
|
|
@@ -2,8 +2,15 @@
|
|
|
2
2
|
* The one page capture over the Chrome sidecar.
|
|
3
3
|
*
|
|
4
4
|
* A capture navigates to a page URL, collects the console errors and the failed requests, waits for the
|
|
5
|
-
* readiness signal of the page, and gives back
|
|
6
|
-
*
|
|
5
|
+
* readiness signal of the page, and gives back base64 screenshots. The eyes tool of a report session reads
|
|
6
|
+
* this module.
|
|
7
|
+
*
|
|
8
|
+
* A short page captures as one full-page shot. A tall page captures as consecutive vertical slices, because
|
|
9
|
+
* one tall picture dies twice on the provider path: a picture past the hard dimension cap rejects the whole
|
|
10
|
+
* request, and a legal tall picture downscales to about 1568 pixels on the long side, which compresses the
|
|
11
|
+
* text past legibility. A slice of about two window heights survives both. The slice budget bounds what one
|
|
12
|
+
* look costs, and a page past the budget truncates with the captured and the total pixels on the coverage,
|
|
13
|
+
* thus the truncation is never silent.
|
|
7
14
|
*
|
|
8
15
|
* The readiness contract is the reason for one shared body. The renderer emits the event name and the
|
|
9
16
|
* sentinel name, and `report-render/page.ts` owns both constants together with the two budgets. This module
|
|
@@ -23,20 +30,41 @@ export interface FailedRequest {
|
|
|
23
30
|
url: string;
|
|
24
31
|
reason: string;
|
|
25
32
|
}
|
|
26
|
-
/**
|
|
33
|
+
/** One picture of a capture: the base64 bytes, and the document rows that a slice holds. */
|
|
34
|
+
export interface CapturedShot {
|
|
35
|
+
readonly base64: string;
|
|
36
|
+
/** The vertical document range of a slice, in CSS pixels. Absent on a whole-document or window picture. */
|
|
37
|
+
readonly range?: {
|
|
38
|
+
readonly fromY: number;
|
|
39
|
+
readonly toY: number;
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* What the pictures hold together. `full` is one shot of the whole document. `tiled` is consecutive vertical
|
|
44
|
+
* slices in document order; `capturedPx` is the height the slices cover and `totalPx` is the document height,
|
|
45
|
+
* thus `capturedPx < totalPx` says that the slice budget ran out and the tail of the page is absent. `viewport`
|
|
46
|
+
* appears when a screenshot threw and the retry at the window passed, thus the one picture shows the top
|
|
47
|
+
* window alone and a section below the fold is absent from it.
|
|
48
|
+
*/
|
|
49
|
+
export type CaptureCoverage = {
|
|
50
|
+
readonly kind: "full";
|
|
51
|
+
} | {
|
|
52
|
+
readonly kind: "tiled";
|
|
53
|
+
readonly capturedPx: number;
|
|
54
|
+
readonly totalPx: number;
|
|
55
|
+
} | {
|
|
56
|
+
readonly kind: "viewport";
|
|
57
|
+
};
|
|
58
|
+
/** The pictures and the faults of one page capture. */
|
|
27
59
|
export interface PageCapture {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
* the first attempt. `viewport` appears when the full-page screenshot threw and the retry at the window
|
|
32
|
-
* passed, thus the picture shows the top window alone and a section below the fold is absent from it.
|
|
33
|
-
*/
|
|
34
|
-
coverage: "full" | "viewport";
|
|
60
|
+
/** The pictures in document order: one shot under `full` and `viewport`, the slices under `tiled`. */
|
|
61
|
+
screenshots: CapturedShot[];
|
|
62
|
+
coverage: CaptureCoverage;
|
|
35
63
|
consoleErrors: string[];
|
|
36
64
|
failedRequests: FailedRequest[];
|
|
37
65
|
}
|
|
38
66
|
/**
|
|
39
|
-
* The capture seam. It navigates to a page URL, and it gives back the
|
|
67
|
+
* The capture seam. It navigates to a page URL, and it gives back the screenshots and the faults. The seam
|
|
40
68
|
* speaks the throw protocol, because the chrome connection does. A test injects a seam that reads no
|
|
41
69
|
* browser, thus a tool orchestration runs with no chrome sidecar.
|
|
42
70
|
*/
|
|
@@ -49,11 +77,12 @@ export interface CaptureOptions {
|
|
|
49
77
|
readonly waitMs?: number;
|
|
50
78
|
}
|
|
51
79
|
/**
|
|
52
|
-
* Capture one page: the
|
|
80
|
+
* Capture one page: the screenshots, the console errors, and the failed requests.
|
|
53
81
|
*
|
|
54
|
-
* The
|
|
55
|
-
*
|
|
56
|
-
*
|
|
82
|
+
* The pictures hold the document, and not the window alone. Thus a caller can judge a section that a reader
|
|
83
|
+
* reaches by a scroll. A short page arrives as one full-page shot, and a tall page arrives as consecutive
|
|
84
|
+
* slices in document order. A refused bitmap degrades to the window, and the coverage of the result names
|
|
85
|
+
* what the pictures hold.
|
|
57
86
|
*
|
|
58
87
|
* The readiness wait is best-effort. A page that never signals still captures at the readiness budget, thus
|
|
59
88
|
* a broken page gives a picture that shows what broke.
|
package/dist/lib/page-capture.js
CHANGED
|
@@ -2,8 +2,15 @@
|
|
|
2
2
|
* The one page capture over the Chrome sidecar.
|
|
3
3
|
*
|
|
4
4
|
* A capture navigates to a page URL, collects the console errors and the failed requests, waits for the
|
|
5
|
-
* readiness signal of the page, and gives back
|
|
6
|
-
*
|
|
5
|
+
* readiness signal of the page, and gives back base64 screenshots. The eyes tool of a report session reads
|
|
6
|
+
* this module.
|
|
7
|
+
*
|
|
8
|
+
* A short page captures as one full-page shot. A tall page captures as consecutive vertical slices, because
|
|
9
|
+
* one tall picture dies twice on the provider path: a picture past the hard dimension cap rejects the whole
|
|
10
|
+
* request, and a legal tall picture downscales to about 1568 pixels on the long side, which compresses the
|
|
11
|
+
* text past legibility. A slice of about two window heights survives both. The slice budget bounds what one
|
|
12
|
+
* look costs, and a page past the budget truncates with the captured and the total pixels on the coverage,
|
|
13
|
+
* thus the truncation is never silent.
|
|
7
14
|
*
|
|
8
15
|
* The readiness contract is the reason for one shared body. The renderer emits the event name and the
|
|
9
16
|
* sentinel name, and `report-render/page.ts` owns both constants together with the two budgets. This module
|
|
@@ -29,6 +36,23 @@ const SELECTOR_TIMEOUT_MS = 5_000;
|
|
|
29
36
|
*/
|
|
30
37
|
const VIEWPORT_WIDTH = 1440;
|
|
31
38
|
const VIEWPORT_HEIGHT = 900;
|
|
39
|
+
/**
|
|
40
|
+
* The height of one capture slice, in CSS pixels, and the bound of a single-shot page.
|
|
41
|
+
*
|
|
42
|
+
* The provider downscales every picture to about 1568 pixels on the long side before the model reads it. Two
|
|
43
|
+
* window heights survive that downscale with readable text, and a taller picture reaches the model compressed
|
|
44
|
+
* past that. Thus a page at this height or under captures as one full-page shot, and a taller page captures
|
|
45
|
+
* as slices of this height.
|
|
46
|
+
*/
|
|
47
|
+
const TILE_HEIGHT_PX = VIEWPORT_HEIGHT * 2;
|
|
48
|
+
/**
|
|
49
|
+
* The most slices of one capture.
|
|
50
|
+
*
|
|
51
|
+
* Each slice is one picture on the tool result, and the budget bounds what one look costs the context. A page
|
|
52
|
+
* taller than the budget covers truncates, and the coverage carries the captured and the total pixels, thus
|
|
53
|
+
* the truncation is never silent.
|
|
54
|
+
*/
|
|
55
|
+
const MAX_TILES = 6;
|
|
32
56
|
/**
|
|
33
57
|
* The body of the readiness wait, in the browser context. The sentinel arm resolves a page that dispatched
|
|
34
58
|
* the event before this wait registered, thus a plain listener never blocks forever. The timer arm bounds a
|
|
@@ -57,37 +81,72 @@ function waitForThemeReady(sentinel, event, timeout) {
|
|
|
57
81
|
function toBase64(shot) {
|
|
58
82
|
return typeof shot === "string" ? shot : Buffer.from(shot).toString("base64");
|
|
59
83
|
}
|
|
84
|
+
/** The body of the height measure, in the browser context. */
|
|
85
|
+
function measureScrollHeight() {
|
|
86
|
+
return document.documentElement.scrollHeight;
|
|
87
|
+
}
|
|
60
88
|
/**
|
|
61
|
-
*
|
|
89
|
+
* Measure the document height, in CSS pixels. A page that refuses the measure reads as a short one, thus the
|
|
90
|
+
* capture takes the one full-page shot and the refusal costs no look.
|
|
91
|
+
*/
|
|
92
|
+
async function measureTotalHeight(page) {
|
|
93
|
+
const measured = await page.evaluate(measureScrollHeight).catch(() => undefined);
|
|
94
|
+
return typeof measured === "number" && Number.isFinite(measured) ? measured : 0;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Capture the slices of one tall page, in document order. Each slice clips {@link TILE_HEIGHT_PX} rows at the
|
|
98
|
+
* reader width, and the last slice ends at the document height or at the budget, whichever comes first.
|
|
99
|
+
*/
|
|
100
|
+
async function captureTiles(page, totalPx) {
|
|
101
|
+
const tileCount = Math.min(Math.ceil(totalPx / TILE_HEIGHT_PX), MAX_TILES);
|
|
102
|
+
const screenshots = [];
|
|
103
|
+
for (let index = 0; index < tileCount; index++) {
|
|
104
|
+
const fromY = index * TILE_HEIGHT_PX;
|
|
105
|
+
const toY = Math.min(fromY + TILE_HEIGHT_PX, totalPx);
|
|
106
|
+
const clip = { x: 0, y: fromY, width: VIEWPORT_WIDTH, height: toY - fromY };
|
|
107
|
+
screenshots.push({ base64: toBase64(await page.screenshot({ encoding: "base64", clip })), range: { fromY, toY } });
|
|
108
|
+
}
|
|
109
|
+
const capturedPx = Math.min(tileCount * TILE_HEIGHT_PX, totalPx);
|
|
110
|
+
return { screenshots, coverage: { kind: "tiled", capturedPx, totalPx } };
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Take the pictures of one settled page.
|
|
114
|
+
*
|
|
115
|
+
* A page at the single-shot bound or under gives one full-page shot. A taller page gives consecutive slices,
|
|
116
|
+
* because the provider path compresses or rejects one tall picture; the module comment carries the account.
|
|
62
117
|
*
|
|
63
|
-
* The compositor can refuse a
|
|
64
|
-
*
|
|
65
|
-
*
|
|
118
|
+
* The compositor can refuse a bitmap of either shape while the bitmap of the window is fine. A degraded
|
|
119
|
+
* picture beats a dead look, thus a refusal retries one time at the window. A second throw names a broken
|
|
120
|
+
* browser and not a tall page, thus it propagates to the caller.
|
|
66
121
|
*/
|
|
67
|
-
async function
|
|
122
|
+
async function captureShots(page) {
|
|
123
|
+
const totalPx = await measureTotalHeight(page);
|
|
68
124
|
try {
|
|
69
|
-
|
|
125
|
+
if (totalPx > TILE_HEIGHT_PX)
|
|
126
|
+
return await captureTiles(page, totalPx);
|
|
127
|
+
return { screenshots: [{ base64: toBase64(await page.screenshot({ encoding: "base64", fullPage: true })) }], coverage: { kind: "full" } };
|
|
70
128
|
}
|
|
71
|
-
catch (
|
|
129
|
+
catch (refused) {
|
|
72
130
|
try {
|
|
73
|
-
return {
|
|
131
|
+
return { screenshots: [{ base64: toBase64(await page.screenshot({ encoding: "base64" })) }], coverage: { kind: "viewport" } };
|
|
74
132
|
}
|
|
75
133
|
catch (refusedViewport) {
|
|
76
134
|
// The two refusals together are the account of a dead look, and the caller reports the second one
|
|
77
135
|
// alone. Thus the first one rides as the cause, and no reader of that report loses half of it.
|
|
78
136
|
if (refusedViewport instanceof Error && refusedViewport.cause === undefined) {
|
|
79
|
-
refusedViewport.cause =
|
|
137
|
+
refusedViewport.cause = refused;
|
|
80
138
|
}
|
|
81
139
|
throw refusedViewport;
|
|
82
140
|
}
|
|
83
141
|
}
|
|
84
142
|
}
|
|
85
143
|
/**
|
|
86
|
-
* Capture one page: the
|
|
144
|
+
* Capture one page: the screenshots, the console errors, and the failed requests.
|
|
87
145
|
*
|
|
88
|
-
* The
|
|
89
|
-
*
|
|
90
|
-
*
|
|
146
|
+
* The pictures hold the document, and not the window alone. Thus a caller can judge a section that a reader
|
|
147
|
+
* reaches by a scroll. A short page arrives as one full-page shot, and a tall page arrives as consecutive
|
|
148
|
+
* slices in document order. A refused bitmap degrades to the window, and the coverage of the result names
|
|
149
|
+
* what the pictures hold.
|
|
91
150
|
*
|
|
92
151
|
* The readiness wait is best-effort. A page that never signals still captures at the readiness budget, thus
|
|
93
152
|
* a broken page gives a picture that shows what broke.
|
|
@@ -124,12 +183,12 @@ export function capturePage(chrome, url, options = {}) {
|
|
|
124
183
|
const settle = options.waitMs;
|
|
125
184
|
await new Promise((resolve) => setTimeout(resolve, settle));
|
|
126
185
|
}
|
|
127
|
-
// The
|
|
186
|
+
// The pictures must show the whole document at the layout that a reader gets. Thus a defect below the
|
|
128
187
|
// fold is visible, and a question about content that never appeared has an answer.
|
|
129
|
-
const
|
|
188
|
+
const shots = await captureShots(page);
|
|
130
189
|
return {
|
|
131
|
-
|
|
132
|
-
coverage:
|
|
190
|
+
screenshots: shots.screenshots,
|
|
191
|
+
coverage: shots.coverage,
|
|
133
192
|
consoleErrors,
|
|
134
193
|
failedRequests,
|
|
135
194
|
};
|
package/dist/loop/run-agent.js
CHANGED
|
@@ -10,7 +10,7 @@ import { classifyProviderError } from "../providers/errors.js";
|
|
|
10
10
|
import { DEFAULT_PROMPT_CACHE, promptCacheProviderOptions } from "../providers/prompt-cache.js";
|
|
11
11
|
import { resultStep } from "./run-step.js";
|
|
12
12
|
import { AskRejectedError, UnavailableAsk } from "../tools/approval/contract.js";
|
|
13
|
-
import { isToolError,
|
|
13
|
+
import { isToolError, readToolResultImages } from "../tools/define-tool.js";
|
|
14
14
|
import { addChatUsage, hasReportedUsage, recordAgentRun } from "./metrics.js";
|
|
15
15
|
import { computeDetail, computeResultDetail } from "./tool-detail.js";
|
|
16
16
|
import { toolOutcomeForOutputType } from "./tool-outcome.js";
|
|
@@ -457,6 +457,16 @@ function imagePlacementFor(capabilities) {
|
|
|
457
457
|
return "user-message";
|
|
458
458
|
return "drop";
|
|
459
459
|
}
|
|
460
|
+
/**
|
|
461
|
+
* The text part that names one deferred picture. The wire holds no structural
|
|
462
|
+
* link between a user message and a tool call, thus the line names the call. A
|
|
463
|
+
* result with several pictures numbers each one, thus the model reads which
|
|
464
|
+
* slice it looks at.
|
|
465
|
+
*/
|
|
466
|
+
function deferredImageLabel(deferred, index) {
|
|
467
|
+
const call = `the tool result ${deferred.toolCallId} of ${deferred.toolName}`;
|
|
468
|
+
return deferred.images.length === 1 ? `The picture of ${call}.` : `The picture ${index + 1} of ${deferred.images.length} of ${call}.`;
|
|
469
|
+
}
|
|
460
470
|
/**
|
|
461
471
|
* Append one user message that carries each deferred picture of the round, then
|
|
462
472
|
* empty the collector.
|
|
@@ -464,10 +474,11 @@ function imagePlacementFor(capabilities) {
|
|
|
464
474
|
* The tool message must come directly after the assistant message with the
|
|
465
475
|
* tool calls. Thus a picture rides a separate message after the whole tool
|
|
466
476
|
* message, and one message batches the round. The parts obey the order of
|
|
467
|
-
* `results`, which is the order of the tool calls
|
|
468
|
-
* result
|
|
469
|
-
*
|
|
470
|
-
*
|
|
477
|
+
* `results`, which is the order of the tool calls, and the pictures of one
|
|
478
|
+
* result keep the order that the tool gave. Each deferred picture has a result
|
|
479
|
+
* of this round, because the collector fills during the dispatch and it empties
|
|
480
|
+
* here. A text part names the tool call of each picture, because the wire holds
|
|
481
|
+
* no structural link between a user message and a tool call.
|
|
471
482
|
*
|
|
472
483
|
* The message carries the synthetic marker, because a `user` message opens a
|
|
473
484
|
* conversation turn. An unmarked one is loop machinery that reads as a turn
|
|
@@ -482,8 +493,10 @@ function appendDeferredImages(messages, results, deferredImages) {
|
|
|
482
493
|
const deferred = byToolCallId.get(result.toolCallId);
|
|
483
494
|
if (deferred === undefined)
|
|
484
495
|
continue;
|
|
485
|
-
|
|
486
|
-
|
|
496
|
+
for (const [index, image] of deferred.images.entries()) {
|
|
497
|
+
content.push({ type: "text", text: deferredImageLabel(deferred, index) });
|
|
498
|
+
content.push({ type: "file", mediaType: image.mediaType, data: { type: "data", data: image.base64 } });
|
|
499
|
+
}
|
|
487
500
|
}
|
|
488
501
|
messages.push(syntheticUserMessage(content));
|
|
489
502
|
deferredImages.length = 0;
|
|
@@ -615,15 +628,16 @@ function jsonValue(value) {
|
|
|
615
628
|
* Encode a tool ok value onto a tool-result message. A value with no picture
|
|
616
629
|
* stays a plain JSON result, byte-identical to a value that never carried one.
|
|
617
630
|
*
|
|
618
|
-
* A value that carries
|
|
619
|
-
* the bytes ride as
|
|
620
|
-
* the JSON text holds no bytes. When
|
|
621
|
-
*
|
|
622
|
-
*
|
|
623
|
-
* reach the
|
|
631
|
+
* A value that carries pictures splits: the JSON data goes to a text part, and
|
|
632
|
+
* the bytes ride as image content blocks after it, in the order that the tool
|
|
633
|
+
* gave. Thus the model sees each picture, and the JSON text holds no bytes. When
|
|
634
|
+
* the wire carries no picture, the blocks drop and the text stays, because
|
|
635
|
+
* base64 text floods the context and the model cannot see it. One warn with the
|
|
636
|
+
* count rides the log, thus an operator sees how many pictures did not reach the
|
|
637
|
+
* model.
|
|
624
638
|
*
|
|
625
639
|
* A wire that renders a picture on a user message only gets the fallback: the
|
|
626
|
-
* result keeps its JSON text, and the
|
|
640
|
+
* result keeps its JSON text, and the pictures go to the collector of the round.
|
|
627
641
|
* The round then sends the bytes after the tool message.
|
|
628
642
|
*/
|
|
629
643
|
function successResult(toolCall, value, encoding) {
|
|
@@ -634,15 +648,18 @@ function successResult(toolCall, value, encoding) {
|
|
|
634
648
|
toolName: toolCall.toolName,
|
|
635
649
|
output: { type: "json", value: jsonData },
|
|
636
650
|
};
|
|
637
|
-
const
|
|
638
|
-
if (
|
|
651
|
+
const images = readToolResultImages(value);
|
|
652
|
+
if (images.length === 0)
|
|
639
653
|
return jsonOnly;
|
|
640
654
|
if (encoding.placement === "drop") {
|
|
641
|
-
encoding.log.warn("the wired provider carries no picture in a tool result, thus the
|
|
655
|
+
encoding.log.warn("the wired provider carries no picture in a tool result, thus the pictures are dropped", {
|
|
656
|
+
toolName: toolCall.toolName,
|
|
657
|
+
imageCount: images.length,
|
|
658
|
+
});
|
|
642
659
|
return jsonOnly;
|
|
643
660
|
}
|
|
644
661
|
if (encoding.placement === "user-message") {
|
|
645
|
-
encoding.deferredImages.push({ toolCallId: toolCall.toolCallId, toolName: toolCall.toolName,
|
|
662
|
+
encoding.deferredImages.push({ toolCallId: toolCall.toolCallId, toolName: toolCall.toolName, images });
|
|
646
663
|
return jsonOnly;
|
|
647
664
|
}
|
|
648
665
|
return {
|
|
@@ -653,7 +670,7 @@ function successResult(toolCall, value, encoding) {
|
|
|
653
670
|
type: "content",
|
|
654
671
|
value: [
|
|
655
672
|
{ type: "text", text: JSON.stringify(jsonData) },
|
|
656
|
-
{ type: "file", mediaType: image.mediaType, data: { type: "data", data: image.base64 } },
|
|
673
|
+
...images.map((image) => ({ type: "file", mediaType: image.mediaType, data: { type: "data", data: image.base64 } })),
|
|
657
674
|
],
|
|
658
675
|
},
|
|
659
676
|
};
|
|
@@ -3,12 +3,10 @@
|
|
|
3
3
|
* `data-presentation`, `data-run-card`, `data-file-reference`).
|
|
4
4
|
*
|
|
5
5
|
* Display tools (`show_plan`, `show_user`, `show_file`) emit these cards live
|
|
6
|
-
* over the chat SSE stream,
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* byte-identical (the deterministic `id` matches, enabling downstream
|
|
11
|
-
* reconciliation).
|
|
6
|
+
* over the chat SSE stream, and the turn persists what it emitted, so a reload
|
|
7
|
+
* replays the built card rather than rebuilding it. These builders therefore run
|
|
8
|
+
* once per card, on the live path only. The `id` stays deterministic all the
|
|
9
|
+
* same: it is what a re-emission of the same card reconciles on.
|
|
12
10
|
*
|
|
13
11
|
* The builders return the FLAT wire `data` payload (no `type` wrapper, no
|
|
14
12
|
* `source`): the tool emits `{ type, source, data }` (the emit pipeline
|
|
@@ -3,12 +3,10 @@
|
|
|
3
3
|
* `data-presentation`, `data-run-card`, `data-file-reference`).
|
|
4
4
|
*
|
|
5
5
|
* Display tools (`show_plan`, `show_user`, `show_file`) emit these cards live
|
|
6
|
-
* over the chat SSE stream,
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* byte-identical (the deterministic `id` matches, enabling downstream
|
|
11
|
-
* reconciliation).
|
|
6
|
+
* over the chat SSE stream, and the turn persists what it emitted, so a reload
|
|
7
|
+
* replays the built card rather than rebuilding it. These builders therefore run
|
|
8
|
+
* once per card, on the live path only. The `id` stays deterministic all the
|
|
9
|
+
* same: it is what a re-emission of the same card reconciles on.
|
|
12
10
|
*
|
|
13
11
|
* The builders return the FLAT wire `data` payload (no `type` wrapper, no
|
|
14
12
|
* `source`): the tool emits `{ type, source, data }` (the emit pipeline
|
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The runtime transcript read: stored display projections → `CortexMessage[]`.
|
|
3
3
|
*
|
|
4
|
-
* `ThreadHistory.
|
|
4
|
+
* `ThreadHistory.loadAll` returns a thread's rows in `seq` order, and every
|
|
5
5
|
* append's first row carries the display projection for that append. Replay is
|
|
6
6
|
* therefore a concatenation: walk the rows, emit each envelope's messages in
|
|
7
7
|
* order, done. No turn grouping, no tool-name recognition, no card rebuilding,
|
|
8
8
|
* no filesystem or database lookup — nothing that could render a reloaded
|
|
9
9
|
* conversation differently from the one the user was shown.
|
|
10
10
|
*
|
|
11
|
-
* A row with no envelope predates the display projection and is skipped
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* A row with no envelope predates the display projection and is skipped — the
|
|
12
|
+
* only handling there is or will be. Reconstructing one from the model
|
|
13
|
+
* transcript instead is what the deleted migration renderer did, and rebuilding
|
|
14
|
+
* a display from a transcript that never carried one makes every later change to
|
|
15
|
+
* a tool or a card silently rewrite history. Skipping keeps the gap observable
|
|
16
|
+
* and the read total in what it consults: the stored projection, nothing else.
|
|
16
17
|
*/
|
|
17
18
|
import type { CortexMessage } from "../contracts/message.js";
|
|
18
19
|
import type { StoredMessage } from "./thread-history.js";
|
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The runtime transcript read: stored display projections → `CortexMessage[]`.
|
|
3
3
|
*
|
|
4
|
-
* `ThreadHistory.
|
|
4
|
+
* `ThreadHistory.loadAll` returns a thread's rows in `seq` order, and every
|
|
5
5
|
* append's first row carries the display projection for that append. Replay is
|
|
6
6
|
* therefore a concatenation: walk the rows, emit each envelope's messages in
|
|
7
7
|
* order, done. No turn grouping, no tool-name recognition, no card rebuilding,
|
|
8
8
|
* no filesystem or database lookup — nothing that could render a reloaded
|
|
9
9
|
* conversation differently from the one the user was shown.
|
|
10
10
|
*
|
|
11
|
-
* A row with no envelope predates the display projection and is skipped
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* A row with no envelope predates the display projection and is skipped — the
|
|
12
|
+
* only handling there is or will be. Reconstructing one from the model
|
|
13
|
+
* transcript instead is what the deleted migration renderer did, and rebuilding
|
|
14
|
+
* a display from a transcript that never carried one makes every later change to
|
|
15
|
+
* a tool or a card silently rewrite history. Skipping keeps the gap observable
|
|
16
|
+
* and the read total in what it consults: the stored projection, nothing else.
|
|
16
17
|
*/
|
|
17
18
|
import { conversationUIToCortexMessages } from "./conversation-display-storage.js";
|
|
18
19
|
export function storedMessagesToCortex(messages) {
|
|
@@ -18,6 +18,7 @@ import { type DataUIPart, type UIMessage } from "ai";
|
|
|
18
18
|
import type { AskPart, ChildSessionStartedPart, FileReferencePart, PlanPart, PresentationPart, ReportRenderedPart, RunCardPart } from "../contracts/chat-parts.js";
|
|
19
19
|
import type { ToolCallOutcome } from "../contracts/chat-events.js";
|
|
20
20
|
import type { CortexMessage, CortexPart, ToolCallPart } from "../contracts/message.js";
|
|
21
|
+
import type { Logger } from "../lib/logger.js";
|
|
21
22
|
export declare const SUPPORTED_DISPLAY_SCHEMA_VERSION = 1;
|
|
22
23
|
type Payload<T extends {
|
|
23
24
|
type: string;
|
|
@@ -52,7 +53,7 @@ export interface StoredDisplayEnvelope {
|
|
|
52
53
|
readonly messages: ConversationUIMessage[];
|
|
53
54
|
}
|
|
54
55
|
export declare function envelopeDisplayMessages(messages: readonly ConversationUIMessage[]): StoredDisplayEnvelope;
|
|
55
|
-
export declare function parseStoredDisplayEnvelope(value: unknown, identity: string): Promise<StoredDisplayEnvelope>;
|
|
56
|
+
export declare function parseStoredDisplayEnvelope(value: unknown, identity: string, logger?: Logger): Promise<StoredDisplayEnvelope>;
|
|
56
57
|
/**
|
|
57
58
|
* Build the stored form of one non-text Cortex part.
|
|
58
59
|
*
|
|
@@ -66,15 +67,6 @@ export declare function parseStoredDisplayEnvelope(value: unknown, identity: str
|
|
|
66
67
|
export declare function conversationDisplayPart(part: Exclude<CortexPart, {
|
|
67
68
|
type: "text";
|
|
68
69
|
}>): DataUIPart<ConversationUIData>;
|
|
69
|
-
/**
|
|
70
|
-
* Project Cortex display messages into the durable representation.
|
|
71
|
-
*
|
|
72
|
-
* The startup backfill's only path into storage: it renders a legacy turn from
|
|
73
|
-
* the model transcript and freezes the result. The live recorder builds the same
|
|
74
|
-
* shape directly from the event stream instead of round-tripping through
|
|
75
|
-
* `CortexMessage`.
|
|
76
|
-
*/
|
|
77
|
-
export declare function cortexMessagesToConversationUI(messages: readonly CortexMessage[]): ConversationUIMessage[];
|
|
78
70
|
/**
|
|
79
71
|
* Read the durable representation back as Cortex display messages.
|
|
80
72
|
*
|