@inflexa-ai/harness 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/sandbox/__fixtures__/deps.js +6 -0
- package/dist/agents/sandbox/data-profiler.js +5 -4
- package/dist/agents/sandbox/shared.js +11 -0
- package/dist/agents/sandbox/types.d.ts +1 -1
- package/dist/app/data-profile-orientation.js +30 -1
- package/dist/contracts/data-profile.d.ts +166 -0
- package/dist/contracts/index.d.ts +1 -0
- package/dist/execution/input-signature.d.ts +33 -0
- package/dist/execution/input-signature.js +0 -0
- package/dist/index.d.ts +5 -2
- package/dist/index.js +10 -3
- package/dist/input-scan/coverage.d.ts +22 -0
- package/dist/input-scan/coverage.js +85 -0
- package/dist/input-scan/enrich.d.ts +41 -0
- package/dist/input-scan/enrich.js +227 -0
- package/dist/input-scan/formats.d.ts +41 -0
- package/dist/input-scan/formats.js +279 -0
- package/dist/input-scan/index.d.ts +12 -0
- package/dist/input-scan/index.js +11 -0
- package/dist/input-scan/scan.d.ts +50 -0
- package/dist/input-scan/scan.js +182 -0
- package/dist/input-scan/shapes.d.ts +46 -0
- package/dist/input-scan/shapes.js +0 -0
- package/dist/input-scan/types.d.ts +172 -0
- package/dist/input-scan/types.js +12 -0
- package/dist/lib/page-capture.d.ts +44 -15
- package/dist/lib/page-capture.js +78 -19
- package/dist/loop/run-agent.js +36 -19
- package/dist/memory/card-builders.d.ts +4 -6
- package/dist/memory/card-builders.js +4 -6
- package/dist/memory/conversation-display-replay.d.ts +7 -6
- package/dist/memory/conversation-display-replay.js +7 -6
- package/dist/memory/conversation-display-storage.d.ts +2 -10
- package/dist/memory/conversation-display-storage.js +79 -32
- package/dist/memory/thread-history.d.ts +31 -20
- package/dist/memory/thread-history.js +45 -45
- package/dist/prompts/report-session.d.ts +1 -1
- package/dist/prompts/report-session.js +22 -13
- package/dist/prompts/sandbox/data-profiler.d.ts +1 -1
- package/dist/prompts/sandbox/data-profiler.js +159 -275
- package/dist/provenance/exec-frame.js +9 -3
- package/dist/runtime/boot.d.ts +7 -8
- package/dist/runtime/boot.js +7 -19
- package/dist/sandbox/client.d.ts +7 -0
- package/dist/sandbox/create-sandbox.js +1 -0
- package/dist/sandbox/docker-client.d.ts +1 -0
- package/dist/sandbox/docker-client.js +20 -15
- package/dist/sandbox/k8s-client.d.ts +6 -5
- package/dist/sandbox/k8s-client.js +55 -39
- package/dist/sandbox/reaper.d.ts +24 -13
- package/dist/sandbox/reaper.js +47 -14
- package/dist/sandbox/types.d.ts +6 -5
- package/dist/schemas/data-profile-schemas.d.ts +62 -0
- package/dist/schemas/data-profile-schemas.js +68 -2
- package/dist/state/data-profile.d.ts +7 -88
- package/dist/state/index.d.ts +1 -1
- package/dist/tasks/data-profile-activity.d.ts +8 -0
- package/dist/tasks/data-profile-activity.js +1 -0
- package/dist/tasks/data-profile-index.d.ts +45 -0
- package/dist/tasks/data-profile-index.js +134 -0
- package/dist/tasks/data-profile.d.ts +8 -23
- package/dist/tasks/data-profile.js +150 -117
- package/dist/tools/define-tool.d.ts +19 -8
- package/dist/tools/define-tool.js +28 -12
- package/dist/tools/report-session/examine-page.d.ts +22 -13
- package/dist/tools/report-session/examine-page.js +15 -10
- package/dist/tools/report-session/index.d.ts +1 -1
- package/dist/tools/research/generate-plan.js +3 -8
- package/dist/tools/research/inspect-data-profile.d.ts +30 -4
- package/dist/tools/research/inspect-data-profile.js +75 -20
- package/dist/tools/workspace/scan-inputs.d.ts +38 -0
- package/dist/tools/workspace/scan-inputs.js +61 -0
- package/dist/tools/workspace/workspace-search.d.ts +1 -1
- package/dist/tools/workspace/workspace-search.js +12 -9
- package/dist/workspace/filesystem.d.ts +27 -0
- package/dist/workspace/filesystem.js +27 -0
- package/package.json +1 -1
- package/dist/app/data-profile-policy.d.ts +0 -36
- package/dist/app/data-profile-policy.js +0 -37
- package/dist/memory/content-to-cortex.d.ts +0 -75
- package/dist/memory/content-to-cortex.js +0 -201
- package/dist/memory/conversation-display-backfill.d.ts +0 -23
- package/dist/memory/conversation-display-backfill.js +0 -91
- package/dist/memory/reconstruct-cards.d.ts +0 -27
- package/dist/memory/reconstruct-cards.js +0 -57
package/dist/loop/run-agent.js
CHANGED
|
@@ -10,7 +10,7 @@ import { classifyProviderError } from "../providers/errors.js";
|
|
|
10
10
|
import { DEFAULT_PROMPT_CACHE, promptCacheProviderOptions } from "../providers/prompt-cache.js";
|
|
11
11
|
import { resultStep } from "./run-step.js";
|
|
12
12
|
import { AskRejectedError, UnavailableAsk } from "../tools/approval/contract.js";
|
|
13
|
-
import { isToolError,
|
|
13
|
+
import { isToolError, readToolResultImages } from "../tools/define-tool.js";
|
|
14
14
|
import { addChatUsage, hasReportedUsage, recordAgentRun } from "./metrics.js";
|
|
15
15
|
import { computeDetail, computeResultDetail } from "./tool-detail.js";
|
|
16
16
|
import { toolOutcomeForOutputType } from "./tool-outcome.js";
|
|
@@ -457,6 +457,16 @@ function imagePlacementFor(capabilities) {
|
|
|
457
457
|
return "user-message";
|
|
458
458
|
return "drop";
|
|
459
459
|
}
|
|
460
|
+
/**
|
|
461
|
+
* The text part that names one deferred picture. The wire holds no structural
|
|
462
|
+
* link between a user message and a tool call, thus the line names the call. A
|
|
463
|
+
* result with several pictures numbers each one, thus the model reads which
|
|
464
|
+
* slice it looks at.
|
|
465
|
+
*/
|
|
466
|
+
function deferredImageLabel(deferred, index) {
|
|
467
|
+
const call = `the tool result ${deferred.toolCallId} of ${deferred.toolName}`;
|
|
468
|
+
return deferred.images.length === 1 ? `The picture of ${call}.` : `The picture ${index + 1} of ${deferred.images.length} of ${call}.`;
|
|
469
|
+
}
|
|
460
470
|
/**
|
|
461
471
|
* Append one user message that carries each deferred picture of the round, then
|
|
462
472
|
* empty the collector.
|
|
@@ -464,10 +474,11 @@ function imagePlacementFor(capabilities) {
|
|
|
464
474
|
* The tool message must come directly after the assistant message with the
|
|
465
475
|
* tool calls. Thus a picture rides a separate message after the whole tool
|
|
466
476
|
* message, and one message batches the round. The parts obey the order of
|
|
467
|
-
* `results`, which is the order of the tool calls
|
|
468
|
-
* result
|
|
469
|
-
*
|
|
470
|
-
*
|
|
477
|
+
* `results`, which is the order of the tool calls, and the pictures of one
|
|
478
|
+
* result keep the order that the tool gave. Each deferred picture has a result
|
|
479
|
+
* of this round, because the collector fills during the dispatch and it empties
|
|
480
|
+
* here. A text part names the tool call of each picture, because the wire holds
|
|
481
|
+
* no structural link between a user message and a tool call.
|
|
471
482
|
*
|
|
472
483
|
* The message carries the synthetic marker, because a `user` message opens a
|
|
473
484
|
* conversation turn. An unmarked one is loop machinery that reads as a turn
|
|
@@ -482,8 +493,10 @@ function appendDeferredImages(messages, results, deferredImages) {
|
|
|
482
493
|
const deferred = byToolCallId.get(result.toolCallId);
|
|
483
494
|
if (deferred === undefined)
|
|
484
495
|
continue;
|
|
485
|
-
|
|
486
|
-
|
|
496
|
+
for (const [index, image] of deferred.images.entries()) {
|
|
497
|
+
content.push({ type: "text", text: deferredImageLabel(deferred, index) });
|
|
498
|
+
content.push({ type: "file", mediaType: image.mediaType, data: { type: "data", data: image.base64 } });
|
|
499
|
+
}
|
|
487
500
|
}
|
|
488
501
|
messages.push(syntheticUserMessage(content));
|
|
489
502
|
deferredImages.length = 0;
|
|
@@ -615,15 +628,16 @@ function jsonValue(value) {
|
|
|
615
628
|
* Encode a tool ok value onto a tool-result message. A value with no picture
|
|
616
629
|
* stays a plain JSON result, byte-identical to a value that never carried one.
|
|
617
630
|
*
|
|
618
|
-
* A value that carries
|
|
619
|
-
* the bytes ride as
|
|
620
|
-
* the JSON text holds no bytes. When
|
|
621
|
-
*
|
|
622
|
-
*
|
|
623
|
-
* reach the
|
|
631
|
+
* A value that carries pictures splits: the JSON data goes to a text part, and
|
|
632
|
+
* the bytes ride as image content blocks after it, in the order that the tool
|
|
633
|
+
* gave. Thus the model sees each picture, and the JSON text holds no bytes. When
|
|
634
|
+
* the wire carries no picture, the blocks drop and the text stays, because
|
|
635
|
+
* base64 text floods the context and the model cannot see it. One warn with the
|
|
636
|
+
* count rides the log, thus an operator sees how many pictures did not reach the
|
|
637
|
+
* model.
|
|
624
638
|
*
|
|
625
639
|
* A wire that renders a picture on a user message only gets the fallback: the
|
|
626
|
-
* result keeps its JSON text, and the
|
|
640
|
+
* result keeps its JSON text, and the pictures go to the collector of the round.
|
|
627
641
|
* The round then sends the bytes after the tool message.
|
|
628
642
|
*/
|
|
629
643
|
function successResult(toolCall, value, encoding) {
|
|
@@ -634,15 +648,18 @@ function successResult(toolCall, value, encoding) {
|
|
|
634
648
|
toolName: toolCall.toolName,
|
|
635
649
|
output: { type: "json", value: jsonData },
|
|
636
650
|
};
|
|
637
|
-
const
|
|
638
|
-
if (
|
|
651
|
+
const images = readToolResultImages(value);
|
|
652
|
+
if (images.length === 0)
|
|
639
653
|
return jsonOnly;
|
|
640
654
|
if (encoding.placement === "drop") {
|
|
641
|
-
encoding.log.warn("the wired provider carries no picture in a tool result, thus the
|
|
655
|
+
encoding.log.warn("the wired provider carries no picture in a tool result, thus the pictures are dropped", {
|
|
656
|
+
toolName: toolCall.toolName,
|
|
657
|
+
imageCount: images.length,
|
|
658
|
+
});
|
|
642
659
|
return jsonOnly;
|
|
643
660
|
}
|
|
644
661
|
if (encoding.placement === "user-message") {
|
|
645
|
-
encoding.deferredImages.push({ toolCallId: toolCall.toolCallId, toolName: toolCall.toolName,
|
|
662
|
+
encoding.deferredImages.push({ toolCallId: toolCall.toolCallId, toolName: toolCall.toolName, images });
|
|
646
663
|
return jsonOnly;
|
|
647
664
|
}
|
|
648
665
|
return {
|
|
@@ -653,7 +670,7 @@ function successResult(toolCall, value, encoding) {
|
|
|
653
670
|
type: "content",
|
|
654
671
|
value: [
|
|
655
672
|
{ type: "text", text: JSON.stringify(jsonData) },
|
|
656
|
-
{ type: "file", mediaType: image.mediaType, data: { type: "data", data: image.base64 } },
|
|
673
|
+
...images.map((image) => ({ type: "file", mediaType: image.mediaType, data: { type: "data", data: image.base64 } })),
|
|
657
674
|
],
|
|
658
675
|
},
|
|
659
676
|
};
|
|
@@ -3,12 +3,10 @@
|
|
|
3
3
|
* `data-presentation`, `data-run-card`, `data-file-reference`).
|
|
4
4
|
*
|
|
5
5
|
* Display tools (`show_plan`, `show_user`, `show_file`) emit these cards live
|
|
6
|
-
* over the chat SSE stream,
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* byte-identical (the deterministic `id` matches, enabling downstream
|
|
11
|
-
* reconciliation).
|
|
6
|
+
* over the chat SSE stream, and the turn persists what it emitted, so a reload
|
|
7
|
+
* replays the built card rather than rebuilding it. These builders therefore run
|
|
8
|
+
* once per card, on the live path only. The `id` stays deterministic all the
|
|
9
|
+
* same: it is what a re-emission of the same card reconciles on.
|
|
12
10
|
*
|
|
13
11
|
* The builders return the FLAT wire `data` payload (no `type` wrapper, no
|
|
14
12
|
* `source`): the tool emits `{ type, source, data }` (the emit pipeline
|
|
@@ -3,12 +3,10 @@
|
|
|
3
3
|
* `data-presentation`, `data-run-card`, `data-file-reference`).
|
|
4
4
|
*
|
|
5
5
|
* Display tools (`show_plan`, `show_user`, `show_file`) emit these cards live
|
|
6
|
-
* over the chat SSE stream,
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* byte-identical (the deterministic `id` matches, enabling downstream
|
|
11
|
-
* reconciliation).
|
|
6
|
+
* over the chat SSE stream, and the turn persists what it emitted, so a reload
|
|
7
|
+
* replays the built card rather than rebuilding it. These builders therefore run
|
|
8
|
+
* once per card, on the live path only. The `id` stays deterministic all the
|
|
9
|
+
* same: it is what a re-emission of the same card reconciles on.
|
|
12
10
|
*
|
|
13
11
|
* The builders return the FLAT wire `data` payload (no `type` wrapper, no
|
|
14
12
|
* `source`): the tool emits `{ type, source, data }` (the emit pipeline
|
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The runtime transcript read: stored display projections → `CortexMessage[]`.
|
|
3
3
|
*
|
|
4
|
-
* `ThreadHistory.
|
|
4
|
+
* `ThreadHistory.loadAll` returns a thread's rows in `seq` order, and every
|
|
5
5
|
* append's first row carries the display projection for that append. Replay is
|
|
6
6
|
* therefore a concatenation: walk the rows, emit each envelope's messages in
|
|
7
7
|
* order, done. No turn grouping, no tool-name recognition, no card rebuilding,
|
|
8
8
|
* no filesystem or database lookup — nothing that could render a reloaded
|
|
9
9
|
* conversation differently from the one the user was shown.
|
|
10
10
|
*
|
|
11
|
-
* A row with no envelope predates the display projection and is skipped
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* A row with no envelope predates the display projection and is skipped — the
|
|
12
|
+
* only handling there is or will be. Reconstructing one from the model
|
|
13
|
+
* transcript instead is what the deleted migration renderer did, and rebuilding
|
|
14
|
+
* a display from a transcript that never carried one makes every later change to
|
|
15
|
+
* a tool or a card silently rewrite history. Skipping keeps the gap observable
|
|
16
|
+
* and the read total in what it consults: the stored projection, nothing else.
|
|
16
17
|
*/
|
|
17
18
|
import type { CortexMessage } from "../contracts/message.js";
|
|
18
19
|
import type { StoredMessage } from "./thread-history.js";
|
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The runtime transcript read: stored display projections → `CortexMessage[]`.
|
|
3
3
|
*
|
|
4
|
-
* `ThreadHistory.
|
|
4
|
+
* `ThreadHistory.loadAll` returns a thread's rows in `seq` order, and every
|
|
5
5
|
* append's first row carries the display projection for that append. Replay is
|
|
6
6
|
* therefore a concatenation: walk the rows, emit each envelope's messages in
|
|
7
7
|
* order, done. No turn grouping, no tool-name recognition, no card rebuilding,
|
|
8
8
|
* no filesystem or database lookup — nothing that could render a reloaded
|
|
9
9
|
* conversation differently from the one the user was shown.
|
|
10
10
|
*
|
|
11
|
-
* A row with no envelope predates the display projection and is skipped
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* A row with no envelope predates the display projection and is skipped — the
|
|
12
|
+
* only handling there is or will be. Reconstructing one from the model
|
|
13
|
+
* transcript instead is what the deleted migration renderer did, and rebuilding
|
|
14
|
+
* a display from a transcript that never carried one makes every later change to
|
|
15
|
+
* a tool or a card silently rewrite history. Skipping keeps the gap observable
|
|
16
|
+
* and the read total in what it consults: the stored projection, nothing else.
|
|
16
17
|
*/
|
|
17
18
|
import { conversationUIToCortexMessages } from "./conversation-display-storage.js";
|
|
18
19
|
export function storedMessagesToCortex(messages) {
|
|
@@ -18,6 +18,7 @@ import { type DataUIPart, type UIMessage } from "ai";
|
|
|
18
18
|
import type { AskPart, ChildSessionStartedPart, FileReferencePart, PlanPart, PresentationPart, ReportRenderedPart, RunCardPart } from "../contracts/chat-parts.js";
|
|
19
19
|
import type { ToolCallOutcome } from "../contracts/chat-events.js";
|
|
20
20
|
import type { CortexMessage, CortexPart, ToolCallPart } from "../contracts/message.js";
|
|
21
|
+
import type { Logger } from "../lib/logger.js";
|
|
21
22
|
export declare const SUPPORTED_DISPLAY_SCHEMA_VERSION = 1;
|
|
22
23
|
type Payload<T extends {
|
|
23
24
|
type: string;
|
|
@@ -52,7 +53,7 @@ export interface StoredDisplayEnvelope {
|
|
|
52
53
|
readonly messages: ConversationUIMessage[];
|
|
53
54
|
}
|
|
54
55
|
export declare function envelopeDisplayMessages(messages: readonly ConversationUIMessage[]): StoredDisplayEnvelope;
|
|
55
|
-
export declare function parseStoredDisplayEnvelope(value: unknown, identity: string): Promise<StoredDisplayEnvelope>;
|
|
56
|
+
export declare function parseStoredDisplayEnvelope(value: unknown, identity: string, logger?: Logger): Promise<StoredDisplayEnvelope>;
|
|
56
57
|
/**
|
|
57
58
|
* Build the stored form of one non-text Cortex part.
|
|
58
59
|
*
|
|
@@ -66,15 +67,6 @@ export declare function parseStoredDisplayEnvelope(value: unknown, identity: str
|
|
|
66
67
|
export declare function conversationDisplayPart(part: Exclude<CortexPart, {
|
|
67
68
|
type: "text";
|
|
68
69
|
}>): DataUIPart<ConversationUIData>;
|
|
69
|
-
/**
|
|
70
|
-
* Project Cortex display messages into the durable representation.
|
|
71
|
-
*
|
|
72
|
-
* The startup backfill's only path into storage: it renders a legacy turn from
|
|
73
|
-
* the model transcript and freezes the result. The live recorder builds the same
|
|
74
|
-
* shape directly from the event stream instead of round-tripping through
|
|
75
|
-
* `CortexMessage`.
|
|
76
|
-
*/
|
|
77
|
-
export declare function cortexMessagesToConversationUI(messages: readonly CortexMessage[]): ConversationUIMessage[];
|
|
78
70
|
/**
|
|
79
71
|
* Read the durable representation back as Cortex display messages.
|
|
80
72
|
*
|
|
@@ -39,23 +39,35 @@ const DisplayEnvelopeHeaderSchema = z.object({
|
|
|
39
39
|
* lifecycle field beside an optional outcome instead would put the meaning of the
|
|
40
40
|
* combination in each reader's head, and readers would disagree.
|
|
41
41
|
*/
|
|
42
|
-
const ToolCallDisplaySchema = z
|
|
43
|
-
.object({
|
|
42
|
+
const ToolCallDisplaySchema = z.object({
|
|
44
43
|
toolCallId: z.string(),
|
|
45
44
|
toolName: z.string(),
|
|
46
45
|
outcome: ToolCallOutcomeSchema,
|
|
47
46
|
detail: z.string().optional(),
|
|
48
|
-
})
|
|
49
|
-
|
|
47
|
+
});
|
|
48
|
+
/**
|
|
49
|
+
* The schema behind each stored part key.
|
|
50
|
+
*
|
|
51
|
+
* Deliberately NOT `.strict()`. These validate rows this package wrote itself,
|
|
52
|
+
* months earlier, in another process — so the only thing an unknown key can mean
|
|
53
|
+
* is that the part shed a field since. Rejecting the row for it would make
|
|
54
|
+
* dropping an optional field, otherwise the safest change available, a
|
|
55
|
+
* read-breaking one. Zod strips the stale key instead, which is what a reader
|
|
56
|
+
* would do with it anyway.
|
|
57
|
+
*
|
|
58
|
+
* A key stays here as long as rows written under it must still render: the
|
|
59
|
+
* vocabulary is append-mostly, and retiring an entry is what
|
|
60
|
+
* {@link parseStoredDisplayEnvelope}'s filter then has to paper over.
|
|
61
|
+
*/
|
|
50
62
|
const dataSchemas = {
|
|
51
63
|
"tool-call": ToolCallDisplaySchema,
|
|
52
|
-
presentation: PresentationPartSchema.omit({ type: true })
|
|
53
|
-
plan: PlanPartSchema.omit({ type: true })
|
|
54
|
-
"run-card": RunCardPartSchema.omit({ type: true })
|
|
55
|
-
"file-reference": FileReferencePartSchema.omit({ type: true })
|
|
56
|
-
ask: AskPartSchema.omit({ type: true })
|
|
57
|
-
"child-session-started": ChildSessionStartedPartSchema.omit({ type: true })
|
|
58
|
-
"report-rendered": ReportRenderedPartSchema.omit({ type: true })
|
|
64
|
+
presentation: PresentationPartSchema.omit({ type: true }),
|
|
65
|
+
plan: PlanPartSchema.omit({ type: true }),
|
|
66
|
+
"run-card": RunCardPartSchema.omit({ type: true }),
|
|
67
|
+
"file-reference": FileReferencePartSchema.omit({ type: true }),
|
|
68
|
+
ask: AskPartSchema.omit({ type: true }),
|
|
69
|
+
"child-session-started": ChildSessionStartedPartSchema.omit({ type: true }),
|
|
70
|
+
"report-rendered": ReportRenderedPartSchema.omit({ type: true }),
|
|
59
71
|
};
|
|
60
72
|
export function envelopeDisplayMessages(messages) {
|
|
61
73
|
if (messages.length === 0)
|
|
@@ -67,14 +79,68 @@ export function envelopeDisplayMessages(messages) {
|
|
|
67
79
|
messages: structuredClone(messages),
|
|
68
80
|
};
|
|
69
81
|
}
|
|
70
|
-
|
|
82
|
+
function isRecord(value) {
|
|
83
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Strip every `data-*` part the current vocabulary cannot render.
|
|
87
|
+
*
|
|
88
|
+
* `validateUIMessages` rejects a whole envelope when one part's key is absent
|
|
89
|
+
* from `dataSchemas`, and exposes no lenient mode, so the filter has to run
|
|
90
|
+
* ahead of it. Two things go: a part whose key was retired, and a part whose
|
|
91
|
+
* payload no longer satisfies the schema still standing behind its key — the
|
|
92
|
+
* second reachable only by making a field required after rows were written,
|
|
93
|
+
* since the schemas strip rather than reject what they don't know.
|
|
94
|
+
*
|
|
95
|
+
* Both are schema evolution, and a transcript missing one card reads better than
|
|
96
|
+
* a thread that will not load at all. That is the whole line: a part this
|
|
97
|
+
* package can identify but no longer understands is dropped; anything it cannot
|
|
98
|
+
* identify AS a part — a non-array `messages`, a part with no `type` — is
|
|
99
|
+
* corruption, passes through untouched, and still fails the validator. There is
|
|
100
|
+
* no partial recovery available for a shape you cannot walk.
|
|
101
|
+
*/
|
|
102
|
+
function dropUnrenderableDataParts(messages, identity, logger) {
|
|
103
|
+
const unknownKey = [];
|
|
104
|
+
const schemaMismatch = [];
|
|
105
|
+
const filtered = messages.map((message) => {
|
|
106
|
+
if (!isRecord(message) || !Array.isArray(message.parts))
|
|
107
|
+
return message;
|
|
108
|
+
const parts = message.parts.filter((part) => {
|
|
109
|
+
if (!isRecord(part) || typeof part.type !== "string" || !part.type.startsWith("data-"))
|
|
110
|
+
return true;
|
|
111
|
+
const key = part.type.slice("data-".length);
|
|
112
|
+
// `Object.hasOwn` before the index: a part typed `data-constructor`
|
|
113
|
+
// would otherwise resolve up the prototype chain to something that is
|
|
114
|
+
// not a schema at all, and throw where it should have been dropped.
|
|
115
|
+
if (!Object.hasOwn(dataSchemas, key)) {
|
|
116
|
+
unknownKey.push(part.type);
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
if (!dataSchemas[key].safeParse(part.data).success) {
|
|
120
|
+
schemaMismatch.push(part.type);
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
return true;
|
|
124
|
+
});
|
|
125
|
+
return parts.length === message.parts.length ? message : { ...message, parts };
|
|
126
|
+
});
|
|
127
|
+
if (unknownKey.length > 0 || schemaMismatch.length > 0) {
|
|
128
|
+
logger?.warn("dropped unrenderable stored display parts", {
|
|
129
|
+
identity,
|
|
130
|
+
...(unknownKey.length > 0 ? { unknownKey: [...new Set(unknownKey)] } : {}),
|
|
131
|
+
...(schemaMismatch.length > 0 ? { schemaMismatch: [...new Set(schemaMismatch)] } : {}),
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
return filtered;
|
|
135
|
+
}
|
|
136
|
+
export async function parseStoredDisplayEnvelope(value, identity, logger) {
|
|
71
137
|
const header = DisplayEnvelopeHeaderSchema.safeParse(value);
|
|
72
138
|
if (!header.success) {
|
|
73
139
|
throw new Error(`Invalid stored conversation display envelope at ${identity}: ${header.error.message}`);
|
|
74
140
|
}
|
|
75
141
|
try {
|
|
76
142
|
const messages = await validateUIMessages({
|
|
77
|
-
messages: header.data.messages,
|
|
143
|
+
messages: dropUnrenderableDataParts(header.data.messages, identity, logger),
|
|
78
144
|
dataSchemas,
|
|
79
145
|
metadataSchema: z.object({ interrupted: z.boolean().optional() }).optional(),
|
|
80
146
|
});
|
|
@@ -110,25 +176,6 @@ export function conversationDisplayPart(part) {
|
|
|
110
176
|
const data = part.type === "tool-call" ? { ...rest, outcome: part.outcome ?? "incomplete" } : rest;
|
|
111
177
|
return { type: displayPartType(part), id: displayPartId(part), data };
|
|
112
178
|
}
|
|
113
|
-
/**
|
|
114
|
-
* Project Cortex display messages into the durable representation.
|
|
115
|
-
*
|
|
116
|
-
* The startup backfill's only path into storage: it renders a legacy turn from
|
|
117
|
-
* the model transcript and freezes the result. The live recorder builds the same
|
|
118
|
-
* shape directly from the event stream instead of round-tripping through
|
|
119
|
-
* `CortexMessage`.
|
|
120
|
-
*/
|
|
121
|
-
export function cortexMessagesToConversationUI(messages) {
|
|
122
|
-
return messages.map((message) => {
|
|
123
|
-
const parts = message.parts.map((part) => part.type === "text" ? { type: "text", text: part.text, state: "done" } : conversationDisplayPart(part));
|
|
124
|
-
return {
|
|
125
|
-
id: message.id,
|
|
126
|
-
role: message.role,
|
|
127
|
-
...(message.interrupted ? { metadata: { interrupted: true } } : {}),
|
|
128
|
-
parts,
|
|
129
|
-
};
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
179
|
/**
|
|
133
180
|
* Read the durable representation back as Cortex display messages.
|
|
134
181
|
*
|
|
@@ -25,9 +25,10 @@ import { ResultAsync } from "neverthrow";
|
|
|
25
25
|
import type { Pool } from "pg";
|
|
26
26
|
import type { TokenUsageRollup } from "../contracts/usage.js";
|
|
27
27
|
import { type DbError } from "../lib/db-result.js";
|
|
28
|
+
import type { Logger } from "../lib/logger.js";
|
|
28
29
|
import { type StoredMessageEnvelope } from "./ai-sdk-message-storage.js";
|
|
29
30
|
import { type ConversationUIMessage, type StoredDisplayEnvelope } from "./conversation-display-storage.js";
|
|
30
|
-
/** One stored message, as returned by the display read (`
|
|
31
|
+
/** One stored message, as returned by the display read (`loadAll`). */
|
|
31
32
|
export interface StoredMessage {
|
|
32
33
|
readonly seq: number;
|
|
33
34
|
readonly envelope: StoredMessageEnvelope;
|
|
@@ -98,18 +99,6 @@ export interface ConversationTurn {
|
|
|
98
99
|
* message the turn-boundary predicates fail to recognise.
|
|
99
100
|
*/
|
|
100
101
|
export declare function conversationRecordTurn(text: string): ConversationTurn;
|
|
101
|
-
/**
|
|
102
|
-
* A page of stored messages plus pagination metadata. Pagination is
|
|
103
|
-
* turn-based: `total` is the thread's turn count and `page`/`perPage` index
|
|
104
|
-
* turns, while `messages` is the flattened rows of the page's turns.
|
|
105
|
-
*/
|
|
106
|
-
export interface MessagePage {
|
|
107
|
-
readonly messages: StoredMessage[];
|
|
108
|
-
readonly total: number;
|
|
109
|
-
readonly page: number;
|
|
110
|
-
readonly perPage: number;
|
|
111
|
-
readonly hasMore: boolean;
|
|
112
|
-
}
|
|
113
102
|
/**
|
|
114
103
|
* The result of `retractLastTurn`. `retracted` carries `messages` — the number
|
|
115
104
|
* of rows removed — so a caller can assert exactly what came off the tail. The
|
|
@@ -170,13 +159,23 @@ export interface ThreadHistory {
|
|
|
170
159
|
*/
|
|
171
160
|
loadRecent(threadId: string, tokenBudget: number, options?: LoadRecentOptions): ResultAsync<ModelMessage[], DbError>;
|
|
172
161
|
/**
|
|
173
|
-
* Return
|
|
174
|
-
* NOT token-windowed
|
|
175
|
-
* eviction.
|
|
176
|
-
*
|
|
177
|
-
*
|
|
162
|
+
* Return a thread's messages oldest-first for UI display, grouped into turns.
|
|
163
|
+
* NOT token-windowed — that is `loadRecent`'s job for the agent loop — and no
|
|
164
|
+
* eviction.
|
|
165
|
+
*
|
|
166
|
+
* There is no paginated form, because a page could never cost less: the read
|
|
167
|
+
* selects and parses every row of the thread and would slice only afterwards,
|
|
168
|
+
* so a page saves no query and no parse. It would only truncate the answer,
|
|
169
|
+
* which is a way to lose turns and not a way to save work. A reader wanting a
|
|
170
|
+
* window takes one here, where it knows what it is bounding.
|
|
171
|
+
*
|
|
172
|
+
* Grouped rather than flat because the grouping is what the read computes and
|
|
173
|
+
* what a turn count means, and `.flat()` recovers the rows in `seq` order for
|
|
174
|
+
* a display replay. The reverse does not hold: a flat return would make a
|
|
175
|
+
* caller that wants the tail turn, or the newest N turns, re-find boundaries
|
|
176
|
+
* this read already found.
|
|
178
177
|
*/
|
|
179
|
-
|
|
178
|
+
loadAll(threadId: string): ResultAsync<StoredMessage[][], DbError>;
|
|
180
179
|
/**
|
|
181
180
|
* Remove the thread's most recent turn — every row from the last
|
|
182
181
|
* genuine-user-start `seq` onward — in a single transaction.
|
|
@@ -209,6 +208,13 @@ export interface ThreadHistory {
|
|
|
209
208
|
* no guarantee, because the tail can move one append later without it.
|
|
210
209
|
*/
|
|
211
210
|
latestSeq(threadId: string): ResultAsync<number | null, DbError>;
|
|
211
|
+
/**
|
|
212
|
+
* When the thread last moved — `MAX(messages.created_at)`, or `null` for a
|
|
213
|
+
* thread with no rows. The thread's ACTIVITY clock, and deliberately not
|
|
214
|
+
* `cortex_analysis_threads.updated_at`: a title rename bumps that column
|
|
215
|
+
* too, so it cannot answer "has a turn settled since <time>".
|
|
216
|
+
*/
|
|
217
|
+
latestTurnAt(threadId: string): ResultAsync<Date | null, DbError>;
|
|
212
218
|
/**
|
|
213
219
|
* The count of the thread turns that a person opened past `seq`. A caller
|
|
214
220
|
* reads it as the new work of the thread past an anchor.
|
|
@@ -260,5 +266,10 @@ export declare const EVICTION_BLOCK_TURNS = 4;
|
|
|
260
266
|
* Create a `ThreadHistory` bound to a Postgres pool — a factory closure
|
|
261
267
|
* capturing `pool` (dependency injection per the harness-durable-runtime spec). The `messages` table is
|
|
262
268
|
* provisioned by the project's state-init DDL.
|
|
269
|
+
*
|
|
270
|
+
* `logger` serves exactly one purpose: reporting a stored display part the
|
|
271
|
+
* current vocabulary had to drop. It is optional because the read returns the
|
|
272
|
+
* same messages either way — the drop reports a deploy, not a failed read — and
|
|
273
|
+
* requiring it would break every call site over a diagnostic.
|
|
263
274
|
*/
|
|
264
|
-
export declare function createThreadHistory(pool: Pool): ThreadHistory;
|
|
275
|
+
export declare function createThreadHistory(pool: Pool, logger?: Logger): ThreadHistory;
|
|
@@ -108,7 +108,7 @@ function serializeEnvelope(message) {
|
|
|
108
108
|
/**
|
|
109
109
|
* Group rows (oldest-first) into turns at genuine-user-start boundaries.
|
|
110
110
|
* Generic over the row shape so the token-windowed read (`loadRecent`) and
|
|
111
|
-
* the display read (`
|
|
111
|
+
* the display read (`loadAll`) share it, each supplying its own start
|
|
112
112
|
* predicate.
|
|
113
113
|
*/
|
|
114
114
|
function groupTurns(rows, isStart) {
|
|
@@ -173,8 +173,13 @@ export const EVICTION_BLOCK_TURNS = 4;
|
|
|
173
173
|
* Create a `ThreadHistory` bound to a Postgres pool — a factory closure
|
|
174
174
|
* capturing `pool` (dependency injection per the harness-durable-runtime spec). The `messages` table is
|
|
175
175
|
* provisioned by the project's state-init DDL.
|
|
176
|
+
*
|
|
177
|
+
* `logger` serves exactly one purpose: reporting a stored display part the
|
|
178
|
+
* current vocabulary had to drop. It is optional because the read returns the
|
|
179
|
+
* same messages either way — the drop reports a deploy, not a failed read — and
|
|
180
|
+
* requiring it would break every call site over a diagnostic.
|
|
176
181
|
*/
|
|
177
|
-
export function createThreadHistory(pool) {
|
|
182
|
+
export function createThreadHistory(pool, logger) {
|
|
178
183
|
function appendTurn(threadId, turn) {
|
|
179
184
|
const { modelMessages: messages, displayMessages, turnUsage, turnDurationMs } = turn;
|
|
180
185
|
if (messages.length === 0)
|
|
@@ -351,52 +356,40 @@ export function createThreadHistory(pool) {
|
|
|
351
356
|
return (keepsFirstTurn ? [turns[0], ...retained] : retained).flat().map((row) => row.message);
|
|
352
357
|
});
|
|
353
358
|
}
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
// `seq::text AS seq` output alias (Postgres resolves an unqualified
|
|
368
|
-
// ORDER BY name to the output column), sorting the bigint as text:
|
|
369
|
-
// "10" before "2". The qualified name forces the bigint column.
|
|
370
|
-
`SELECT seq::text AS seq, message_envelope, display_envelope, reported_usage, turn_duration_ms::text AS turn_duration_ms
|
|
359
|
+
// Display reads are turn-bounded, not row-bounded. A serial-tool assistant
|
|
360
|
+
// turn is persisted as one row per step (plus its tool_result `user` rows),
|
|
361
|
+
// so a row-windowed page could split a turn — truncating the trailing
|
|
362
|
+
// report card/text out of the page the UI fetches. Read the thread and group
|
|
363
|
+
// into turns, so a turn always reloads intact. Threads are conversation-scoped
|
|
364
|
+
// and bounded, so reading every row here matches `loadRecent`'s existing
|
|
365
|
+
async function readTurns(threadId) {
|
|
366
|
+
const { rows } = await pool.query(
|
|
367
|
+
// ORDER BY must qualify `messages.seq` — a bare `seq` would bind to the
|
|
368
|
+
// `seq::text AS seq` output alias (Postgres resolves an unqualified
|
|
369
|
+
// ORDER BY name to the output column), sorting the bigint as text:
|
|
370
|
+
// "10" before "2". The qualified name forces the bigint column.
|
|
371
|
+
`SELECT seq::text AS seq, message_envelope, display_envelope, reported_usage, turn_duration_ms::text AS turn_duration_ms
|
|
371
372
|
FROM messages WHERE thread_id = $1
|
|
372
373
|
ORDER BY messages.seq ASC`, [threadId]);
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
return {
|
|
380
|
-
seq: Number(r.seq),
|
|
381
|
-
envelope,
|
|
382
|
-
message: envelope.message,
|
|
383
|
-
...(displayEnvelope ? { displayEnvelope } : {}),
|
|
384
|
-
...(r.reported_usage === null ? {} : { usage: r.reported_usage }),
|
|
385
|
-
...(r.turn_duration_ms === null ? {} : { durationMs: Number(r.turn_duration_ms) }),
|
|
386
|
-
};
|
|
387
|
-
}));
|
|
388
|
-
const turns = groupTurns(stored, (row) => isGenuineUserStart(row.message));
|
|
389
|
-
const total = turns.length;
|
|
390
|
-
const offset = safePage * safePerPage;
|
|
391
|
-
const pageTurns = turns.slice(offset, offset + safePerPage);
|
|
374
|
+
// Spread rather than `usage: r.reported_usage ?? undefined`, so a row with
|
|
375
|
+
// no rollup carries no `usage` KEY at all — absent, not present-and-undefined.
|
|
376
|
+
// The duration obeys the same rule, and it reads from its own column.
|
|
377
|
+
const stored = await Promise.all(rows.map(async (r) => {
|
|
378
|
+
const envelope = parseStoredMessageEnvelope(r.message_envelope, `${threadId}/${r.seq}`);
|
|
379
|
+
const displayEnvelope = r.display_envelope == null ? undefined : await parseStoredDisplayEnvelope(r.display_envelope, `${threadId}/${r.seq}/display`, logger);
|
|
392
380
|
return {
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
381
|
+
seq: Number(r.seq),
|
|
382
|
+
envelope,
|
|
383
|
+
message: envelope.message,
|
|
384
|
+
...(displayEnvelope ? { displayEnvelope } : {}),
|
|
385
|
+
...(r.reported_usage === null ? {} : { usage: r.reported_usage }),
|
|
386
|
+
...(r.turn_duration_ms === null ? {} : { durationMs: Number(r.turn_duration_ms) }),
|
|
398
387
|
};
|
|
399
|
-
});
|
|
388
|
+
}));
|
|
389
|
+
return groupTurns(stored, (row) => isGenuineUserStart(row.message));
|
|
390
|
+
}
|
|
391
|
+
function loadAll(threadId) {
|
|
392
|
+
return tryQuery("thread-history.loadAll", () => readTurns(threadId));
|
|
400
393
|
}
|
|
401
394
|
function retractLastTurn(threadId) {
|
|
402
395
|
return withTransaction(pool, "thread-history.retractLastTurn", (client) =>
|
|
@@ -452,6 +445,13 @@ export function createThreadHistory(pool) {
|
|
|
452
445
|
return maxSeq === null ? null : Number(maxSeq);
|
|
453
446
|
});
|
|
454
447
|
}
|
|
448
|
+
function latestTurnAt(threadId) {
|
|
449
|
+
// `MAX(created_at)` over no rows is SQL NULL, so a thread with no messages
|
|
450
|
+
// reads as `null` rather than as a time. This is the thread's ACTIVITY
|
|
451
|
+
// clock — distinct from `cortex_analysis_threads.updated_at`, which a
|
|
452
|
+
// rename also bumps, and so cannot answer "did a turn settle after X".
|
|
453
|
+
return tryQuery("thread-history.latestTurnAt", () => pool.query("SELECT MAX(created_at) AS latest_at FROM messages WHERE thread_id = $1", [threadId])).map(({ rows }) => rows[0]?.latest_at ?? null);
|
|
454
|
+
}
|
|
455
455
|
function countUserTurnsAfter(threadId, seq) {
|
|
456
456
|
// `GENUINE_USER_START_SQL` is the stored-envelope twin of
|
|
457
457
|
// `isGenuineUserStart`, and the tail retraction cuts on the same text.
|
|
@@ -468,5 +468,5 @@ export function createThreadHistory(pool) {
|
|
|
468
468
|
AND seq > $2::bigint
|
|
469
469
|
AND ${GENUINE_USER_START_SQL}`, [threadId, seq])).map(({ rows }) => Number(rows[0].turns));
|
|
470
470
|
}
|
|
471
|
-
return { appendTurn, loadRecent,
|
|
471
|
+
return { appendTurn, loadRecent, loadAll, retractLastTurn, latestSeq, latestTurnAt, countUserTurnsAfter };
|
|
472
472
|
}
|