@inflexa-ai/harness 0.24.0 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/dist/agents/sandbox/__fixtures__/deps.js +6 -0
  2. package/dist/agents/sandbox/data-profiler.js +5 -4
  3. package/dist/agents/sandbox/shared.js +11 -0
  4. package/dist/agents/sandbox/types.d.ts +1 -1
  5. package/dist/app/data-profile-orientation.js +30 -1
  6. package/dist/contracts/data-profile.d.ts +166 -0
  7. package/dist/contracts/index.d.ts +1 -0
  8. package/dist/execution/input-signature.d.ts +33 -0
  9. package/dist/execution/input-signature.js +0 -0
  10. package/dist/index.d.ts +5 -2
  11. package/dist/index.js +10 -3
  12. package/dist/input-scan/coverage.d.ts +22 -0
  13. package/dist/input-scan/coverage.js +85 -0
  14. package/dist/input-scan/enrich.d.ts +41 -0
  15. package/dist/input-scan/enrich.js +227 -0
  16. package/dist/input-scan/formats.d.ts +41 -0
  17. package/dist/input-scan/formats.js +279 -0
  18. package/dist/input-scan/index.d.ts +12 -0
  19. package/dist/input-scan/index.js +11 -0
  20. package/dist/input-scan/scan.d.ts +50 -0
  21. package/dist/input-scan/scan.js +182 -0
  22. package/dist/input-scan/shapes.d.ts +46 -0
  23. package/dist/input-scan/shapes.js +0 -0
  24. package/dist/input-scan/types.d.ts +172 -0
  25. package/dist/input-scan/types.js +12 -0
  26. package/dist/lib/page-capture.d.ts +44 -15
  27. package/dist/lib/page-capture.js +78 -19
  28. package/dist/loop/run-agent.js +36 -19
  29. package/dist/memory/card-builders.d.ts +4 -6
  30. package/dist/memory/card-builders.js +4 -6
  31. package/dist/memory/conversation-display-replay.d.ts +7 -6
  32. package/dist/memory/conversation-display-replay.js +7 -6
  33. package/dist/memory/conversation-display-storage.d.ts +2 -10
  34. package/dist/memory/conversation-display-storage.js +79 -32
  35. package/dist/memory/thread-history.d.ts +31 -20
  36. package/dist/memory/thread-history.js +45 -45
  37. package/dist/prompts/report-session.d.ts +1 -1
  38. package/dist/prompts/report-session.js +22 -13
  39. package/dist/prompts/sandbox/data-profiler.d.ts +1 -1
  40. package/dist/prompts/sandbox/data-profiler.js +159 -275
  41. package/dist/provenance/exec-frame.js +9 -3
  42. package/dist/runtime/boot.d.ts +7 -8
  43. package/dist/runtime/boot.js +7 -19
  44. package/dist/sandbox/client.d.ts +7 -0
  45. package/dist/sandbox/create-sandbox.js +1 -0
  46. package/dist/sandbox/docker-client.d.ts +1 -0
  47. package/dist/sandbox/docker-client.js +20 -15
  48. package/dist/sandbox/k8s-client.d.ts +6 -5
  49. package/dist/sandbox/k8s-client.js +55 -39
  50. package/dist/sandbox/reaper.d.ts +24 -13
  51. package/dist/sandbox/reaper.js +47 -14
  52. package/dist/sandbox/types.d.ts +6 -5
  53. package/dist/schemas/data-profile-schemas.d.ts +62 -0
  54. package/dist/schemas/data-profile-schemas.js +68 -2
  55. package/dist/state/data-profile.d.ts +7 -88
  56. package/dist/state/index.d.ts +1 -1
  57. package/dist/tasks/data-profile-activity.d.ts +8 -0
  58. package/dist/tasks/data-profile-activity.js +1 -0
  59. package/dist/tasks/data-profile-index.d.ts +45 -0
  60. package/dist/tasks/data-profile-index.js +134 -0
  61. package/dist/tasks/data-profile.d.ts +8 -23
  62. package/dist/tasks/data-profile.js +150 -117
  63. package/dist/tools/define-tool.d.ts +19 -8
  64. package/dist/tools/define-tool.js +28 -12
  65. package/dist/tools/report-session/examine-page.d.ts +22 -13
  66. package/dist/tools/report-session/examine-page.js +15 -10
  67. package/dist/tools/report-session/index.d.ts +1 -1
  68. package/dist/tools/research/generate-plan.js +3 -8
  69. package/dist/tools/research/inspect-data-profile.d.ts +30 -4
  70. package/dist/tools/research/inspect-data-profile.js +75 -20
  71. package/dist/tools/workspace/scan-inputs.d.ts +38 -0
  72. package/dist/tools/workspace/scan-inputs.js +61 -0
  73. package/dist/tools/workspace/workspace-search.d.ts +1 -1
  74. package/dist/tools/workspace/workspace-search.js +12 -9
  75. package/dist/workspace/filesystem.d.ts +27 -0
  76. package/dist/workspace/filesystem.js +27 -0
  77. package/package.json +1 -1
  78. package/dist/app/data-profile-policy.d.ts +0 -36
  79. package/dist/app/data-profile-policy.js +0 -37
  80. package/dist/memory/content-to-cortex.d.ts +0 -75
  81. package/dist/memory/content-to-cortex.js +0 -201
  82. package/dist/memory/conversation-display-backfill.d.ts +0 -23
  83. package/dist/memory/conversation-display-backfill.js +0 -91
  84. package/dist/memory/reconstruct-cards.d.ts +0 -27
  85. package/dist/memory/reconstruct-cards.js +0 -57
@@ -10,7 +10,7 @@ import { classifyProviderError } from "../providers/errors.js";
10
10
  import { DEFAULT_PROMPT_CACHE, promptCacheProviderOptions } from "../providers/prompt-cache.js";
11
11
  import { resultStep } from "./run-step.js";
12
12
  import { AskRejectedError, UnavailableAsk } from "../tools/approval/contract.js";
13
- import { isToolError, readToolResultImage } from "../tools/define-tool.js";
13
+ import { isToolError, readToolResultImages } from "../tools/define-tool.js";
14
14
  import { addChatUsage, hasReportedUsage, recordAgentRun } from "./metrics.js";
15
15
  import { computeDetail, computeResultDetail } from "./tool-detail.js";
16
16
  import { toolOutcomeForOutputType } from "./tool-outcome.js";
@@ -457,6 +457,16 @@ function imagePlacementFor(capabilities) {
457
457
  return "user-message";
458
458
  return "drop";
459
459
  }
460
+ /**
461
+ * The text part that names one deferred picture. The wire holds no structural
462
+ * link between a user message and a tool call, thus the line names the call. A
463
+ * result with several pictures numbers each one, thus the model reads which
464
+ * slice it looks at.
465
+ */
466
+ function deferredImageLabel(deferred, index) {
467
+ const call = `the tool result ${deferred.toolCallId} of ${deferred.toolName}`;
468
+ return deferred.images.length === 1 ? `The picture of ${call}.` : `The picture ${index + 1} of ${deferred.images.length} of ${call}.`;
469
+ }
460
470
  /**
461
471
  * Append one user message that carries each deferred picture of the round, then
462
472
  * empty the collector.
@@ -464,10 +474,11 @@ function imagePlacementFor(capabilities) {
464
474
  * The tool message must come directly after the assistant message with the
465
475
  * tool calls. Thus a picture rides a separate message after the whole tool
466
476
  * message, and one message batches the round. The parts obey the order of
467
- * `results`, which is the order of the tool calls. Each deferred picture has a
468
- * result of this round, because the collector fills during the dispatch and it
469
- * empties here. A text part names the tool call of each picture, because the wire
470
- * holds no structural link between a user message and a tool call.
477
+ * `results`, which is the order of the tool calls, and the pictures of one
478
+ * result keep the order that the tool gave. Each deferred picture has a result
479
+ * of this round, because the collector fills during the dispatch and it empties
480
+ * here. A text part names the tool call of each picture, because the wire holds
481
+ * no structural link between a user message and a tool call.
471
482
  *
472
483
  * The message carries the synthetic marker, because a `user` message opens a
473
484
  * conversation turn. An unmarked one is loop machinery that reads as a turn
@@ -482,8 +493,10 @@ function appendDeferredImages(messages, results, deferredImages) {
482
493
  const deferred = byToolCallId.get(result.toolCallId);
483
494
  if (deferred === undefined)
484
495
  continue;
485
- content.push({ type: "text", text: `The picture of the tool result ${deferred.toolCallId} of ${deferred.toolName}.` });
486
- content.push({ type: "file", mediaType: deferred.image.mediaType, data: { type: "data", data: deferred.image.base64 } });
496
+ for (const [index, image] of deferred.images.entries()) {
497
+ content.push({ type: "text", text: deferredImageLabel(deferred, index) });
498
+ content.push({ type: "file", mediaType: image.mediaType, data: { type: "data", data: image.base64 } });
499
+ }
487
500
  }
488
501
  messages.push(syntheticUserMessage(content));
489
502
  deferredImages.length = 0;
@@ -615,15 +628,16 @@ function jsonValue(value) {
615
628
  * Encode a tool ok value onto a tool-result message. A value with no picture
616
629
  * stays a plain JSON result, byte-identical to a value that never carried one.
617
630
  *
618
- * A value that carries a picture splits: the JSON data goes to a text part, and
619
- * the bytes ride as an image content block. Thus the model sees the picture, and
620
- * the JSON text holds no bytes. When the wire carries no picture, the block drops
621
- * and the text stays, because base64 text floods the context and the model cannot
622
- * see it. The drop rides the log, thus an operator sees that the picture did not
623
- * reach the model.
631
+ * A value that carries pictures splits: the JSON data goes to a text part, and
632
+ * the bytes ride as image content blocks after it, in the order that the tool
633
+ * gave. Thus the model sees each picture, and the JSON text holds no bytes. When
634
+ * the wire carries no picture, the blocks drop and the text stays, because
635
+ * base64 text floods the context and the model cannot see it. One warn with the
636
+ * count rides the log, thus an operator sees how many pictures did not reach the
637
+ * model.
624
638
  *
625
639
  * A wire that renders a picture on a user message only gets the fallback: the
626
- * result keeps its JSON text, and the picture goes to the collector of the round.
640
+ * result keeps its JSON text, and the pictures go to the collector of the round.
627
641
  * The round then sends the bytes after the tool message.
628
642
  */
629
643
  function successResult(toolCall, value, encoding) {
@@ -634,15 +648,18 @@ function successResult(toolCall, value, encoding) {
634
648
  toolName: toolCall.toolName,
635
649
  output: { type: "json", value: jsonData },
636
650
  };
637
- const image = readToolResultImage(value);
638
- if (image === undefined)
651
+ const images = readToolResultImages(value);
652
+ if (images.length === 0)
639
653
  return jsonOnly;
640
654
  if (encoding.placement === "drop") {
641
- encoding.log.warn("the wired provider carries no picture in a tool result, thus the picture is dropped", { toolName: toolCall.toolName });
655
+ encoding.log.warn("the wired provider carries no picture in a tool result, thus the pictures are dropped", {
656
+ toolName: toolCall.toolName,
657
+ imageCount: images.length,
658
+ });
642
659
  return jsonOnly;
643
660
  }
644
661
  if (encoding.placement === "user-message") {
645
- encoding.deferredImages.push({ toolCallId: toolCall.toolCallId, toolName: toolCall.toolName, image });
662
+ encoding.deferredImages.push({ toolCallId: toolCall.toolCallId, toolName: toolCall.toolName, images });
646
663
  return jsonOnly;
647
664
  }
648
665
  return {
@@ -653,7 +670,7 @@ function successResult(toolCall, value, encoding) {
653
670
  type: "content",
654
671
  value: [
655
672
  { type: "text", text: JSON.stringify(jsonData) },
656
- { type: "file", mediaType: image.mediaType, data: { type: "data", data: image.base64 } },
673
+ ...images.map((image) => ({ type: "file", mediaType: image.mediaType, data: { type: "data", data: image.base64 } })),
657
674
  ],
658
675
  },
659
676
  };
@@ -3,12 +3,10 @@
3
3
  * `data-presentation`, `data-run-card`, `data-file-reference`).
4
4
  *
5
5
  * Display tools (`show_plan`, `show_user`, `show_file`) emit these cards live
6
- * over the chat SSE stream, but the persisted turn keeps only the Anthropic
7
- * transcript (text / tool_use / tool_result) — the cards are not stored. To
8
- * re-render a card on reload, `content-to-cortex` reconstructs it from the
9
- * tool_use block via these same builders, so the live and replayed cards are
10
- * byte-identical (the deterministic `id` matches, enabling downstream
11
- * reconciliation).
6
+ * over the chat SSE stream, and the turn persists what it emitted, so a reload
7
+ * replays the built card rather than rebuilding it. These builders therefore run
8
+ * once per card, on the live path only. The `id` stays deterministic all the
9
+ * same: it is what a re-emission of the same card reconciles on.
12
10
  *
13
11
  * The builders return the FLAT wire `data` payload (no `type` wrapper, no
14
12
  * `source`): the tool emits `{ type, source, data }` (the emit pipeline
@@ -3,12 +3,10 @@
3
3
  * `data-presentation`, `data-run-card`, `data-file-reference`).
4
4
  *
5
5
  * Display tools (`show_plan`, `show_user`, `show_file`) emit these cards live
6
- * over the chat SSE stream, but the persisted turn keeps only the Anthropic
7
- * transcript (text / tool_use / tool_result) — the cards are not stored. To
8
- * re-render a card on reload, `content-to-cortex` reconstructs it from the
9
- * tool_use block via these same builders, so the live and replayed cards are
10
- * byte-identical (the deterministic `id` matches, enabling downstream
11
- * reconciliation).
6
+ * over the chat SSE stream, and the turn persists what it emitted, so a reload
7
+ * replays the built card rather than rebuilding it. These builders therefore run
8
+ * once per card, on the live path only. The `id` stays deterministic all the
9
+ * same: it is what a re-emission of the same card reconciles on.
12
10
  *
13
11
  * The builders return the FLAT wire `data` payload (no `type` wrapper, no
14
12
  * `source`): the tool emits `{ type, source, data }` (the emit pipeline
@@ -1,18 +1,19 @@
1
1
  /**
2
2
  * The runtime transcript read: stored display projections → `CortexMessage[]`.
3
3
  *
4
- * `ThreadHistory.loadPage` returns a thread's rows in `seq` order, and every
4
+ * `ThreadHistory.loadAll` returns a thread's rows in `seq` order, and every
5
5
  * append's first row carries the display projection for that append. Replay is
6
6
  * therefore a concatenation: walk the rows, emit each envelope's messages in
7
7
  * order, done. No turn grouping, no tool-name recognition, no card rebuilding,
8
8
  * no filesystem or database lookup — nothing that could render a reloaded
9
9
  * conversation differently from the one the user was shown.
10
10
  *
11
- * A row with no envelope predates the display projection and is skipped. The
12
- * startup backfill (`conversation-display-backfill.ts`) writes one for every such
13
- * row before the runtime serves traffic, so in a booted harness there are none;
14
- * skipping rather than falling back to the migration renderer is what keeps the
15
- * reconstruction path out of the runtime entirely.
11
+ * A row with no envelope predates the display projection and is skipped — the
12
+ * only handling there is or will be. Reconstructing one from the model
13
+ * transcript instead is what the deleted migration renderer did, and rebuilding
14
+ * a display from a transcript that never carried one makes every later change to
15
+ * a tool or a card silently rewrite history. Skipping keeps the gap observable
16
+ * and the read total in what it consults: the stored projection, nothing else.
16
17
  */
17
18
  import type { CortexMessage } from "../contracts/message.js";
18
19
  import type { StoredMessage } from "./thread-history.js";
@@ -1,18 +1,19 @@
1
1
  /**
2
2
  * The runtime transcript read: stored display projections → `CortexMessage[]`.
3
3
  *
4
- * `ThreadHistory.loadPage` returns a thread's rows in `seq` order, and every
4
+ * `ThreadHistory.loadAll` returns a thread's rows in `seq` order, and every
5
5
  * append's first row carries the display projection for that append. Replay is
6
6
  * therefore a concatenation: walk the rows, emit each envelope's messages in
7
7
  * order, done. No turn grouping, no tool-name recognition, no card rebuilding,
8
8
  * no filesystem or database lookup — nothing that could render a reloaded
9
9
  * conversation differently from the one the user was shown.
10
10
  *
11
- * A row with no envelope predates the display projection and is skipped. The
12
- * startup backfill (`conversation-display-backfill.ts`) writes one for every such
13
- * row before the runtime serves traffic, so in a booted harness there are none;
14
- * skipping rather than falling back to the migration renderer is what keeps the
15
- * reconstruction path out of the runtime entirely.
11
+ * A row with no envelope predates the display projection and is skipped — the
12
+ * only handling there is or will be. Reconstructing one from the model
13
+ * transcript instead is what the deleted migration renderer did, and rebuilding
14
+ * a display from a transcript that never carried one makes every later change to
15
+ * a tool or a card silently rewrite history. Skipping keeps the gap observable
16
+ * and the read total in what it consults: the stored projection, nothing else.
16
17
  */
17
18
  import { conversationUIToCortexMessages } from "./conversation-display-storage.js";
18
19
  export function storedMessagesToCortex(messages) {
@@ -18,6 +18,7 @@ import { type DataUIPart, type UIMessage } from "ai";
18
18
  import type { AskPart, ChildSessionStartedPart, FileReferencePart, PlanPart, PresentationPart, ReportRenderedPart, RunCardPart } from "../contracts/chat-parts.js";
19
19
  import type { ToolCallOutcome } from "../contracts/chat-events.js";
20
20
  import type { CortexMessage, CortexPart, ToolCallPart } from "../contracts/message.js";
21
+ import type { Logger } from "../lib/logger.js";
21
22
  export declare const SUPPORTED_DISPLAY_SCHEMA_VERSION = 1;
22
23
  type Payload<T extends {
23
24
  type: string;
@@ -52,7 +53,7 @@ export interface StoredDisplayEnvelope {
52
53
  readonly messages: ConversationUIMessage[];
53
54
  }
54
55
  export declare function envelopeDisplayMessages(messages: readonly ConversationUIMessage[]): StoredDisplayEnvelope;
55
- export declare function parseStoredDisplayEnvelope(value: unknown, identity: string): Promise<StoredDisplayEnvelope>;
56
+ export declare function parseStoredDisplayEnvelope(value: unknown, identity: string, logger?: Logger): Promise<StoredDisplayEnvelope>;
56
57
  /**
57
58
  * Build the stored form of one non-text Cortex part.
58
59
  *
@@ -66,15 +67,6 @@ export declare function parseStoredDisplayEnvelope(value: unknown, identity: str
66
67
  export declare function conversationDisplayPart(part: Exclude<CortexPart, {
67
68
  type: "text";
68
69
  }>): DataUIPart<ConversationUIData>;
69
- /**
70
- * Project Cortex display messages into the durable representation.
71
- *
72
- * The startup backfill's only path into storage: it renders a legacy turn from
73
- * the model transcript and freezes the result. The live recorder builds the same
74
- * shape directly from the event stream instead of round-tripping through
75
- * `CortexMessage`.
76
- */
77
- export declare function cortexMessagesToConversationUI(messages: readonly CortexMessage[]): ConversationUIMessage[];
78
70
  /**
79
71
  * Read the durable representation back as Cortex display messages.
80
72
  *
@@ -39,23 +39,35 @@ const DisplayEnvelopeHeaderSchema = z.object({
39
39
  * lifecycle field beside an optional outcome instead would put the meaning of the
40
40
  * combination in each reader's head, and readers would disagree.
41
41
  */
42
- const ToolCallDisplaySchema = z
43
- .object({
42
+ const ToolCallDisplaySchema = z.object({
44
43
  toolCallId: z.string(),
45
44
  toolName: z.string(),
46
45
  outcome: ToolCallOutcomeSchema,
47
46
  detail: z.string().optional(),
48
- })
49
- .strict();
47
+ });
48
+ /**
49
+ * The schema behind each stored part key.
50
+ *
51
+ * Deliberately NOT `.strict()`. These validate rows this package wrote itself,
52
+ * months earlier, in another process — so the only thing an unknown key can mean
53
+ * is that the part shed a field since. Rejecting the row for it would make
54
+ * dropping an optional field, otherwise the safest change available, a
55
+ * read-breaking one. Zod strips the stale key instead, which is what a reader
56
+ * would do with it anyway.
57
+ *
58
+ * A key stays here as long as rows written under it must still render: the
59
+ * vocabulary is append-mostly, and retiring an entry is what
60
+ * {@link parseStoredDisplayEnvelope}'s filter then has to paper over.
61
+ */
50
62
  const dataSchemas = {
51
63
  "tool-call": ToolCallDisplaySchema,
52
- presentation: PresentationPartSchema.omit({ type: true }).strict(),
53
- plan: PlanPartSchema.omit({ type: true }).strict(),
54
- "run-card": RunCardPartSchema.omit({ type: true }).strict(),
55
- "file-reference": FileReferencePartSchema.omit({ type: true }).strict(),
56
- ask: AskPartSchema.omit({ type: true }).strict(),
57
- "child-session-started": ChildSessionStartedPartSchema.omit({ type: true }).strict(),
58
- "report-rendered": ReportRenderedPartSchema.omit({ type: true }).strict(),
64
+ presentation: PresentationPartSchema.omit({ type: true }),
65
+ plan: PlanPartSchema.omit({ type: true }),
66
+ "run-card": RunCardPartSchema.omit({ type: true }),
67
+ "file-reference": FileReferencePartSchema.omit({ type: true }),
68
+ ask: AskPartSchema.omit({ type: true }),
69
+ "child-session-started": ChildSessionStartedPartSchema.omit({ type: true }),
70
+ "report-rendered": ReportRenderedPartSchema.omit({ type: true }),
59
71
  };
60
72
  export function envelopeDisplayMessages(messages) {
61
73
  if (messages.length === 0)
@@ -67,14 +79,68 @@ export function envelopeDisplayMessages(messages) {
67
79
  messages: structuredClone(messages),
68
80
  };
69
81
  }
70
- export async function parseStoredDisplayEnvelope(value, identity) {
82
+ function isRecord(value) {
83
+ return typeof value === "object" && value !== null && !Array.isArray(value);
84
+ }
85
+ /**
86
+ * Strip every `data-*` part the current vocabulary cannot render.
87
+ *
88
+ * `validateUIMessages` rejects a whole envelope when one part's key is absent
89
+ * from `dataSchemas`, and exposes no lenient mode, so the filter has to run
90
+ * ahead of it. Two things go: a part whose key was retired, and a part whose
91
+ * payload no longer satisfies the schema still standing behind its key — the
92
+ * second reachable only by making a field required after rows were written,
93
+ * since the schemas strip rather than reject what they don't know.
94
+ *
95
+ * Both are schema evolution, and a transcript missing one card reads better than
96
+ * a thread that will not load at all. That is the whole line: a part this
97
+ * package can identify but no longer understands is dropped; anything it cannot
98
+ * identify AS a part — a non-array `messages`, a part with no `type` — is
99
+ * corruption, passes through untouched, and still fails the validator. There is
100
+ * no partial recovery available for a shape you cannot walk.
101
+ */
102
+ function dropUnrenderableDataParts(messages, identity, logger) {
103
+ const unknownKey = [];
104
+ const schemaMismatch = [];
105
+ const filtered = messages.map((message) => {
106
+ if (!isRecord(message) || !Array.isArray(message.parts))
107
+ return message;
108
+ const parts = message.parts.filter((part) => {
109
+ if (!isRecord(part) || typeof part.type !== "string" || !part.type.startsWith("data-"))
110
+ return true;
111
+ const key = part.type.slice("data-".length);
112
+ // `Object.hasOwn` before the index: a part typed `data-constructor`
113
+ // would otherwise resolve up the prototype chain to something that is
114
+ // not a schema at all, and throw where it should have been dropped.
115
+ if (!Object.hasOwn(dataSchemas, key)) {
116
+ unknownKey.push(part.type);
117
+ return false;
118
+ }
119
+ if (!dataSchemas[key].safeParse(part.data).success) {
120
+ schemaMismatch.push(part.type);
121
+ return false;
122
+ }
123
+ return true;
124
+ });
125
+ return parts.length === message.parts.length ? message : { ...message, parts };
126
+ });
127
+ if (unknownKey.length > 0 || schemaMismatch.length > 0) {
128
+ logger?.warn("dropped unrenderable stored display parts", {
129
+ identity,
130
+ ...(unknownKey.length > 0 ? { unknownKey: [...new Set(unknownKey)] } : {}),
131
+ ...(schemaMismatch.length > 0 ? { schemaMismatch: [...new Set(schemaMismatch)] } : {}),
132
+ });
133
+ }
134
+ return filtered;
135
+ }
136
+ export async function parseStoredDisplayEnvelope(value, identity, logger) {
71
137
  const header = DisplayEnvelopeHeaderSchema.safeParse(value);
72
138
  if (!header.success) {
73
139
  throw new Error(`Invalid stored conversation display envelope at ${identity}: ${header.error.message}`);
74
140
  }
75
141
  try {
76
142
  const messages = await validateUIMessages({
77
- messages: header.data.messages,
143
+ messages: dropUnrenderableDataParts(header.data.messages, identity, logger),
78
144
  dataSchemas,
79
145
  metadataSchema: z.object({ interrupted: z.boolean().optional() }).optional(),
80
146
  });
@@ -110,25 +176,6 @@ export function conversationDisplayPart(part) {
110
176
  const data = part.type === "tool-call" ? { ...rest, outcome: part.outcome ?? "incomplete" } : rest;
111
177
  return { type: displayPartType(part), id: displayPartId(part), data };
112
178
  }
113
- /**
114
- * Project Cortex display messages into the durable representation.
115
- *
116
- * The startup backfill's only path into storage: it renders a legacy turn from
117
- * the model transcript and freezes the result. The live recorder builds the same
118
- * shape directly from the event stream instead of round-tripping through
119
- * `CortexMessage`.
120
- */
121
- export function cortexMessagesToConversationUI(messages) {
122
- return messages.map((message) => {
123
- const parts = message.parts.map((part) => part.type === "text" ? { type: "text", text: part.text, state: "done" } : conversationDisplayPart(part));
124
- return {
125
- id: message.id,
126
- role: message.role,
127
- ...(message.interrupted ? { metadata: { interrupted: true } } : {}),
128
- parts,
129
- };
130
- });
131
- }
132
179
  /**
133
180
  * Read the durable representation back as Cortex display messages.
134
181
  *
@@ -25,9 +25,10 @@ import { ResultAsync } from "neverthrow";
25
25
  import type { Pool } from "pg";
26
26
  import type { TokenUsageRollup } from "../contracts/usage.js";
27
27
  import { type DbError } from "../lib/db-result.js";
28
+ import type { Logger } from "../lib/logger.js";
28
29
  import { type StoredMessageEnvelope } from "./ai-sdk-message-storage.js";
29
30
  import { type ConversationUIMessage, type StoredDisplayEnvelope } from "./conversation-display-storage.js";
30
- /** One stored message, as returned by the display read (`loadPage`). */
31
+ /** One stored message, as returned by the display read (`loadAll`). */
31
32
  export interface StoredMessage {
32
33
  readonly seq: number;
33
34
  readonly envelope: StoredMessageEnvelope;
@@ -98,18 +99,6 @@ export interface ConversationTurn {
98
99
  * message the turn-boundary predicates fail to recognise.
99
100
  */
100
101
  export declare function conversationRecordTurn(text: string): ConversationTurn;
101
- /**
102
- * A page of stored messages plus pagination metadata. Pagination is
103
- * turn-based: `total` is the thread's turn count and `page`/`perPage` index
104
- * turns, while `messages` is the flattened rows of the page's turns.
105
- */
106
- export interface MessagePage {
107
- readonly messages: StoredMessage[];
108
- readonly total: number;
109
- readonly page: number;
110
- readonly perPage: number;
111
- readonly hasMore: boolean;
112
- }
113
102
  /**
114
103
  * The result of `retractLastTurn`. `retracted` carries `messages` — the number
115
104
  * of rows removed — so a caller can assert exactly what came off the tail. The
@@ -170,13 +159,23 @@ export interface ThreadHistory {
170
159
  */
171
160
  loadRecent(threadId: string, tokenBudget: number, options?: LoadRecentOptions): ResultAsync<ModelMessage[], DbError>;
172
161
  /**
173
- * Return one page of a thread's messages oldest-first for UI display —
174
- * NOT token-windowed (that is `loadRecent`'s job for the agent loop). No
175
- * eviction. Paginated by whole turns: `page`, `perPage`, and `total` count
176
- * turns, not rows, so a multi-row turn always reloads intact. `messages`
177
- * holds the flattened rows of the selected turns.
162
+ * Return a thread's messages oldest-first for UI display, grouped into turns.
163
+ * NOT token-windowed — that is `loadRecent`'s job for the agent loop — and no
164
+ * eviction.
165
+ *
166
+ * There is no paginated form, because a page could never cost less: the read
167
+ * selects and parses every row of the thread and would slice only afterwards,
168
+ * so a page saves no query and no parse. It would only truncate the answer,
169
+ * which is a way to lose turns and not a way to save work. A reader wanting a
170
+ * window takes one here, where it knows what it is bounding.
171
+ *
172
+ * Grouped rather than flat because the grouping is what the read computes and
173
+ * what a turn count means, and `.flat()` recovers the rows in `seq` order for
174
+ * a display replay. The reverse does not hold: a flat return would make a
175
+ * caller that wants the tail turn, or the newest N turns, re-find boundaries
176
+ * this read already found.
178
177
  */
179
- loadPage(threadId: string, page: number, perPage: number): ResultAsync<MessagePage, DbError>;
178
+ loadAll(threadId: string): ResultAsync<StoredMessage[][], DbError>;
180
179
  /**
181
180
  * Remove the thread's most recent turn — every row from the last
182
181
  * genuine-user-start `seq` onward — in a single transaction.
@@ -209,6 +208,13 @@ export interface ThreadHistory {
209
208
  * no guarantee, because the tail can move one append later without it.
210
209
  */
211
210
  latestSeq(threadId: string): ResultAsync<number | null, DbError>;
211
+ /**
212
+ * When the thread last moved — `MAX(messages.created_at)`, or `null` for a
213
+ * thread with no rows. The thread's ACTIVITY clock, and deliberately not
214
+ * `cortex_analysis_threads.updated_at`: a title rename bumps that column
215
+ * too, so it cannot answer "has a turn settled since <time>".
216
+ */
217
+ latestTurnAt(threadId: string): ResultAsync<Date | null, DbError>;
212
218
  /**
213
219
  * The count of the thread turns that a person opened past `seq`. A caller
214
220
  * reads it as the new work of the thread past an anchor.
@@ -260,5 +266,10 @@ export declare const EVICTION_BLOCK_TURNS = 4;
260
266
  * Create a `ThreadHistory` bound to a Postgres pool — a factory closure
261
267
  * capturing `pool` (dependency injection per the harness-durable-runtime spec). The `messages` table is
262
268
  * provisioned by the project's state-init DDL.
269
+ *
270
+ * `logger` serves exactly one purpose: reporting a stored display part the
271
+ * current vocabulary had to drop. It is optional because the read returns the
272
+ * same messages either way — the drop reports a deploy, not a failed read — and
273
+ * requiring it would break every call site over a diagnostic.
263
274
  */
264
- export declare function createThreadHistory(pool: Pool): ThreadHistory;
275
+ export declare function createThreadHistory(pool: Pool, logger?: Logger): ThreadHistory;
@@ -108,7 +108,7 @@ function serializeEnvelope(message) {
108
108
  /**
109
109
  * Group rows (oldest-first) into turns at genuine-user-start boundaries.
110
110
  * Generic over the row shape so the token-windowed read (`loadRecent`) and
111
- * the display read (`loadPage`) share it, each supplying its own start
111
+ * the display read (`loadAll`) share it, each supplying its own start
112
112
  * predicate.
113
113
  */
114
114
  function groupTurns(rows, isStart) {
@@ -173,8 +173,13 @@ export const EVICTION_BLOCK_TURNS = 4;
173
173
  * Create a `ThreadHistory` bound to a Postgres pool — a factory closure
174
174
  * capturing `pool` (dependency injection per the harness-durable-runtime spec). The `messages` table is
175
175
  * provisioned by the project's state-init DDL.
176
+ *
177
+ * `logger` serves exactly one purpose: reporting a stored display part the
178
+ * current vocabulary had to drop. It is optional because the read returns the
179
+ * same messages either way — the drop reports a deploy, not a failed read — and
180
+ * requiring it would break every call site over a diagnostic.
176
181
  */
177
- export function createThreadHistory(pool) {
182
+ export function createThreadHistory(pool, logger) {
178
183
  function appendTurn(threadId, turn) {
179
184
  const { modelMessages: messages, displayMessages, turnUsage, turnDurationMs } = turn;
180
185
  if (messages.length === 0)
@@ -351,52 +356,40 @@ export function createThreadHistory(pool) {
351
356
  return (keepsFirstTurn ? [turns[0], ...retained] : retained).flat().map((row) => row.message);
352
357
  });
353
358
  }
354
- function loadPage(threadId, page, perPage) {
355
- const safePerPage = Math.min(Math.max(perPage, 1), 200);
356
- const safePage = Math.max(page, 0);
357
- return tryQuery("thread-history.loadPage", async () => {
358
- // Display pages are turn-bounded, not row-bounded. A serial-tool assistant
359
- // turn is persisted as one row per step (plus its tool_result `user` rows),
360
- // so a row-windowed page could split a turn — truncating the trailing
361
- // report card/text out of the page the UI fetches. Read the thread, group
362
- // into turns, and slice whole turns so a turn always reloads intact.
363
- // Threads are conversation-scoped and bounded, so reading every row here
364
- // matches `loadRecent`'s existing whole-thread read.
365
- const { rows } = await pool.query(
366
- // ORDER BY must qualify `messages.seq` — a bare `seq` would bind to the
367
- // `seq::text AS seq` output alias (Postgres resolves an unqualified
368
- // ORDER BY name to the output column), sorting the bigint as text:
369
- // "10" before "2". The qualified name forces the bigint column.
370
- `SELECT seq::text AS seq, message_envelope, display_envelope, reported_usage, turn_duration_ms::text AS turn_duration_ms
359
+ // Display reads are turn-bounded, not row-bounded. A serial-tool assistant
360
+ // turn is persisted as one row per step (plus its tool_result `user` rows),
361
+ // so a row-windowed page could split a turn — truncating the trailing
362
+ // report card/text out of the page the UI fetches. Read the thread and group
363
+ // into turns, so a turn always reloads intact. Threads are conversation-scoped
364
+ // and bounded, so reading every row here matches `loadRecent`'s existing
365
+ async function readTurns(threadId) {
366
+ const { rows } = await pool.query(
367
+ // ORDER BY must qualify `messages.seq` — a bare `seq` would bind to the
368
+ // `seq::text AS seq` output alias (Postgres resolves an unqualified
369
+ // ORDER BY name to the output column), sorting the bigint as text:
370
+ // "10" before "2". The qualified name forces the bigint column.
371
+ `SELECT seq::text AS seq, message_envelope, display_envelope, reported_usage, turn_duration_ms::text AS turn_duration_ms
371
372
  FROM messages WHERE thread_id = $1
372
373
  ORDER BY messages.seq ASC`, [threadId]);
373
- // Spread rather than `usage: r.reported_usage ?? undefined`, so a row with
374
- // no rollup carries no `usage` KEY at all — absent, not present-and-undefined.
375
- // The duration obeys the same rule, and it reads from its own column.
376
- const stored = await Promise.all(rows.map(async (r) => {
377
- const envelope = parseStoredMessageEnvelope(r.message_envelope, `${threadId}/${r.seq}`);
378
- const displayEnvelope = r.display_envelope == null ? undefined : await parseStoredDisplayEnvelope(r.display_envelope, `${threadId}/${r.seq}/display`);
379
- return {
380
- seq: Number(r.seq),
381
- envelope,
382
- message: envelope.message,
383
- ...(displayEnvelope ? { displayEnvelope } : {}),
384
- ...(r.reported_usage === null ? {} : { usage: r.reported_usage }),
385
- ...(r.turn_duration_ms === null ? {} : { durationMs: Number(r.turn_duration_ms) }),
386
- };
387
- }));
388
- const turns = groupTurns(stored, (row) => isGenuineUserStart(row.message));
389
- const total = turns.length;
390
- const offset = safePage * safePerPage;
391
- const pageTurns = turns.slice(offset, offset + safePerPage);
374
+ // Spread rather than `usage: r.reported_usage ?? undefined`, so a row with
375
+ // no rollup carries no `usage` KEY at all — absent, not present-and-undefined.
376
+ // The duration obeys the same rule, and it reads from its own column.
377
+ const stored = await Promise.all(rows.map(async (r) => {
378
+ const envelope = parseStoredMessageEnvelope(r.message_envelope, `${threadId}/${r.seq}`);
379
+ const displayEnvelope = r.display_envelope == null ? undefined : await parseStoredDisplayEnvelope(r.display_envelope, `${threadId}/${r.seq}/display`, logger);
392
380
  return {
393
- messages: pageTurns.flat(),
394
- total,
395
- page: safePage,
396
- perPage: safePerPage,
397
- hasMore: offset + pageTurns.length < total,
381
+ seq: Number(r.seq),
382
+ envelope,
383
+ message: envelope.message,
384
+ ...(displayEnvelope ? { displayEnvelope } : {}),
385
+ ...(r.reported_usage === null ? {} : { usage: r.reported_usage }),
386
+ ...(r.turn_duration_ms === null ? {} : { durationMs: Number(r.turn_duration_ms) }),
398
387
  };
399
- });
388
+ }));
389
+ return groupTurns(stored, (row) => isGenuineUserStart(row.message));
390
+ }
391
+ function loadAll(threadId) {
392
+ return tryQuery("thread-history.loadAll", () => readTurns(threadId));
400
393
  }
401
394
  function retractLastTurn(threadId) {
402
395
  return withTransaction(pool, "thread-history.retractLastTurn", (client) =>
@@ -452,6 +445,13 @@ export function createThreadHistory(pool) {
452
445
  return maxSeq === null ? null : Number(maxSeq);
453
446
  });
454
447
  }
448
+ function latestTurnAt(threadId) {
449
+ // `MAX(created_at)` over no rows is SQL NULL, so a thread with no messages
450
+ // reads as `null` rather than as a time. This is the thread's ACTIVITY
451
+ // clock — distinct from `cortex_analysis_threads.updated_at`, which a
452
+ // rename also bumps, and so cannot answer "did a turn settle after X".
453
+ return tryQuery("thread-history.latestTurnAt", () => pool.query("SELECT MAX(created_at) AS latest_at FROM messages WHERE thread_id = $1", [threadId])).map(({ rows }) => rows[0]?.latest_at ?? null);
454
+ }
455
455
  function countUserTurnsAfter(threadId, seq) {
456
456
  // `GENUINE_USER_START_SQL` is the stored-envelope twin of
457
457
  // `isGenuineUserStart`, and the tail retraction cuts on the same text.
@@ -468,5 +468,5 @@ export function createThreadHistory(pool) {
468
468
  AND seq > $2::bigint
469
469
  AND ${GENUINE_USER_START_SQL}`, [threadId, seq])).map(({ rows }) => Number(rows[0].turns));
470
470
  }
471
- return { appendTurn, loadRecent, loadPage, retractLastTurn, latestSeq, countUserTurnsAfter };
471
+ return { appendTurn, loadRecent, loadAll, retractLastTurn, latestSeq, latestTurnAt, countUserTurnsAfter };
472
472
  }