@inflexa-ai/harness 0.14.2 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/README.md +2 -1
  2. package/dist/agents/conversation-agent.d.ts +7 -0
  3. package/dist/agents/conversation-agent.js +5 -3
  4. package/dist/app/synthesize-run.d.ts +3 -0
  5. package/dist/app/synthesize-run.js +2 -1
  6. package/dist/billing/noop-usage-recorder.d.ts +7 -0
  7. package/dist/billing/noop-usage-recorder.js +10 -0
  8. package/dist/billing/usage-recorder.d.ts +63 -0
  9. package/dist/billing/usage-recorder.js +15 -0
  10. package/dist/contracts/chat-events.d.ts +44 -3
  11. package/dist/contracts/chat-parts.d.ts +38 -2
  12. package/dist/contracts/chat-parts.js +1 -1
  13. package/dist/contracts/data-profile.d.ts +26 -0
  14. package/dist/contracts/data-profile.js +26 -0
  15. package/dist/contracts/index.d.ts +3 -1
  16. package/dist/contracts/index.js +1 -0
  17. package/dist/contracts/message.d.ts +21 -1
  18. package/dist/contracts/part-registry.js +1 -0
  19. package/dist/contracts/schemas/chat-events.d.ts +48 -2
  20. package/dist/contracts/schemas/chat-events.js +8 -1
  21. package/dist/contracts/schemas/chat-parts.d.ts +46 -3
  22. package/dist/contracts/schemas/chat-parts.js +13 -0
  23. package/dist/contracts/schemas/index.d.ts +2 -1
  24. package/dist/contracts/schemas/index.js +2 -1
  25. package/dist/contracts/schemas/usage.d.ts +11 -0
  26. package/dist/contracts/schemas/usage.js +18 -0
  27. package/dist/contracts/usage.d.ts +51 -0
  28. package/dist/contracts/usage.js +41 -0
  29. package/dist/execution/artifact-metadata.d.ts +3 -0
  30. package/dist/execution/artifact-metadata.js +1 -0
  31. package/dist/execution/dbos-workflow-purger.d.ts +30 -0
  32. package/dist/execution/dbos-workflow-purger.js +193 -0
  33. package/dist/execution/post-step-pipeline.d.ts +3 -0
  34. package/dist/execution/post-step-pipeline.js +2 -0
  35. package/dist/execution/reconcile-manifest.js +28 -15
  36. package/dist/execution/report-runner.d.ts +16 -0
  37. package/dist/execution/report-runner.js +4 -0
  38. package/dist/execution/run-synthesis.d.ts +3 -0
  39. package/dist/execution/run-synthesis.js +2 -0
  40. package/dist/execution/step-summary.d.ts +3 -0
  41. package/dist/execution/step-summary.js +1 -0
  42. package/dist/execution/workflow-purger.d.ts +38 -0
  43. package/dist/execution/workflow-purger.js +24 -0
  44. package/dist/index.d.ts +24 -2
  45. package/dist/index.js +27 -0
  46. package/dist/lib/metrics.js +2 -1
  47. package/dist/loop/metrics.d.ts +18 -0
  48. package/dist/loop/metrics.js +18 -1
  49. package/dist/loop/run-agent.d.ts +35 -0
  50. package/dist/loop/run-agent.js +149 -23
  51. package/dist/loop/run-to-terminal.d.ts +5 -2
  52. package/dist/loop/run-to-terminal.js +36 -2
  53. package/dist/loop/tool-detail.d.ts +54 -0
  54. package/dist/loop/tool-detail.js +105 -0
  55. package/dist/loop/tool-outcome.d.ts +21 -0
  56. package/dist/loop/tool-outcome.js +25 -0
  57. package/dist/loop/types.d.ts +15 -1
  58. package/dist/memory/content-to-cortex.d.ts +28 -4
  59. package/dist/memory/content-to-cortex.js +70 -14
  60. package/dist/memory/thread-history.d.ts +21 -1
  61. package/dist/memory/thread-history.js +29 -7
  62. package/dist/memory/thread-store.d.ts +65 -8
  63. package/dist/memory/thread-store.js +81 -17
  64. package/dist/provenance/collector.d.ts +5 -3
  65. package/dist/provenance/collector.js +5 -3
  66. package/dist/providers/ai-sdk.js +91 -16
  67. package/dist/providers/streaming-chat.js +5 -0
  68. package/dist/providers/types.d.ts +23 -0
  69. package/dist/providers/types.js +1 -0
  70. package/dist/runtime/assemble.d.ts +25 -8
  71. package/dist/runtime/assemble.js +7 -4
  72. package/dist/sandbox/sandbox-step-translate.d.ts +22 -1
  73. package/dist/sandbox/sandbox-step-translate.js +23 -36
  74. package/dist/schemas/run-synthesis.d.ts +2 -2
  75. package/dist/state/index.d.ts +2 -0
  76. package/dist/state/index.js +3 -0
  77. package/dist/state/init.js +28 -0
  78. package/dist/state/purge-analysis.d.ts +74 -0
  79. package/dist/state/purge-analysis.js +205 -0
  80. package/dist/state/vector-store.d.ts +6 -0
  81. package/dist/state/vector-store.js +22 -2
  82. package/dist/tasks/data-profile-activity.d.ts +14 -2
  83. package/dist/tasks/data-profile-activity.js +1 -1
  84. package/dist/tasks/data-profile.d.ts +3 -0
  85. package/dist/tasks/data-profile.js +9 -3
  86. package/dist/tools/bio/pubmed.js +15 -0
  87. package/dist/tools/bio/search-gene.js +12 -0
  88. package/dist/tools/define-tool.d.ts +41 -3
  89. package/dist/tools/define-tool.js +4 -1
  90. package/dist/tools/detail-resolver.d.ts +35 -0
  91. package/dist/tools/detail-resolver.js +39 -0
  92. package/dist/tools/execute-analysis.js +3 -0
  93. package/dist/tools/iterate-report.d.ts +3 -0
  94. package/dist/tools/iterate-report.js +3 -0
  95. package/dist/tools/memory/update-working-memory.js +8 -0
  96. package/dist/tools/research/generate-analogy-report.d.ts +3 -0
  97. package/dist/tools/research/generate-analogy-report.js +6 -0
  98. package/dist/tools/research/generate-plan.d.ts +3 -0
  99. package/dist/tools/research/generate-plan.js +6 -0
  100. package/dist/tools/research/inspect-run.js +4 -0
  101. package/dist/tools/research/literature-reviewer.d.ts +3 -0
  102. package/dist/tools/research/literature-reviewer.js +6 -0
  103. package/dist/tools/workspace/edit-file.js +2 -0
  104. package/dist/tools/workspace/execute-command.js +25 -0
  105. package/dist/tools/workspace/grep.js +3 -0
  106. package/dist/tools/workspace/list-files.js +2 -0
  107. package/dist/tools/workspace/read-file.js +7 -0
  108. package/dist/tools/workspace/workspace-search.js +1 -0
  109. package/dist/tools/workspace/write-file.js +3 -0
  110. package/dist/workflows/execute-analysis.d.ts +7 -0
  111. package/dist/workflows/execute-analysis.js +31 -2
  112. package/dist/workflows/execute-target-assessment.d.ts +3 -0
  113. package/dist/workflows/sandbox-step.d.ts +23 -0
  114. package/dist/workflows/sandbox-step.js +33 -2
  115. package/dist/workflows/target-assessment/lib/clinical-consequence-annotator.d.ts +3 -0
  116. package/dist/workflows/target-assessment/lib/clinical-consequence-annotator.js +1 -0
  117. package/package.json +1 -1
@@ -14,22 +14,46 @@
14
14
  *
15
15
  * Display cards are emitted live over the chat SSE stream but never persisted
16
16
  * (storage holds only the AI SDK model-message transcript). `resolveCard`
17
- * rebuilds them from the persisted tool-call part so they reappear on reload.
17
+ * rebuilds them from the persisted tool-call part so they reappear on reload,
18
+ * and `resolveDetail` rebuilds each call's one-line detail the same way.
19
+ *
20
+ * A `tool-result` block is not rendered, but it is not ignored either: it is the
21
+ * only record of how its call ended. The blocks are indexed by `toolCallId`
22
+ * before conversion so each tool-call part can report its own outcome — without
23
+ * that pass every reloaded call reported success, including the failed ones.
18
24
  *
19
25
  * A row that yields no renderable parts (e.g. a `tool`-role message carrying
20
26
  * only tool-result continuation parts) is omitted entirely, so the rendered
21
27
  * conversation has no empty bubbles. Storage is never mutated.
22
28
  */
23
29
  import type { CortexMessage } from "@inflexa-ai/harness/contracts/message.js";
30
+ import type { ToolDetailResolver } from "../tools/detail-resolver.js";
24
31
  import type { ToolCardResolver } from "./reconstruct-cards.js";
25
32
  import type { StoredMessage } from "./thread-history.js";
33
+ /**
34
+ * The resolvers a caller may supply, as one object.
35
+ *
36
+ * An options bag rather than a growing positional list: both are optional, both
37
+ * are function-typed, and a second positional parameter of the same shape is a
38
+ * call site that silently does the wrong thing when the two are swapped.
39
+ */
40
+ export interface ContentToCortexOptions {
41
+ /** Rebuilds a display card (`data-plan`, `data-presentation`) from a persisted tool call. */
42
+ readonly resolveCard?: ToolCardResolver;
43
+ /** Rebuilds a call's one-line detail from its tool name and persisted input. */
44
+ readonly resolveDetail?: ToolDetailResolver;
45
+ }
26
46
  /**
27
47
  * Map stored messages (oldest-first) to `CortexMessage[]`. Rows that produce
28
48
  * no renderable parts are dropped, and consecutive assistant rows are coalesced
29
49
  * into one message. The first row's `seq` becomes the stable message id.
30
- * `resolveCard` (optional) reconstructs display cards from `tool_use` blocks.
50
+ * `options.resolveCard` reconstructs display cards from `tool_use` blocks and
51
+ * `options.resolveDetail` reconstructs each call's one-line detail; both are
52
+ * optional and each tool-call part reports its outcome either way.
31
53
  * A row carrying the interruption marker sets `interrupted: true` on the message
32
- * it lands in, including when coalesced into an assistant run.
54
+ * it lands in, including when coalesced into an assistant run; a row carrying a
55
+ * stored turn rollup sets `usage` the same way, so a reloaded conversation shows
56
+ * the cost the live turn showed without a second query.
33
57
  *
34
58
  * Coalescing mirrors the live SSE shape. The agent loop persists each assistant
35
59
  * step (typically a single `tool_use`) as its own row, and the `tool_result`
@@ -46,4 +70,4 @@ import type { StoredMessage } from "./thread-history.js";
46
70
  * turn's lone user message followed by the next turn's), and merging them would
47
71
  * fabricate one bubble from two messages the user sent separately.
48
72
  */
49
- export declare function contentToCortexMessages(messages: readonly StoredMessage[], resolveCard?: ToolCardResolver): Promise<CortexMessage[]>;
73
+ export declare function contentToCortexMessages(messages: readonly StoredMessage[], options?: ContentToCortexOptions): Promise<CortexMessage[]>;
@@ -14,39 +14,74 @@
14
14
  *
15
15
  * Display cards are emitted live over the chat SSE stream but never persisted
16
16
  * (storage holds only the AI SDK model-message transcript). `resolveCard`
17
- * rebuilds them from the persisted tool-call part so they reappear on reload.
17
+ * rebuilds them from the persisted tool-call part so they reappear on reload,
18
+ * and `resolveDetail` rebuilds each call's one-line detail the same way.
19
+ *
20
+ * A `tool-result` block is not rendered, but it is not ignored either: it is the
21
+ * only record of how its call ended. The blocks are indexed by `toolCallId`
22
+ * before conversion so each tool-call part can report its own outcome — without
23
+ * that pass every reloaded call reported success, including the failed ones.
18
24
  *
19
25
  * A row that yields no renderable parts (e.g. a `tool`-role message carrying
20
26
  * only tool-result continuation parts) is omitted entirely, so the rendered
21
27
  * conversation has no empty bubbles. Storage is never mutated.
22
28
  */
29
+ import { toolOutcomeForOutputType } from "../loop/tool-outcome.js";
23
30
  import { isInterruptedMessage, isSyntheticRecordMessage, isSyntheticUserMessage } from "./ai-sdk-message-storage.js";
24
- function genericToolCall(block) {
31
+ /**
32
+ * How each stored call ended, keyed by tool-call id.
33
+ *
34
+ * A whole page is indexed before conversion because the pairing crosses rows: a
35
+ * call rides an `assistant` row and its result rides the `tool` row after it.
36
+ */
37
+ function indexOutcomes(messages) {
38
+ const outcomes = new Map();
39
+ for (const { message } of messages) {
40
+ const content = message.content;
41
+ if (typeof content === "string")
42
+ continue;
43
+ for (const block of content) {
44
+ if (block.type !== "tool-result")
45
+ continue;
46
+ outcomes.set(block.toolCallId, toolOutcomeForOutputType(block.output.type));
47
+ }
48
+ }
49
+ return outcomes;
50
+ }
51
+ function genericToolCall(block, outcomes, resolveDetail) {
52
+ // No paired result is the shape of a transcript whose `tool` row was never
53
+ // appended (an aborted turn). Nothing failed, so `ok` is the honest report.
54
+ const outcome = outcomes.get(block.toolCallId) ?? "ok";
55
+ const detail = resolveDetail?.(block.toolName, block.input);
25
56
  return {
26
57
  type: "tool-call",
27
58
  toolCallId: block.toolCallId,
28
59
  toolName: block.toolName,
29
60
  status: "finished",
61
+ outcome,
62
+ ...(detail === undefined ? {} : { detail }),
30
63
  };
31
64
  }
32
- async function blockToPart(block, resolveCard) {
65
+ async function blockToPart(block, outcomes, options) {
33
66
  if (block.type === "text")
34
67
  return { type: "text", text: block.text };
35
68
  if (block.type === "tool-call") {
36
- const card = resolveCard ? await resolveCard({ type: "tool_use", id: block.toolCallId, name: block.toolName, input: block.input }) : null;
37
- return card ?? genericToolCall(block);
69
+ const card = options.resolveCard
70
+ ? await options.resolveCard({ type: "tool_use", id: block.toolCallId, name: block.toolName, input: block.input })
71
+ : null;
72
+ return card ?? genericToolCall(block, outcomes, options.resolveDetail);
38
73
  }
39
74
  // reasoning, tool_result, file, and any other block — not rendered.
40
75
  return null;
41
76
  }
42
- async function rowToParts(message, resolveCard) {
77
+ async function rowToParts(message, outcomes, options) {
43
78
  const content = message.content;
44
79
  if (typeof content === "string") {
45
80
  return content.length > 0 ? [{ type: "text", text: content }] : [];
46
81
  }
47
82
  const parts = [];
48
83
  for (const block of content) {
49
- const part = await blockToPart(block, resolveCard);
84
+ const part = await blockToPart(block, outcomes, options);
50
85
  if (part)
51
86
  parts.push(part);
52
87
  }
@@ -56,9 +91,13 @@ async function rowToParts(message, resolveCard) {
56
91
  * Map stored messages (oldest-first) to `CortexMessage[]`. Rows that produce
57
92
  * no renderable parts are dropped, and consecutive assistant rows are coalesced
58
93
  * into one message. The first row's `seq` becomes the stable message id.
59
- * `resolveCard` (optional) reconstructs display cards from `tool_use` blocks.
94
+ * `options.resolveCard` reconstructs display cards from `tool_use` blocks and
95
+ * `options.resolveDetail` reconstructs each call's one-line detail; both are
96
+ * optional and each tool-call part reports its outcome either way.
60
97
  * A row carrying the interruption marker sets `interrupted: true` on the message
61
- * it lands in, including when coalesced into an assistant run.
98
+ * it lands in, including when coalesced into an assistant run; a row carrying a
99
+ * stored turn rollup sets `usage` the same way, so a reloaded conversation shows
100
+ * the cost the live turn showed without a second query.
62
101
  *
63
102
  * Coalescing mirrors the live SSE shape. The agent loop persists each assistant
64
103
  * step (typically a single `tool_use`) as its own row, and the `tool_result`
@@ -75,7 +114,8 @@ async function rowToParts(message, resolveCard) {
75
114
  * turn's lone user message followed by the next turn's), and merging them would
76
115
  * fabricate one bubble from two messages the user sent separately.
77
116
  */
78
- export async function contentToCortexMessages(messages, resolveCard) {
117
+ export async function contentToCortexMessages(messages, options = {}) {
118
+ const outcomes = indexOutcomes(messages);
79
119
  const out = [];
80
120
  for (const message of messages) {
81
121
  // The loop's truncation nudge carries the `user` role only for the wire
@@ -94,17 +134,25 @@ export async function contentToCortexMessages(messages, resolveCard) {
94
134
  const isRecord = isSyntheticRecordMessage(message.message);
95
135
  if (isSyntheticUserMessage(message.message) && !isRecord)
96
136
  continue;
97
- const parts = await rowToParts(message.message, resolveCard);
137
+ const parts = await rowToParts(message.message, outcomes, options);
98
138
  // The marker is read before the zero-parts drop so the flag can never be lost to it:
99
139
  // a marked row carries an interruption fact regardless of whether it renders any parts.
100
140
  // When such a row contributes no bubble of its own, the interruption belongs to the
101
141
  // assistant run it trailed, so fold the flag onto the previous emitted assistant.
102
142
  const interrupted = isInterruptedMessage(message.message);
103
143
  if (parts.length === 0) {
104
- if (interrupted) {
105
- const prev = out[out.length - 1];
106
- if (prev && prev.role === "assistant")
144
+ // A row's stored rollup is a fact about the turn, not about what the row
145
+ // renders — so like the interruption marker it is read before the
146
+ // zero-parts drop and folded onto the assistant run it trailed. Without
147
+ // this, a turn whose last assistant row held only unrendered blocks
148
+ // (reasoning alone) would lose the figure to the drop, which is the exact
149
+ // disappearance persisting it exists to fix.
150
+ const prev = out[out.length - 1];
151
+ if (prev && prev.role === "assistant") {
152
+ if (interrupted)
107
153
  prev.interrupted = true;
154
+ if (message.usage)
155
+ prev.usage = message.usage;
108
156
  }
109
157
  continue;
110
158
  }
@@ -122,11 +170,19 @@ export async function contentToCortexMessages(messages, resolveCard) {
122
170
  prev.parts.push(...parts);
123
171
  if (interrupted)
124
172
  prev.interrupted = true;
173
+ // Coalescing is what rebuilds the one-bubble-per-turn shape, and the rollup
174
+ // rides the turn's LAST assistant row — so the run keeps whichever of its
175
+ // rows carried one. Assigning rather than merging is deliberate: two rollups
176
+ // in one run would mean two turns were merged, which coalescing never does.
177
+ if (message.usage)
178
+ prev.usage = message.usage;
125
179
  continue;
126
180
  }
127
181
  const cortex = { id: String(message.seq), role, parts };
128
182
  if (interrupted)
129
183
  cortex.interrupted = true;
184
+ if (message.usage)
185
+ cortex.usage = message.usage;
130
186
  out.push(cortex);
131
187
  }
132
188
  return out;
@@ -23,6 +23,7 @@
23
23
  import type { ModelMessage } from "ai";
24
24
  import { ResultAsync } from "neverthrow";
25
25
  import type { Pool } from "pg";
26
+ import type { TokenUsageRollup } from "../contracts/usage.js";
26
27
  import { type DbError } from "../lib/db-result.js";
27
28
  import { type StoredMessageEnvelope } from "./ai-sdk-message-storage.js";
28
29
  /** One stored message, as returned by the display read (`loadPage`). */
@@ -30,6 +31,15 @@ export interface StoredMessage {
30
31
  readonly seq: number;
31
32
  readonly envelope: StoredMessageEnvelope;
32
33
  readonly message: ModelMessage;
34
+ /**
35
+ * What providers reported for the whole TURN this row completed — present
36
+ * only on the last assistant row of a turn appended with a rollup, absent
37
+ * everywhere else. Not a per-row figure, and unrelated to the `tokens` count
38
+ * `loadRecent` windows by: that is an offline estimate stamped on every row,
39
+ * this is a provider's own report (see the `reported_usage` column comment in
40
+ * the state-init DDL).
41
+ */
42
+ readonly usage?: TokenUsageRollup;
33
43
  }
34
44
  /**
35
45
  * A page of stored messages plus pagination metadata. Pagination is
@@ -66,8 +76,18 @@ export interface ThreadHistory {
66
76
  /**
67
77
  * Append one conversation turn — every message written in a single
68
78
  * transaction with a `seq` monotonically increasing per thread.
79
+ *
80
+ * `turnUsage` is the turn's reported rollup, as `AgentFinish.turnUsage`
81
+ * carries it out of the loop and the `finish` event carries it to a live
82
+ * surface. It is stored on the LAST assistant message of the turn — the row a
83
+ * reader associates with the reply — so a reloaded transcript renders the
84
+ * figure the live turn showed. Optional at every layer: omitting it, and
85
+ * supplying one that reports no quantity at all, both leave the row without
86
+ * one, because a turn that reported nothing must not read back as a turn that
87
+ * cost zero. A turn writing no assistant message stores none and still
88
+ * succeeds — there is no row on which the figure would mean anything.
69
89
  */
70
- appendTurn(threadId: string, messages: readonly ModelMessage[]): ResultAsync<void, DbError>;
90
+ appendTurn(threadId: string, messages: readonly ModelMessage[], turnUsage?: TokenUsageRollup): ResultAsync<void, DbError>;
71
91
  /**
72
92
  * Return a recent-turns window that fits `tokenBudget`, oldest-first,
73
93
  * snapped to a valid AI SDK model-message sequence. The window START advances
@@ -24,6 +24,7 @@ import { metrics } from "@opentelemetry/api";
24
24
  import { ResultAsync, ok, okAsync } from "neverthrow";
25
25
  import { stripNulCharacters } from "../input-sanitization.js";
26
26
  import { tryMutation, tryQuery, withTransaction } from "../lib/db-result.js";
27
+ import { hasReportedUsage } from "../loop/metrics.js";
27
28
  import { countTokens } from "./count-tokens.js";
28
29
  import { HARNESS_PROVIDER_NAMESPACE, SYNTHETIC_MESSAGE_KEY, envelopeMessage, isSyntheticUserMessage, parseStoredMessageEnvelope, } from "./ai-sdk-message-storage.js";
29
30
  /** A resolved `ok(undefined)` ResultAsync — the empty/seed transaction step. */
@@ -155,9 +156,20 @@ export const EVICTION_BLOCK_TURNS = 4;
155
156
  * provisioned by the project's state-init DDL.
156
157
  */
157
158
  export function createThreadHistory(pool) {
158
- function appendTurn(threadId, messages) {
159
+ function appendTurn(threadId, messages, turnUsage) {
159
160
  if (messages.length === 0)
160
161
  return okVoid();
162
+ // A rollup that reports no quantity is stored as absent, not as a rollup of
163
+ // absences, so "no figure" has exactly one representation in storage.
164
+ // `hasReportedUsage` is the loop's own predicate for the same question,
165
+ // reused rather than restated so the write and the loop cannot drift.
166
+ const rollup = hasReportedUsage(turnUsage) ? JSON.stringify(turnUsage) : null;
167
+ // The rollup describes the turn, and the assistant reply is the row a reader
168
+ // associates with the answer. The LAST assistant row, not any of them: a
169
+ // serial-tool turn writes one assistant row per step and only the last ends
170
+ // the turn. -1 (a turn that persisted no reply — an abort before any output)
171
+ // matches no index below, so nothing is written and the append still succeeds.
172
+ const rollupRow = messages.reduce((last, m, i) => (m.role === "assistant" ? i : last), -1);
161
173
  return withTransaction(pool, "thread-history.appendTurn", (client) =>
162
174
  // Serialize concurrent appends on this thread — without the lock, two
163
175
  // transactions can both read the same MAX(seq) and collide on the
@@ -173,12 +185,14 @@ export function createThreadHistory(pool) {
173
185
  // re-throws it to force ROLLBACK (a returned `err` that does not
174
186
  // reach `withTransaction` would COMMIT silently).
175
187
  messages.reduce((chain, message, i) => chain.andThen(() => tryMutation("thread-history.appendTurn.insert", () => client.query(
176
- // `::json`, never `::jsonb` — see the column comment in the state-init DDL.
177
- `INSERT INTO messages (thread_id, seq, message_envelope, tokens)
178
- VALUES ($1, $2, $3::json, $4)
188
+ // `message_envelope::json`, never `::jsonb`, and `reported_usage::jsonb`,
189
+ // never `::json` — see the column comments in the state-init DDL.
190
+ `INSERT INTO messages (thread_id, seq, message_envelope, tokens, reported_usage)
191
+ VALUES ($1, $2, $3::json, $4, $5::jsonb)
179
192
  ON CONFLICT (thread_id, seq) DO UPDATE
180
193
  SET message_envelope = EXCLUDED.message_envelope,
181
- tokens = EXCLUDED.tokens`, [threadId, startSeq + i, serializeEnvelope(message), countTokens(message.content)])).map(() => undefined)), okVoid()))
194
+ tokens = EXCLUDED.tokens,
195
+ reported_usage = EXCLUDED.reported_usage`, [threadId, startSeq + i, serializeEnvelope(message), countTokens(message.content), i === rollupRow ? rollup : null])).map(() => undefined)), okVoid()))
182
196
  .andThen(() =>
183
197
  // Thread listings sort on `cortex_analysis_threads.updated_at`, and
184
198
  // the only other writer of it is the title update — so without this
@@ -226,6 +240,12 @@ export function createThreadHistory(pool) {
226
240
  // ORDER BY name to the output column), sorting the bigint as text:
227
241
  // "10" before "2". Scrambled order splits a tool-call/tool-result pair
228
242
  // across an intervening turn. The qualified name forces the bigint column.
243
+ //
244
+ // `reported_usage` is deliberately NOT selected. Windowing budgets on
245
+ // `tokens` — an offline estimate every row carries — and a turn that
246
+ // reported nothing has no rollup at all, so budgeting on the rollup
247
+ // would silently stop evicting on exactly those threads. The two are
248
+ // different measurements sharing a unit; see the column comments.
229
249
  `SELECT seq::text AS seq, message_envelope, tokens
230
250
  FROM messages WHERE thread_id = $1 ORDER BY messages.seq ASC`, [threadId]);
231
251
  const parsed = rows.map((row) => ({
@@ -291,12 +311,14 @@ export function createThreadHistory(pool) {
291
311
  // `seq::text AS seq` output alias (Postgres resolves an unqualified
292
312
  // ORDER BY name to the output column), sorting the bigint as text:
293
313
  // "10" before "2". The qualified name forces the bigint column.
294
- `SELECT seq::text AS seq, message_envelope
314
+ `SELECT seq::text AS seq, message_envelope, reported_usage
295
315
  FROM messages WHERE thread_id = $1
296
316
  ORDER BY messages.seq ASC`, [threadId]);
297
317
  const stored = rows.map((r) => {
298
318
  const envelope = parseStoredMessageEnvelope(r.message_envelope, `${threadId}/${r.seq}`);
299
- return { seq: Number(r.seq), envelope, message: envelope.message };
319
+ // Spread rather than `usage: r.reported_usage ?? undefined`, so a row with
320
+ // no rollup carries no `usage` KEY at all — absent, not present-and-undefined.
321
+ return { seq: Number(r.seq), envelope, message: envelope.message, ...(r.reported_usage === null ? {} : { usage: r.reported_usage }) };
300
322
  });
301
323
  const turns = groupTurns(stored, (row) => isGenuineUserStart(row.message));
302
324
  const total = turns.length;
@@ -10,9 +10,29 @@
10
10
  * the vocabulary is conversation-shaped on purpose — reaching for it inside
11
11
  * a workflow step feels immediately wrong.
12
12
  *
13
- * Delete is soft: `deleteThread` sets `deleted_at`; `getThread` and
14
- * `listThreads` filter `deleted_at IS NULL`, so the row (and its messages)
15
- * survive a delete and a tombstoned thread is indistinguishable from absent.
13
+ * Three lifecycle verbs with distinct guarantees. `archiveThread` is
14
+ * recoverable: it stamps `deleted_at`, and because `getThread` and the default
15
+ * `listThreads` filter `deleted_at IS NULL` an archived thread is
16
+ * indistinguishable from an absent one while its row and every one of its
17
+ * `messages` rows stay in storage. `unarchiveThread` clears the stamp and the
18
+ * thread reads as it did; a listing widened with `includeArchived` is the only
19
+ * way to obtain the id it takes, so the recovery is reachable by a host that
20
+ * holds no thread ids it obtained elsewhere.
21
+ * `purgeThread` is not recoverable: it removes the `messages` rows and the
22
+ * metadata row in one transaction, so a failure partway leaves both — never a
23
+ * thread stripped of its transcript, nor a transcript with nothing naming it.
24
+ * It is the thread-scoped member of this package's reclamation vocabulary —
25
+ * `purgeAnalysis` reclaims an analysis's whole persisted footprint,
26
+ * `purgeThread` one thread's.
27
+ *
28
+ * A purge creates no orphan of its own, but it is NOT serialized against a
29
+ * concurrent `appendTurn`: `messages` carries no foreign key to
30
+ * `cortex_analysis_threads`, and the append deliberately tolerates a missing
31
+ * metadata row. A turn committing after the purge therefore persists messages
32
+ * under a `thread_id` that no longer resolves to an analysis, leaving them
33
+ * unreachable by any later thread- or analysis-scoped reclamation. Stop writes
34
+ * to a thread — unbind it from any live conversation — before purging it; this
35
+ * store cannot observe a host's in-flight turns, so it does not enforce that.
16
36
  */
17
37
  import { type ResultAsync } from "neverthrow";
18
38
  import type { Pool } from "pg";
@@ -24,6 +44,13 @@ export interface Thread {
24
44
  readonly title: string | null;
25
45
  readonly createdAt: Date;
26
46
  readonly updatedAt: Date;
47
+ /**
48
+ * The archive tombstone — `null` on a live thread, the moment it left view
49
+ * on an archived one. Required rather than optional: a row is always one or
50
+ * the other, so a caller listing archived threads has no `undefined` case to
51
+ * fall through and render a tombstoned conversation as live.
52
+ */
53
+ readonly deletedAt: Date | null;
27
54
  }
28
55
  export interface CreateThreadInput {
29
56
  readonly threadId: string;
@@ -34,6 +61,13 @@ export interface ListThreadsInput {
34
61
  readonly analysisId: string;
35
62
  readonly page?: number;
36
63
  readonly perPage?: number;
64
+ /**
65
+ * Widen the listing to archived threads alongside live ones. It widens, it
66
+ * does not switch: a caller wanting only the archived rows filters the
67
+ * result on `deletedAt`, which an archived-only listing could not be
68
+ * widened back out of.
69
+ */
70
+ readonly includeArchived?: boolean;
37
71
  }
38
72
  export interface ThreadPage {
39
73
  readonly threads: Thread[];
@@ -49,13 +83,36 @@ export interface ThreadStore {
49
83
  * `created_at`). Returns the live row.
50
84
  */
51
85
  createThread(input: CreateThreadInput): ResultAsync<Thread, DbError>;
52
- /** The live thread by id, or `null` if absent or soft-deleted. */
86
+ /** The live thread by id, or `null` if absent or archived. */
53
87
  getThread(threadId: string): ResultAsync<Thread | null, DbError>;
54
- /** Set only the title (and bump `updated_at`). No-op on a missing/deleted row. */
88
+ /**
89
+ * Set only the title, bumping `updated_at` forward — never behind the stamp
90
+ * the row already carries. No-op on a missing/archived row.
91
+ */
55
92
  updateTitle(threadId: string, title: string): ResultAsync<Thread | null, DbError>;
56
- /** Soft-delete: set `deleted_at`. The row and its messages remain. */
57
- deleteThread(threadId: string): ResultAsync<void, DbError>;
58
- /** Live threads for one analysis, newest-updated first, paginated. */
93
+ /**
94
+ * Soft-delete: stamp `deleted_at` so the thread leaves `getThread` and
95
+ * `listThreads`. The row and every one of its messages remain, and an
96
+ * already-archived thread keeps the stamp it got the first time.
97
+ */
98
+ archiveThread(threadId: string): ResultAsync<void, DbError>;
99
+ /**
100
+ * Clear `deleted_at`, returning the thread and its messages to view exactly
101
+ * as they read before the archive. No-op on a live or absent thread.
102
+ */
103
+ unarchiveThread(threadId: string): ResultAsync<void, DbError>;
104
+ /**
105
+ * Reclaim the thread's whole footprint: its `messages` rows and its
106
+ * `cortex_analysis_threads` row, removed together in one transaction.
107
+ * Unrecoverable — nothing of the thread survives, and no tombstone marks
108
+ * that it existed. A `thread_id` with no row succeeds as a no-op.
109
+ */
110
+ purgeThread(threadId: string): ResultAsync<void, DbError>;
111
+ /**
112
+ * Threads for one analysis, newest-updated first, paginated. Live threads
113
+ * only unless `includeArchived` widens the set; `total` and `hasMore`
114
+ * always describe whichever set the page was drawn from.
115
+ */
59
116
  listThreads(input: ListThreadsInput): ResultAsync<ThreadPage, DbError>;
60
117
  }
61
118
  /**
@@ -10,12 +10,32 @@
10
10
  * the vocabulary is conversation-shaped on purpose — reaching for it inside
11
11
  * a workflow step feels immediately wrong.
12
12
  *
13
- * Delete is soft: `deleteThread` sets `deleted_at`; `getThread` and
14
- * `listThreads` filter `deleted_at IS NULL`, so the row (and its messages)
15
- * survive a delete and a tombstoned thread is indistinguishable from absent.
13
+ * Three lifecycle verbs with distinct guarantees. `archiveThread` is
14
+ * recoverable: it stamps `deleted_at`, and because `getThread` and the default
15
+ * `listThreads` filter `deleted_at IS NULL` an archived thread is
16
+ * indistinguishable from an absent one while its row and every one of its
17
+ * `messages` rows stay in storage. `unarchiveThread` clears the stamp and the
18
+ * thread reads as it did; a listing widened with `includeArchived` is the only
19
+ * way to obtain the id it takes, so the recovery is reachable by a host that
20
+ * holds no thread ids it obtained elsewhere.
21
+ * `purgeThread` is not recoverable: it removes the `messages` rows and the
22
+ * metadata row in one transaction, so a failure partway leaves both — never a
23
+ * thread stripped of its transcript, nor a transcript with nothing naming it.
24
+ * It is the thread-scoped member of this package's reclamation vocabulary —
25
+ * `purgeAnalysis` reclaims an analysis's whole persisted footprint,
26
+ * `purgeThread` one thread's.
27
+ *
28
+ * A purge creates no orphan of its own, but it is NOT serialized against a
29
+ * concurrent `appendTurn`: `messages` carries no foreign key to
30
+ * `cortex_analysis_threads`, and the append deliberately tolerates a missing
31
+ * metadata row. A turn committing after the purge therefore persists messages
32
+ * under a `thread_id` that no longer resolves to an analysis, leaving them
33
+ * unreachable by any later thread- or analysis-scoped reclamation. Stop writes
34
+ * to a thread — unbind it from any live conversation — before purging it; this
35
+ * store cannot observe a host's in-flight turns, so it does not enforce that.
16
36
  */
17
37
  import { okAsync } from "neverthrow";
18
- import { tryMutation, tryQuery } from "../lib/db-result.js";
38
+ import { tryMutation, tryQuery, withTransaction } from "../lib/db-result.js";
19
39
  function toThread(row) {
20
40
  return {
21
41
  threadId: row.thread_id,
@@ -23,6 +43,7 @@ function toThread(row) {
23
43
  title: row.title,
24
44
  createdAt: row.created_at,
25
45
  updatedAt: row.updated_at,
46
+ deletedAt: row.deleted_at,
26
47
  };
27
48
  }
28
49
  const DEFAULT_PER_PAGE = 100;
@@ -37,43 +58,86 @@ export function createThreadStore(pool) {
37
58
  return tryMutation("thread-store.createThread.insert", () => pool.query(`INSERT INTO cortex_analysis_threads (thread_id, analysis_id, title)
38
59
  VALUES ($1, $2, $3)
39
60
  ON CONFLICT (thread_id) DO NOTHING
40
- RETURNING thread_id, analysis_id, title, created_at, updated_at`, [input.threadId, input.analysisId, input.title ?? null])).andThen(({ rows }) => {
61
+ RETURNING thread_id, analysis_id, title, created_at, updated_at, deleted_at`, [input.threadId, input.analysisId, input.title ?? null])).andThen(({ rows }) => {
41
62
  if (rows[0])
42
63
  return okAsync(toThread(rows[0]));
43
64
  // Row already existed — ON CONFLICT DO NOTHING returns nothing. Read it
44
- // back (it may be soft-deleted; return it regardless so the caller sees
65
+ // back (it may be archived; return it regardless so the caller sees
45
66
  // the existing row's identity).
46
- return tryQuery("thread-store.createThread.readback", () => pool.query(`SELECT thread_id, analysis_id, title, created_at, updated_at
67
+ return tryQuery("thread-store.createThread.readback", () => pool.query(`SELECT thread_id, analysis_id, title, created_at, updated_at, deleted_at
47
68
  FROM cortex_analysis_threads WHERE thread_id = $1`, [input.threadId])).map(({ rows: existing }) => toThread(existing[0]));
48
69
  });
49
70
  }
50
71
  function getThread(threadId) {
51
- return tryQuery("thread-store.getThread", () => pool.query(`SELECT thread_id, analysis_id, title, created_at, updated_at
72
+ return tryQuery("thread-store.getThread", () => pool.query(`SELECT thread_id, analysis_id, title, created_at, updated_at, deleted_at
52
73
  FROM cortex_analysis_threads
53
74
  WHERE thread_id = $1 AND deleted_at IS NULL`, [threadId])).map(({ rows }) => (rows[0] ? toThread(rows[0]) : null));
54
75
  }
55
76
  function updateTitle(threadId, title) {
56
- return tryMutation("thread-store.updateTitle", () => pool.query(`UPDATE cortex_analysis_threads
57
- SET title = $2, updated_at = NOW()
77
+ return tryMutation("thread-store.updateTitle", () => pool.query(
78
+ // `updated_at` is the listing's activity clock, and a rename is not its
79
+ // only writer — a turn append touches the same column, and a rename that
80
+ // starts before one can still reach the row after it. Taking the LATER of
81
+ // the two makes the bump forward-only, so a rename can never pull the
82
+ // column back behind a stamp the row already carries, no matter which
83
+ // writer set it or how long this statement waited to run. `NOW()` cannot
84
+ // give that: it reads this transaction's START time, not the moment the
85
+ // row is actually written.
86
+ `UPDATE cortex_analysis_threads
87
+ SET title = $2, updated_at = GREATEST(updated_at, clock_timestamp())
58
88
  WHERE thread_id = $1 AND deleted_at IS NULL
59
- RETURNING thread_id, analysis_id, title, created_at, updated_at`, [threadId, title])).map(({ rows }) => (rows[0] ? toThread(rows[0]) : null));
89
+ RETURNING thread_id, analysis_id, title, created_at, updated_at, deleted_at`, [threadId, title])).map(({ rows }) => (rows[0] ? toThread(rows[0]) : null));
60
90
  }
61
- function deleteThread(threadId) {
62
- return tryMutation("thread-store.deleteThread", () => pool.query(`UPDATE cortex_analysis_threads
91
+ function archiveThread(threadId) {
92
+ return tryMutation("thread-store.archiveThread", () => pool.query(
93
+ // The tombstone guard is what makes a second archive change nothing:
94
+ // the stamp records when the thread left view, and re-stamping it
95
+ // would push that moment forward on every repeat.
96
+ `UPDATE cortex_analysis_threads
63
97
  SET deleted_at = NOW()
64
98
  WHERE thread_id = $1 AND deleted_at IS NULL`, [threadId])).map(() => undefined);
65
99
  }
100
+ function unarchiveThread(threadId) {
101
+ return tryMutation("thread-store.unarchiveThread", () => pool.query(
102
+ // `updated_at` is untouched on purpose: it orders the listing by
103
+ // conversation activity, and restoring a thread to view is not
104
+ // activity — a restored thread lands back where its last turn left it.
105
+ `UPDATE cortex_analysis_threads
106
+ SET deleted_at = NULL
107
+ WHERE thread_id = $1 AND deleted_at IS NOT NULL`, [threadId])).map(() => undefined);
108
+ }
109
+ function purgeThread(threadId) {
110
+ // One transaction for both statements: `messages` is attributable to an
111
+ // analysis only by joining through `cortex_analysis_threads`, so a metadata
112
+ // row removed without its messages strands them beyond the reach of any
113
+ // later reclamation. Either the pair goes or neither does.
114
+ return withTransaction(pool, "thread-store.purgeThread", (client) => tryMutation("thread-store.purgeThread.messages", () => client.query("DELETE FROM messages WHERE thread_id = $1", [threadId]))
115
+ .andThen(() => tryMutation("thread-store.purgeThread.thread", () => client.query("DELETE FROM cortex_analysis_threads WHERE thread_id = $1", [threadId])))
116
+ .map(() => undefined));
117
+ }
66
118
  function listThreads(input) {
67
119
  const perPage = Math.min(Math.max(input.perPage ?? DEFAULT_PER_PAGE, 1), MAX_PER_PAGE);
68
120
  const page = Math.max(input.page ?? 0, 0);
69
121
  const offset = page * perPage;
122
+ // The count and the page are two statements over what has to be one set:
123
+ // a total drawn from a different predicate than the page's would report a
124
+ // size the caller can never page to. Writing the row scope once — one of
125
+ // two fixed fragments, never caller text — makes them unable to disagree.
126
+ //
127
+ // The widened scope is the one that cannot use the table's index, which is
128
+ // partial on `deleted_at IS NULL`: dropping the predicate drops the index
129
+ // with it, so an archived-inclusive listing scans. Deliberate — the live
130
+ // listing is the hot path and keeps its index, while widening is a rare,
131
+ // user-initiated recovery. A deployment that made it routine would want a
132
+ // plain `analysis_id` index beside the partial one, not this listing changed.
133
+ const scope = input.includeArchived ? "analysis_id = $1" : "analysis_id = $1 AND deleted_at IS NULL";
70
134
  return tryQuery("thread-store.listThreads.count", () => pool.query(`SELECT COUNT(*)::text AS count
71
135
  FROM cortex_analysis_threads
72
- WHERE analysis_id = $1 AND deleted_at IS NULL`, [input.analysisId])).andThen((totalResult) => {
136
+ WHERE ${scope}`, [input.analysisId])).andThen((totalResult) => {
73
137
  const total = Number(totalResult.rows[0].count);
74
- return tryQuery("thread-store.listThreads.page", () => pool.query(`SELECT thread_id, analysis_id, title, created_at, updated_at
138
+ return tryQuery("thread-store.listThreads.page", () => pool.query(`SELECT thread_id, analysis_id, title, created_at, updated_at, deleted_at
75
139
  FROM cortex_analysis_threads
76
- WHERE analysis_id = $1 AND deleted_at IS NULL
140
+ WHERE ${scope}
77
141
  ORDER BY updated_at DESC, thread_id
78
142
  LIMIT $2 OFFSET $3`, [input.analysisId, perPage, offset])).map(({ rows }) => ({
79
143
  threads: rows.map(toThread),
@@ -84,5 +148,5 @@ export function createThreadStore(pool) {
84
148
  }));
85
149
  });
86
150
  }
87
- return { createThread, getThread, updateTitle, deleteThread, listThreads };
151
+ return { createThread, getThread, updateTitle, archiveThread, unarchiveThread, purgeThread, listThreads };
88
152
  }
@@ -165,9 +165,11 @@ export declare class ProvenanceCollector {
165
165
  * are shared objects). `reconcileManifestWithDisk` calls this for a read that
166
166
  * is not a content-attestable file of this analysis: one resolving to a
167
167
  * directory (e.g. an `ls` / `list.files` of a mount, which the inotify frame
168
- * tracks as an open), or one resolving outside the analysis tree (e.g.
169
- * `/{resourceId}/..`, the container root). Neither may reach registration.
170
- * Mirrors the output-side non-file and out-of-bounds drops.
168
+ * tracks as an open), one resolving outside the analysis tree (e.g.
169
+ * `/{resourceId}/..`, the container root), or one naming a path that is not
170
+ * there at reconcile (a capture layer reports attempted opens, so a read
171
+ * that failed is reported like one that succeeded). None may reach
172
+ * registration. Mirrors the output-side non-file and out-of-bounds drops.
171
173
  */
172
174
  dropInput(ref: InputRef): void;
173
175
  /**
@@ -278,9 +278,11 @@ export class ProvenanceCollector {
278
278
  * are shared objects). `reconcileManifestWithDisk` calls this for a read that
279
279
  * is not a content-attestable file of this analysis: one resolving to a
280
280
  * directory (e.g. an `ls` / `list.files` of a mount, which the inotify frame
281
- * tracks as an open), or one resolving outside the analysis tree (e.g.
282
- * `/{resourceId}/..`, the container root). Neither may reach registration.
283
- * Mirrors the output-side non-file and out-of-bounds drops.
281
+ * tracks as an open), one resolving outside the analysis tree (e.g.
282
+ * `/{resourceId}/..`, the container root), or one naming a path that is not
283
+ * there at reconcile (a capture layer reports attempted opens, so a read
284
+ * that failed is reported like one that succeeded). None may reach
285
+ * registration. Mirrors the output-side non-file and out-of-bounds drops.
284
286
  */
285
287
  dropInput(ref) {
286
288
  for (const [key, val] of this.inputAccesses) {