@larkup/workspace-server 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/LICENSE +176 -0
  2. package/README.md +2 -2
  3. package/dist-types/app/api/chat/feedback/route.d.ts +4 -0
  4. package/dist-types/app/api/chat/feedback/route.d.ts.map +1 -0
  5. package/dist-types/app/api/chat/route.d.ts +3 -0
  6. package/dist-types/app/api/chat/route.d.ts.map +1 -0
  7. package/dist-types/app/api/chat/tools.d.ts +67 -0
  8. package/dist-types/app/api/chat/tools.d.ts.map +1 -0
  9. package/dist-types/app/api/config/test-embedding/route.d.ts +3 -0
  10. package/dist-types/app/api/config/test-embedding/route.d.ts.map +1 -0
  11. package/dist-types/app/api/config/verify/route.d.ts +3 -0
  12. package/dist-types/app/api/config/verify/route.d.ts.map +1 -0
  13. package/dist-types/app/api/describe-image/route.d.ts +4 -0
  14. package/dist-types/app/api/describe-image/route.d.ts.map +1 -0
  15. package/dist-types/app/api/doc-editor/edit/route.d.ts +10 -0
  16. package/dist-types/app/api/doc-editor/edit/route.d.ts.map +1 -0
  17. package/dist-types/app/api/doc-editor/export/route.d.ts +9 -0
  18. package/dist-types/app/api/doc-editor/export/route.d.ts.map +1 -0
  19. package/dist-types/app/api/doc-editor/parse/route.d.ts +10 -0
  20. package/dist-types/app/api/doc-editor/parse/route.d.ts.map +1 -0
  21. package/dist-types/app/api/doc-editor/restore/route.d.ts +2 -0
  22. package/dist-types/app/api/doc-editor/restore/route.d.ts.map +1 -0
  23. package/dist-types/app/api/document/sign/route.d.ts +2 -0
  24. package/dist-types/app/api/document/sign/route.d.ts.map +1 -0
  25. package/dist-types/app/api/files/remote/route.d.ts +5 -0
  26. package/dist-types/app/api/files/remote/route.d.ts.map +1 -0
  27. package/dist-types/app/api/mcp/[connectionId]/route.d.ts +9 -0
  28. package/dist-types/app/api/mcp/[connectionId]/route.d.ts.map +1 -0
  29. package/dist-types/app/api/mcp/[connectionId]/test/route.d.ts +8 -0
  30. package/dist-types/app/api/mcp/[connectionId]/test/route.d.ts.map +1 -0
  31. package/dist-types/app/api/mcp/route.d.ts +4 -0
  32. package/dist-types/app/api/mcp/route.d.ts.map +1 -0
  33. package/dist-types/app/api/parse-file/route.d.ts +6 -0
  34. package/dist-types/app/api/parse-file/route.d.ts.map +1 -0
  35. package/dist-types/app/api/pdf-page/route.d.ts +4 -0
  36. package/dist-types/app/api/pdf-page/route.d.ts.map +1 -0
  37. package/dist-types/app/api/proxy/verify/route.d.ts +4 -0
  38. package/dist-types/app/api/proxy/verify/route.d.ts.map +1 -0
  39. package/dist-types/app/api/search/bing/route.d.ts +5 -0
  40. package/dist-types/app/api/search/bing/route.d.ts.map +1 -0
  41. package/dist-types/app/api/search/brave/route.d.ts +5 -0
  42. package/dist-types/app/api/search/brave/route.d.ts.map +1 -0
  43. package/dist-types/app/api/search/exa/route.d.ts +5 -0
  44. package/dist-types/app/api/search/exa/route.d.ts.map +1 -0
  45. package/dist-types/app/api/search/google/route.d.ts +30 -0
  46. package/dist-types/app/api/search/google/route.d.ts.map +1 -0
  47. package/dist-types/app/api/search/route.d.ts +7 -0
  48. package/dist-types/app/api/search/route.d.ts.map +1 -0
  49. package/dist-types/app/api/search/tavily/route.d.ts +5 -0
  50. package/dist-types/app/api/search/tavily/route.d.ts.map +1 -0
  51. package/dist-types/app/api/search/verify/route.d.ts +4 -0
  52. package/dist-types/app/api/search/verify/route.d.ts.map +1 -0
  53. package/dist-types/app/api/tabular/route.d.ts +9 -0
  54. package/dist-types/app/api/tabular/route.d.ts.map +1 -0
  55. package/dist-types/lib/chat/assistant-source-scope.d.ts +10 -0
  56. package/dist-types/lib/chat/assistant-source-scope.d.ts.map +1 -0
  57. package/dist-types/lib/chat/conversation-memory.d.ts +80 -0
  58. package/dist-types/lib/chat/conversation-memory.d.ts.map +1 -0
  59. package/dist-types/lib/chat/gateway-fallbacks.d.ts +7 -0
  60. package/dist-types/lib/chat/gateway-fallbacks.d.ts.map +1 -0
  61. package/dist-types/lib/chat/image-description.d.ts +10 -0
  62. package/dist-types/lib/chat/image-description.d.ts.map +1 -0
  63. package/dist-types/lib/chat/media-retrieval-routing.d.ts +14 -0
  64. package/dist-types/lib/chat/media-retrieval-routing.d.ts.map +1 -0
  65. package/dist-types/lib/chat/media-source-routing.d.ts +53 -0
  66. package/dist-types/lib/chat/media-source-routing.d.ts.map +1 -0
  67. package/dist-types/lib/chat/message-input.d.ts +8 -0
  68. package/dist-types/lib/chat/message-input.d.ts.map +1 -0
  69. package/dist-types/lib/chat/model-provider.d.ts +23 -0
  70. package/dist-types/lib/chat/model-provider.d.ts.map +1 -0
  71. package/dist-types/lib/chat/response-style.d.ts +2 -0
  72. package/dist-types/lib/chat/response-style.d.ts.map +1 -0
  73. package/dist-types/lib/chat/retrieval-ranking.d.ts +32 -0
  74. package/dist-types/lib/chat/retrieval-ranking.d.ts.map +1 -0
  75. package/dist-types/lib/chat/retrieval-routing.d.ts +39 -0
  76. package/dist-types/lib/chat/retrieval-routing.d.ts.map +1 -0
  77. package/dist-types/lib/chat/tabular-dataset-routing.d.ts +15 -0
  78. package/dist-types/lib/chat/tabular-dataset-routing.d.ts.map +1 -0
  79. package/dist-types/lib/chat/tabular-query-plan.d.ts +12 -0
  80. package/dist-types/lib/chat/tabular-query-plan.d.ts.map +1 -0
  81. package/dist-types/lib/chat/tabular-routing.d.ts +41 -0
  82. package/dist-types/lib/chat/tabular-routing.d.ts.map +1 -0
  83. package/dist-types/lib/chat/tabular-sandbox-files.d.ts +33 -0
  84. package/dist-types/lib/chat/tabular-sandbox-files.d.ts.map +1 -0
  85. package/dist-types/lib/chat/tabular-visualization.d.ts +31 -0
  86. package/dist-types/lib/chat/tabular-visualization.d.ts.map +1 -0
  87. package/dist-types/lib/chat/tool-call-repair.d.ts +11 -0
  88. package/dist-types/lib/chat/tool-call-repair.d.ts.map +1 -0
  89. package/dist-types/lib/chat/tool-context.d.ts +72 -0
  90. package/dist-types/lib/chat/tool-context.d.ts.map +1 -0
  91. package/dist-types/lib/chat/tool-registry.d.ts +3 -0
  92. package/dist-types/lib/chat/tool-registry.d.ts.map +1 -0
  93. package/dist-types/lib/chat/video-rag-routing.d.ts +33 -0
  94. package/dist-types/lib/chat/video-rag-routing.d.ts.map +1 -0
  95. package/dist-types/lib/chat/visual-routing.d.ts +53 -0
  96. package/dist-types/lib/chat/visual-routing.d.ts.map +1 -0
  97. package/dist-types/lib/enterprise-client.d.ts +31 -0
  98. package/dist-types/lib/enterprise-client.d.ts.map +1 -0
  99. package/dist-types/lib/marketplace/tool-runtime-config.d.ts +7 -0
  100. package/dist-types/lib/marketplace/tool-runtime-config.d.ts.map +1 -0
  101. package/dist-types/lib/media/knowledge.d.ts +47 -0
  102. package/dist-types/lib/media/knowledge.d.ts.map +1 -0
  103. package/dist-types/lib/media/pdf-inspection.d.ts +30 -0
  104. package/dist-types/lib/media/pdf-inspection.d.ts.map +1 -0
  105. package/dist-types/lib/media/source-identity.d.ts +12 -0
  106. package/dist-types/lib/media/source-identity.d.ts.map +1 -0
  107. package/dist-types/lib/media/source-utils.d.ts +157 -0
  108. package/dist-types/lib/media/source-utils.d.ts.map +1 -0
  109. package/dist-types/lib/media/video/quick-look.d.ts +60 -0
  110. package/dist-types/lib/media/video/quick-look.d.ts.map +1 -0
  111. package/dist-types/lib/media/video-intelligence-adapter.d.ts +521 -0
  112. package/dist-types/lib/media/video-intelligence-adapter.d.ts.map +1 -0
  113. package/dist-types/lib/media/video-intelligence-connection.d.ts +32 -0
  114. package/dist-types/lib/media/video-intelligence-connection.d.ts.map +1 -0
  115. package/dist-types/lib/remote-files.d.ts +12 -0
  116. package/dist-types/lib/remote-files.d.ts.map +1 -0
  117. package/dist-types/lib/search/provider-verification.d.ts +5 -0
  118. package/dist-types/lib/search/provider-verification.d.ts.map +1 -0
  119. package/dist-types/runtime-host.d.ts +20 -0
  120. package/dist-types/runtime-host.d.ts.map +1 -0
  121. package/package.json +338 -3
  122. package/src/app/api/chat/feedback/route.ts +77 -0
  123. package/src/app/api/chat/route.ts +2277 -0
  124. package/src/app/api/chat/tools.ts +3563 -0
  125. package/src/app/api/config/test-embedding/route.ts +39 -0
  126. package/src/app/api/config/verify/route.ts +309 -0
  127. package/src/app/api/describe-image/route.ts +108 -0
  128. package/src/app/api/doc-editor/edit/route.ts +87 -0
  129. package/src/app/api/doc-editor/export/route.ts +32 -0
  130. package/src/app/api/doc-editor/parse/route.ts +74 -0
  131. package/src/app/api/doc-editor/restore/route.ts +70 -0
  132. package/src/app/api/document/sign/route.ts +55 -0
  133. package/src/app/api/files/remote/route.ts +32 -0
  134. package/src/app/api/mcp/[connectionId]/route.ts +29 -0
  135. package/src/app/api/mcp/[connectionId]/test/route.ts +13 -0
  136. package/src/app/api/mcp/route.ts +22 -0
  137. package/src/app/api/parse-file/route.ts +86 -0
  138. package/src/app/api/pdf-page/route.ts +40 -0
  139. package/src/app/api/proxy/verify/route.ts +78 -0
  140. package/src/app/api/search/bing/route.ts +86 -0
  141. package/src/app/api/search/brave/route.ts +88 -0
  142. package/src/app/api/search/exa/route.ts +100 -0
  143. package/src/app/api/search/google/route.ts +147 -0
  144. package/src/app/api/search/route.ts +28 -0
  145. package/src/app/api/search/tavily/route.ts +93 -0
  146. package/src/app/api/search/verify/route.ts +27 -0
  147. package/src/app/api/tabular/route.ts +68 -0
  148. package/src/lib/chat/assistant-source-scope.ts +32 -0
  149. package/src/lib/chat/conversation-memory.ts +621 -0
  150. package/src/lib/chat/gateway-fallbacks.ts +12 -0
  151. package/src/lib/chat/image-description.ts +21 -0
  152. package/src/lib/chat/media-retrieval-routing.ts +37 -0
  153. package/src/lib/chat/media-source-routing.ts +100 -0
  154. package/src/lib/chat/message-input.ts +48 -0
  155. package/src/lib/chat/model-provider.ts +157 -0
  156. package/src/lib/chat/response-style.ts +3 -0
  157. package/src/lib/chat/retrieval-ranking.ts +126 -0
  158. package/src/lib/chat/retrieval-routing.ts +221 -0
  159. package/src/lib/chat/tabular-dataset-routing.ts +93 -0
  160. package/src/lib/chat/tabular-query-plan.ts +195 -0
  161. package/src/lib/chat/tabular-routing.ts +122 -0
  162. package/src/lib/chat/tabular-sandbox-files.ts +78 -0
  163. package/src/lib/chat/tabular-visualization.ts +165 -0
  164. package/src/lib/chat/tool-call-repair.ts +84 -0
  165. package/src/lib/chat/tool-context.ts +1222 -0
  166. package/src/lib/chat/tool-registry.ts +6 -0
  167. package/src/lib/chat/video-rag-routing.ts +149 -0
  168. package/src/lib/chat/visual-routing.ts +225 -0
  169. package/src/lib/enterprise-client.ts +96 -0
  170. package/src/lib/marketplace/tool-runtime-config.ts +155 -0
  171. package/src/lib/media/knowledge.ts +489 -0
  172. package/src/lib/media/pdf-inspection.ts +280 -0
  173. package/src/lib/media/source-identity.ts +50 -0
  174. package/src/lib/media/source-utils.ts +1156 -0
  175. package/src/lib/media/video/quick-look.ts +368 -0
  176. package/src/lib/media/video-intelligence-adapter.ts +1946 -0
  177. package/src/lib/media/video-intelligence-connection.ts +151 -0
  178. package/src/lib/remote-files.ts +182 -0
  179. package/src/lib/search/provider-verification.ts +95 -0
  180. package/src/runtime-host.ts +31 -0
@@ -0,0 +1,1222 @@
1
+ import { formatTabularVisualizationAnswer } from './tabular-visualization';
2
+
3
+ type ToolResultPart = {
4
+ type?: string;
5
+ toolName?: string;
6
+ output?: unknown;
7
+ result?: unknown;
8
+ [key: string]: unknown;
9
+ };
10
+
11
+ /**
12
+ * Interpret the generic media-evidence verification protocol.
13
+ *
14
+ * `directlyEstablished` describes whether one observation states the answer.
15
+ * It is deliberately false for an answer established by a chronological trail,
16
+ * which is still answer-level evidence and must not be downgraded to a refusal.
17
+ */
18
+ export function mediaClaimNeedsCorroboration(value: unknown): boolean {
19
+ if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
20
+ const verification = value as {
21
+ status?: unknown;
22
+ directlyEstablished?: unknown;
23
+ requiresCorroboration?: unknown;
24
+ };
25
+ if (verification.status === 'needs-corroboration') return true;
26
+ if (
27
+ verification.status === 'directly-established' ||
28
+ verification.status === 'established-by-trail'
29
+ ) {
30
+ return false;
31
+ }
32
+ if (typeof verification.requiresCorroboration === 'boolean') {
33
+ return verification.requiresCorroboration;
34
+ }
35
+ // Backward compatibility for evidence providers that predate `status`.
36
+ return verification.directlyEstablished === false;
37
+ }
38
+
39
+ /** True when the evidence provider says the claim can be answered now. */
40
+ export function mediaClaimIsAnswerLevel(value: unknown): boolean {
41
+ if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
42
+ const verification = value as {
43
+ status?: unknown;
44
+ directlyEstablished?: unknown;
45
+ requiresCorroboration?: unknown;
46
+ };
47
+ if (
48
+ verification.status === 'directly-established' ||
49
+ verification.status === 'established-by-trail'
50
+ ) {
51
+ return true;
52
+ }
53
+ if (verification.status === 'needs-corroboration') return false;
54
+ if (verification.requiresCorroboration === true) return false;
55
+ return verification.directlyEstablished === true;
56
+ }
57
+
58
+ /** Find an answer-level media claim even when it is nested in a search result. */
59
+ export function containsAnswerLevelMediaEvidence(value: unknown): boolean {
60
+ if (typeof value === 'string') {
61
+ try {
62
+ return containsAnswerLevelMediaEvidence(JSON.parse(value));
63
+ } catch {
64
+ return false;
65
+ }
66
+ }
67
+ if (Array.isArray(value)) return value.some(containsAnswerLevelMediaEvidence);
68
+ if (!value || typeof value !== 'object') return false;
69
+ const record = value as Record<string, unknown>;
70
+ if (
71
+ record.success === true &&
72
+ mediaClaimIsAnswerLevel(record.claimVerification) &&
73
+ (Array.isArray(record.evidence) || record.directObservation !== undefined)
74
+ ) {
75
+ return true;
76
+ }
77
+ return Object.values(record).some(containsAnswerLevelMediaEvidence);
78
+ }
79
+
80
+ function questionSimilarity(left: string, right: string) {
81
+ const terms = (value: string) =>
82
+ new Set(
83
+ value
84
+ .normalize('NFKC')
85
+ .toLocaleLowerCase()
86
+ .match(/[\p{Letter}\p{Number}]+/gu) ?? [],
87
+ );
88
+ const leftTerms = terms(left);
89
+ const rightTerms = terms(right);
90
+ if (leftTerms.size === 0 || rightTerms.size === 0) return 0;
91
+ const weight = (values: Set<string>) =>
92
+ [...values].reduce((total, term) => total + term.length, 0);
93
+ const shared = new Set([...leftTerms].filter((term) => rightTerms.has(term)));
94
+ return weight(shared) / Math.max(weight(leftTerms), weight(rightTerms));
95
+ }
96
+
97
+ /** Extract only direct claims verified for the current question. */
98
+ export function collectQuestionMatchedDirectClaims(value: unknown, question: string): string[] {
99
+ const claims = new Set<string>();
100
+ const inspectText = (text: string) => {
101
+ for (const block of text.split(/(?=Claim question:)/gi)) {
102
+ const claimedQuestion = block.match(/Claim question:\s*([^\n]+)/i)?.[1]?.trim();
103
+ const answer = block.match(/Claim answer:\s*([^\n]+)/i)?.[1]?.trim();
104
+ if (
105
+ claimedQuestion &&
106
+ answer &&
107
+ /Claim verdict:\s*direct/i.test(block) &&
108
+ questionSimilarity(claimedQuestion, question) >= 0.55
109
+ ) {
110
+ claims.add(answer);
111
+ }
112
+ }
113
+ };
114
+ const visit = (candidate: unknown) => {
115
+ if (typeof candidate === 'string') return inspectText(candidate);
116
+ if (Array.isArray(candidate)) return candidate.forEach(visit);
117
+ if (candidate && typeof candidate === 'object') Object.values(candidate).forEach(visit);
118
+ };
119
+ visit(value);
120
+ return [...claims].slice(0, 8);
121
+ }
122
+
123
+ /**
124
+ * Last-resort visible text when a provider closes after tools without writing
125
+ * its answer. Only answer-level media results qualify; locators and
126
+ * needs-corroboration results must never be turned into claims here.
127
+ */
128
+ export function collectAnswerLevelMediaStatements(value: unknown): string[] {
129
+ const statements: string[] = [];
130
+ const seen = new Set<string>();
131
+ let exhaustive = false;
132
+ const textOf = (payload: unknown) => {
133
+ if (typeof payload === 'string') return payload;
134
+ if (payload && typeof payload === 'object' && !Array.isArray(payload)) {
135
+ const text = (payload as { text?: unknown }).text;
136
+ if (typeof text === 'string') return text;
137
+ }
138
+ return '';
139
+ };
140
+ const visit = (candidate: unknown) => {
141
+ if (typeof candidate === 'string') {
142
+ try {
143
+ visit(JSON.parse(candidate));
144
+ } catch {
145
+ /* ordinary text is not a verified media result */
146
+ }
147
+ return;
148
+ }
149
+ if (Array.isArray(candidate)) return candidate.forEach(visit);
150
+ if (!candidate || typeof candidate !== 'object') return;
151
+ const record = candidate as Record<string, unknown>;
152
+ if (
153
+ record.continuation &&
154
+ typeof record.continuation === 'object' &&
155
+ (record.continuation as { exhaustive?: unknown }).exhaustive === true
156
+ ) {
157
+ exhaustive = true;
158
+ }
159
+ if (
160
+ record.success === true &&
161
+ mediaClaimIsAnswerLevel(record.claimVerification) &&
162
+ Array.isArray(record.evidence)
163
+ ) {
164
+ for (const item of record.evidence as Array<{ payload?: unknown }>) {
165
+ const sourceText = textOf(item?.payload);
166
+ // Individual source readings can explicitly state that they are only
167
+ // partial context. A top-level scan being complete does not upgrade a
168
+ // partial reading into an answer to a different question.
169
+ if (
170
+ /^Observed context \(not a complete answer\):/i.test(sourceText.trim()) ||
171
+ /\nClaim verdict:\s*(?:partial|not-established)\b/i.test(sourceText)
172
+ ) {
173
+ continue;
174
+ }
175
+ const statement = sourceText
176
+ .split(/\nClaim question:/i, 1)[0]
177
+ .replace(/^(?:Reconciled|Indexed)\s+(?:state|event|context|overview):\s*/i, '')
178
+ .trim()
179
+ .slice(0, 700);
180
+ const key = statement.normalize('NFKC').toLocaleLowerCase();
181
+ if (!statement || seen.has(key)) continue;
182
+ seen.add(key);
183
+ statements.push(statement);
184
+ }
185
+ return;
186
+ }
187
+ Object.values(record).forEach(visit);
188
+ };
189
+ visit(value);
190
+ return exhaustive ? statements.slice(0, 2_000) : statements.slice(-8);
191
+ }
192
+
193
+ /**
194
+ * Collect the visibility protocol without trying to interpret the user's
195
+ * language. The reply model receives this separately from ranked snippets so
196
+ * a direct re-watch of one moment cannot hide the reconciled recurrence data.
197
+ */
198
+ export function collectObservedSubjectLedger(value: unknown): string[] {
199
+ type Appearance = { startSecs: number; endSecs: number };
200
+ const lines = new Set<string>();
201
+ const visit = (candidate: unknown) => {
202
+ if (typeof candidate === 'string') {
203
+ try {
204
+ visit(JSON.parse(candidate));
205
+ } catch {
206
+ /* ordinary text is not a structured evidence result */
207
+ }
208
+ return;
209
+ }
210
+ if (Array.isArray(candidate)) return candidate.forEach(visit);
211
+ if (!candidate || typeof candidate !== 'object') return;
212
+ const record = candidate as Record<string, unknown>;
213
+ if (record.success === true && mediaClaimIsAnswerLevel(record.claimVerification)) {
214
+ const observedSubjects = record.observedSubjects;
215
+ if (!Array.isArray(observedSubjects)) return;
216
+ for (const subject of observedSubjects) {
217
+ if (!subject || typeof subject !== 'object') continue;
218
+ const identity =
219
+ typeof (subject as { identity?: unknown }).identity === 'string'
220
+ ? (subject as { identity: string }).identity.trim()
221
+ : '';
222
+ const identityBasis =
223
+ typeof (subject as { identityBasis?: unknown }).identityBasis === 'string'
224
+ ? (subject as { identityBasis: string }).identityBasis.trim()
225
+ : '';
226
+ const appearances = Array.isArray((subject as { appearances?: unknown }).appearances)
227
+ ? (subject as { appearances: unknown[] }).appearances
228
+ .map((appearance) => {
229
+ if (!appearance || typeof appearance !== 'object') return undefined;
230
+ const startSecs = Number((appearance as { startSecs?: unknown }).startSecs);
231
+ const endSecs = Number((appearance as { endSecs?: unknown }).endSecs);
232
+ return Number.isFinite(startSecs) &&
233
+ Number.isFinite(endSecs) &&
234
+ endSecs >= startSecs
235
+ ? ({ startSecs, endSecs } satisfies Appearance)
236
+ : undefined;
237
+ })
238
+ .filter((appearance): appearance is Appearance => Boolean(appearance))
239
+ : [];
240
+ if (!identity || !identityBasis || appearances.length === 0) continue;
241
+ lines.add(
242
+ `Reconciled visible subject: ${identity}\n` +
243
+ `Identity basis: ${identityBasis}\n` +
244
+ `Observed appearances: ${appearances
245
+ .map((appearance) => `${appearance.startSecs}-${appearance.endSecs}s`)
246
+ .join('; ')}`,
247
+ );
248
+ }
249
+ return;
250
+ }
251
+ Object.values(record).forEach(visit);
252
+ };
253
+ visit(value);
254
+ return [...lines].slice(0, 200);
255
+ }
256
+
257
+ /**
258
+ * Render a source-position-to-recurrence answer when the local visual subject
259
+ * is described but not named. This protocol guard prevents a prose model from
260
+ * borrowing a proper name from another timestamp and attaching it to the
261
+ * requested moment.
262
+ */
263
+ export function formatLocatedObservedSubjectAnswer(value: unknown): string | undefined {
264
+ type Appearance = { startSecs: number; endSecs: number };
265
+ type Subject = {
266
+ identity: string;
267
+ identityBasis: string;
268
+ appearances: Appearance[];
269
+ };
270
+ type LocalSubject = Subject & { localAppearance: Appearance };
271
+ type LocatedSubject = LocalSubject & { directClaim?: string };
272
+ const localSubjects: LocatedSubject[] = [];
273
+ const visit = (candidate: unknown) => {
274
+ if (typeof candidate === 'string') {
275
+ try {
276
+ visit(JSON.parse(candidate));
277
+ } catch {
278
+ /* only structured media evidence establishes this protocol */
279
+ }
280
+ return;
281
+ }
282
+ if (Array.isArray(candidate)) return candidate.forEach(visit);
283
+ if (!candidate || typeof candidate !== 'object') return;
284
+ const record = candidate as Record<string, unknown>;
285
+ if (record.success !== true || !mediaClaimIsAnswerLevel(record.claimVerification)) {
286
+ Object.values(record).forEach(visit);
287
+ return;
288
+ }
289
+ const directive = (record.investigation as { directive?: unknown } | undefined)?.directive as
290
+ { scope?: unknown; recordSet?: unknown; timeRange?: unknown } | undefined;
291
+ const requestedRange = directive?.timeRange as
292
+ { startSecs?: unknown; endSecs?: unknown } | undefined;
293
+ const startSecs = Number(requestedRange?.startSecs);
294
+ const endSecs = Number(requestedRange?.endSecs);
295
+ if (
296
+ directive?.scope !== 'temporal' ||
297
+ directive.recordSet !== 'observed' ||
298
+ !Number.isFinite(startSecs) ||
299
+ !Number.isFinite(endSecs) ||
300
+ endSecs < startSecs ||
301
+ !Array.isArray(record.evidence) ||
302
+ !Array.isArray(record.observedSubjects)
303
+ ) {
304
+ return;
305
+ }
306
+ const ledger = (record.observedSubjects as unknown[])
307
+ .map((item): Subject | undefined => {
308
+ if (!item || typeof item !== 'object') return undefined;
309
+ const identity =
310
+ typeof (item as { identity?: unknown }).identity === 'string'
311
+ ? (item as { identity: string }).identity.trim()
312
+ : '';
313
+ const identityBasis =
314
+ typeof (item as { identityBasis?: unknown }).identityBasis === 'string'
315
+ ? (item as { identityBasis: string }).identityBasis.trim()
316
+ : '';
317
+ const appearances = Array.isArray((item as { appearances?: unknown }).appearances)
318
+ ? (item as { appearances: unknown[] }).appearances
319
+ .map((appearance): Appearance | undefined => {
320
+ if (!appearance || typeof appearance !== 'object') return undefined;
321
+ const startSecs = Number((appearance as { startSecs?: unknown }).startSecs);
322
+ const endSecs = Number((appearance as { endSecs?: unknown }).endSecs);
323
+ return Number.isFinite(startSecs) &&
324
+ Number.isFinite(endSecs) &&
325
+ endSecs >= startSecs
326
+ ? { startSecs, endSecs }
327
+ : undefined;
328
+ })
329
+ .filter((appearance): appearance is Appearance => Boolean(appearance))
330
+ : [];
331
+ return identity && identityBasis && appearances.length > 0
332
+ ? { identity, identityBasis, appearances }
333
+ : undefined;
334
+ })
335
+ .filter((subject): subject is Subject => Boolean(subject));
336
+ for (const item of record.evidence as unknown[]) {
337
+ if (!item || typeof item !== 'object') continue;
338
+ const payload = (item as { payload?: unknown }).payload;
339
+ const text =
340
+ typeof payload === 'string'
341
+ ? payload
342
+ : payload && typeof payload === 'object' && !Array.isArray(payload)
343
+ ? (payload as { text?: unknown }).text
344
+ : undefined;
345
+ if (typeof text !== 'string') continue;
346
+ const encoded = text.match(/^Visible subject:\s*(\{.+\})$/imu)?.[1];
347
+ if (!encoded) continue;
348
+ try {
349
+ const parsed = JSON.parse(encoded) as {
350
+ identity?: unknown;
351
+ identityBasis?: unknown;
352
+ startMs?: unknown;
353
+ endMs?: unknown;
354
+ };
355
+ const identity = typeof parsed.identity === 'string' ? parsed.identity.trim() : '';
356
+ const identityBasis =
357
+ typeof parsed.identityBasis === 'string' ? parsed.identityBasis.trim() : '';
358
+ const localStartSecs = Number(parsed.startMs) / 1_000;
359
+ const localEndSecs = Number(parsed.endMs) / 1_000;
360
+ if (
361
+ !identity ||
362
+ !['source-described', 'unresolved'].includes(identityBasis) ||
363
+ !Number.isFinite(localStartSecs) ||
364
+ !Number.isFinite(localEndSecs) ||
365
+ localEndSecs < localStartSecs ||
366
+ localStartSecs > endSecs ||
367
+ localEndSecs < startSecs
368
+ ) {
369
+ continue;
370
+ }
371
+ const matched = ledger.find(
372
+ (subject) => subject.identity === identity && subject.identityBasis === identityBasis,
373
+ );
374
+ localSubjects.push({
375
+ identity,
376
+ identityBasis,
377
+ appearances: matched?.appearances ?? [
378
+ { startSecs: localStartSecs, endSecs: localEndSecs },
379
+ ],
380
+ localAppearance: { startSecs: localStartSecs, endSecs: localEndSecs },
381
+ ...(text.match(/\nClaim answer:\s*([^\n]+)/i)?.[1]?.trim()
382
+ ? { directClaim: text.match(/\nClaim answer:\s*([^\n]+)/i)![1]!.trim() }
383
+ : {}),
384
+ });
385
+ } catch {
386
+ /* a malformed protocol record cannot produce a deterministic answer */
387
+ }
388
+ }
389
+ };
390
+ visit(value);
391
+ const subject = localSubjects[0];
392
+ if (!subject) return undefined;
393
+ const formatTime = (seconds: number) => displayTimecode(seconds);
394
+ const uniqueAppearances = subject.appearances.filter(
395
+ (appearance, index, appearances) =>
396
+ index === 0 ||
397
+ appearance.startSecs !== appearances[index - 1]?.startSecs ||
398
+ appearance.endSecs !== appearances[index - 1]?.endSecs,
399
+ );
400
+ const ranges = uniqueAppearances
401
+ .map((appearance) =>
402
+ appearance.endSecs > appearance.startSecs
403
+ ? `${formatTime(appearance.startSecs)}–${formatTime(appearance.endSecs)}`
404
+ : formatTime(appearance.startSecs),
405
+ )
406
+ .join(', ');
407
+ const separatelyDescribed = localSubjects.filter(
408
+ (candidate) =>
409
+ candidate.identity !== subject.identity || candidate.identityBasis !== subject.identityBasis,
410
+ );
411
+ const separateLabels = separatelyDescribed
412
+ .map((candidate) => {
413
+ const appearances = candidate.appearances
414
+ .map((appearance) =>
415
+ appearance.endSecs > appearance.startSecs
416
+ ? `${formatTime(appearance.startSecs)}–${formatTime(appearance.endSecs)}`
417
+ : formatTime(appearance.startSecs),
418
+ )
419
+ .join(', ');
420
+ return `${candidate.identity} (${candidate.identityBasis}) at ${appearances}`;
421
+ })
422
+ .join('; ');
423
+ const identityBasisArticle = /^[aeiou]/i.test(subject.identityBasis) ? 'an' : 'a';
424
+ return (
425
+ (subject.directClaim ? `${subject.directClaim} ` : '') +
426
+ `I saw ${subject.identity} at ${formatTime(subject.localAppearance.startSecs)}–${formatTime(
427
+ subject.localAppearance.endSecs,
428
+ )}. ` +
429
+ `This is ${identityBasisArticle} ${subject.identityBasis} identification, so I cannot safely equate it with a separately named appearance. ` +
430
+ `The recorded interval${uniqueAppearances.length === 1 ? '' : 's'} for this exact description: ${ranges}.` +
431
+ (separateLabels
432
+ ? ` The source also uses separate visible-subject labels: ${separateLabels}. It does not establish that those labels are the same identity.`
433
+ : '')
434
+ );
435
+ }
436
+
437
+ /**
438
+ * Render a repeated visual appearance directly from the evidence protocol.
439
+ * This is intentionally protocol-driven: a subject can be a person, animal,
440
+ * object, character, or anything else the visual index describes. It avoids
441
+ * asking a prose model to reinterpret explicit discrete moments as one
442
+ * continuous appearance or as an absence claim.
443
+ */
444
+ export function formatObservedAppearanceAnswer(value: unknown): string | undefined {
445
+ type Appearance = { startSecs: number; endSecs: number };
446
+ type Subject = { identity: string; basis: string; appearances: Appearance[] };
447
+ const subjects = new Map<string, Subject>();
448
+ const textOf = (payload: unknown) => {
449
+ if (typeof payload === 'string') return payload;
450
+ if (payload && typeof payload === 'object' && !Array.isArray(payload)) {
451
+ const text = (payload as { text?: unknown }).text;
452
+ if (typeof text === 'string') return text;
453
+ }
454
+ return '';
455
+ };
456
+ const parseAppearances = (line: string): Appearance[] => {
457
+ const milliseconds = [...line.matchAll(/(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)ms\b/gu)].map(
458
+ (match) => ({ startSecs: Number(match[1]) / 1_000, endSecs: Number(match[2]) / 1_000 }),
459
+ );
460
+ if (milliseconds.length > 0) return milliseconds;
461
+ return [...line.matchAll(/(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)s\b/gu)].map((match) => ({
462
+ startSecs: Number(match[1]),
463
+ endSecs: Number(match[2]),
464
+ }));
465
+ };
466
+ const addEvidence = (evidence: unknown) => {
467
+ if (!Array.isArray(evidence)) return;
468
+ for (const item of evidence) {
469
+ if (!item || typeof item !== 'object') continue;
470
+ const text = textOf((item as { payload?: unknown }).payload);
471
+ const match = text.match(
472
+ /^Reconciled visible subject:\s*([^\n]+)\nIdentity basis:\s*([^\n]+)\nObserved appearances:\s*([^\n]+)/im,
473
+ );
474
+ if (!match) continue;
475
+ const identity = match[1]?.trim();
476
+ const basis = match[2]?.trim();
477
+ const appearances = parseAppearances(match[3] ?? '').filter(
478
+ (range) => Number.isFinite(range.startSecs) && Number.isFinite(range.endSecs),
479
+ );
480
+ if (!identity || !basis || appearances.length === 0) continue;
481
+ const key = `${basis}\u0000${identity}`;
482
+ const subject = subjects.get(key) ?? { identity, basis, appearances: [] };
483
+ subject.appearances.push(...appearances);
484
+ subjects.set(key, subject);
485
+ }
486
+ };
487
+ const visit = (candidate: unknown) => {
488
+ if (typeof candidate === 'string') {
489
+ try {
490
+ visit(JSON.parse(candidate));
491
+ } catch {
492
+ /* plain text is not a structured answer-level media result */
493
+ }
494
+ return;
495
+ }
496
+ if (Array.isArray(candidate)) return candidate.forEach(visit);
497
+ if (!candidate || typeof candidate !== 'object') return;
498
+ const record = candidate as Record<string, unknown>;
499
+ if (record.success === true && mediaClaimIsAnswerLevel(record.claimVerification)) {
500
+ addEvidence(record.evidence);
501
+ return;
502
+ }
503
+ Object.values(record).forEach(visit);
504
+ };
505
+ visit(value);
506
+ const formatTime = (seconds: number) => {
507
+ const minutes = Math.floor(seconds / 60);
508
+ const remainder = seconds - minutes * 60;
509
+ return `${String(minutes).padStart(2, '0')}:${remainder.toFixed(3).padStart(6, '0')}`;
510
+ };
511
+ const rendered = [...subjects.values()]
512
+ .map((subject) => ({
513
+ ...subject,
514
+ appearances: subject.appearances
515
+ .sort((left, right) => left.startSecs - right.startSecs || left.endSecs - right.endSecs)
516
+ .filter(
517
+ (range, index, ranges) =>
518
+ index === 0 ||
519
+ range.startSecs !== ranges[index - 1]?.startSecs ||
520
+ range.endSecs !== ranges[index - 1]?.endSecs,
521
+ ),
522
+ }))
523
+ .filter((subject) => subject.appearances.length > 1)
524
+ .map(
525
+ (subject) =>
526
+ `I saw ${subject.identity} at ${subject.appearances
527
+ .map((range) =>
528
+ range.endSecs > range.startSecs
529
+ ? `${formatTime(range.startSecs)}–${formatTime(range.endSecs)}`
530
+ : formatTime(range.startSecs),
531
+ )
532
+ .join(' and ')}.`,
533
+ );
534
+ // Several subjects in the selected evidence are source context, not proof
535
+ // that a visibility inventory is the requested answer. A single recurring
536
+ // subject is unambiguous and can safely bypass prose re-interpretation.
537
+ return rendered.length === 1
538
+ ? `${rendered[0]} These are separate observed moments, not continuous visibility.`
539
+ : undefined;
540
+ }
541
+
542
+ /** Render a final two-sided numeric outcome without waiting on a slow prose model. */
543
+ export function formatOutcomeMediaAnswer(
544
+ value: unknown,
545
+ investigation: VideoInvestigationDirective,
546
+ ): string | undefined {
547
+ if (investigation.goal !== 'trace') return undefined;
548
+ type Reading = { at: number; text: string };
549
+ const readings: Reading[] = [];
550
+ const visit = (candidate: unknown) => {
551
+ if (typeof candidate === 'string') {
552
+ try {
553
+ visit(JSON.parse(candidate));
554
+ } catch {
555
+ /* only structured verified evidence qualifies */
556
+ }
557
+ return;
558
+ }
559
+ if (Array.isArray(candidate)) return candidate.forEach(visit);
560
+ if (!candidate || typeof candidate !== 'object') return;
561
+ const record = candidate as Record<string, unknown>;
562
+ if (record.success === true && mediaClaimIsAnswerLevel(record.claimVerification)) {
563
+ const temporal = record.temporalContext as { readings?: unknown } | undefined;
564
+ if (Array.isArray(temporal?.readings)) {
565
+ for (const item of temporal.readings) {
566
+ if (!item || typeof item !== 'object') continue;
567
+ const reading = item as { atSecs?: unknown; text?: unknown };
568
+ if (typeof reading.text !== 'string') continue;
569
+ readings.push({ at: Number(reading.atSecs ?? 0), text: reading.text });
570
+ }
571
+ }
572
+ return;
573
+ }
574
+ Object.values(record).forEach(visit);
575
+ };
576
+ visit(value);
577
+
578
+ const cleanLabel = (label: string) =>
579
+ label
580
+ .replace(/^.*?(?:(?:final\s+)?(?:result|score)|النتيجة(?:\s+النهائية)?)\s*:\s*/iu, '')
581
+ .replace(/^(?:Reconciled|Indexed)\s+state:\s*/i, '')
582
+ .replace(/^(?:(?:final\s+)?(?:result|score)|النتيجة(?:\s+النهائية)?)\s*/iu, '')
583
+ .replace(/^[\s:،؛,.'"“”‘’_-]+|[\s:،؛,.'"“”‘’_-]+$/gu, '')
584
+ .trim();
585
+ // Scoreboards commonly render `left side 2 - 1 right side`. Preserve that
586
+ // left/right binding; reducing the score to two unlabeled numbers can swap
587
+ // the winner when a nearby summary lists the teams in another order.
588
+ const scorelinePattern =
589
+ /^\s*(.+?\p{L})\s+(\d+(?:[.,]\d+)?)\s*[-–—−]\s*(\d+(?:[.,]\d+)?)\s+(\p{L}.*?)\s*$/u;
590
+ const scorelineCandidates = readings
591
+ .sort((left, right) => left.at - right.at)
592
+ .flatMap((reading) => {
593
+ const quoted = [...reading.text.matchAll(/["'“”‘’]([^"'“”‘’\n]{2,180})["'“”‘’]/gu)].map(
594
+ (match) => match[1],
595
+ );
596
+ const segments = quoted.length > 0 ? quoted : reading.text.split('\n');
597
+ return segments.flatMap((segment) => {
598
+ const match = segment.match(scorelinePattern);
599
+ if (!match) return [];
600
+ const left = { label: cleanLabel(match[1]), value: Number(match[2].replace(',', '.')) };
601
+ const right = { label: cleanLabel(match[4]), value: Number(match[3].replace(',', '.')) };
602
+ return left.label &&
603
+ right.label &&
604
+ Number.isFinite(left.value) &&
605
+ Number.isFinite(right.value)
606
+ ? [{ ...reading, pairs: [left, right] }]
607
+ : [];
608
+ });
609
+ });
610
+ const pairPattern =
611
+ /([^=,;،؛\n]{2,100}?)(?:\s*=\s*|\s*:\s*|\s+)(-?\d+(?:[.,]\d+)?)(?=$|[,;،؛])/gu;
612
+ const candidates = readings
613
+ .sort((left, right) => left.at - right.at)
614
+ .map((reading) => {
615
+ const pairs = [...reading.text.matchAll(pairPattern)].map((match) => ({
616
+ label: cleanLabel(match[1] ?? ''),
617
+ value: Number((match[2] ?? '').replace(',', '.')),
618
+ }));
619
+ return {
620
+ ...reading,
621
+ pairs: pairs.filter((pair) => pair.label && Number.isFinite(pair.value)),
622
+ };
623
+ })
624
+ .filter((reading) => reading.pairs.length === 2);
625
+ const settled = scorelineCandidates.at(-1) ?? candidates.at(-1);
626
+ if (!settled) return undefined;
627
+ const [left, right] = settled.pairs;
628
+ if (left.value === right.value) {
629
+ return `It ended level at ${left.value}–${right.value}.`;
630
+ }
631
+ const winner = left.value > right.value ? left : right;
632
+ const runnerUp = winner === left ? right : left;
633
+ return `${winner.label} won, ${winner.value}–${runnerUp.value} over ${runnerUp.label}.`;
634
+ }
635
+
636
+ /** Use the strongest bounded source reading directly when it already settled the question. */
637
+ export function formatDirectObservationAnswer(value: unknown, question = ''): string | undefined {
638
+ if (mediaEvidenceRequestsOnlySpeech(value)) return undefined;
639
+ const readings: Array<{ found: string; confidence: string; at: string }> = [];
640
+ const visit = (candidate: unknown) => {
641
+ if (typeof candidate === 'string') {
642
+ try {
643
+ visit(JSON.parse(candidate));
644
+ } catch {
645
+ /* only structured verified evidence qualifies */
646
+ }
647
+ return;
648
+ }
649
+ if (Array.isArray(candidate)) return candidate.forEach(visit);
650
+ if (!candidate || typeof candidate !== 'object') return;
651
+ const record = candidate as Record<string, unknown>;
652
+ if (record.success === true && mediaClaimIsAnswerLevel(record.claimVerification)) {
653
+ const direct = record.directObservation as { readings?: unknown } | undefined;
654
+ if (Array.isArray(direct?.readings)) {
655
+ for (const item of direct.readings) {
656
+ if (!item || typeof item !== 'object') continue;
657
+ const reading = item as {
658
+ found?: unknown;
659
+ confidence?: unknown;
660
+ at?: unknown;
661
+ settlesQuestion?: unknown;
662
+ coverageComplete?: unknown;
663
+ };
664
+ if (typeof reading.found !== 'string' || !reading.found.trim()) continue;
665
+ // This is a visible fallback, so it must only use a re-watch that
666
+ // explicitly answered the exact question. A related observation is
667
+ // still useful tool context for the answer model, but never a final
668
+ // answer by itself.
669
+ if (reading.settlesQuestion !== true) continue;
670
+ if (reading.coverageComplete === false) continue;
671
+ if (/^Observed context \(not a complete answer\):/i.test(reading.found.trim())) {
672
+ continue;
673
+ }
674
+ readings.push({
675
+ found: reading.found.trim(),
676
+ confidence: String(reading.confidence ?? ''),
677
+ at: String(reading.at ?? ''),
678
+ });
679
+ }
680
+ }
681
+ return;
682
+ }
683
+ Object.values(record).forEach(visit);
684
+ };
685
+ visit(value);
686
+ const asksForGroupAppearance =
687
+ /\b(?:each|every|all|both)\b[\s\S]{0,50}\b(?:wear|wearing|wore|dressed|clothes|clothing|outfit|appearance)\b|\b(?:wear|wearing|wore|dressed|clothes|clothing|outfit|appearance)\b[\s\S]{0,50}\b(?:each|every|all|both)\b/i.test(
688
+ question,
689
+ );
690
+ const best = readings.sort(
691
+ (left, right) =>
692
+ (asksForGroupAppearance
693
+ ? Number(/@[\w.-]+/.test(left.found)) - Number(/@[\w.-]+/.test(right.found))
694
+ : 0) ||
695
+ Number(right.confidence === 'high') - Number(left.confidence === 'high') ||
696
+ Math.min(right.found.length, 600) - Math.min(left.found.length, 600),
697
+ )[0];
698
+ if (!best) return undefined;
699
+ if (!asksForGroupAppearance) return best.found;
700
+ return best.found
701
+ .replace(/\s*\([^)]*@[\w.-]+[^)]*\)/g, '')
702
+ .split(/(?<=[.!?])\s+/)
703
+ .filter((sentence) => !/@[\w.-]+/.test(sentence))
704
+ .join(' ')
705
+ .trim();
706
+ }
707
+
708
+ /** Keep speech-only requests out of deterministic visual answer renderers. */
709
+ export function mediaEvidenceRequestsOnlySpeech(value: unknown): boolean {
710
+ const visit = (candidate: unknown): boolean => {
711
+ if (typeof candidate === 'string') {
712
+ try {
713
+ return visit(JSON.parse(candidate));
714
+ } catch {
715
+ return false;
716
+ }
717
+ }
718
+ if (Array.isArray(candidate)) return candidate.some(visit);
719
+ if (!candidate || typeof candidate !== 'object') return false;
720
+ const record = candidate as Record<string, unknown>;
721
+ const requestedEvidence = ((
722
+ record.plan as { investigation?: { evidence?: unknown } } | undefined
723
+ )?.investigation?.evidence ??
724
+ (record.investigation as { directive?: { evidence?: unknown } } | undefined)?.directive
725
+ ?.evidence) as unknown;
726
+ if (
727
+ Array.isArray(requestedEvidence) &&
728
+ requestedEvidence.length > 0 &&
729
+ requestedEvidence.every((item) => item === 'speech')
730
+ ) {
731
+ return true;
732
+ }
733
+ return Object.values(record).some(visit);
734
+ };
735
+ return visit(value);
736
+ }
737
+
738
+ /** Render a compact, source-grounded participant roster without relying on prose-model formatting. */
739
+ export function formatParticipantInventory(value: unknown, question: string): string | undefined {
740
+ const participants: Array<{ name: string; description: string; at: number }> = [];
741
+ const seen = new Set<string>();
742
+ const textOf = (payload: unknown) => {
743
+ if (typeof payload === 'string') return payload;
744
+ if (payload && typeof payload === 'object' && !Array.isArray(payload)) {
745
+ const text = (payload as { text?: unknown }).text;
746
+ if (typeof text === 'string') return text;
747
+ }
748
+ return '';
749
+ };
750
+ const visit = (candidate: unknown) => {
751
+ if (typeof candidate === 'string') {
752
+ try {
753
+ visit(JSON.parse(candidate));
754
+ } catch {
755
+ /* only structured verified media results qualify */
756
+ }
757
+ return;
758
+ }
759
+ if (Array.isArray(candidate)) return candidate.forEach(visit);
760
+ if (!candidate || typeof candidate !== 'object') return;
761
+ const record = candidate as Record<string, unknown>;
762
+ if (
763
+ record.success === true &&
764
+ mediaClaimIsAnswerLevel(record.claimVerification) &&
765
+ Array.isArray(record.evidence)
766
+ ) {
767
+ for (const item of record.evidence as Array<Record<string, unknown>>) {
768
+ const sourceText = textOf(item.payload) || (typeof item.text === 'string' ? item.text : '');
769
+ const match = sourceText.match(
770
+ /^Reconciled participant:\s*([^—\n]{1,120})\s*—\s*([^\n]+)/imu,
771
+ );
772
+ if (!match) continue;
773
+ const name = match[1]!.trim();
774
+ const description = match[2]!.trim();
775
+ const range = (item.timeRange ?? {}) as { startSecs?: unknown };
776
+ const at = Number(range.startSecs ?? item.startSecs ?? 0);
777
+ const key = name.normalize('NFKC').toLocaleLowerCase();
778
+ if (!name || !description || seen.has(key)) continue;
779
+ seen.add(key);
780
+ participants.push({ name, description, at: Number.isFinite(at) ? at : 0 });
781
+ }
782
+ return;
783
+ }
784
+ Object.values(record).forEach(visit);
785
+ };
786
+ visit(value);
787
+ if (participants.length === 0) return undefined;
788
+
789
+ const ordered = participants.sort((left, right) => left.at - right.at);
790
+ const escapeCell = (cell: string) => cell.replace(/\|/g, '\\|');
791
+ return `| Participant | Description | Timestamp |\n| --- | --- | --- |\n${ordered
792
+ .map(
793
+ (participant) =>
794
+ `| ${escapeCell(participant.name)} | ${escapeCell(participant.description)} | [${displayTimecode(participant.at)}] |`,
795
+ )
796
+ .join('\n')}`;
797
+ }
798
+
799
+ function displayTimecode(seconds: number) {
800
+ const total = Math.max(0, Math.floor(seconds));
801
+ const hours = Math.floor(total / 3_600);
802
+ const minutes = Math.floor((total % 3_600) / 60);
803
+ const rest = total % 60;
804
+ return hours > 0
805
+ ? `${hours}:${String(minutes).padStart(2, '0')}:${String(rest).padStart(2, '0')}`
806
+ : `${minutes}:${String(rest).padStart(2, '0')}`;
807
+ }
808
+
809
+ /**
810
+ * Render a verified complete media inventory without putting hundreds of
811
+ * already-final source items through another model context window.
812
+ */
813
+ export function formatExhaustiveMediaAnswer(value: unknown, _question: string): string | undefined {
814
+ type Item = {
815
+ at: number;
816
+ text: string;
817
+ sourceInventory: boolean;
818
+ answer?: string;
819
+ respondent?: string;
820
+ };
821
+ let complete: Item[] | undefined;
822
+ let inventorySummary: string | undefined;
823
+ const textOf = (payload: unknown) => {
824
+ if (typeof payload === 'string') return payload;
825
+ if (payload && typeof payload === 'object' && !Array.isArray(payload)) {
826
+ const text = (payload as { text?: unknown }).text;
827
+ if (typeof text === 'string') return text;
828
+ }
829
+ return '';
830
+ };
831
+ const clean = (raw: string) => {
832
+ const sourceLine = raw
833
+ .split(
834
+ /\n(?=(?:Context|Answer|Answered by|Source answer|Source respondent|Source task id|Claim question|Claim verdict|Claim answer):)/i,
835
+ 1,
836
+ )[0]
837
+ .replace(/^(?:Reconciled|Indexed)\s+(?:state|event|context|overview):\s*/i, '')
838
+ .replace(/^(?:Source (?:question|item|structure)(?:\s*\([^)]*\))?|Question):\s*/i, '')
839
+ .trim();
840
+ return sourceLine;
841
+ };
842
+ const visit = (candidate: unknown) => {
843
+ if (typeof candidate === 'string') {
844
+ try {
845
+ visit(JSON.parse(candidate));
846
+ } catch {
847
+ /* only structured verified evidence qualifies */
848
+ }
849
+ return;
850
+ }
851
+ if (Array.isArray(candidate)) return candidate.forEach(visit);
852
+ if (!candidate || typeof candidate !== 'object') return;
853
+ const record = candidate as Record<string, unknown>;
854
+ const continuation = record.continuation as
855
+ { exhaustive?: unknown; hasMore?: unknown; contextLimitReached?: unknown } | undefined;
856
+ const inventory = record.inventory as
857
+ { recordSet?: unknown; coverage?: { complete?: unknown; reason?: unknown } } | undefined;
858
+ const directive = (record.investigation as { directive?: unknown } | undefined)?.directive as
859
+ { scope?: unknown; goal?: unknown } | undefined;
860
+ // An exhaustive evidence page is a retrieval property, not an instruction
861
+ // to recite it. Modern video evidence carries the agent's investigation
862
+ // directive: only an explicit source inventory may bypass answer synthesis.
863
+ const isExplicitSourceInventory =
864
+ directive?.scope === 'source' && directive.goal === 'enumerate';
865
+ // A structured inventory is already a final data result. It must never be
866
+ // handed back to a prose model just because the source correctly reports
867
+ // partial coverage. The UI has the complete returned table; this message
868
+ // only explains its scope and truth status.
869
+ if (
870
+ record.success === true &&
871
+ continuation?.exhaustive === true &&
872
+ continuation.hasMore !== true &&
873
+ continuation.contextLimitReached !== true &&
874
+ inventory &&
875
+ typeof inventory === 'object' &&
876
+ typeof inventory.recordSet === 'string' &&
877
+ Array.isArray(record.rows)
878
+ ) {
879
+ const count = record.rows.length;
880
+ const completeCoverage = inventory.coverage?.complete === true;
881
+ const reason =
882
+ typeof inventory.coverage?.reason === 'string' ? inventory.coverage.reason.trim() : '';
883
+ inventorySummary = completeCoverage
884
+ ? `I completed the source-wide ${inventory.recordSet} inventory: ${count} records. The full searchable and downloadable table is above.`
885
+ : `I completed a scan of ${count} indexed ${inventory.recordSet} records. The table above contains every matching record currently available, but it is not a guarantee of complete source coverage${reason ? `: ${reason}` : '.'}`;
886
+ return;
887
+ }
888
+ if (
889
+ record.success === true &&
890
+ continuation?.exhaustive === true &&
891
+ continuation.hasMore !== true &&
892
+ mediaClaimIsAnswerLevel(record.claimVerification) &&
893
+ (!directive || isExplicitSourceInventory) &&
894
+ Array.isArray(record.evidence)
895
+ ) {
896
+ complete = (record.evidence as Array<Record<string, unknown>>)
897
+ .map((item) => {
898
+ const range = (item.timeRange ?? {}) as { startSecs?: unknown };
899
+ const at = Number(range.startSecs ?? item.startSecs ?? 0);
900
+ const raw = textOf(item.payload);
901
+ const answer = raw.match(/^(?:Source answer|Answer):\s*(.+)$/imu)?.[1]?.trim();
902
+ const respondent = raw
903
+ .match(/^(?:Source respondent|Answered by):\s*(.+)$/imu)?.[1]
904
+ ?.trim();
905
+ return {
906
+ at: Number.isFinite(at) ? at : 0,
907
+ text: clean(raw),
908
+ ...(answer ? { answer } : {}),
909
+ ...(respondent ? { respondent } : {}),
910
+ sourceInventory:
911
+ /^(?:Source (?:question|item|structure)(?:\s*\([^)]*\))?|Question):/i.test(
912
+ raw.trim(),
913
+ ),
914
+ };
915
+ })
916
+ .filter((item) => item.text)
917
+ .sort((left, right) => left.at - right.at);
918
+ return;
919
+ }
920
+ Object.values(record).forEach(visit);
921
+ };
922
+ visit(value);
923
+ if (inventorySummary) return inventorySummary;
924
+ if (!complete || complete.length === 0) return undefined;
925
+ const seen = new Set<string>();
926
+ const items = complete.filter((item) => {
927
+ const key = `${Math.floor(item.at)}:${item.text.normalize('NFKC').toLocaleLowerCase()}`;
928
+ if (seen.has(key)) return false;
929
+ seen.add(key);
930
+ return true;
931
+ });
932
+ // Small ordinary inventories (a roster or a handful of states) benefit from
933
+ // natural synthesis. Source-authored inventory records are already the exact
934
+ // requested units, and a large result must avoid another context window.
935
+ if (items.length <= 48 && !items.every((item) => item.sourceInventory)) return undefined;
936
+ // Legacy tool records have no structured `rows` field. When they carry
937
+ // answer columns, render their data shape directly; this is independent of
938
+ // the user's vocabulary or language. Newer records take the data-table path
939
+ // above and never reach this compatibility branch.
940
+ if (items.every((item) => item.sourceInventory) && items.some((item) => item.respondent)) {
941
+ const escapeCell = (cell: string | undefined) => (cell || '—').replace(/\|/g, '\\|');
942
+ const header = '| Question | Answered by | Answer |';
943
+ const divider = '| --- | --- | --- |';
944
+ return `${header}\n${divider}\n${items
945
+ .map(
946
+ (item) =>
947
+ `| [${displayTimecode(item.at)}] ${escapeCell(item.text)} | ${escapeCell(
948
+ item.respondent,
949
+ )} | ${escapeCell(item.answer)} |`,
950
+ )
951
+ .join('\n')}`;
952
+ }
953
+ return `Here is the complete list in chronological order:\n\n${items
954
+ .map((item) => `- [${displayTimecode(item.at)}] ${item.text}`)
955
+ .join('\n')}`;
956
+ }
957
+
958
+ /** Guarantee a visible, normally-closing UI answer without delaying AI SDK lifecycle events. */
959
+ export function recoverEmptyUIMessageStream(fallbackText: string) {
960
+ const fallback =
961
+ fallbackText.trim() ||
962
+ 'I found the relevant source, but the answer could not be completed. Please try again.';
963
+
964
+ return () => {
965
+ let hasText = false;
966
+ let recoveryEmitted = false;
967
+
968
+ const emitRecovery = (
969
+ controller: TransformStreamDefaultController<any>,
970
+ recoveryText = fallback,
971
+ ) => {
972
+ if (hasText || recoveryEmitted) return;
973
+ const id = `answer-${crypto.randomUUID()}`;
974
+ controller.enqueue({ type: 'text-start', id });
975
+ controller.enqueue({ type: 'text-delta', id, delta: recoveryText });
976
+ controller.enqueue({ type: 'text-end', id });
977
+ hasText = true;
978
+ recoveryEmitted = true;
979
+ };
980
+
981
+ return new TransformStream<any, any>({
982
+ transform(chunk, controller) {
983
+ if (!chunk || typeof chunk !== 'object') return;
984
+ if (chunk.type === 'text-delta' && String(chunk.delta ?? '').trim()) hasText = true;
985
+ if (chunk.type === 'error') {
986
+ emitRecovery(controller, String(chunk.errorText ?? '').trim() || fallback);
987
+ // A recovered error is ordinary assistant text. Do not also leave the
988
+ // client in its error state with a duplicate red technical banner.
989
+ return;
990
+ }
991
+ if (chunk.type === 'finish') emitRecovery(controller);
992
+ controller.enqueue(chunk);
993
+ },
994
+ flush(controller) {
995
+ emitRecovery(controller);
996
+ },
997
+ });
998
+ };
999
+ }
1000
+
1001
+ function compactMediaEvidence(value: unknown): unknown {
1002
+ if (!value || typeof value !== 'object' || Array.isArray(value)) return value;
1003
+ const result = value as Record<string, any>;
1004
+ if (typeof result.mediaAssetId !== 'string' || !Array.isArray(result.evidence)) return value;
1005
+ const exhaustive = result.continuation?.exhaustive === true;
1006
+ const evidenceLimit = exhaustive ? 240 : 48;
1007
+ const payloadLimit = exhaustive ? 260 : 700;
1008
+ return {
1009
+ success: result.success,
1010
+ mediaAssetId: result.mediaAssetId,
1011
+ fileName: result.fileName,
1012
+ supportingClip: result.supportingClip,
1013
+ claimVerification: result.claimVerification,
1014
+ investigation: result.investigation,
1015
+ observedSubjects: Array.isArray(result.observedSubjects)
1016
+ ? result.observedSubjects.slice(0, 200)
1017
+ : undefined,
1018
+ continuation: result.continuation,
1019
+ directObservation: result.directObservation
1020
+ ? {
1021
+ rule: result.directObservation.rule,
1022
+ readings: Array.isArray(result.directObservation.readings)
1023
+ ? result.directObservation.readings.slice(0, 12).map((reading: any) => ({
1024
+ range: reading.range,
1025
+ at: reading.at,
1026
+ found: reading.found,
1027
+ read: Array.isArray(reading.read) ? reading.read.slice(0, 12) : reading.read,
1028
+ confidence: reading.confidence,
1029
+ }))
1030
+ : [],
1031
+ }
1032
+ : undefined,
1033
+ temporalContext: result.temporalContext
1034
+ ? {
1035
+ rule: result.temporalContext.rule,
1036
+ readings: Array.isArray(result.temporalContext.readings)
1037
+ ? result.temporalContext.readings.slice(-48)
1038
+ : [],
1039
+ }
1040
+ : undefined,
1041
+ evidence: result.evidence.slice(0, evidenceLimit).map((item: any) => ({
1042
+ id: item.id ?? item.evidenceId,
1043
+ modality: item.modality,
1044
+ timeRange: item.timeRange ?? {
1045
+ startSecs: item.startSecs,
1046
+ endSecs: item.endSecs,
1047
+ precision: item.precision,
1048
+ },
1049
+ payload:
1050
+ typeof item.payload === 'string'
1051
+ ? item.payload.slice(0, payloadLimit)
1052
+ : (item.payload ??
1053
+ (typeof item.text === 'string' ? item.text.slice(0, payloadLimit) : item.text)),
1054
+ confidence: item.confidence,
1055
+ conflicted: item.conflicted,
1056
+ })),
1057
+ };
1058
+ }
1059
+
1060
+ /** Keep page-local text and visual readings when they travel with a search result. */
1061
+ function compactPdfPageEvidence(value: unknown): Record<string, unknown> | undefined {
1062
+ if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined;
1063
+ const result = value as Record<string, any>;
1064
+ if (!Array.isArray(result.pages)) return undefined;
1065
+ return {
1066
+ success: result.success,
1067
+ documentId: result.documentId,
1068
+ title: result.title,
1069
+ totalPages: result.totalPages,
1070
+ pages: result.pages.slice(0, 3).map((page: any) => ({
1071
+ pageNumber: page.pageNumber,
1072
+ text: typeof page.text === 'string' ? page.text.slice(0, 1_600) : undefined,
1073
+ tables: Array.isArray(page.tables) ? page.tables.slice(0, 2) : undefined,
1074
+ analysis: typeof page.analysis === 'string' ? page.analysis.slice(0, 3_000) : undefined,
1075
+ previewUrl: page.previewUrl,
1076
+ })),
1077
+ };
1078
+ }
1079
+
1080
+ function compactValue(value: unknown, toolName?: string): unknown {
1081
+ const jsonValue =
1082
+ typeof value === 'object' && value !== null && (value as { type?: string }).type === 'json'
1083
+ ? (value as { value?: unknown }).value
1084
+ : value;
1085
+ const isJsonEnvelope = jsonValue !== value;
1086
+ const parsed = typeof jsonValue === 'string' ? tryParse(jsonValue) : jsonValue;
1087
+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return value;
1088
+
1089
+ const result = parsed as Record<string, any>;
1090
+ let compact: Record<string, any> | undefined;
1091
+ if (toolName === 'searchKnowledgeBase' && Array.isArray(result.hits)) {
1092
+ compact = {
1093
+ query: result.query,
1094
+ hits: result.hits.slice(0, 4).map((hit: any) => ({
1095
+ documentId: hit.documentId,
1096
+ title: hit.title,
1097
+ url: hit.url,
1098
+ score: hit.score,
1099
+ text: typeof hit.text === 'string' ? hit.text.slice(0, 1_200) : hit.text,
1100
+ images: Array.isArray(hit.images)
1101
+ ? hit.images.slice(0, 4).map((image: any) => ({
1102
+ imageUrl: image.imageUrl,
1103
+ pageNumber: image.pageNumber,
1104
+ index: image.index,
1105
+ description:
1106
+ typeof image.description === 'string'
1107
+ ? image.description.slice(0, 1_600)
1108
+ : undefined,
1109
+ }))
1110
+ : undefined,
1111
+ metadata: hit.metadata
1112
+ ? {
1113
+ mediaAssetId: hit.metadata.mediaAssetId,
1114
+ pageNumber: hit.metadata.pageNumber,
1115
+ contentKind: hit.metadata.contentKind,
1116
+ }
1117
+ : undefined,
1118
+ })),
1119
+ // The evidence-first shortcut embeds a completed video investigation
1120
+ // directly on the search result (see searchKnowledgeBase's execute()
1121
+ // in chat/tools.ts) so a model that stops after one tool call still
1122
+ // has what it needs. Dropping it here left the model with nothing to
1123
+ // answer from on the next step -- observed live as a completely empty
1124
+ // final response despite the video having been found correctly.
1125
+ ...(result.videoEvidence !== undefined
1126
+ ? { videoEvidence: compactMediaEvidence(result.videoEvidence) }
1127
+ : {}),
1128
+ ...(compactPdfPageEvidence(result.pdfInspection)
1129
+ ? { pdfInspection: compactPdfPageEvidence(result.pdfInspection) }
1130
+ : {}),
1131
+ ...(compactPdfPageEvidence(result.pdfVisualAnalysis)
1132
+ ? { pdfVisualAnalysis: compactPdfPageEvidence(result.pdfVisualAnalysis) }
1133
+ : {}),
1134
+ };
1135
+ } else if (typeof result.mediaAssetId === 'string' && Array.isArray(result.evidence)) {
1136
+ compact = compactMediaEvidence(result) as Record<string, any>;
1137
+ } else if (toolName === 'queryTabularData' && Array.isArray(result.rows)) {
1138
+ compact = result.visualization
1139
+ ? {
1140
+ columns: result.columns,
1141
+ totalRows: result.totalRows,
1142
+ aggregationResults: result.aggregationResults,
1143
+ chartSynopsis: formatTabularVisualizationAnswer(result),
1144
+ }
1145
+ : {
1146
+ columns: result.columns,
1147
+ rows: result.rows.slice(0, 50),
1148
+ totalRows: result.totalRows,
1149
+ aggregationResults: result.aggregationResults,
1150
+ };
1151
+ } else if (toolName === 'executeAnalysis') {
1152
+ compact = {
1153
+ stdout: typeof result.stdout === 'string' ? result.stdout.slice(0, 4_000) : result.stdout,
1154
+ stderr: typeof result.stderr === 'string' ? result.stderr.slice(0, 1_000) : result.stderr,
1155
+ exitCode: result.exitCode,
1156
+ };
1157
+ } else if (
1158
+ (toolName === 'inspectPdfPages' || toolName === 'analyzePdfPages') &&
1159
+ Array.isArray(result.pages)
1160
+ ) {
1161
+ compact = compactPdfPageEvidence(result);
1162
+ }
1163
+
1164
+ if (!compact) return value;
1165
+ const serialized = typeof jsonValue === 'string' ? JSON.stringify(compact) : compact;
1166
+ return isJsonEnvelope ? { ...(value as Record<string, unknown>), value: serialized } : serialized;
1167
+ }
1168
+
1169
+ function tryParse(value: string): unknown {
1170
+ try {
1171
+ return JSON.parse(value);
1172
+ } catch {
1173
+ return undefined;
1174
+ }
1175
+ }
1176
+
1177
+ /**
1178
+ * Removing every tool for the final answer step (activeTools: [],
1179
+ * toolChoice: 'none') isn't always enough signal on its own -- observed
1180
+ * live with a reasoning-tagged model (Claude Sonnet 4.6) on a large,
1181
+ * evidence-heavy video investigation: it burned a few output tokens and
1182
+ * finished with no tool call and no text at all, leaving the turn looking
1183
+ * answered (citations rendered from the tool result) but silently empty.
1184
+ * An explicit nudge naming the moment ("you have every tool result now,
1185
+ * answer") reliably produced a real answer where the bare absence of tools
1186
+ * didn't -- reproduced and confirmed live before and after this fix.
1187
+ */
1188
+ export function withFinalAnswerNudge(messages: readonly any[]): any[] {
1189
+ return [
1190
+ ...messages,
1191
+ {
1192
+ role: 'user',
1193
+ content: [
1194
+ {
1195
+ type: 'text',
1196
+ text: 'You have every tool result gathered above. Answer the original question now, directly, using that evidence -- do not call another tool.',
1197
+ },
1198
+ ],
1199
+ },
1200
+ ];
1201
+ }
1202
+
1203
+ /** Keep rendered tool outputs intact while bounding the next model call's evidence. */
1204
+ export function compactToolContextForModel(messages: readonly any[]): any[] {
1205
+ return messages.map((message) => {
1206
+ if (!Array.isArray(message?.content)) return message;
1207
+ return {
1208
+ ...message,
1209
+ content: message.content.map((part: ToolResultPart) => {
1210
+ if (part.type !== 'tool-result') return part;
1211
+ if ('output' in part) {
1212
+ return { ...part, output: compactValue(part.output, part.toolName) };
1213
+ }
1214
+ if ('result' in part) {
1215
+ return { ...part, result: compactValue(part.result, part.toolName) };
1216
+ }
1217
+ return part;
1218
+ }),
1219
+ };
1220
+ });
1221
+ }
1222
+ import type { VideoInvestigationDirective } from '@larkup/core/video-knowledge/query-planner';