@vellumai/assistant 0.11.5 → 0.11.6-staging.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/AGENTS.md +5 -1
  2. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
  3. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
  4. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
  5. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
  6. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +70 -0
  7. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +16 -1
  8. package/node_modules/@vellumai/gateway-client/src/index.ts +6 -2
  9. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +121 -61
  10. package/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
  11. package/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
  12. package/openapi.yaml +421 -15
  13. package/package.json +1 -1
  14. package/scripts/sync-web-search-catalog.ts +6 -0
  15. package/src/__tests__/app-pin-store.test.ts +149 -0
  16. package/src/__tests__/channel-availability-routes.test.ts +23 -1
  17. package/src/__tests__/channel-readiness-discord.test.ts +231 -0
  18. package/src/__tests__/channel-readiness-service.test.ts +126 -0
  19. package/src/__tests__/channel-readiness-slack-remote.test.ts +141 -0
  20. package/src/__tests__/channel-reply-delivery.test.ts +4 -4
  21. package/src/__tests__/client-os-metadata-persistence.test.ts +23 -10
  22. package/src/__tests__/conversation-delete-watch-timeline.test.ts +231 -0
  23. package/src/__tests__/conversation-error.test.ts +17 -0
  24. package/src/__tests__/conversation-seed-composer.test.ts +8 -0
  25. package/src/__tests__/conversation-slash-commands.test.ts +8 -0
  26. package/src/__tests__/disk-pressure-policy.test.ts +6 -0
  27. package/src/__tests__/gemini-provider.test.ts +138 -0
  28. package/src/__tests__/history-repair.test.ts +105 -3
  29. package/src/__tests__/identity-routes.test.ts +1 -0
  30. package/src/__tests__/llm-catalog-parity.test.ts +45 -0
  31. package/src/__tests__/migration-import-from-path.test.ts +349 -0
  32. package/src/__tests__/notification-telegram-adapter.test.ts +102 -0
  33. package/src/__tests__/oauth-commands-routes.test.ts +89 -0
  34. package/src/__tests__/oauth-provider-profiles.test.ts +7 -6
  35. package/src/__tests__/openai-provider.test.ts +18 -0
  36. package/src/__tests__/openai-responses-provider.test.ts +18 -0
  37. package/src/__tests__/platform-callback-registration.test.ts +184 -0
  38. package/src/__tests__/plugin-api-store-credential.test.ts +71 -3
  39. package/src/__tests__/pricing.test.ts +2 -2
  40. package/src/__tests__/public-ingress-urls.test.ts +36 -0
  41. package/src/__tests__/resolve-trust-class.test.ts +0 -48
  42. package/src/__tests__/sanitize-config-for-transfer.test.ts +28 -0
  43. package/src/__tests__/secret-routes-platform-proxy.test.ts +49 -0
  44. package/src/__tests__/settings-routes.test.ts +85 -3
  45. package/src/__tests__/web-search-catalog-parity.test.ts +8 -0
  46. package/src/agent/history-repair/history-repair.ts +45 -14
  47. package/src/agent/loop.ts +4 -1
  48. package/src/api/constants/profile-config-validation.ts +60 -0
  49. package/src/api/events/tool-result.ts +6 -1
  50. package/src/api/events/watch-retro-completed.ts +52 -0
  51. package/src/api/index.ts +11 -0
  52. package/src/apps/app-pin-reconciler.ts +92 -0
  53. package/src/apps/app-pin-store.ts +125 -0
  54. package/src/channels/gateway-channel-socket-health.ts +32 -0
  55. package/src/channels/gateway-discord-admission.ts +32 -0
  56. package/src/channels/types.ts +20 -0
  57. package/src/cli/commands/__tests__/conversations-slack.test.ts +1 -1
  58. package/src/cli/commands/__tests__/inference-profiles.test.ts +16 -4
  59. package/src/cli/commands/__tests__/inference-providers.test.ts +67 -2
  60. package/src/cli/commands/channels/__tests__/channels.test.ts +85 -0
  61. package/src/cli/commands/channels/index.ts +45 -31
  62. package/src/cli/commands/inference-profiles.ts +56 -3
  63. package/src/cli/commands/inference-providers.ts +28 -2
  64. package/src/cli/commands/oauth/index.help.ts +7 -1
  65. package/src/cli/commands/oauth/request.test.ts +290 -0
  66. package/src/cli/commands/oauth/request.ts +57 -41
  67. package/src/cli/lib/bundled-marketplace.json +14 -1
  68. package/src/cli/lib/open-browser.test.ts +67 -0
  69. package/src/cli/lib/open-browser.ts +24 -5
  70. package/src/config/__tests__/profile-materialization.test.ts +26 -0
  71. package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +6 -0
  72. package/src/config/bundled-skills/schedule/SKILL.md +1 -1
  73. package/src/config/feature-flag-registry.json +17 -1
  74. package/src/config/profile-materialization.ts +29 -0
  75. package/src/config/sanitize-for-transfer.ts +16 -0
  76. package/src/config/schemas/llm.ts +7 -0
  77. package/src/config/schemas/services.ts +6 -0
  78. package/src/context/outbound-sanitize.ts +6 -0
  79. package/src/daemon/__tests__/lifecycle-watch-timeline-sweep.test.ts +98 -0
  80. package/src/daemon/conversation-error.ts +24 -2
  81. package/src/daemon/conversation-slash.ts +6 -15
  82. package/src/daemon/daemon-control.ts +1 -0
  83. package/src/daemon/disk-pressure-policy.ts +7 -1
  84. package/src/daemon/handlers/__tests__/config-ingress-tunnel-records.test.ts +208 -0
  85. package/src/daemon/handlers/config-ingress.ts +115 -5
  86. package/src/daemon/lifecycle.ts +26 -0
  87. package/src/daemon/message-types/web-activity.ts +3 -2
  88. package/src/daemon/trust-context.ts +0 -37
  89. package/src/inbound/__tests__/tunnel-probe.test.ts +448 -0
  90. package/src/inbound/platform-callback-registration.ts +28 -2
  91. package/src/inbound/public-ingress-urls.ts +12 -0
  92. package/src/inbound/tunnel-probe.ts +261 -0
  93. package/src/live-voice/__tests__/live-voice-connection.test.ts +25 -0
  94. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +8 -2
  95. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +212 -10
  96. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +5 -2
  97. package/src/live-voice/live-voice-connection.ts +46 -8
  98. package/src/live-voice/live-voice-manager.ts +25 -0
  99. package/src/live-voice/live-voice-session-manager.ts +318 -2
  100. package/src/live-voice/live-voice-session.ts +52 -2
  101. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +126 -68
  102. package/src/messaging/providers/channel-transport.ts +64 -47
  103. package/src/messaging/providers/discord/send.test.ts +46 -1
  104. package/src/messaging/providers/discord/send.ts +51 -0
  105. package/src/messaging/providers/discord/transport.ts +26 -3
  106. package/src/messaging/providers/index.ts +22 -47
  107. package/src/messaging/providers/slack/send.test.ts +83 -26
  108. package/src/messaging/providers/slack/send.ts +120 -51
  109. package/src/messaging/providers/slack/stream-tasks.test.ts +26 -0
  110. package/src/messaging/providers/slack/stream-tasks.ts +39 -0
  111. package/src/messaging/providers/slack/transport.ts +24 -22
  112. package/src/messaging/providers/telegram-bot/send.test.ts +109 -12
  113. package/src/messaging/providers/telegram-bot/send.ts +43 -0
  114. package/src/messaging/providers/telegram-bot/transport.ts +25 -8
  115. package/src/notifications/__tests__/assistant-reply-producer.test.ts +30 -7
  116. package/src/notifications/adapters/telegram.ts +48 -1
  117. package/src/notifications/assistant-reply-producer.ts +7 -7
  118. package/src/notifications/conversation-seed-composer.ts +7 -2
  119. package/src/oauth/byo-connection.test.ts +63 -0
  120. package/src/oauth/byo-connection.ts +16 -15
  121. package/src/oauth/connection.test.ts +111 -0
  122. package/src/oauth/connection.ts +142 -1
  123. package/src/oauth/platform-connection.test.ts +34 -0
  124. package/src/oauth/platform-connection.ts +28 -5
  125. package/src/oauth/seed-providers.ts +15 -1
  126. package/src/permissions/types.ts +3 -1
  127. package/src/persistence/conversation-crud.ts +46 -0
  128. package/src/persistence/conversation-types.ts +11 -9
  129. package/src/persistence/db-async-query.ts +2 -1
  130. package/src/persistence/db-maintenance.ts +15 -0
  131. package/src/persistence/embeddings/qdrant-manager.ts +1 -0
  132. package/src/persistence/migrations/367-create-watch-timeline-entries.ts +46 -0
  133. package/src/persistence/migrations/368-watch-timeline-screenshot-blob.ts +33 -0
  134. package/src/persistence/migrations/369-create-app-pins.ts +37 -0
  135. package/src/persistence/migrations/__tests__/367-create-watch-timeline-entries.test.ts +98 -0
  136. package/src/persistence/migrations/__tests__/368-watch-timeline-screenshot-blob.test.ts +98 -0
  137. package/src/persistence/schema/index.ts +1 -0
  138. package/src/persistence/schema/infrastructure.ts +17 -0
  139. package/src/persistence/schema/watch.ts +29 -0
  140. package/src/persistence/steps.ts +6 -0
  141. package/src/plugins/mtime-cache.ts +11 -0
  142. package/src/providers/__tests__/retry-network-error.test.ts +84 -0
  143. package/src/providers/connection-resolution.ts +23 -1
  144. package/src/providers/content-blocks.ts +9 -0
  145. package/src/providers/fetch-provider-catalog.ts +19 -0
  146. package/src/providers/gemini/client.ts +13 -5
  147. package/src/providers/inference/__tests__/endpoint-probe.test.ts +92 -0
  148. package/src/providers/inference/__tests__/profile-config-validation.test.ts +39 -0
  149. package/src/providers/inference/__tests__/profile-probe-classify.test.ts +66 -0
  150. package/src/providers/inference/adapter-factory.ts +0 -9
  151. package/src/providers/inference/credential-rotation.ts +61 -0
  152. package/src/providers/inference/endpoint-probe.ts +115 -0
  153. package/src/providers/inference/profile-probe.ts +256 -0
  154. package/src/providers/model-catalog.ts +170 -125
  155. package/src/providers/openai/__tests__/api-error-normalization.test.ts +17 -1
  156. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +42 -60
  157. package/src/providers/openai/__tests__/connection-error-wrap.test.ts +44 -0
  158. package/src/providers/openai/__tests__/orphan-tool-result-guard.test.ts +34 -2
  159. package/src/providers/openai/api-error-normalization.ts +16 -2
  160. package/src/providers/openai/chat-completions-provider.ts +75 -29
  161. package/src/providers/openai/responses-provider.ts +5 -2
  162. package/src/providers/openrouter/client.ts +0 -1
  163. package/src/providers/provider-send-message.ts +11 -0
  164. package/src/providers/retry.ts +6 -0
  165. package/src/providers/search-provider-catalog.ts +20 -0
  166. package/src/providers/vercel-ai-gateway/client.ts +0 -1
  167. package/src/runtime/AGENTS.md +1 -0
  168. package/src/runtime/__tests__/desktop-presence.test.ts +27 -4
  169. package/src/runtime/__tests__/host-observe.test.ts +302 -0
  170. package/src/runtime/channel-readiness-service.ts +214 -14
  171. package/src/runtime/channel-readiness-types.ts +49 -2
  172. package/src/runtime/channel-reply-delivery.ts +2 -2
  173. package/src/runtime/desktop-presence.ts +24 -21
  174. package/src/runtime/host-observe.ts +246 -0
  175. package/src/runtime/http-server.ts +181 -1
  176. package/src/runtime/migrations/__tests__/staged-import-path.test.ts +104 -0
  177. package/src/runtime/migrations/staged-import-path.ts +116 -0
  178. package/src/runtime/routes/__tests__/app-pin-routes.test.ts +383 -0
  179. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +80 -0
  180. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +118 -0
  181. package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +20 -0
  182. package/src/runtime/routes/__tests__/ingress-status-routes.test.ts +508 -0
  183. package/src/runtime/routes/__tests__/plugins-routes.test.ts +35 -56
  184. package/src/runtime/routes/__tests__/watch-routes-guardian-cache.test.ts +139 -0
  185. package/src/runtime/routes/__tests__/watch-routes.test.ts +598 -0
  186. package/src/runtime/routes/app-management-routes.ts +140 -29
  187. package/src/runtime/routes/channel-availability-routes.ts +1 -0
  188. package/src/runtime/routes/channel-readiness-routes.ts +14 -2
  189. package/src/runtime/routes/conversation-query-routes.ts +10 -0
  190. package/src/runtime/routes/guardian-approval-interception.ts +24 -33
  191. package/src/runtime/routes/host-cu-routes.ts +18 -0
  192. package/src/runtime/routes/identity-routes.ts +2 -0
  193. package/src/runtime/routes/inbound-message-handler.ts +10 -7
  194. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +166 -308
  195. package/src/runtime/routes/inbound-stages/background-dispatch.ts +158 -335
  196. package/src/runtime/routes/index.ts +2 -0
  197. package/src/runtime/routes/inference-profiles-routes.ts +232 -31
  198. package/src/runtime/routes/inference-provider-connection-routes.ts +24 -4
  199. package/src/runtime/routes/ingress-status-routes.ts +180 -0
  200. package/src/runtime/routes/live-voice-routes.test.ts +40 -1
  201. package/src/runtime/routes/live-voice-routes.ts +34 -0
  202. package/src/runtime/routes/migration-routes.ts +218 -10
  203. package/src/runtime/routes/oauth-commands-routes.ts +23 -16
  204. package/src/runtime/routes/plugins-routes.ts +12 -28
  205. package/src/runtime/routes/question-routes.ts +6 -0
  206. package/src/runtime/routes/secret-routes.ts +7 -27
  207. package/src/runtime/routes/settings-routes.ts +9 -6
  208. package/src/runtime/routes/watch-routes.ts +807 -0
  209. package/src/runtime/slack-reply-session.test.ts +230 -121
  210. package/src/runtime/slack-reply-session.ts +113 -81
  211. package/src/runtime/{slack-task-progress.test.ts → task-progress.test.ts} +1 -28
  212. package/src/runtime/{slack-task-progress.ts → task-progress.ts} +30 -51
  213. package/src/security/__tests__/untrusted-content.test.ts +42 -0
  214. package/src/security/untrusted-content.ts +28 -9
  215. package/src/telemetry/__tests__/live-voice-funnel.test.ts +108 -0
  216. package/src/telemetry/live-voice-funnel.ts +75 -8
  217. package/src/tools/credentials/store.ts +18 -6
  218. package/src/tools/network/__tests__/firecrawl-compat.test.ts +77 -0
  219. package/src/tools/network/__tests__/web-fetch-fastcrw.test.ts +169 -0
  220. package/src/tools/network/__tests__/web-search.test.ts +97 -2
  221. package/src/tools/network/firecrawl-compat.ts +90 -0
  222. package/src/tools/network/web-fetch.ts +142 -62
  223. package/src/tools/network/web-search.ts +141 -55
  224. package/src/tools/types.ts +2 -1
  225. package/src/util/oauth-request-body.test.ts +74 -0
  226. package/src/util/oauth-request-body.ts +60 -0
  227. package/src/util/worker-process.ts +1 -0
  228. package/src/watch/__tests__/watch-retro.test.ts +665 -0
  229. package/src/watch/__tests__/watch-session-manager.test.ts +566 -0
  230. package/src/watch/__tests__/watch-timeline.test.ts +670 -0
  231. package/src/watch/watch-retro.ts +480 -0
  232. package/src/watch/watch-session-manager.ts +575 -0
  233. package/src/watch/watch-timeline.ts +848 -0
@@ -0,0 +1,670 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { describe, expect, test } from "bun:test";
3
+
4
+ import {
5
+ createConversation,
6
+ deleteConversation,
7
+ } from "../../persistence/conversation-crud.js";
8
+ import { getSqlite } from "../../persistence/db-connection.js";
9
+ import { initializeDb } from "../../persistence/db-init.js";
10
+ import {
11
+ appendNarration,
12
+ appendObservation,
13
+ DEFAULT_MAX_AX_TREES,
14
+ DEFAULT_MAX_ENTRIES,
15
+ DEFAULT_MAX_RENDER_BYTES,
16
+ drainOrphanedWatchTimelineEntries,
17
+ purgeAllWatchTimelines,
18
+ purgeWatchTimelineForConversation,
19
+ readWatchScreenshot,
20
+ renderWatchTimeline,
21
+ sweepOrphanedWatchTimelineEntries,
22
+ WATCH_SCREENSHOT_MIME,
23
+ } from "../watch-timeline.js";
24
+
25
+ await initializeDb();
26
+
27
+ /** A 1x1 JPEG, the shape the host hands over on every observe. */
28
+ const SCREENSHOT_BASE64 =
29
+ "/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAABAAEDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD3+iiigD//2Q==";
30
+
31
+ const SCREENSHOT_BYTES = Buffer.from(SCREENSHOT_BASE64, "base64");
32
+
33
+ /**
34
+ * A fresh session against a real conversation row, so entries from one test are
35
+ * invisible to the next and appends are not refused as orphans.
36
+ */
37
+ function newSession() {
38
+ const conversation = createConversation(`watch-${randomUUID()}`);
39
+ return { sessionId: randomUUID(), conversationId: conversation.id };
40
+ }
41
+
42
+ /** The SQL every statement drizzle prepares while `run` executes. */
43
+ function captureSql<T>(run: () => T): { result: T; statements: string[] } {
44
+ const sqlite = getSqlite();
45
+ const statements: string[] = [];
46
+ const original = sqlite.prepare.bind(sqlite);
47
+ const patched = sqlite as unknown as { prepare: typeof original };
48
+ patched.prepare = ((sql: string, ...rest: unknown[]) => {
49
+ statements.push(sql);
50
+ return (original as (...args: unknown[]) => unknown)(sql, ...rest);
51
+ }) as typeof original;
52
+ try {
53
+ return { result: run(), statements };
54
+ } finally {
55
+ patched.prepare = original;
56
+ }
57
+ }
58
+
59
+ describe("watch timeline", () => {
60
+ test("reads entries back in timestamp order, not append order", () => {
61
+ const { sessionId, conversationId } = newSession();
62
+
63
+ appendObservation(sessionId, {
64
+ conversationId,
65
+ atMs: 4_000,
66
+ observation: { axTree: "window Mail" },
67
+ });
68
+ appendNarration(sessionId, {
69
+ conversationId,
70
+ atMs: 2_000,
71
+ text: "this is the inbox",
72
+ });
73
+ appendNarration(sessionId, {
74
+ conversationId,
75
+ atMs: 12_000,
76
+ text: "and this is where I file it",
77
+ });
78
+
79
+ const rendered = renderWatchTimeline(sessionId);
80
+
81
+ expect(rendered.entries.map((e) => e.atMs)).toEqual([2_000, 4_000, 12_000]);
82
+ expect(rendered.totalEntries).toBe(3);
83
+ expect(rendered.truncated).toBe(false);
84
+ expect(rendered.text).toContain("[t+00:02] narration: this is the inbox");
85
+ expect(rendered.text).toContain("[t+00:04] screen:");
86
+ expect(rendered.text).toContain("<ax-tree>\nwindow Mail\n</ax-tree>");
87
+ expect(rendered.text.indexOf("[t+00:04]")).toBeGreaterThan(
88
+ rendered.text.indexOf("[t+00:02]"),
89
+ );
90
+ expect(rendered.text.indexOf("[t+00:12]")).toBeGreaterThan(
91
+ rendered.text.indexOf("[t+00:04]"),
92
+ );
93
+ });
94
+
95
+ test("renders hours only once the session has any", () => {
96
+ const { sessionId, conversationId } = newSession();
97
+ appendNarration(sessionId, {
98
+ conversationId,
99
+ atMs: 3_723_000,
100
+ text: "still going",
101
+ });
102
+
103
+ expect(renderWatchTimeline(sessionId).text).toContain("[t+01:02:03]");
104
+ });
105
+
106
+ test("appends nothing for an observation that failed", () => {
107
+ const { sessionId, conversationId } = newSession();
108
+
109
+ const result = appendObservation(sessionId, {
110
+ conversationId,
111
+ atMs: 1_000,
112
+ observation: {
113
+ axTree: "window Mail",
114
+ screenshot: SCREENSHOT_BASE64,
115
+ executionError: "screen recording permission denied",
116
+ },
117
+ attachScreenshot: true,
118
+ });
119
+
120
+ expect(result).toEqual({ ok: false, reason: "observation_failed" });
121
+ expect(renderWatchTimeline(sessionId).totalEntries).toBe(0);
122
+ });
123
+
124
+ test("appends nothing for a screenshot the caller did not ask to keep", () => {
125
+ const { sessionId, conversationId } = newSession();
126
+
127
+ const result = appendObservation(sessionId, {
128
+ conversationId,
129
+ atMs: 1_000,
130
+ observation: { screenshot: SCREENSHOT_BASE64 },
131
+ });
132
+
133
+ expect(result).toEqual({ ok: false, reason: "empty" });
134
+ expect(renderWatchTimeline(sessionId).totalEntries).toBe(0);
135
+ });
136
+
137
+ test("keeps a screenshot-only observation the host fell back to", () => {
138
+ // No focused window means no AX tree, and the host answers with a bare
139
+ // screenshot. Discarding that would leave a session spent in an
140
+ // inaccessible app with an entirely empty timeline.
141
+ const { sessionId, conversationId } = newSession();
142
+
143
+ const result = appendObservation(sessionId, {
144
+ conversationId,
145
+ atMs: 1_000,
146
+ observation: { screenshot: SCREENSHOT_BASE64 },
147
+ attachScreenshot: true,
148
+ });
149
+
150
+ expect(result.ok).toBe(true);
151
+
152
+ const rendered = renderWatchTimeline(sessionId);
153
+ expect(rendered.totalEntries).toBe(1);
154
+ expect(rendered.screenshotEntryIds).toHaveLength(1);
155
+ expect(rendered.text).toContain("[t+00:01] screen:");
156
+ // The retrospective can tell a screen it cannot read from a screen with
157
+ // nothing on it, and knows an image of it exists.
158
+ expect(rendered.text).toContain("<ax-tree-unavailable />");
159
+ expect(rendered.text).toContain(
160
+ "a screenshot of this moment was captured.",
161
+ );
162
+ });
163
+
164
+ test("appends nothing for a narration that is only whitespace", () => {
165
+ const { sessionId, conversationId } = newSession();
166
+
167
+ const result = appendNarration(sessionId, {
168
+ conversationId,
169
+ atMs: 1_000,
170
+ text: " \n ",
171
+ });
172
+
173
+ expect(result).toEqual({ ok: false, reason: "empty" });
174
+ expect(renderWatchTimeline(sessionId).totalEntries).toBe(0);
175
+ });
176
+
177
+ test("stores a screenshot only when attachScreenshot asks for it", () => {
178
+ const { sessionId, conversationId } = newSession();
179
+ const observation = {
180
+ axTree: "window Mail",
181
+ screenshot: SCREENSHOT_BASE64,
182
+ };
183
+
184
+ appendObservation(sessionId, { conversationId, atMs: 1_000, observation });
185
+ appendObservation(sessionId, {
186
+ conversationId,
187
+ atMs: 2_000,
188
+ observation,
189
+ attachScreenshot: false,
190
+ });
191
+ appendObservation(sessionId, {
192
+ conversationId,
193
+ atMs: 3_000,
194
+ observation,
195
+ attachScreenshot: true,
196
+ });
197
+
198
+ const rendered = renderWatchTimeline(sessionId);
199
+ expect(rendered.entries.map((e) => e.screenshotBytes)).toEqual([
200
+ null,
201
+ null,
202
+ SCREENSHOT_BYTES.length,
203
+ ]);
204
+ expect(rendered.screenshotEntryIds).toEqual([
205
+ rendered.entries[2]?.id as string,
206
+ ]);
207
+ expect(rendered.text).toContain(
208
+ "a screenshot of this moment was captured.",
209
+ );
210
+ });
211
+
212
+ test("hands back the exact frame the observation carried", () => {
213
+ const { sessionId, conversationId } = newSession();
214
+ appendObservation(sessionId, {
215
+ conversationId,
216
+ atMs: 1_000,
217
+ observation: { axTree: "window Mail", screenshot: SCREENSHOT_BASE64 },
218
+ attachScreenshot: true,
219
+ });
220
+
221
+ const [entryId] = renderWatchTimeline(sessionId).screenshotEntryIds;
222
+
223
+ const frame = readWatchScreenshot(entryId as string);
224
+ expect(frame?.mimeType).toBe(WATCH_SCREENSHOT_MIME);
225
+ expect(Buffer.from(frame?.bytes as Buffer)).toEqual(SCREENSHOT_BYTES);
226
+ });
227
+
228
+ test("has no frame to hand back for an entry that carries none", () => {
229
+ const { sessionId, conversationId } = newSession();
230
+ appendNarration(sessionId, { conversationId, atMs: 1_000, text: "hello" });
231
+
232
+ const [entry] = renderWatchTimeline(sessionId).entries;
233
+
234
+ expect(readWatchScreenshot(entry?.id as string)).toBeNull();
235
+ expect(readWatchScreenshot("no-such-entry")).toBeNull();
236
+ });
237
+
238
+ test("drops a frame over the per-entry size cap and keeps the rest", () => {
239
+ const { sessionId, conversationId } = newSession();
240
+ const oversized = Buffer.alloc(2_000_001, 0x41).toString("base64");
241
+
242
+ const result = appendObservation(sessionId, {
243
+ conversationId,
244
+ atMs: 1_000,
245
+ observation: { axTree: "window Mail", screenshot: oversized },
246
+ attachScreenshot: true,
247
+ });
248
+
249
+ expect(result.ok).toBe(true);
250
+ const rendered = renderWatchTimeline(sessionId);
251
+ expect(rendered.entries[0]?.screenshotBytes).toBeNull();
252
+ expect(rendered.screenshotEntryIds).toHaveLength(0);
253
+ expect(rendered.text).toContain("<ax-tree>\nwindow Mail\n</ax-tree>");
254
+ });
255
+
256
+ test("refuses an observation whose only content is an oversized frame", () => {
257
+ const { sessionId, conversationId } = newSession();
258
+ const oversized = Buffer.alloc(2_000_001, 0x41).toString("base64");
259
+
260
+ const result = appendObservation(sessionId, {
261
+ conversationId,
262
+ atMs: 1_000,
263
+ observation: { screenshot: oversized },
264
+ attachScreenshot: true,
265
+ });
266
+
267
+ expect(result).toEqual({ ok: false, reason: "empty" });
268
+ expect(renderWatchTimeline(sessionId).totalEntries).toBe(0);
269
+ });
270
+
271
+ test("bounds the rendered entries by default and reports the truncation", () => {
272
+ const { sessionId, conversationId } = newSession();
273
+ const total = DEFAULT_MAX_ENTRIES + 5;
274
+ for (let i = 0; i < total; i++) {
275
+ appendNarration(sessionId, {
276
+ conversationId,
277
+ atMs: i * 1_000,
278
+ text: `line ${i}`,
279
+ });
280
+ }
281
+
282
+ const rendered = renderWatchTimeline(sessionId);
283
+
284
+ expect(rendered.totalEntries).toBe(total);
285
+ expect(rendered.entries).toHaveLength(DEFAULT_MAX_ENTRIES);
286
+ expect(rendered.truncated).toBe(true);
287
+ expect(rendered.entries[0]?.text).toBe("line 5");
288
+ expect(rendered.text).not.toContain("line 4\n");
289
+ expect(rendered.text).toContain(`line ${total - 1}`);
290
+
291
+ const unbounded = renderWatchTimeline(sessionId, { maxEntries: total });
292
+ expect(unbounded.entries).toHaveLength(total);
293
+ expect(unbounded.truncated).toBe(false);
294
+ });
295
+
296
+ test("spells out only the most recent AX trees and collapses the rest", () => {
297
+ const { sessionId, conversationId } = newSession();
298
+ const treeCount = DEFAULT_MAX_AX_TREES + 2;
299
+ for (let i = 0; i < treeCount; i++) {
300
+ appendObservation(sessionId, {
301
+ conversationId,
302
+ atMs: i * 1_000,
303
+ observation: { axTree: `tree-${i}`, axDiff: `diff-${i}` },
304
+ });
305
+ }
306
+
307
+ const rendered = renderWatchTimeline(sessionId);
308
+
309
+ for (let i = 0; i < treeCount; i++) {
310
+ const spelledOut = i >= treeCount - DEFAULT_MAX_AX_TREES;
311
+ expect(rendered.text.includes(`<ax-tree>\ntree-${i}\n</ax-tree>`)).toBe(
312
+ spelledOut,
313
+ );
314
+ // The offset and the diff survive a collapsed tree, so a compacted entry
315
+ // still says when it happened and what moved.
316
+ expect(rendered.text).toContain(`diff-${i}`);
317
+ }
318
+ expect(rendered.text).toContain("<ax-tree-omitted />");
319
+ });
320
+
321
+ test("escapes a closing ax-tree tag inside captured content", () => {
322
+ const { sessionId, conversationId } = newSession();
323
+ appendObservation(sessionId, {
324
+ conversationId,
325
+ atMs: 1_000,
326
+ observation: { axTree: "text </ax-tree> more" },
327
+ });
328
+
329
+ const { text } = renderWatchTimeline(sessionId);
330
+ expect(text).toContain("text &lt;/ax-tree&gt; more");
331
+ expect(text.match(/<\/ax-tree>/g)).toHaveLength(1);
332
+ });
333
+
334
+ test("bounds the rendered text by bytes, keeping the newest entries", () => {
335
+ const { sessionId, conversationId } = newSession();
336
+ const total = 20;
337
+ for (let i = 0; i < total; i++) {
338
+ appendNarration(sessionId, {
339
+ conversationId,
340
+ atMs: i * 1_000,
341
+ text: `entry-${i} ${"x".repeat(2_000)}`,
342
+ });
343
+ }
344
+
345
+ const rendered = renderWatchTimeline(sessionId, {
346
+ maxEntries: total,
347
+ maxRenderBytes: 10_000,
348
+ });
349
+
350
+ expect(rendered.totalEntries).toBe(total);
351
+ expect(Buffer.byteLength(rendered.text, "utf8")).toBeLessThanOrEqual(
352
+ 10_000,
353
+ );
354
+ // The count bound admitted every entry, so the byte budget is the only
355
+ // thing that could have dropped one.
356
+ expect(rendered.entries.length).toBeLessThan(total);
357
+ expect(rendered.truncated).toBe(true);
358
+ // Spent newest first, so the last thing that happened always survives.
359
+ expect(rendered.entries.at(-1)?.atMs).toBe((total - 1) * 1_000);
360
+ expect(rendered.text).toContain(`entry-${total - 1}`);
361
+ expect(rendered.text).not.toContain("entry-0 ");
362
+ });
363
+
364
+ test("a single oversized AX tree cannot blow the budget", () => {
365
+ const { sessionId, conversationId } = newSession();
366
+
367
+ appendObservation(sessionId, {
368
+ conversationId,
369
+ atMs: 1_000,
370
+ observation: { axTree: "e".repeat(2_000_000) },
371
+ });
372
+
373
+ const rendered = renderWatchTimeline(sessionId);
374
+
375
+ expect(rendered.entries).toHaveLength(1);
376
+ expect(Buffer.byteLength(rendered.text, "utf8")).toBeLessThanOrEqual(
377
+ DEFAULT_MAX_RENDER_BYTES,
378
+ );
379
+ expect(rendered.truncated).toBe(true);
380
+ // Clipped in place rather than dropped, and the wrapper still closes so the
381
+ // block reads as one tree.
382
+ expect(rendered.text).toContain("[truncated]");
383
+ expect(rendered.text).toContain("<ax-tree>");
384
+ expect(rendered.text.match(/<\/ax-tree>/g)).toHaveLength(1);
385
+ });
386
+
387
+ test("gives up the tree before the offset and the diff", () => {
388
+ const { sessionId, conversationId } = newSession();
389
+
390
+ appendObservation(sessionId, {
391
+ conversationId,
392
+ atMs: 1_000,
393
+ observation: { axTree: "e".repeat(50_000), axDiff: "row added" },
394
+ });
395
+
396
+ const rendered = renderWatchTimeline(sessionId, { maxRenderBytes: 320 });
397
+
398
+ expect(rendered.text).toContain("[t+00:01] screen:");
399
+ expect(rendered.text).toContain("row added");
400
+ expect(rendered.text).not.toContain("e".repeat(1_000));
401
+ expect(rendered.truncated).toBe(true);
402
+ });
403
+
404
+ test("reports nothing truncated when everything fits", () => {
405
+ const { sessionId, conversationId } = newSession();
406
+ appendNarration(sessionId, {
407
+ conversationId,
408
+ atMs: 1_000,
409
+ text: "short enough",
410
+ });
411
+
412
+ expect(renderWatchTimeline(sessionId).truncated).toBe(false);
413
+ });
414
+
415
+ test("bounds the read itself, not a slice of the whole session", () => {
416
+ const { sessionId, conversationId } = newSession();
417
+ const total = 12;
418
+ for (let i = 0; i < total; i++) {
419
+ appendNarration(sessionId, {
420
+ conversationId,
421
+ atMs: i * 1_000,
422
+ text: `line ${i}`,
423
+ });
424
+ }
425
+
426
+ const { result: rendered, statements } = captureSql(() =>
427
+ renderWatchTimeline(sessionId, { maxEntries: 3 }),
428
+ );
429
+
430
+ // The entries the render reasoned over are the newest three, and the count
431
+ // it reports is the session's real one rather than what it read.
432
+ expect(rendered.entries.map((e) => e.text)).toEqual([
433
+ "line 9",
434
+ "line 10",
435
+ "line 11",
436
+ ]);
437
+ expect(rendered.totalEntries).toBe(total);
438
+ expect(rendered.truncated).toBe(true);
439
+
440
+ // The bound is in the query, so the rows never read are never hydrated.
441
+ const selects = statements.filter((sql) =>
442
+ sql.includes("watch_timeline_entries"),
443
+ );
444
+ const rowRead = selects.find((sql) => sql.includes('"ax_tree"'));
445
+ expect(rowRead).toBeDefined();
446
+ expect(rowRead).toContain("limit");
447
+ // The screenshot is measured rather than selected, so reading a session
448
+ // never pulls its pixels into memory.
449
+ expect(rowRead).toContain("length(");
450
+ expect(selects.some((sql) => sql.includes("count("))).toBe(true);
451
+ });
452
+
453
+ test("reports the true total when the count bound admits nothing", () => {
454
+ const { sessionId, conversationId } = newSession();
455
+ appendNarration(sessionId, { conversationId, atMs: 1_000, text: "hello" });
456
+
457
+ const rendered = renderWatchTimeline(sessionId, { maxEntries: 0 });
458
+
459
+ expect(rendered.entries).toHaveLength(0);
460
+ expect(rendered.totalEntries).toBe(1);
461
+ expect(rendered.truncated).toBe(true);
462
+ });
463
+
464
+ test("refuses an append whose conversation is gone", () => {
465
+ const { sessionId, conversationId } = newSession();
466
+ deleteConversation(conversationId);
467
+
468
+ const result = appendNarration(sessionId, {
469
+ conversationId,
470
+ atMs: 1_000,
471
+ text: "too late",
472
+ });
473
+
474
+ expect(result).toEqual({ ok: false, reason: "conversation_missing" });
475
+ expect(renderWatchTimeline(sessionId).totalEntries).toBe(0);
476
+ });
477
+
478
+ test("purges a conversation's rows and the frames they carry", () => {
479
+ const { sessionId, conversationId } = newSession();
480
+ const other = newSession();
481
+
482
+ appendObservation(sessionId, {
483
+ conversationId,
484
+ atMs: 1_000,
485
+ observation: { axTree: "window Mail", screenshot: SCREENSHOT_BASE64 },
486
+ attachScreenshot: true,
487
+ });
488
+ appendNarration(sessionId, {
489
+ conversationId,
490
+ atMs: 2_000,
491
+ text: "filing it here",
492
+ });
493
+ appendNarration(other.sessionId, {
494
+ conversationId: other.conversationId,
495
+ atMs: 1_000,
496
+ text: "a different conversation",
497
+ });
498
+
499
+ const [entryId] = renderWatchTimeline(sessionId).screenshotEntryIds;
500
+ expect(readWatchScreenshot(entryId as string)).not.toBeNull();
501
+
502
+ expect(purgeWatchTimelineForConversation(conversationId)).toBe(2);
503
+
504
+ expect(renderWatchTimeline(sessionId).totalEntries).toBe(0);
505
+ expect(readWatchScreenshot(entryId as string)).toBeNull();
506
+ expect(renderWatchTimeline(other.sessionId).totalEntries).toBe(1);
507
+ });
508
+
509
+ test("purges every session's rows and frames", () => {
510
+ const first = newSession();
511
+ const second = newSession();
512
+
513
+ for (const session of [first, second]) {
514
+ appendObservation(session.sessionId, {
515
+ conversationId: session.conversationId,
516
+ atMs: 1_000,
517
+ observation: { axTree: "window Mail", screenshot: SCREENSHOT_BASE64 },
518
+ attachScreenshot: true,
519
+ });
520
+ }
521
+
522
+ const entryIds = [first, second].map(
523
+ (session) => renderWatchTimeline(session.sessionId).screenshotEntryIds[0],
524
+ );
525
+ for (const id of entryIds) {
526
+ expect(readWatchScreenshot(id as string)).not.toBeNull();
527
+ }
528
+
529
+ expect(purgeAllWatchTimelines()).toBeGreaterThanOrEqual(2);
530
+
531
+ expect(renderWatchTimeline(first.sessionId).totalEntries).toBe(0);
532
+ expect(renderWatchTimeline(second.sessionId).totalEntries).toBe(0);
533
+ for (const id of entryIds) {
534
+ expect(readWatchScreenshot(id as string)).toBeNull();
535
+ }
536
+ });
537
+ });
538
+
539
+ describe("watch timeline orphan sweep", () => {
540
+ /**
541
+ * Delete the conversation row without going through `deleteConversation`,
542
+ * which purges the timeline on its way out. This is the state a purge that
543
+ * failed after the row was committed as deleted leaves behind, and the state
544
+ * a crash between the two writes leaves behind.
545
+ */
546
+ function orphanConversation(conversationId: string): void {
547
+ getSqlite()
548
+ .query("DELETE FROM conversations WHERE id = ?")
549
+ .run(conversationId);
550
+ }
551
+
552
+ test("drains a backlog larger than one sweep page", async () => {
553
+ // A single sweep is deliberately one page. Startup drains, because on an
554
+ // install where database maintenance never runs it is the only pass the
555
+ // residue will ever see.
556
+ await drainOrphanedWatchTimelineEntries();
557
+
558
+ const { sessionId, conversationId } = newSession();
559
+ const PAGE = 5_000;
560
+ // More than two pages on purpose: with a single leftover page, a `drain`
561
+ // that never looped would still clear the remainder in its one sweep and
562
+ // the test would pass without exercising the loop at all.
563
+ const TOTAL = PAGE * 2 + 25;
564
+ const sqlite = getSqlite();
565
+ const insert = sqlite.query(
566
+ "INSERT INTO watch_timeline_entries (id, session_id, conversation_id, at_ms, kind, text, created_at) VALUES (?, ?, ?, ?, 'narration', ?, ?)",
567
+ );
568
+ sqlite.exec("BEGIN");
569
+ for (let i = 0; i < TOTAL; i += 1) {
570
+ insert.run(`drain-${i}`, sessionId, conversationId, i, "step", i);
571
+ }
572
+ sqlite.exec("COMMIT");
573
+ orphanConversation(conversationId);
574
+
575
+ // One page, and the remainder is still there: this is what a single
576
+ // startup sweep would have left behind for good.
577
+ expect(sweepOrphanedWatchTimelineEntries()).toBe(PAGE);
578
+ expect(
579
+ sqlite
580
+ .query(
581
+ "SELECT count(*) AS n FROM watch_timeline_entries WHERE conversation_id = ?",
582
+ )
583
+ .get(conversationId) as { n: number },
584
+ ).toEqual({ n: TOTAL - PAGE });
585
+
586
+ // The drain finishes the job, which takes two more pages.
587
+ expect(await drainOrphanedWatchTimelineEntries()).toBe(TOTAL - PAGE);
588
+ expect(
589
+ sqlite
590
+ .query(
591
+ "SELECT count(*) AS n FROM watch_timeline_entries WHERE conversation_id = ?",
592
+ )
593
+ .get(conversationId) as { n: number },
594
+ ).toEqual({ n: 0 });
595
+ });
596
+
597
+ test("sweeps entries whose conversation is gone, frames included", () => {
598
+ // Drain anything earlier tests orphaned so the count below is this
599
+ // session's rows and nothing else.
600
+ sweepOrphanedWatchTimelineEntries();
601
+
602
+ const { sessionId, conversationId } = newSession();
603
+ appendObservation(sessionId, {
604
+ conversationId,
605
+ atMs: 1_000,
606
+ observation: { axTree: "window Mail", screenshot: SCREENSHOT_BASE64 },
607
+ attachScreenshot: true,
608
+ });
609
+ appendNarration(sessionId, {
610
+ conversationId,
611
+ atMs: 2_000,
612
+ text: "filing it here",
613
+ });
614
+ const [entryId] = renderWatchTimeline(sessionId).screenshotEntryIds;
615
+ expect(readWatchScreenshot(entryId as string)).not.toBeNull();
616
+
617
+ orphanConversation(conversationId);
618
+
619
+ expect(sweepOrphanedWatchTimelineEntries()).toBe(2);
620
+ expect(renderWatchTimeline(sessionId).totalEntries).toBe(0);
621
+ expect(readWatchScreenshot(entryId as string)).toBeNull();
622
+ });
623
+
624
+ test("leaves entries whose conversation still exists", () => {
625
+ sweepOrphanedWatchTimelineEntries();
626
+
627
+ const live = newSession();
628
+ const orphaned = newSession();
629
+ appendNarration(live.sessionId, {
630
+ conversationId: live.conversationId,
631
+ atMs: 1_000,
632
+ text: "still here",
633
+ });
634
+ appendNarration(orphaned.sessionId, {
635
+ conversationId: orphaned.conversationId,
636
+ atMs: 1_000,
637
+ text: "conversation deleted",
638
+ });
639
+
640
+ orphanConversation(orphaned.conversationId);
641
+
642
+ expect(sweepOrphanedWatchTimelineEntries()).toBe(1);
643
+ expect(renderWatchTimeline(orphaned.sessionId).totalEntries).toBe(0);
644
+ expect(renderWatchTimeline(live.sessionId).totalEntries).toBe(1);
645
+ });
646
+
647
+ test("reports nothing swept when there is nothing to sweep", () => {
648
+ sweepOrphanedWatchTimelineEntries();
649
+
650
+ expect(sweepOrphanedWatchTimelineEntries()).toBe(0);
651
+ });
652
+
653
+ test("reports nothing swept rather than throwing when the table is gone", () => {
654
+ const sqlite = getSqlite();
655
+ sqlite
656
+ .query(
657
+ "ALTER TABLE watch_timeline_entries RENAME TO watch_timeline_entries_hidden",
658
+ )
659
+ .run();
660
+ try {
661
+ expect(sweepOrphanedWatchTimelineEntries()).toBe(0);
662
+ } finally {
663
+ sqlite
664
+ .query(
665
+ "ALTER TABLE watch_timeline_entries_hidden RENAME TO watch_timeline_entries",
666
+ )
667
+ .run();
668
+ }
669
+ });
670
+ });