@vellumai/assistant 0.5.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (347) hide show
  1. package/ARCHITECTURE.md +54 -54
  2. package/docs/architecture/integrations.md +62 -67
  3. package/docs/credential-execution-service.md +3 -3
  4. package/package.json +1 -1
  5. package/src/__tests__/agent-loop.test.ts +111 -0
  6. package/src/__tests__/always-loaded-tools-guard.test.ts +3 -4
  7. package/src/__tests__/app-builder-tool-scripts.test.ts +13 -151
  8. package/src/__tests__/app-dir-path-guard.test.ts +78 -0
  9. package/src/__tests__/app-executors.test.ts +1 -291
  10. package/src/__tests__/app-git-history.test.ts +4 -4
  11. package/src/__tests__/app-routes-csp.test.ts +1 -0
  12. package/src/__tests__/app-store-dir-names.test.ts +426 -0
  13. package/src/__tests__/assistant-feature-flags-integration.test.ts +7 -9
  14. package/src/__tests__/attachments-store.test.ts +169 -21
  15. package/src/__tests__/attachments.test.ts +115 -1
  16. package/src/__tests__/btw-routes.test.ts +1 -0
  17. package/src/__tests__/canonical-guardian-store.test.ts +38 -0
  18. package/src/__tests__/channel-reply-delivery.test.ts +55 -0
  19. package/src/__tests__/checker.test.ts +54 -0
  20. package/src/__tests__/claude-code-skill-regression.test.ts +2 -0
  21. package/src/__tests__/claude-code-tool-profiles.test.ts +2 -0
  22. package/src/__tests__/compaction.benchmark.test.ts +2 -1
  23. package/src/__tests__/config-schema-cmd.test.ts +68 -21
  24. package/src/__tests__/config-schema.test.ts +1 -1
  25. package/src/__tests__/conversation-agent-loop-overflow.test.ts +149 -5
  26. package/src/__tests__/conversation-agent-loop.test.ts +290 -2
  27. package/src/__tests__/conversation-attachments.test.ts +17 -19
  28. package/src/__tests__/conversation-disk-view-integration.test.ts +277 -0
  29. package/src/__tests__/conversation-disk-view.test.ts +810 -0
  30. package/src/__tests__/conversation-error.test.ts +1 -1
  31. package/src/__tests__/conversation-fork-crud.test.ts +551 -0
  32. package/src/__tests__/conversation-fork-route.test.ts +386 -0
  33. package/src/__tests__/conversation-history-web-search.test.ts +1 -1
  34. package/src/__tests__/conversation-key-store-disk-view.test.ts +130 -0
  35. package/src/__tests__/conversation-media-retry.test.ts +8 -2
  36. package/src/__tests__/conversation-queue.test.ts +36 -1
  37. package/src/__tests__/conversation-routes-disk-view.test.ts +439 -0
  38. package/src/__tests__/conversation-routes-guardian-reply.test.ts +2 -2
  39. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -7
  40. package/src/__tests__/conversation-runtime-assembly.test.ts +17 -2
  41. package/src/__tests__/conversation-skill-tools.test.ts +4 -9
  42. package/src/__tests__/conversation-slash-commands.test.ts +149 -0
  43. package/src/__tests__/conversation-store.test.ts +24 -21
  44. package/src/__tests__/conversation-surfaces-state-update.test.ts +246 -0
  45. package/src/__tests__/conversation-surfaces-task-progress.test.ts +1 -0
  46. package/src/__tests__/conversation-title-service.test.ts +137 -0
  47. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +25 -315
  48. package/src/__tests__/conversation-tool-setup-memory-scope.test.ts +1 -0
  49. package/src/__tests__/conversation-tool-setup-side-effect-flag.test.ts +1 -0
  50. package/src/__tests__/conversation-workspace-cache-state.test.ts +44 -2
  51. package/src/__tests__/conversation-workspace-injection.test.ts +11 -0
  52. package/src/__tests__/credential-execution-feature-gates.test.ts +3 -3
  53. package/src/__tests__/credential-security-invariants.test.ts +3 -0
  54. package/src/__tests__/credential-vault-unit.test.ts +5 -10
  55. package/src/__tests__/cu-unified-flow.test.ts +1 -0
  56. package/src/__tests__/db-conversation-fork-lineage-migration.test.ts +241 -0
  57. package/src/__tests__/db-llm-request-log-provider-migration.test.ts +214 -0
  58. package/src/__tests__/diagnostics-export.test.ts +70 -1
  59. package/src/__tests__/filesystem-tools.test.ts +4 -2
  60. package/src/__tests__/first-greeting.test.ts +80 -0
  61. package/src/__tests__/gateway-only-guard.test.ts +1 -0
  62. package/src/__tests__/handlers-user-message-approval-consumption.test.ts +3 -7
  63. package/src/__tests__/history-repair.test.ts +103 -10
  64. package/src/__tests__/http-conversation-lineage.test.ts +251 -0
  65. package/src/__tests__/image-source-path-reinject.test.ts +136 -0
  66. package/src/__tests__/llm-context-normalization.test.ts +1116 -0
  67. package/src/__tests__/llm-context-route-provider.test.ts +217 -0
  68. package/src/__tests__/llm-request-log-turn-query.test.ts +270 -0
  69. package/src/__tests__/media-generate-image.test.ts +47 -94
  70. package/src/__tests__/memory-lifecycle-e2e.test.ts +3 -1
  71. package/src/__tests__/memory-recall-quality.test.ts +5 -5
  72. package/src/__tests__/migration-cross-version-compatibility.test.ts +4 -1
  73. package/src/__tests__/migration-export-http.test.ts +3 -1
  74. package/src/__tests__/migration-import-commit-http.test.ts +18 -4
  75. package/src/__tests__/migration-import-preflight-http.test.ts +1 -3
  76. package/src/__tests__/mime-builder.test.ts +3 -2
  77. package/src/__tests__/non-member-access-request.test.ts +12 -1
  78. package/src/__tests__/notification-decision-identity.test.ts +52 -0
  79. package/src/__tests__/oauth-apps-routes.test.ts +103 -0
  80. package/src/__tests__/oauth-store.test.ts +115 -0
  81. package/src/__tests__/provider-error-scenarios.test.ts +1 -3
  82. package/src/__tests__/provider-failover-actual-provider.test.ts +66 -0
  83. package/src/__tests__/recording-handler.test.ts +17 -0
  84. package/src/__tests__/registry.test.ts +3 -8
  85. package/src/__tests__/relay-server.test.ts +1 -1
  86. package/src/__tests__/runtime-attachment-metadata.test.ts +7 -3
  87. package/src/__tests__/schema-transforms.test.ts +165 -5
  88. package/src/__tests__/server-history-render.test.ts +2 -2
  89. package/src/__tests__/skill-feature-flags-integration.test.ts +18 -17
  90. package/src/__tests__/skill-feature-flags.test.ts +13 -13
  91. package/src/__tests__/skill-load-feature-flag.test.ts +4 -4
  92. package/src/__tests__/slack-app-setup-skill-regression.test.ts +3 -1
  93. package/src/__tests__/slack-inbound-verification.test.ts +2 -2
  94. package/src/__tests__/starter-task-flow.test.ts +1 -0
  95. package/src/__tests__/suggestion-routes.test.ts +443 -0
  96. package/src/__tests__/swarm-conversation-integration.test.ts +1 -0
  97. package/src/__tests__/swarm-recursion.test.ts +1 -0
  98. package/src/__tests__/swarm-tool.test.ts +1 -0
  99. package/src/__tests__/system-prompt.test.ts +8 -0
  100. package/src/__tests__/tool-execution-abort-cleanup.test.ts +1 -0
  101. package/src/__tests__/tool-preview-lifecycle.test.ts +32 -5
  102. package/src/__tests__/top-level-renderer.test.ts +22 -0
  103. package/src/__tests__/turn-boundary-resolution.test.ts +243 -0
  104. package/src/__tests__/web-fetch.test.ts +6 -2
  105. package/src/__tests__/workspace-migration-006-services-config.test.ts +335 -0
  106. package/src/__tests__/workspace-migration-007-web-search-provider-rename.test.ts +312 -0
  107. package/src/__tests__/workspace-migration-009-backfill-conversation-disk-view.test.ts +278 -0
  108. package/src/__tests__/workspace-migration-010-app-dir-rename.test.ts +275 -0
  109. package/src/__tests__/workspace-migration-012-rename-conversation-disk-view-dirs.test.ts +77 -0
  110. package/src/__tests__/workspace-migration-013-repair-conversation-disk-view.test.ts +401 -0
  111. package/src/__tests__/workspace-migration-backfill-installation-id.test.ts +328 -0
  112. package/src/__tests__/workspace-migration-seed-device-id.test.ts +6 -10
  113. package/src/agent/attachments.ts +27 -1
  114. package/src/agent/loop.ts +29 -1
  115. package/src/avatar/traits-png-sync.ts +80 -25
  116. package/src/bundler/app-bundler.ts +4 -4
  117. package/src/calls/call-domain.ts +1 -0
  118. package/src/calls/voice-session-bridge.ts +1 -0
  119. package/src/cli/commands/auth.ts +92 -0
  120. package/src/cli/commands/avatar.ts +7 -6
  121. package/src/cli/commands/config.ts +2 -0
  122. package/src/cli/commands/oauth/providers.ts +29 -0
  123. package/src/cli/program.ts +12 -0
  124. package/src/cli.ts +15 -48
  125. package/src/config/bundled-skills/app-builder/SKILL.md +103 -28
  126. package/src/config/bundled-skills/app-builder/TOOLS.json +5 -199
  127. package/src/config/bundled-skills/app-builder/tools/{app-query.ts → app-refresh.ts} +2 -2
  128. package/src/config/bundled-skills/contacts/tools/google-contacts.ts +2 -3
  129. package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +6 -9
  130. package/src/config/bundled-skills/gmail/tools/gmail-attachments.ts +4 -6
  131. package/src/config/bundled-skills/gmail/tools/gmail-draft.ts +2 -3
  132. package/src/config/bundled-skills/gmail/tools/gmail-filters.ts +2 -3
  133. package/src/config/bundled-skills/gmail/tools/gmail-follow-up.ts +2 -3
  134. package/src/config/bundled-skills/gmail/tools/gmail-forward.ts +2 -3
  135. package/src/config/bundled-skills/gmail/tools/gmail-label.ts +4 -6
  136. package/src/config/bundled-skills/gmail/tools/gmail-outreach-scan.ts +2 -3
  137. package/src/config/bundled-skills/gmail/tools/gmail-send-draft.ts +2 -3
  138. package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +2 -3
  139. package/src/config/bundled-skills/gmail/tools/gmail-trash.ts +2 -3
  140. package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts +2 -3
  141. package/src/config/bundled-skills/gmail/tools/gmail-vacation.ts +2 -3
  142. package/src/config/bundled-skills/google-calendar/tools/shared.ts +1 -1
  143. package/src/config/bundled-skills/image-studio/SKILL.md +2 -2
  144. package/src/config/bundled-skills/image-studio/TOOLS.json +2 -2
  145. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +45 -72
  146. package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +2 -2
  147. package/src/config/bundled-skills/messaging/tools/shared.ts +1 -1
  148. package/src/config/bundled-skills/settings/tools/voice-config-update.ts +19 -3
  149. package/src/config/bundled-skills/skill-management/TOOLS.json +2 -2
  150. package/src/config/bundled-skills/slack/tools/shared.ts +19 -4
  151. package/src/config/bundled-skills/slack/tools/slack-scan-digest.ts +2 -3
  152. package/src/config/bundled-skills/transcribe/SKILL.md +1 -1
  153. package/src/config/bundled-skills/transcribe/TOOLS.json +2 -6
  154. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +19 -83
  155. package/src/config/bundled-tool-registry.ts +2 -14
  156. package/src/config/feature-flag-registry.json +16 -0
  157. package/src/config/loader.ts +64 -0
  158. package/src/config/raw-config-utils.ts +30 -0
  159. package/src/config/schema-utils.ts +28 -7
  160. package/src/config/schema.ts +8 -0
  161. package/src/config/schemas/elevenlabs.ts +18 -0
  162. package/src/config/schemas/memory-lifecycle.ts +4 -2
  163. package/src/config/schemas/memory-storage.ts +1 -1
  164. package/src/config/schemas/services.ts +8 -6
  165. package/src/contacts/contact-store.ts +13 -6
  166. package/src/contacts/contacts-write.ts +0 -1
  167. package/src/context/window-manager.ts +13 -2
  168. package/src/daemon/conversation-agent-loop-handlers.ts +46 -42
  169. package/src/daemon/conversation-agent-loop.ts +56 -19
  170. package/src/daemon/conversation-attachments.ts +18 -36
  171. package/src/daemon/conversation-error.ts +2 -1
  172. package/src/daemon/conversation-history.ts +18 -4
  173. package/src/daemon/conversation-lifecycle.ts +39 -15
  174. package/src/daemon/conversation-messaging.ts +70 -26
  175. package/src/daemon/conversation-process.ts +58 -34
  176. package/src/daemon/conversation-runtime-assembly.ts +21 -38
  177. package/src/daemon/conversation-slash.ts +121 -256
  178. package/src/daemon/conversation-surfaces.ts +143 -20
  179. package/src/daemon/conversation-tool-setup.ts +0 -6
  180. package/src/daemon/conversation-workspace.ts +21 -1
  181. package/src/daemon/conversation.ts +51 -29
  182. package/src/daemon/first-greeting.ts +35 -0
  183. package/src/daemon/handlers/config-embeddings.ts +148 -0
  184. package/src/daemon/handlers/config-model.ts +71 -26
  185. package/src/daemon/handlers/conversations.ts +0 -23
  186. package/src/daemon/handlers/recording.ts +26 -21
  187. package/src/daemon/history-repair.ts +28 -8
  188. package/src/daemon/host-cu-proxy.ts +2 -2
  189. package/src/daemon/lifecycle.ts +106 -64
  190. package/src/daemon/message-protocol.ts +3 -0
  191. package/src/daemon/message-types/conversations.ts +19 -0
  192. package/src/daemon/message-types/messages.ts +1 -0
  193. package/src/daemon/message-types/shared.ts +2 -0
  194. package/src/daemon/message-types/surfaces.ts +2 -0
  195. package/src/daemon/message-types/upgrades.ts +23 -0
  196. package/src/daemon/server.ts +83 -12
  197. package/src/daemon/shutdown-handlers.ts +8 -5
  198. package/src/daemon/startup-error.ts +9 -0
  199. package/src/daemon/tool-side-effects.ts +11 -28
  200. package/src/events/tool-permission-telemetry-listener.ts +1 -3
  201. package/src/instrument.ts +0 -4
  202. package/src/media/app-icon-generator.ts +2 -2
  203. package/src/memory/app-git-service.ts +28 -16
  204. package/src/memory/app-store.ts +230 -41
  205. package/src/memory/attachments-store.ts +558 -130
  206. package/src/memory/conversation-attention-store.ts +70 -0
  207. package/src/memory/conversation-crud.ts +442 -3
  208. package/src/memory/conversation-directories.ts +125 -0
  209. package/src/memory/conversation-disk-view.ts +390 -0
  210. package/src/memory/conversation-key-store.ts +17 -5
  211. package/src/memory/conversation-queries.ts +5 -1
  212. package/src/memory/conversation-title-service.ts +21 -49
  213. package/src/memory/db-init.ts +28 -0
  214. package/src/memory/embedding-backend.ts +42 -53
  215. package/src/memory/embedding-gemini.test.ts +4 -4
  216. package/src/memory/embedding-local.ts +1 -3
  217. package/src/memory/embedding-ollama.ts +1 -3
  218. package/src/memory/embedding-openai.ts +1 -3
  219. package/src/memory/indexer.ts +9 -7
  220. package/src/memory/items-extractor.ts +42 -13
  221. package/src/memory/job-handlers/conversation-starters.ts +6 -1
  222. package/src/memory/job-handlers/embedding.test.ts +1 -4
  223. package/src/memory/llm-request-log-store.ts +100 -1
  224. package/src/memory/migrations/102-alter-table-columns.ts +5 -0
  225. package/src/memory/migrations/146-schedule-oneshot-routing.ts +3 -3
  226. package/src/memory/migrations/147-migrate-reminders-to-schedules.ts +66 -70
  227. package/src/memory/migrations/148-drop-reminders-table.ts +5 -9
  228. package/src/memory/migrations/160-drop-loopback-port-column.ts +1 -3
  229. package/src/memory/migrations/174-rename-thread-starters-table.ts +0 -7
  230. package/src/memory/migrations/178-oauth-providers-managed-service-config-key.ts +15 -0
  231. package/src/memory/migrations/179-llm-request-log-message-id.ts +16 -0
  232. package/src/memory/migrations/180-backfill-inline-attachments-to-disk.ts +66 -0
  233. package/src/memory/migrations/181-rename-thread-starters-checkpoints.ts +46 -0
  234. package/src/memory/migrations/182-oauth-providers-display-metadata.ts +20 -0
  235. package/src/memory/migrations/183-add-conversation-fork-lineage.ts +22 -0
  236. package/src/memory/migrations/184-llm-request-log-provider.ts +12 -0
  237. package/src/memory/migrations/index.ts +7 -0
  238. package/src/memory/migrations/registry.ts +13 -0
  239. package/src/memory/retriever.test.ts +601 -2
  240. package/src/memory/retriever.ts +85 -9
  241. package/src/memory/schema/conversations.ts +6 -0
  242. package/src/memory/schema/infrastructure.ts +13 -7
  243. package/src/memory/schema/oauth.ts +6 -0
  244. package/src/messaging/providers/gmail/mime-builder.ts +3 -1
  245. package/src/notifications/copy-composer.ts +26 -0
  246. package/src/notifications/decision-engine.ts +14 -1
  247. package/src/notifications/emit-signal.ts +1 -1
  248. package/src/notifications/signal.ts +36 -0
  249. package/src/oauth/byo-connection.test.ts +1 -45
  250. package/src/oauth/byo-connection.ts +2 -8
  251. package/src/oauth/connect-orchestrator.ts +15 -11
  252. package/src/oauth/connection-resolver.test.ts +191 -0
  253. package/src/oauth/connection-resolver.ts +66 -38
  254. package/src/oauth/connection.ts +0 -1
  255. package/src/oauth/oauth-store.ts +97 -47
  256. package/src/oauth/platform-connection.test.ts +0 -1
  257. package/src/oauth/platform-connection.ts +11 -3
  258. package/src/oauth/seed-providers.ts +78 -3
  259. package/src/oauth/token-persistence.ts +16 -10
  260. package/src/permissions/checker.ts +62 -19
  261. package/src/prompts/system-prompt.ts +2 -0
  262. package/src/prompts/templates/BOOTSTRAP.md +2 -0
  263. package/src/providers/anthropic/client.ts +8 -1
  264. package/src/providers/failover.ts +4 -1
  265. package/src/providers/gemini/client.ts +50 -0
  266. package/src/providers/model-catalog.ts +92 -0
  267. package/src/providers/model-intents.ts +29 -20
  268. package/src/providers/openai/client.ts +49 -0
  269. package/src/providers/types.ts +2 -0
  270. package/src/runtime/access-request-helper.ts +16 -7
  271. package/src/runtime/auth/credential-service.ts +3 -1
  272. package/src/runtime/auth/route-policy.ts +14 -1
  273. package/src/runtime/btw-sidechain.ts +101 -0
  274. package/src/runtime/channel-reply-delivery.ts +17 -1
  275. package/src/runtime/http-router.ts +3 -1
  276. package/src/runtime/http-server.ts +196 -141
  277. package/src/runtime/http-types.ts +1 -0
  278. package/src/runtime/migrations/vbundle-builder.ts +5 -1
  279. package/src/runtime/routes/access-request-decision.ts +41 -0
  280. package/src/runtime/routes/app-management-routes.ts +6 -3
  281. package/src/runtime/routes/app-routes.ts +7 -3
  282. package/src/runtime/routes/approval-routes.ts +1 -0
  283. package/src/runtime/routes/approval-strategies/guardian-callback-strategy.ts +34 -2
  284. package/src/runtime/routes/attachment-routes.ts +45 -15
  285. package/src/runtime/routes/btw-routes.ts +21 -61
  286. package/src/runtime/routes/conversation-management-routes.ts +68 -0
  287. package/src/runtime/routes/conversation-query-routes.ts +180 -10
  288. package/src/runtime/routes/conversation-routes.ts +222 -28
  289. package/src/runtime/routes/conversation-starter-routes.ts +9 -11
  290. package/src/runtime/routes/diagnostics-routes.ts +1 -0
  291. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +2 -2
  292. package/src/runtime/routes/llm-context-normalization.ts +1199 -0
  293. package/src/runtime/routes/log-export-routes.ts +3 -0
  294. package/src/runtime/routes/memory-item-routes.test.ts +34 -0
  295. package/src/runtime/routes/memory-item-routes.ts +4 -0
  296. package/src/runtime/routes/migration-routes.ts +4 -1
  297. package/src/runtime/routes/oauth-apps.ts +291 -0
  298. package/src/runtime/routes/secret-routes.ts +28 -1
  299. package/src/runtime/routes/settings-routes.ts +14 -0
  300. package/src/runtime/routes/trace-event-routes.ts +4 -1
  301. package/src/schedule/schedule-store.ts +9 -21
  302. package/src/security/secure-keys.ts +21 -0
  303. package/src/signals/bash.ts +1 -1
  304. package/src/swarm/backend-claude-code.ts +3 -6
  305. package/src/telemetry/usage-telemetry-reporter.test.ts +3 -2
  306. package/src/telemetry/usage-telemetry-reporter.ts +3 -1
  307. package/src/tools/AGENTS.md +6 -10
  308. package/src/tools/apps/executors.ts +17 -232
  309. package/src/tools/claude-code/claude-code.ts +2 -3
  310. package/src/tools/credentials/vault.ts +7 -12
  311. package/src/tools/host-filesystem/read.ts +13 -10
  312. package/src/tools/network/__tests__/web-search.test.ts +4 -2
  313. package/src/tools/schedule/list.ts +2 -7
  314. package/src/tools/schema-transforms.ts +5 -0
  315. package/src/tools/shared/filesystem/format-diff.ts +4 -21
  316. package/src/tools/skills/execute.ts +1 -1
  317. package/src/tools/tool-manifest.ts +0 -6
  318. package/src/tools/ui-surface/definitions.ts +2 -2
  319. package/src/util/device-id.ts +28 -5
  320. package/src/util/platform.ts +6 -0
  321. package/src/util/pricing.ts +1 -0
  322. package/src/util/retry.ts +1 -3
  323. package/src/workspace/migrations/002-backfill-installation-id.ts +23 -12
  324. package/src/workspace/migrations/003-seed-device-id.ts +3 -4
  325. package/src/workspace/migrations/006-services-config.ts +5 -0
  326. package/src/workspace/migrations/008-voice-timeout-and-max-steps.ts +12 -0
  327. package/src/workspace/migrations/009-backfill-conversation-disk-view.ts +10 -0
  328. package/src/workspace/migrations/010-app-dir-rename.ts +223 -0
  329. package/src/workspace/migrations/012-rename-conversation-disk-view-dirs.ts +64 -0
  330. package/src/workspace/migrations/013-repair-conversation-disk-view.ts +11 -0
  331. package/src/workspace/migrations/rebuild-conversation-disk-view.ts +186 -0
  332. package/src/workspace/migrations/registry.ts +10 -0
  333. package/src/workspace/top-level-renderer.ts +12 -0
  334. package/src/__tests__/asset-materialize-tool.test.ts +0 -523
  335. package/src/__tests__/asset-search-tool.test.ts +0 -536
  336. package/src/__tests__/fixtures/media-reuse-fixtures.ts +0 -56
  337. package/src/__tests__/media-reuse-story.e2e.test.ts +0 -762
  338. package/src/__tests__/media-visibility-policy.test.ts +0 -190
  339. package/src/config/bundled-skills/app-builder/tools/app-file-edit.ts +0 -14
  340. package/src/config/bundled-skills/app-builder/tools/app-file-list.ts +0 -13
  341. package/src/config/bundled-skills/app-builder/tools/app-file-read.ts +0 -21
  342. package/src/config/bundled-skills/app-builder/tools/app-file-write.ts +0 -14
  343. package/src/config/bundled-skills/app-builder/tools/app-list.ts +0 -13
  344. package/src/config/bundled-skills/app-builder/tools/app-update.ts +0 -23
  345. package/src/daemon/media-visibility-policy.ts +0 -59
  346. package/src/tools/assets/materialize.ts +0 -248
  347. package/src/tools/assets/search.ts +0 -400
@@ -1,11 +1,8 @@
1
1
  import {
2
+ attachInlineAttachmentToMessage,
2
3
  AttachmentUploadError,
3
- FILE_BACKED_THRESHOLD_BYTES,
4
- linkAttachmentToMessage,
4
+ getFilePathForAttachment,
5
5
  setAttachmentThumbnail,
6
- uploadAttachment,
7
- uploadFileBackedAttachment,
8
- writeAttachmentToDisk,
9
6
  } from "../memory/attachments-store.js";
10
7
  import {
11
8
  check,
@@ -107,12 +104,6 @@ export async function approveHostAttachmentRead(
107
104
  return isAllowDecision(response.decision);
108
105
  }
109
106
 
110
- export function formatAttachmentWarnings(warnings: string[]): string | null {
111
- if (warnings.length === 0) return null;
112
- const lines = warnings.map((warning) => `Attachment warning: ${warning}`);
113
- return `\n\n${lines.join("\n")}`;
114
- }
115
-
116
107
  export interface AttachmentResolutionResult {
117
108
  assistantAttachments: AssistantAttachmentDraft[];
118
109
  emittedAttachments: UserMessageAttachment[];
@@ -211,28 +202,16 @@ export async function resolveAssistantAttachments(
211
202
  if (assistantAttachments.length > 0 && lastAssistantMessageId) {
212
203
  for (let i = 0; i < assistantAttachments.length; i++) {
213
204
  const draft = assistantAttachments[i];
214
- const isFileBacked = draft.sizeBytes > FILE_BACKED_THRESHOLD_BYTES;
215
205
  let stored;
216
- let diskFilePath: string | undefined;
217
206
  try {
218
- if (isFileBacked) {
219
- diskFilePath = writeAttachmentToDisk(
220
- draft.dataBase64,
221
- draft.filename,
222
- );
223
- stored = uploadFileBackedAttachment(
224
- draft.filename,
225
- draft.mimeType,
226
- diskFilePath,
227
- draft.sizeBytes,
228
- );
229
- } else {
230
- stored = uploadAttachment(
231
- draft.filename,
232
- draft.mimeType,
233
- draft.dataBase64,
234
- );
235
- }
207
+ stored = attachInlineAttachmentToMessage(
208
+ lastAssistantMessageId,
209
+ i,
210
+ draft.filename,
211
+ draft.mimeType,
212
+ draft.dataBase64,
213
+ { skipSizeLimit: true },
214
+ );
236
215
  } catch (err) {
237
216
  if (err instanceof AttachmentUploadError) {
238
217
  log.warn(
@@ -246,11 +225,11 @@ export async function resolveAssistantAttachments(
246
225
  }
247
226
  throw err;
248
227
  }
249
- linkAttachmentToMessage(lastAssistantMessageId, stored.id, i);
250
228
  const isVideo = draft.mimeType.startsWith("video/");
251
- const omitData =
252
- isFileBacked ||
253
- (isVideo && draft.dataBase64.length > MAX_INLINE_B64_SIZE);
229
+ // Only omit data for videos — they have an end-to-end lazy-load path
230
+ // via /v1/attachments/:id/content. Other types (images, PDFs) still need
231
+ // inline data for thumbnails, preview, and file-save in the client.
232
+ const omitData = isVideo && draft.dataBase64.length > MAX_INLINE_B64_SIZE;
254
233
 
255
234
  // Generate and persist a thumbnail for video attachments.
256
235
  let thumbnailData: string | undefined;
@@ -259,6 +238,7 @@ export async function resolveAssistantAttachments(
259
238
  if (existing) {
260
239
  thumbnailData = existing;
261
240
  } else {
241
+ const diskFilePath = getFilePathForAttachment(stored.id);
262
242
  const generated = diskFilePath
263
243
  ? await generateVideoThumbnailFromPath(diskFilePath)
264
244
  : await generateVideoThumbnail(draft.dataBase64);
@@ -274,8 +254,9 @@ export async function resolveAssistantAttachments(
274
254
  filename: draft.filename,
275
255
  mimeType: draft.mimeType,
276
256
  data: omitData ? "" : draft.dataBase64,
257
+ sourceType: draft.sourceType,
277
258
  ...(omitData ? { sizeBytes: draft.sizeBytes } : {}),
278
- ...(isFileBacked ? { fileBacked: true } : {}),
259
+ fileBacked: true,
279
260
  ...(thumbnailData ? { thumbnailData } : {}),
280
261
  });
281
262
  }
@@ -285,6 +266,7 @@ export async function resolveAssistantAttachments(
285
266
  filename: draft.filename,
286
267
  mimeType: draft.mimeType,
287
268
  data: draft.dataBase64,
269
+ sourceType: draft.sourceType,
288
270
  });
289
271
  }
290
272
  }
@@ -390,7 +390,8 @@ function classifyByMessage(
390
390
  if (isStreamingError(message)) {
391
391
  return {
392
392
  code: "PROVIDER_API",
393
- userMessage: "The AI provider's response was interrupted. Please try again.",
393
+ userMessage:
394
+ "The AI provider's response was interrupted. Please try again.",
394
395
  retryable: true,
395
396
  errorCategory: "stream_corruption",
396
397
  };
@@ -29,6 +29,16 @@ function isToolResultBlock(
29
29
  );
30
30
  }
31
31
 
32
+ function isSystemNoticeBlock(
33
+ block: ContentBlock | Record<string, unknown>,
34
+ ): boolean {
35
+ if (block.type !== "text") return false;
36
+ const text = (block as { text?: string }).text ?? "";
37
+ return (
38
+ text.startsWith("<system_notice>") && text.endsWith("</system_notice>")
39
+ );
40
+ }
41
+
32
42
  function isUndoableUserMessage(message: Message): boolean {
33
43
  if (message.role !== "user") return false;
34
44
  if (getSummaryFromContextMessage(message) != null) return false;
@@ -37,8 +47,9 @@ function isUndoableUserMessage(message: Message): boolean {
37
47
  // responses) are not undoable. Messages that have both tool_result and text blocks
38
48
  // (e.g. after repairHistory merges a tool_result turn with a user prompt) are still
39
49
  // undoable because they contain real user content.
50
+ // System notice text blocks (retry nudges, progress checks) are not user content.
40
51
  const hasNonToolResultContent = message.content.some(
41
- (block) => !isToolResultBlock(block),
52
+ (block) => !isToolResultBlock(block) && !isSystemNoticeBlock(block),
42
53
  );
43
54
  if (!hasNonToolResultContent) return false;
44
55
  return true;
@@ -131,10 +142,10 @@ export async function cleanupQdrantVectors(
131
142
  export function consolidateAssistantMessages(
132
143
  conversationId: string,
133
144
  userMessageId: string,
134
- ): void {
145
+ ): boolean {
135
146
  const allMessages = getMessages(conversationId);
136
147
  const userMsgIndex = allMessages.findIndex((m) => m.id === userMessageId);
137
- if (userMsgIndex === -1) return;
148
+ if (userMsgIndex === -1) return false;
138
149
 
139
150
  const messagesToConsolidate: typeof allMessages = [];
140
151
  const internalToolResultMessages: typeof allMessages = [];
@@ -171,12 +182,14 @@ export function consolidateAssistantMessages(
171
182
 
172
183
  // Only consolidate if there are multiple assistant messages
173
184
  if (messagesToConsolidate.length <= 1) {
185
+ let didMutate = false;
174
186
  // Still delete internal tool_result messages even if only one assistant message,
175
187
  // and collect IDs for vector cleanup
176
188
  const allSegmentIds: string[] = [];
177
189
  const allOrphanedItemIds: string[] = [];
178
190
  for (const id of messagesToDelete) {
179
191
  const deleted = deleteMessageById(id);
192
+ didMutate = true;
180
193
  allSegmentIds.push(...deleted.segmentIds);
181
194
  allOrphanedItemIds.push(...deleted.orphanedItemIds);
182
195
  }
@@ -194,7 +207,7 @@ export function consolidateAssistantMessages(
194
207
  );
195
208
  });
196
209
  }
197
- return;
210
+ return didMutate;
198
211
  }
199
212
 
200
213
  log.info(
@@ -339,6 +352,7 @@ export function consolidateAssistantMessages(
339
352
  },
340
353
  "Assistant messages consolidated",
341
354
  );
355
+ return true;
342
356
  }
343
357
 
344
358
  // ── Undo ─────────────────────────────────────────────────────────────
@@ -68,6 +68,39 @@ function filterMessagesForUntrustedActor(messages: MessageRow[]): MessageRow[] {
68
68
  });
69
69
  }
70
70
 
71
+ /**
72
+ * Re-inject image source path annotations into message content blocks.
73
+ *
74
+ * When the desktop client attaches images from local files, the source paths
75
+ * are stored in `metadata.imageSourcePaths` (keyed by filename). The LLM-facing
76
+ * content omits these paths at persistence time, so we re-inject them when
77
+ * loading history from the DB. Only user messages are annotated.
78
+ */
79
+ export function reinjectImageSourcePaths(
80
+ content: ContentBlock[],
81
+ role: string,
82
+ metadataJson: string | null,
83
+ ): ContentBlock[] {
84
+ if (role !== "user" || !metadataJson) return content;
85
+ try {
86
+ const meta = JSON.parse(metadataJson);
87
+ if (!meta.imageSourcePaths || typeof meta.imageSourcePaths !== "object") {
88
+ return content;
89
+ }
90
+ const paths = Object.values(meta.imageSourcePaths).filter(
91
+ (v): v is string => typeof v === "string",
92
+ );
93
+ if (paths.length === 0) return content;
94
+ const annotation = paths
95
+ .map((p) => `[Attached image source: ${p}]`)
96
+ .join("\n");
97
+ return [...content, { type: "text" as const, text: annotation }];
98
+ } catch {
99
+ // metadata parse failure — skip annotation, not critical
100
+ return content;
101
+ }
102
+ }
103
+
71
104
  // ── Context Interfaces ───────────────────────────────────────────────
72
105
 
73
106
  export interface LoadFromDbContext {
@@ -78,7 +111,6 @@ export interface LoadFromDbContext {
78
111
  contextCompactedAt: number | null;
79
112
  trustContext?: { trustClass: TrustClass };
80
113
  loadedHistoryTrustClass?: TrustClass;
81
- hasAttachments?: boolean;
82
114
  }
83
115
 
84
116
  export interface AbortContext {
@@ -93,6 +125,7 @@ export interface AbortContext {
93
125
  string,
94
126
  { surfaceType: SurfaceType; data: SurfaceData; title?: string }
95
127
  >;
128
+ accumulatedSurfaceState: Map<string, Record<string, unknown>>;
96
129
  readonly queue: MessageQueue;
97
130
  }
98
131
 
@@ -151,6 +184,9 @@ export async function loadFromDb(ctx: LoadFromDbContext): Promise<void> {
151
184
  );
152
185
  content = [{ type: "text", text: m.content }];
153
186
  }
187
+
188
+ content = reinjectImageSourcePaths(content, role, m.metadata);
189
+
154
190
  return { role, content };
155
191
  });
156
192
 
@@ -182,20 +218,6 @@ export async function loadFromDb(ctx: LoadFromDbContext): Promise<void> {
182
218
 
183
219
  ctx.loadedHistoryTrustClass = trustClass;
184
220
 
185
- // Scan ALL db messages (including compacted ones) for attachments so that
186
- // asset tools remain available after context compaction.
187
- if (
188
- ctx.contextCompactedMessageCount > 0 &&
189
- dbMessages.some(
190
- (m) =>
191
- m.role === "user" &&
192
- (m.content.includes('"type":"image"') ||
193
- m.content.includes('"type":"file"')),
194
- )
195
- ) {
196
- ctx.hasAttachments = true;
197
- }
198
-
199
221
  log.info(
200
222
  { conversationId: ctx.conversationId, count: ctx.messages.length },
201
223
  "Loaded messages from DB",
@@ -216,6 +238,7 @@ export function abortConversation(ctx: AbortContext): void {
216
238
  ctx.pendingSurfaceActions.clear();
217
239
  ctx.surfaceActionRequestIds.clear();
218
240
  ctx.surfaceState.clear();
241
+ ctx.accumulatedSurfaceState.clear();
219
242
  unregisterWatchNotifiers(ctx.conversationId);
220
243
  for (const queued of ctx.queue) {
221
244
  queued.onEvent({
@@ -247,6 +270,7 @@ export function disposeConversation(ctx: DisposeContext): void {
247
270
  ctx.pendingSurfaceActions.clear();
248
271
  ctx.surfaceActionRequestIds.clear();
249
272
  ctx.surfaceState.clear();
273
+ ctx.accumulatedSurfaceState.clear();
250
274
  ctx.lastSurfaceAction.clear();
251
275
  ctx.workspaceTopLevelContext = null;
252
276
  }
@@ -7,6 +7,7 @@
7
7
 
8
8
  import { v4 as uuid } from "uuid";
9
9
 
10
+ import { enrichMessageWithSourcePaths } from "../agent/attachments.js";
10
11
  import { createUserMessage } from "../agent/message-types.js";
11
12
  import type {
12
13
  TurnChannelContext,
@@ -14,17 +15,23 @@ import type {
14
15
  } from "../channels/types.js";
15
16
  import { parseChannelId, parseInterfaceId } from "../channels/types.js";
16
17
  import {
18
+ attachInlineAttachmentToMessage,
19
+ attachmentExists,
17
20
  AttachmentUploadError,
18
21
  linkAttachmentToMessage,
19
- uploadAttachment,
20
22
  validateAttachmentUpload,
21
23
  } from "../memory/attachments-store.js";
22
24
  import {
23
25
  addMessage,
26
+ getConversation,
24
27
  provenanceFromTrustContext,
25
28
  setConversationOriginChannelIfUnset,
26
29
  setConversationOriginInterfaceIfUnset,
27
30
  } from "../memory/conversation-crud.js";
31
+ import {
32
+ syncMessageToDisk,
33
+ updateMetaFile,
34
+ } from "../memory/conversation-disk-view.js";
28
35
  import type { SecretPrompter } from "../permissions/secret-prompter.js";
29
36
  import type { Message } from "../providers/types.js";
30
37
  import { getLogger } from "../util/logger.js";
@@ -276,17 +283,20 @@ export async function persistUserMessage(
276
283
  ctx.processing = true;
277
284
  ctx.abortController = new AbortController();
278
285
 
279
- const userMessage = createUserMessage(
280
- content,
281
- attachments.map((attachment) => ({
282
- id: attachment.id,
283
- filename: attachment.filename,
284
- mimeType: attachment.mimeType,
285
- data: attachment.data,
286
- extractedText: attachment.extractedText,
287
- })),
286
+ const attachmentInputs = attachments.map((attachment) => ({
287
+ id: attachment.id,
288
+ filename: attachment.filename,
289
+ mimeType: attachment.mimeType,
290
+ data: attachment.data,
291
+ extractedText: attachment.extractedText,
292
+ filePath: attachment.filePath,
293
+ }));
294
+ const cleanMessage = createUserMessage(content, attachmentInputs);
295
+ const llmMessage = enrichMessageWithSourcePaths(
296
+ cleanMessage,
297
+ attachmentInputs,
288
298
  );
289
- ctx.messages.push(userMessage);
299
+ ctx.messages.push(llmMessage);
290
300
 
291
301
  try {
292
302
  const turnCtx =
@@ -294,6 +304,14 @@ export async function persistUserMessage(
294
304
  const turnIfCtx =
295
305
  extractTurnInterfaceContext(metadata) ?? ctx.getTurnInterfaceContext();
296
306
  const provenance = provenanceFromTrustContext(ctx.trustContext);
307
+ const imageSourcePaths: Record<string, string> = {};
308
+ for (let i = 0; i < attachments.length; i++) {
309
+ const a = attachments[i];
310
+ if (a.filePath && a.mimeType.toLowerCase().startsWith("image/")) {
311
+ imageSourcePaths[`${i}:${a.filename}`] = a.filePath;
312
+ }
313
+ }
314
+
297
315
  const mergedMetadata = {
298
316
  ...(metadata ?? {}),
299
317
  ...provenance,
@@ -309,6 +327,7 @@ export async function persistUserMessage(
309
327
  assistantMessageInterface: turnIfCtx.assistantMessageInterface,
310
328
  }
311
329
  : {}),
330
+ ...(Object.keys(imageSourcePaths).length > 0 ? { imageSourcePaths } : {}),
312
331
  };
313
332
 
314
333
  // When displayContent is provided (e.g. original text before recording
@@ -317,18 +336,9 @@ export async function persistUserMessage(
317
336
  // the stripped content.
318
337
  const contentToPersist = displayContent
319
338
  ? JSON.stringify(
320
- createUserMessage(
321
- displayContent,
322
- attachments.map((a) => ({
323
- id: a.id,
324
- filename: a.filename,
325
- mimeType: a.mimeType,
326
- data: a.data,
327
- extractedText: a.extractedText,
328
- })),
329
- ).content,
339
+ createUserMessage(displayContent, attachmentInputs).content,
330
340
  )
331
- : JSON.stringify(userMessage.content);
341
+ : JSON.stringify(cleanMessage.content);
332
342
  const persistedUserMessage = await addMessage(
333
343
  ctx.conversationId,
334
344
  "user",
@@ -349,15 +359,33 @@ export async function persistUserMessage(
349
359
  );
350
360
  }
351
361
 
362
+ // Rewrite meta.json so the on-disk metadata reflects the origin channel
363
+ if (turnCtx || turnIfCtx) {
364
+ const convForMeta = getConversation(ctx.conversationId);
365
+ if (convForMeta) {
366
+ updateMetaFile(convForMeta);
367
+ }
368
+ }
369
+
352
370
  if (!persistedUserMessage.id) {
353
371
  throw new Error("Failed to persist user message");
354
372
  }
355
373
 
356
- // Index user attachments in the attachments table so asset_search can find them.
374
+ // Index user attachments in the attachments table for later retrieval.
357
375
  for (let i = 0; i < attachments.length; i++) {
358
376
  const a = attachments[i];
359
- if (!a.data) continue;
360
377
  try {
378
+ // If the attachment already exists in the store (e.g. file-backed
379
+ // attachments uploaded separately), link it directly without
380
+ // re-uploading. This handles the case where data is empty because
381
+ // the attachment content lives on disk.
382
+ if (a.id && attachmentExists(a.id)) {
383
+ linkAttachmentToMessage(persistedUserMessage.id, a.id, i);
384
+ continue;
385
+ }
386
+
387
+ if (!a.data) continue;
388
+
361
389
  const validation = validateAttachmentUpload(a.filename, a.mimeType);
362
390
  if (!validation.ok) {
363
391
  log.warn(
@@ -366,8 +394,14 @@ export async function persistUserMessage(
366
394
  );
367
395
  continue;
368
396
  }
369
- const stored = uploadAttachment(a.filename, a.mimeType, a.data);
370
- linkAttachmentToMessage(persistedUserMessage.id, stored.id, i);
397
+ attachInlineAttachmentToMessage(
398
+ persistedUserMessage.id,
399
+ i,
400
+ a.filename,
401
+ a.mimeType,
402
+ a.data,
403
+ { sourcePath: a.filePath },
404
+ );
371
405
  } catch (err) {
372
406
  if (err instanceof AttachmentUploadError) {
373
407
  log.warn(
@@ -383,6 +417,16 @@ export async function persistUserMessage(
383
417
  }
384
418
  }
385
419
 
420
+ // Sync the persisted user message (with attachments) to the disk view
421
+ const conv = getConversation(ctx.conversationId);
422
+ if (conv) {
423
+ syncMessageToDisk(
424
+ ctx.conversationId,
425
+ persistedUserMessage.id,
426
+ conv.createdAt,
427
+ );
428
+ }
429
+
386
430
  return persistedUserMessage.id;
387
431
  } catch (err) {
388
432
  ctx.messages.pop();
@@ -6,6 +6,7 @@
6
6
  * used by conversation-history.ts.
7
7
  */
8
8
 
9
+ import { enrichMessageWithSourcePaths } from "../agent/attachments.js";
9
10
  import {
10
11
  createAssistantMessage,
11
12
  createUserMessage,
@@ -25,18 +26,14 @@ import {
25
26
  } from "../memory/conversation-crud.js";
26
27
  import { extractPreferences } from "../notifications/preference-extractor.js";
27
28
  import { createPreference } from "../notifications/preferences-store.js";
28
- import { getConfiguredProviders } from "../providers/provider-availability.js";
29
29
  import type { Message } from "../providers/types.js";
30
30
  import { routeGuardianReply } from "../runtime/guardian-reply-router.js";
31
31
  import { getLogger } from "../util/logger.js";
32
32
  import type { MessageQueue } from "./conversation-queue-manager.js";
33
33
  import type { QueueDrainReason } from "./conversation-queue-manager.js";
34
34
  import type { TrustContext } from "./conversation-runtime-assembly.js";
35
- import {
36
- isProviderShortcut,
37
- resolveSlash,
38
- type SlashContext,
39
- } from "./conversation-slash.js";
35
+ import { resolveSlash, type SlashContext } from "./conversation-slash.js";
36
+ import { getModelInfo } from "./handlers/config-model.js";
40
37
  import type {
41
38
  ServerMessage,
42
39
  UsageStats,
@@ -49,23 +46,12 @@ const log = getLogger("conversation-process");
49
46
 
50
47
  /** Build a model_info event with fresh config data. */
51
48
  export async function buildModelInfoEvent(): Promise<ServerMessage> {
52
- const config = getConfig();
53
- return {
54
- type: "model_info",
55
- model: config.services.inference.model,
56
- provider: config.services.inference.provider,
57
- configuredProviders: await getConfiguredProviders(),
58
- };
49
+ return { type: "model_info", ...(await getModelInfo()) };
59
50
  }
60
51
 
61
- /** True when the trimmed content is a /model or /models slash command. */
52
+ /** True when the trimmed content is the /models slash command. */
62
53
  export function isModelSlashCommand(content: string): boolean {
63
- const trimmed = content.trim();
64
- return (
65
- trimmed === "/model" ||
66
- trimmed === "/models" ||
67
- trimmed.startsWith("/model ")
68
- );
54
+ return content.trim() === "/models";
69
55
  }
70
56
 
71
57
  // ── Context Interface ────────────────────────────────────────────────
@@ -196,6 +182,7 @@ function buildSlashContext(
196
182
  conversation: ProcessConversationContext,
197
183
  ): SlashContext {
198
184
  const config = getConfig();
185
+ const turnInterface = conversation.getTurnInterfaceContext();
199
186
  return {
200
187
  messageCount: conversation.messages.length,
201
188
  inputTokens: conversation.usageStats.inputTokens,
@@ -204,6 +191,7 @@ function buildSlashContext(
204
191
  model: config.services.inference.model,
205
192
  provider: config.services.inference.provider,
206
193
  estimatedCost: conversation.usageStats.estimatedCost,
194
+ userMessageInterface: turnInterface?.userMessageInterface,
207
195
  };
208
196
  }
209
197
 
@@ -308,6 +296,13 @@ export async function drainQueue(
308
296
  const drainProvenance = provenanceFromTrustContext(
309
297
  conversation.trustContext,
310
298
  );
299
+ const drainImageSourcePaths: Record<string, string> = {};
300
+ for (let i = 0; i < next.attachments.length; i++) {
301
+ const a = next.attachments[i];
302
+ if (a.filePath && a.mimeType.toLowerCase().startsWith("image/")) {
303
+ drainImageSourcePaths[`${i}:${a.filename}`] = a.filePath;
304
+ }
305
+ }
311
306
  const drainChannelMeta = {
312
307
  ...drainProvenance,
313
308
  ...(queuedTurnCtx
@@ -324,8 +319,15 @@ export async function drainQueue(
324
319
  }
325
320
  : {}),
326
321
  ...(next.metadata?.automated ? { automated: true } : {}),
322
+ ...(Object.keys(drainImageSourcePaths).length > 0
323
+ ? { imageSourcePaths: drainImageSourcePaths }
324
+ : {}),
327
325
  };
328
- const userMsg = createUserMessage(next.content, next.attachments);
326
+ const cleanUserMsg = createUserMessage(next.content, next.attachments);
327
+ const llmUserMsg = enrichMessageWithSourcePaths(
328
+ cleanUserMsg,
329
+ next.attachments,
330
+ );
329
331
  // When displayContent is provided (e.g. original text before recording
330
332
  // intent stripping), persist that to DB so users see the full message.
331
333
  // The in-memory userMessage (sent to the LLM) still uses the stripped content.
@@ -333,14 +335,14 @@ export async function drainQueue(
333
335
  ? JSON.stringify(
334
336
  createUserMessage(next.displayContent, next.attachments).content,
335
337
  )
336
- : JSON.stringify(userMsg.content);
338
+ : JSON.stringify(cleanUserMsg.content);
337
339
  await addMessage(
338
340
  conversation.conversationId,
339
341
  "user",
340
342
  contentToPersist,
341
343
  drainChannelMeta,
342
344
  );
343
- conversation.messages.push(userMsg);
345
+ conversation.messages.push(llmUserMsg);
344
346
 
345
347
  const assistantMsg = createAssistantMessage(slashResult.message);
346
348
  await addMessage(
@@ -366,10 +368,7 @@ export async function drainQueue(
366
368
 
367
369
  // Emit fresh model info before the text delta so the client has
368
370
  // up-to-date configuredProviders when rendering /model or /models UI.
369
- if (
370
- isModelSlashCommand(next.content) ||
371
- isProviderShortcut(next.content)
372
- ) {
371
+ if (isModelSlashCommand(next.content)) {
373
372
  next.onEvent(await buildModelInfoEvent());
374
373
  }
375
374
  next.onEvent({ type: "assistant_text_delta", text: slashResult.message });
@@ -602,6 +601,13 @@ export async function processMessage(
602
601
 
603
602
  if (routerResult.consumed) {
604
603
  const guardianIfCtx = conversation.getTurnInterfaceContext();
604
+ const guardianImageSourcePaths: Record<string, string> = {};
605
+ for (let i = 0; i < attachments.length; i++) {
606
+ const a = attachments[i];
607
+ if (a.filePath && a.mimeType.toLowerCase().startsWith("image/")) {
608
+ guardianImageSourcePaths[`${i}:${a.filename}`] = a.filePath;
609
+ }
610
+ }
605
611
  const routerChannelMeta = {
606
612
  userMessageChannel: "vellum" as const,
607
613
  assistantMessageChannel: "vellum" as const,
@@ -609,16 +615,23 @@ export async function processMessage(
609
615
  assistantMessageInterface:
610
616
  guardianIfCtx?.assistantMessageInterface ?? "vellum",
611
617
  provenanceTrustClass: "guardian" as const,
618
+ ...(Object.keys(guardianImageSourcePaths).length > 0
619
+ ? { imageSourcePaths: guardianImageSourcePaths }
620
+ : {}),
612
621
  };
613
622
 
614
- const userMsg = createUserMessage(content, attachments);
623
+ const cleanUserMsg = createUserMessage(content, attachments);
624
+ const llmUserMsg = enrichMessageWithSourcePaths(
625
+ cleanUserMsg,
626
+ attachments,
627
+ );
615
628
  const persisted = await addMessage(
616
629
  conversation.conversationId,
617
630
  "user",
618
- JSON.stringify(userMsg.content),
631
+ JSON.stringify(cleanUserMsg.content),
619
632
  routerChannelMeta,
620
633
  );
621
- conversation.messages.push(userMsg);
634
+ conversation.messages.push(llmUserMsg);
622
635
 
623
636
  const replyText =
624
637
  routerResult.replyText ??
@@ -666,6 +679,13 @@ export async function processMessage(
666
679
  const pmTurnCtx = conversation.getTurnChannelContext();
667
680
  const pmInterfaceCtx = conversation.getTurnInterfaceContext();
668
681
  const pmProvenance = provenanceFromTrustContext(conversation.trustContext);
682
+ const pmImageSourcePaths: Record<string, string> = {};
683
+ for (let i = 0; i < attachments.length; i++) {
684
+ const a = attachments[i];
685
+ if (a.filePath && a.mimeType.toLowerCase().startsWith("image/")) {
686
+ pmImageSourcePaths[`${i}:${a.filename}`] = a.filePath;
687
+ }
688
+ }
669
689
  const pmChannelMeta = {
670
690
  ...pmProvenance,
671
691
  ...(pmTurnCtx
@@ -680,21 +700,25 @@ export async function processMessage(
680
700
  assistantMessageInterface: pmInterfaceCtx.assistantMessageInterface,
681
701
  }
682
702
  : {}),
703
+ ...(Object.keys(pmImageSourcePaths).length > 0
704
+ ? { imageSourcePaths: pmImageSourcePaths }
705
+ : {}),
683
706
  };
684
- const userMsg = createUserMessage(content, attachments);
707
+ const cleanUserMsg = createUserMessage(content, attachments);
708
+ const llmUserMsg = enrichMessageWithSourcePaths(cleanUserMsg, attachments);
685
709
  // When displayContent is provided (e.g. original text before recording
686
710
  // intent stripping), persist that to DB so users see the full message.
687
711
  // The in-memory userMessage (sent to the LLM) still uses the stripped content.
688
712
  const contentToPersist = displayContent
689
713
  ? JSON.stringify(createUserMessage(displayContent, attachments).content)
690
- : JSON.stringify(userMsg.content);
714
+ : JSON.stringify(cleanUserMsg.content);
691
715
  const persisted = await addMessage(
692
716
  conversation.conversationId,
693
717
  "user",
694
718
  contentToPersist,
695
719
  pmChannelMeta,
696
720
  );
697
- conversation.messages.push(userMsg);
721
+ conversation.messages.push(llmUserMsg);
698
722
 
699
723
  const assistantMsg = createAssistantMessage(slashResult.message);
700
724
  await addMessage(
@@ -720,7 +744,7 @@ export async function processMessage(
720
744
 
721
745
  // Emit fresh model info before the text delta so the client has
722
746
  // up-to-date configuredProviders when rendering /model or /models UI.
723
- if (isModelSlashCommand(content) || isProviderShortcut(content)) {
747
+ if (isModelSlashCommand(content)) {
724
748
  onEvent(await buildModelInfoEvent());
725
749
  }
726
750
  onEvent({ type: "assistant_text_delta", text: slashResult.message });