@vellumai/assistant 0.11.4 → 0.11.5-dev.202608220019.a778cf0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (457) hide show
  1. package/AGENTS.md +6 -2
  2. package/ARCHITECTURE.md +29 -4
  3. package/README.md +1 -1
  4. package/docs/architecture/memory.md +17 -5
  5. package/docs/architecture/security.md +96 -152
  6. package/docs/guardian-request-flow.md +13 -2
  7. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
  8. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
  9. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/url-normalization.test.ts +135 -0
  10. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  11. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
  12. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/url-normalization.ts +107 -0
  13. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
  14. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
  15. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/url-normalization.test.ts +135 -0
  16. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  17. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
  18. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/url-normalization.ts +107 -0
  19. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +166 -1
  20. package/node_modules/@vellumai/gateway-client/src/http-delivery.ts +7 -14
  21. package/node_modules/@vellumai/gateway-client/src/index.ts +2 -0
  22. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +30 -32
  23. package/node_modules/@vellumai/service-contracts/package.json +1 -0
  24. package/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
  25. package/node_modules/@vellumai/service-contracts/src/__tests__/url-normalization.test.ts +135 -0
  26. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  27. package/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
  28. package/node_modules/@vellumai/service-contracts/src/url-normalization.ts +107 -0
  29. package/openapi.yaml +487 -11
  30. package/package.json +1 -1
  31. package/scripts/sync-web-search-catalog.ts +6 -0
  32. package/src/__tests__/anthropic-provider.test.ts +10 -3
  33. package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
  34. package/src/__tests__/background-workers-disk-pressure.test.ts +1 -1
  35. package/src/__tests__/channel-approval-routes.test.ts +4 -0
  36. package/src/__tests__/channel-inbound-disk-pressure.test.ts +5 -6
  37. package/src/__tests__/channel-readiness-service.test.ts +126 -0
  38. package/src/__tests__/channel-reply-delivery.test.ts +30 -17
  39. package/src/__tests__/checker.test.ts +20 -314
  40. package/src/__tests__/cli-memory-v2-reembed-skills.test.ts +6 -2
  41. package/src/__tests__/client-os-metadata-persistence.test.ts +16 -0
  42. package/src/__tests__/compactor-retained-image-validation.test.ts +142 -0
  43. package/src/__tests__/config-loader-backfill.test.ts +19 -10
  44. package/src/__tests__/container-cpu-sampler.test.ts +153 -0
  45. package/src/__tests__/context-search-memory-v2-source.test.ts +0 -1
  46. package/src/__tests__/conversation-attachments.test.ts +0 -1
  47. package/src/__tests__/conversation-delete-watch-timeline.test.ts +231 -0
  48. package/src/__tests__/conversation-error.test.ts +17 -0
  49. package/src/__tests__/conversation-lifecycle.test.ts +20 -26
  50. package/src/__tests__/conversation-pairing.test.ts +186 -0
  51. package/src/__tests__/conversation-runtime-assembly.test.ts +21 -2
  52. package/src/__tests__/credential-broker-server-use.test.ts +24 -5
  53. package/src/__tests__/credential-routes.test.ts +22 -3
  54. package/src/__tests__/credential-security-invariants.test.ts +2 -1
  55. package/src/__tests__/daemon-credential-client.test.ts +88 -0
  56. package/src/__tests__/default-plugin-names-guard.test.ts +12 -1
  57. package/src/__tests__/dm-backfill.test.ts +63 -0
  58. package/src/__tests__/dynamic-skill-background-guard.test.ts +0 -2
  59. package/src/__tests__/edit-propagation.test.ts +107 -4
  60. package/src/__tests__/events-client-registration.test.ts +28 -0
  61. package/src/__tests__/file-write-tool.test.ts +4 -2
  62. package/src/__tests__/guardian-prompt-notice-privacy.test.ts +133 -0
  63. package/src/__tests__/guardian-verify-setup-skill-regression.test.ts +143 -97
  64. package/src/__tests__/helpers/gateway-classify-mock.ts +27 -6
  65. package/src/__tests__/host-proxy-interface.test.ts +11 -1
  66. package/src/__tests__/host-shell-tool.test.ts +23 -4
  67. package/src/__tests__/image-conversion.test.ts +143 -1
  68. package/src/__tests__/inline-skill-load-permissions.test.ts +47 -36
  69. package/src/__tests__/input-repairs.test.ts +207 -0
  70. package/src/__tests__/live-workspace-guard.test.ts +75 -0
  71. package/src/__tests__/managed-profile-guard.test.ts +4 -2
  72. package/src/__tests__/mcp-abort-signal.test.ts +1 -1
  73. package/src/__tests__/mcp-client-auth.test.ts +1 -1
  74. package/src/__tests__/mcp-tool-annotations-risk.test.ts +1 -1
  75. package/src/__tests__/media-resolve-image-validation.test.ts +310 -0
  76. package/src/__tests__/mtime-cache.test.ts +2 -0
  77. package/src/__tests__/notification-vellum-adapter.test.ts +124 -1
  78. package/src/__tests__/openai-provider.test.ts +40 -0
  79. package/src/__tests__/openai-responses-provider.test.ts +18 -0
  80. package/src/__tests__/personal-memory-auth-bypass.test.ts +22 -6
  81. package/src/__tests__/platform-bash-auto-approve.test.ts +0 -4
  82. package/src/__tests__/platform.test.ts +16 -1
  83. package/src/__tests__/plugin-api-resolve-credential.test.ts +22 -13
  84. package/src/__tests__/plugin-api-shim.test.ts +5 -0
  85. package/src/__tests__/plugin-api-store-credential.test.ts +268 -0
  86. package/src/__tests__/plugin-config-data-migration.test.ts +8 -1
  87. package/src/__tests__/plugin-disabled-state.test.ts +2 -0
  88. package/src/__tests__/plugin-effective-enabled-set.test.ts +55 -3
  89. package/src/__tests__/plugin-execution-context.test.ts +73 -0
  90. package/src/__tests__/plugin-import-boundary-guard.test.ts +0 -1
  91. package/src/__tests__/plugin-import-boundary-reverse-guard.test.ts +0 -1
  92. package/src/__tests__/reaction-persistence.test.ts +200 -7
  93. package/src/__tests__/require-fresh-approval.test.ts +0 -4
  94. package/src/__tests__/resource-pressure-guard.test.ts +428 -0
  95. package/src/__tests__/resource-pressure-routes.test.ts +113 -0
  96. package/src/__tests__/risk-classification-boundary-guard.test.ts +115 -0
  97. package/src/__tests__/run-conversation-turn-persistence.test.ts +27 -1
  98. package/src/__tests__/sanitize-config-for-transfer.test.ts +28 -0
  99. package/src/__tests__/secret-routes-platform-proxy.test.ts +7 -0
  100. package/src/__tests__/settings-routes.test.ts +85 -3
  101. package/src/__tests__/skill-tool-factory.test.ts +161 -0
  102. package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +1 -1
  103. package/src/__tests__/tool-executor-lifecycle-events.test.ts +132 -5
  104. package/src/__tests__/tool-executor.test.ts +63 -38
  105. package/src/__tests__/tool-policy.test.ts +97 -1
  106. package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +7 -4
  107. package/src/__tests__/user-plugin-loader.test.ts +2 -0
  108. package/src/__tests__/validate-input.test.ts +243 -7
  109. package/src/__tests__/verification-control-plane-policy.test.ts +0 -2
  110. package/src/__tests__/voice-scoped-grant-consumer.test.ts +2 -0
  111. package/src/__tests__/voice-session-bridge.test.ts +42 -0
  112. package/src/__tests__/web-search-catalog-parity.test.ts +8 -0
  113. package/src/acp/__tests__/acp-claude-oauth.test.ts +97 -29
  114. package/src/acp/__tests__/prepare-agent-env.test.ts +376 -153
  115. package/src/acp/acp-claude-oauth.ts +38 -21
  116. package/src/acp/prepare-agent-env.ts +146 -55
  117. package/src/agent/loop-tool-dedup.test.ts +161 -0
  118. package/src/agent/loop.ts +68 -22
  119. package/src/api/constants/app-tools.ts +34 -0
  120. package/src/api/constants/profile-config-validation.ts +60 -0
  121. package/src/api/events/resource-pressure-status-changed.ts +53 -0
  122. package/src/api/events/tool-result.ts +6 -1
  123. package/src/api/events/watch-retro-completed.ts +52 -0
  124. package/src/api/index.ts +30 -0
  125. package/src/api/responses/resource-pressure-status.ts +25 -0
  126. package/src/approvals/guardian-channel-delivery.ts +70 -6
  127. package/src/approvals/guardian-request-resolvers.ts +32 -66
  128. package/src/calls/__tests__/voice-session-bridge.test.ts +114 -0
  129. package/src/calls/voice-session-bridge.ts +60 -6
  130. package/src/channels/__tests__/message-audience.test.ts +49 -0
  131. package/src/channels/__tests__/types.test.ts +17 -3
  132. package/src/channels/message-audience.ts +40 -0
  133. package/src/channels/types.ts +16 -12
  134. package/src/cli/__tests__/catalog-search-help.test.ts +50 -0
  135. package/src/cli/commands/__tests__/channel-verification-sessions.test.ts +31 -0
  136. package/src/cli/commands/__tests__/conversations-search.test.ts +289 -0
  137. package/src/cli/commands/__tests__/conversations-slack.test.ts +2 -1
  138. package/src/cli/commands/__tests__/inference-profiles.test.ts +16 -4
  139. package/src/cli/commands/__tests__/inference-providers.test.ts +135 -2
  140. package/src/cli/commands/__tests__/keys.test.ts +89 -8
  141. package/src/cli/commands/__tests__/plugins.test.ts +57 -1
  142. package/src/cli/commands/channel-verification-sessions.help.ts +11 -9
  143. package/src/cli/commands/channel-verification-sessions.ts +11 -21
  144. package/src/cli/commands/conversations.help.ts +34 -0
  145. package/src/cli/commands/conversations.ts +125 -0
  146. package/src/cli/commands/inference-profiles.ts +56 -3
  147. package/src/cli/commands/inference-providers.ts +37 -6
  148. package/src/cli/commands/inference.help.ts +9 -4
  149. package/src/cli/commands/keys.help.ts +13 -1
  150. package/src/cli/commands/keys.ts +26 -15
  151. package/src/cli/commands/memory/__tests__/memory-v2.test.ts +57 -7
  152. package/src/cli/commands/memory/index.help.ts +41 -27
  153. package/src/cli/commands/memory/index.ts +2 -0
  154. package/src/cli/commands/memory/memory-v2.ts +58 -54
  155. package/src/cli/commands/memory/memory-validate.ts +18 -0
  156. package/src/cli/commands/monitoring.ts +1 -0
  157. package/src/cli/commands/oauth/request.test.ts +93 -0
  158. package/src/cli/commands/oauth/request.ts +23 -9
  159. package/src/cli/commands/plugins.help.ts +28 -31
  160. package/src/cli/commands/plugins.ts +10 -0
  161. package/src/cli/commands/skills.help.ts +13 -16
  162. package/src/cli/commands/trust.ts +3 -14
  163. package/src/cli/lib/__tests__/install-from-github.test.ts +38 -0
  164. package/src/cli/lib/__tests__/merge-plugin-tree.test.ts +34 -0
  165. package/src/cli/lib/__tests__/plugin-surfaces.test.ts +32 -1
  166. package/src/cli/lib/__tests__/toggle-plugin.test.ts +2 -1
  167. package/src/cli/lib/__tests__/upgrade-plugin.test.ts +64 -0
  168. package/src/cli/lib/bundled-marketplace.json +26 -0
  169. package/src/cli/lib/daemon-credential-client.ts +25 -3
  170. package/src/cli/lib/install-from-github.ts +35 -24
  171. package/src/cli/lib/merge-plugin-tree.ts +22 -4
  172. package/src/cli/lib/open-browser.test.ts +67 -0
  173. package/src/cli/lib/open-browser.ts +24 -5
  174. package/src/cli/lib/plugin-surfaces.ts +27 -0
  175. package/src/config/__tests__/balanced-model-experiment.test.ts +7 -7
  176. package/src/config/__tests__/default-profile-catalog.test.ts +3 -3
  177. package/src/config/__tests__/profile-materialization.test.ts +26 -0
  178. package/src/config/default-profile-catalog.ts +5 -6
  179. package/src/config/feature-flag-registry.json +36 -28
  180. package/src/config/profile-materialization.ts +29 -0
  181. package/src/config/sanitize-for-transfer.ts +16 -0
  182. package/src/config/schemas/llm.ts +7 -0
  183. package/src/config/schemas/services.ts +6 -0
  184. package/src/config/webhook-routing.ts +8 -0
  185. package/src/context/compactor.ts +31 -2
  186. package/src/context/outbound-sanitize.ts +6 -0
  187. package/src/daemon/__tests__/lifecycle-watch-timeline-sweep.test.ts +98 -0
  188. package/src/daemon/__tests__/provider-rejection-log-fields.test.ts +272 -0
  189. package/src/daemon/conversation-agent-loop-handlers.ts +4 -1
  190. package/src/daemon/conversation-error.ts +19 -2
  191. package/src/daemon/conversation-messaging.ts +22 -2
  192. package/src/daemon/conversation-process.ts +11 -0
  193. package/src/daemon/conversation-store.ts +10 -0
  194. package/src/daemon/conversation-surfaces.ts +21 -8
  195. package/src/daemon/conversation.ts +12 -10
  196. package/src/daemon/daemon-control.ts +1 -0
  197. package/src/daemon/handlers/__tests__/config-ingress-tunnel-records.test.ts +208 -0
  198. package/src/daemon/handlers/config-ingress.ts +115 -5
  199. package/src/daemon/lifecycle.ts +30 -2
  200. package/src/daemon/message-types/conversations.ts +5 -7
  201. package/src/daemon/message-types/web-activity.ts +3 -2
  202. package/src/daemon/provider-rejection-log-fields.ts +124 -0
  203. package/src/daemon/resource-pressure-guard-lifecycle.ts +66 -0
  204. package/src/daemon/resource-pressure-guard.ts +387 -0
  205. package/src/daemon/shutdown-handlers.ts +2 -0
  206. package/src/daemon/startup-error.ts +16 -0
  207. package/src/daemon/trust-context.ts +14 -10
  208. package/src/daemon/unsendable-image-notice.ts +76 -0
  209. package/src/hooks/registry.ts +65 -51
  210. package/src/inbound/__tests__/tunnel-probe.test.ts +448 -0
  211. package/src/inbound/tunnel-probe.ts +261 -0
  212. package/src/ipc/__tests__/socket-path.test.ts +15 -7
  213. package/src/ipc/gateway-client.test.ts +1 -1
  214. package/src/ipc/gateway-client.ts +14 -21
  215. package/src/ipc/socket-cleanup.ts +2 -11
  216. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +1 -1
  217. package/src/messaging/provider-message-metadata.ts +128 -0
  218. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +138 -73
  219. package/src/messaging/providers/channel-transport.ts +100 -16
  220. package/src/messaging/providers/discord/send.ts +16 -0
  221. package/src/messaging/providers/discord/transport.ts +23 -1
  222. package/src/messaging/providers/index.ts +84 -16
  223. package/src/messaging/providers/slack/message-metadata.ts +45 -0
  224. package/src/messaging/providers/slack/render-transcript.ts +1 -4
  225. package/src/messaging/providers/slack/send.test.ts +88 -34
  226. package/src/messaging/providers/slack/send.ts +99 -50
  227. package/src/messaging/providers/slack/transport.ts +35 -37
  228. package/src/messaging/providers/telegram-bot/send.test.ts +88 -18
  229. package/src/messaging/providers/telegram-bot/send.ts +36 -0
  230. package/src/messaging/providers/telegram-bot/transport.ts +22 -8
  231. package/src/messaging/read-provider-metadata.test.ts +157 -0
  232. package/src/messaging/read-provider-metadata.ts +51 -0
  233. package/src/notifications/__tests__/assistant-reply-producer.test.ts +94 -0
  234. package/src/notifications/__tests__/edit-notification.test.ts +174 -2
  235. package/src/notifications/__tests__/home-feed-side-effect.test.ts +536 -4
  236. package/src/notifications/adapters/macos.ts +55 -0
  237. package/src/notifications/adapters/slack.ts +10 -5
  238. package/src/notifications/assistant-reply-producer.ts +37 -4
  239. package/src/notifications/conversation-pairing.ts +111 -21
  240. package/src/notifications/edit-notification.ts +38 -8
  241. package/src/notifications/emit-signal.ts +12 -14
  242. package/src/notifications/home-feed-side-effect.ts +218 -15
  243. package/src/notifications/types.ts +5 -0
  244. package/src/permissions/AGENTS.md +16 -0
  245. package/src/permissions/checker.test.ts +75 -122
  246. package/src/permissions/checker.ts +44 -560
  247. package/src/permissions/confirmation-guardian-request.test.ts +28 -3
  248. package/src/permissions/confirmation-guardian-request.ts +5 -1
  249. package/src/persistence/conversation-crud.ts +69 -20
  250. package/src/persistence/conversation-queries.ts +14 -1
  251. package/src/persistence/conversation-types.ts +22 -0
  252. package/src/persistence/db-async-query.ts +2 -1
  253. package/src/persistence/db-maintenance.ts +15 -0
  254. package/src/persistence/delivery-crud.ts +122 -15
  255. package/src/persistence/embeddings/qdrant-manager.ts +1 -0
  256. package/src/persistence/migrations/367-create-watch-timeline-entries.ts +46 -0
  257. package/src/persistence/migrations/368-watch-timeline-screenshot-blob.ts +33 -0
  258. package/src/persistence/migrations/__tests__/367-create-watch-timeline-entries.test.ts +98 -0
  259. package/src/persistence/migrations/__tests__/368-watch-timeline-screenshot-blob.test.ts +98 -0
  260. package/src/persistence/schema/index.ts +1 -0
  261. package/src/persistence/schema/watch.ts +29 -0
  262. package/src/persistence/steps.ts +4 -0
  263. package/src/plugin-api/__tests__/import-graph-partial-mock.test.ts +67 -0
  264. package/src/plugin-api/conversation-turn.ts +27 -0
  265. package/src/plugin-api/credential-scope.test.ts +15 -0
  266. package/src/plugin-api/credential-scope.ts +14 -0
  267. package/src/plugin-api/index.ts +19 -1
  268. package/src/plugin-api/resolve-credential.ts +15 -10
  269. package/src/plugin-api/store-credential.ts +148 -0
  270. package/src/plugin-api/system-card.ts +36 -0
  271. package/src/plugin-api/vision-support.test.ts +82 -0
  272. package/src/plugin-api/vision-support.ts +14 -2
  273. package/src/plugins/defaults/image-fallback/__tests__/image-fallback.test.ts +361 -110
  274. package/src/plugins/defaults/image-fallback/__tests__/vision-recovery.test.ts +4 -0
  275. package/src/plugins/defaults/image-fallback/hooks/user-prompt-submit.ts +58 -0
  276. package/src/plugins/defaults/image-fallback/src/caption-blocks.ts +64 -23
  277. package/src/plugins/defaults/image-recovery/detect.ts +24 -1
  278. package/src/plugins/defaults/main.ts +15 -0
  279. package/src/plugins/defaults/memory/AGENTS.md +11 -3
  280. package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +0 -1
  281. package/src/plugins/defaults/memory/graph-topology/pending-buffer.ts +9 -12
  282. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
  283. package/src/plugins/defaults/memory/src/memory-v2-routes.ts +12 -10
  284. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +114 -0
  285. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +8 -0
  286. package/src/plugins/defaults/memory/substrate/__tests__/edge-index.test.ts +0 -30
  287. package/src/plugins/defaults/memory/substrate/__tests__/ingest.test.ts +73 -1
  288. package/src/plugins/defaults/memory/substrate/__tests__/page-index.test.ts +37 -0
  289. package/src/plugins/defaults/memory/substrate/__tests__/page-links.test.ts +208 -0
  290. package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +158 -1
  291. package/src/plugins/defaults/memory/substrate/__tests__/static-context.test.ts +1 -61
  292. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +68 -6
  293. package/src/plugins/defaults/memory/substrate/edge-index.ts +0 -31
  294. package/src/plugins/defaults/memory/substrate/ingest.ts +59 -0
  295. package/src/plugins/defaults/memory/substrate/page-index.ts +35 -8
  296. package/src/plugins/defaults/memory/substrate/page-links.ts +133 -0
  297. package/src/plugins/defaults/memory/substrate/page-store.ts +10 -0
  298. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +124 -34
  299. package/src/plugins/defaults/memory/substrate/static-context.ts +0 -29
  300. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +3 -1
  301. package/src/plugins/defaults/memory/v3/card.ts +9 -9
  302. package/src/plugins/defaults/memory/v3/core-set.test.ts +7 -0
  303. package/src/plugins/defaults/memory/v3/core-set.ts +5 -1
  304. package/src/plugins/defaults/memory/v3/edge.ts +13 -57
  305. package/src/plugins/mtime-cache.ts +1 -1
  306. package/src/plugins/pipeline.ts +14 -7
  307. package/src/plugins/plugin-execution-context.ts +35 -11
  308. package/src/plugins/plugin-tree-walk.ts +63 -4
  309. package/src/providers/__tests__/retry-network-error.test.ts +84 -0
  310. package/src/providers/anthropic/__tests__/pause-turn-continuation.test.ts +170 -0
  311. package/src/providers/anthropic/client.ts +332 -241
  312. package/src/providers/connection-resolution.ts +25 -2
  313. package/src/providers/content-blocks.ts +22 -0
  314. package/src/providers/fetch-provider-catalog.ts +19 -0
  315. package/src/providers/gemini/client.ts +5 -2
  316. package/src/providers/inference/__tests__/adapter-factory-litellm.test.ts +47 -1
  317. package/src/providers/inference/__tests__/adapter-factory-ollama.test.ts +83 -0
  318. package/src/providers/inference/__tests__/adapter-factory-openai-compatible.test.ts +98 -1
  319. package/src/providers/inference/__tests__/base-url-route-validation.test.ts +38 -0
  320. package/src/providers/inference/__tests__/base-url-security.test.ts +10 -0
  321. package/src/providers/inference/__tests__/connections-ollama.test.ts +90 -0
  322. package/src/providers/inference/__tests__/endpoint-probe.test.ts +92 -0
  323. package/src/providers/inference/__tests__/missing-credential-guard.test.ts +117 -0
  324. package/src/providers/inference/__tests__/profile-config-validation.test.ts +39 -0
  325. package/src/providers/inference/__tests__/profile-probe-classify.test.ts +66 -0
  326. package/src/providers/inference/adapter-factory.ts +33 -2
  327. package/src/providers/inference/auth.ts +13 -2
  328. package/src/providers/inference/credential-usage.ts +37 -0
  329. package/src/providers/inference/endpoint-probe.ts +115 -0
  330. package/src/providers/inference/missing-credential-guard.ts +110 -0
  331. package/src/providers/inference/profile-probe.ts +256 -0
  332. package/src/providers/inference/resolve-auth.ts +7 -7
  333. package/src/providers/media-resolve.ts +176 -15
  334. package/src/providers/model-catalog.ts +10 -0
  335. package/src/providers/openai/__tests__/api-error-normalization.test.ts +17 -1
  336. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +576 -46
  337. package/src/providers/openai/__tests__/connection-error-wrap.test.ts +44 -0
  338. package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +125 -2
  339. package/src/providers/openai/api-error-normalization.ts +16 -2
  340. package/src/providers/openai/chat-completions-provider.ts +326 -44
  341. package/src/providers/openai/responses-provider.ts +5 -2
  342. package/src/providers/provider-send-message.ts +11 -0
  343. package/src/providers/retry.ts +6 -0
  344. package/src/providers/search-provider-catalog.ts +20 -0
  345. package/src/routes/route-host-protocol.ts +7 -0
  346. package/src/routes/worker.ts +31 -10
  347. package/src/runtime/AGENTS.md +36 -0
  348. package/src/runtime/__tests__/host-observe.test.ts +302 -0
  349. package/src/runtime/__tests__/runtime-http-port-collision.test.ts +131 -0
  350. package/src/runtime/__tests__/web-presence.test.ts +234 -0
  351. package/src/runtime/assistant-event-hub.ts +38 -0
  352. package/src/runtime/channel-readiness-service.ts +39 -13
  353. package/src/runtime/channel-readiness-types.ts +49 -2
  354. package/src/runtime/channel-reply-delivery.ts +55 -32
  355. package/src/runtime/channel-retry-sweep.ts +6 -4
  356. package/src/runtime/effective-capabilities.test.ts +19 -0
  357. package/src/runtime/effective-capabilities.ts +25 -0
  358. package/src/runtime/guardian-reply-router.ts +6 -2
  359. package/src/runtime/host-observe.ts +246 -0
  360. package/src/runtime/http-errors.ts +1 -0
  361. package/src/runtime/http-server.ts +185 -6
  362. package/src/runtime/routes/__tests__/client-routes.test.ts +150 -0
  363. package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +133 -0
  364. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +80 -0
  365. package/src/runtime/routes/__tests__/credential-delete-in-use.test.ts +155 -0
  366. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +118 -0
  367. package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +72 -1
  368. package/src/runtime/routes/__tests__/ingress-status-routes.test.ts +508 -0
  369. package/src/runtime/routes/__tests__/monitoring-routes.test.ts +1 -0
  370. package/src/runtime/routes/__tests__/pending-interactions-route.test.ts +175 -0
  371. package/src/runtime/routes/__tests__/plugins-routes.test.ts +103 -71
  372. package/src/runtime/routes/__tests__/surface-action-routes.test.ts +55 -6
  373. package/src/runtime/routes/__tests__/system-card-turn-termination.test.ts +107 -0
  374. package/src/runtime/routes/__tests__/user-route-dispatcher-host.test.ts +52 -1
  375. package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +75 -4
  376. package/src/runtime/routes/__tests__/watch-routes-guardian-cache.test.ts +139 -0
  377. package/src/runtime/routes/__tests__/watch-routes.test.ts +598 -0
  378. package/src/runtime/routes/approval-routes.ts +37 -4
  379. package/src/runtime/routes/approval-strategies/guardian-text-engine-strategy.ts +16 -22
  380. package/src/runtime/routes/canned-message-complete.ts +68 -29
  381. package/src/runtime/routes/channel-readiness-routes.ts +14 -2
  382. package/src/runtime/routes/client-routes.ts +113 -0
  383. package/src/runtime/routes/conversation-list-routes.ts +22 -7
  384. package/src/runtime/routes/conversation-management-routes.ts +15 -9
  385. package/src/runtime/routes/conversation-query-routes.ts +10 -0
  386. package/src/runtime/routes/conversation-routes.ts +36 -22
  387. package/src/runtime/routes/credential-in-use.ts +85 -0
  388. package/src/runtime/routes/credential-routes.ts +55 -85
  389. package/src/runtime/routes/guardian-approval-interception.ts +44 -53
  390. package/src/runtime/routes/guardian-approval-reply-helpers.ts +14 -12
  391. package/src/runtime/routes/host-cu-routes.ts +18 -0
  392. package/src/runtime/routes/identity-routes.ts +5 -121
  393. package/src/runtime/routes/inbound-message-handler.ts +74 -30
  394. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +16 -17
  395. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +182 -297
  396. package/src/runtime/routes/inbound-stages/background-dispatch.ts +173 -386
  397. package/src/runtime/routes/inbound-stages/edit-intercept.ts +101 -77
  398. package/src/runtime/routes/inbound-stages/guardian-reply-intercept.ts +17 -8
  399. package/src/runtime/routes/inbound-stages/reaction-intercept.test.ts +91 -7
  400. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +72 -70
  401. package/src/runtime/routes/index.ts +4 -0
  402. package/src/runtime/routes/inference-profiles-routes.ts +232 -31
  403. package/src/runtime/routes/inference-provider-connection-routes.ts +33 -11
  404. package/src/runtime/routes/ingress-status-routes.ts +180 -0
  405. package/src/runtime/routes/monitoring-routes.ts +9 -1
  406. package/src/runtime/routes/oauth-commands-routes.ts +1 -1
  407. package/src/runtime/routes/plugins-routes.ts +28 -32
  408. package/src/runtime/routes/resource-pressure-routes.ts +22 -0
  409. package/src/runtime/routes/secret-routes.ts +39 -8
  410. package/src/runtime/routes/settings-routes.ts +15 -12
  411. package/src/runtime/routes/surface-action-routes.ts +20 -19
  412. package/src/runtime/routes/user-route-dispatcher.ts +31 -2
  413. package/src/runtime/routes/user-route-resolution.ts +18 -1
  414. package/src/runtime/routes/watch-routes.ts +807 -0
  415. package/src/runtime/slack-reply-session.test.ts +23 -13
  416. package/src/runtime/slack-reply-session.ts +38 -55
  417. package/src/runtime/web-presence.ts +90 -0
  418. package/src/security/__tests__/untrusted-content.test.ts +42 -0
  419. package/src/security/untrusted-content.ts +28 -9
  420. package/src/skills/validate-input.ts +177 -40
  421. package/src/subagent/__tests__/consult-context-gating.test.ts +6 -10
  422. package/src/subagent/consult-context.ts +7 -23
  423. package/src/tools/credentials/broker.ts +24 -79
  424. package/src/tools/credentials/ref-parse.ts +35 -0
  425. package/src/tools/credentials/resolve.ts +4 -10
  426. package/src/tools/credentials/store.ts +168 -0
  427. package/src/tools/credentials/tool-policy.ts +67 -0
  428. package/src/tools/executor.ts +53 -15
  429. package/src/tools/network/__tests__/firecrawl-compat.test.ts +77 -0
  430. package/src/tools/network/__tests__/web-fetch-fastcrw.test.ts +169 -0
  431. package/src/tools/network/__tests__/web-search.test.ts +138 -3
  432. package/src/tools/network/firecrawl-compat.ts +90 -0
  433. package/src/tools/network/url-safety.ts +9 -16
  434. package/src/tools/network/web-fetch.ts +142 -62
  435. package/src/tools/network/web-search.ts +162 -58
  436. package/src/tools/permission-checker.ts +43 -52
  437. package/src/tools/schema-transforms.ts +40 -5
  438. package/src/tools/shared/input-repairs.ts +160 -0
  439. package/src/tools/skills/skill-tool-factory.ts +18 -4
  440. package/src/tools/subagent/spawn.ts +0 -1
  441. package/src/tools/tool-approval-handler.ts +16 -10
  442. package/src/tools/tool-types.ts +26 -13
  443. package/src/tools/types.ts +6 -6
  444. package/src/util/__tests__/cgroup-memory.test.ts +3 -0
  445. package/src/util/cgroup-memory.ts +3 -0
  446. package/src/util/container-cpu-sampler.ts +250 -0
  447. package/src/util/image-conversion.ts +178 -14
  448. package/src/util/platform.ts +93 -10
  449. package/src/util/worker-process.ts +1 -0
  450. package/src/watch/__tests__/watch-retro.test.ts +665 -0
  451. package/src/watch/__tests__/watch-session-manager.test.ts +566 -0
  452. package/src/watch/__tests__/watch-timeline.test.ts +670 -0
  453. package/src/watch/watch-retro.ts +480 -0
  454. package/src/watch/watch-session-manager.ts +575 -0
  455. package/src/watch/watch-timeline.ts +848 -0
  456. package/src/permissions/ipc-risk-types.ts +0 -143
  457. package/src/permissions/risk-types.ts +0 -76
@@ -0,0 +1,256 @@
1
+ /**
2
+ * Live save-time probe for a user-authored inference profile: one minimal
3
+ * request dispatched through the profile's own resolved model and connection,
4
+ * so a wrong model name, a rejected key, or an impossible token budget
5
+ * surfaces when the profile is saved instead of on the first chat message.
6
+ *
7
+ * The verdict is advisory and classified by which object the user should fix
8
+ * (this profile vs its provider connection), using the semantic
9
+ * `ProviderErrorReason` the provider adapters stamp at their throw sites.
10
+ * Managed and routing-identity profiles are never probed: their pins are
11
+ * hand-validated, and a probe there would spend managed credits on every
12
+ * save.
13
+ */
14
+
15
+ import type { ResolutionFallbackReason } from "../../config/llm-resolver.js";
16
+ import {
17
+ resolveCallSiteConfig,
18
+ selectWinningProfile,
19
+ } from "../../config/llm-resolver.js";
20
+ import { getConfigReadOnly } from "../../config/loader.js";
21
+ import { getDb } from "../../persistence/db-connection.js";
22
+ import { ProviderError, type ProviderErrorReason } from "../../util/errors.js";
23
+ import { getLogger } from "../../util/logger.js";
24
+ import { dispatchProviderResolvable } from "../connection-resolution.js";
25
+ import {
26
+ createTimeout,
27
+ getConfiguredProvider,
28
+ userMessage,
29
+ } from "../provider-send-message.js";
30
+ import { ROUTING_IDENTITY_PROVIDERS } from "./auth.js";
31
+ import { getConnection, MANAGED_CONNECTION_NAMES } from "./connections.js";
32
+ import { PROBE_TIMEOUT_MS } from "./endpoint-probe.js";
33
+
34
+ const log = getLogger("inference-profile-probe");
35
+
36
+ /** Which object the user should fix when the probe fails. */
37
+ export type ProfileCheckBlame =
38
+ | "profile"
39
+ | "provider"
40
+ | "transient"
41
+ | "unknown";
42
+
43
+ export interface ProfileCheck {
44
+ ok: boolean;
45
+ blame?: ProfileCheckBlame;
46
+ /** Semantic reason (ProviderErrorReason, or "resolution"/"not_configured"). */
47
+ reason?: string;
48
+ /** Upstream or resolver error text, verbatim. */
49
+ detail?: string;
50
+ /** The connection to inspect when `blame` is "provider". */
51
+ connection?: string;
52
+ /** English summary for non-localized surfaces (the CLI). */
53
+ message?: string;
54
+ }
55
+
56
+ const PROVIDER_BLAME_REASONS: ReadonlySet<ProviderErrorReason> = new Set([
57
+ "invalid_credentials",
58
+ "insufficient_credits",
59
+ "daily_limit_reached",
60
+ "network_error",
61
+ "server_error",
62
+ ]);
63
+
64
+ const TRANSIENT_BLAME_REASONS: ReadonlySet<ProviderErrorReason> = new Set([
65
+ "rate_limited",
66
+ "overloaded",
67
+ ]);
68
+
69
+ /**
70
+ * Map a failed probe dispatch onto the object the user should fix. Reasons
71
+ * the adapters stamp about the request itself (model unknown or restricted,
72
+ * parameter and token-budget rejections) blame the profile; credential,
73
+ * billing, and reachability reasons blame the provider connection.
74
+ */
75
+ export function classifyProbeFailure(err: unknown): {
76
+ blame: ProfileCheckBlame;
77
+ reason?: string;
78
+ detail: string;
79
+ } {
80
+ const detail = err instanceof Error ? err.message : String(err);
81
+ if (!(err instanceof ProviderError) || err.reason === undefined) {
82
+ return { blame: "unknown", detail };
83
+ }
84
+ if (PROVIDER_BLAME_REASONS.has(err.reason)) {
85
+ return { blame: "provider", reason: err.reason, detail };
86
+ }
87
+ if (TRANSIENT_BLAME_REASONS.has(err.reason)) {
88
+ return { blame: "transient", reason: err.reason, detail };
89
+ }
90
+ if (err.reason === "unknown") {
91
+ return { blame: "unknown", reason: err.reason, detail };
92
+ }
93
+ return { blame: "profile", reason: err.reason, detail };
94
+ }
95
+
96
+ function checkMessage(check: {
97
+ blame: ProfileCheckBlame;
98
+ detail: string;
99
+ connection?: string;
100
+ }): string {
101
+ switch (check.blame) {
102
+ case "profile":
103
+ return `A test request through this profile failed: ${check.detail} Check the profile's model and parameters.`;
104
+ case "provider":
105
+ return (
106
+ `A test request through this profile failed: ${check.detail} ` +
107
+ (check.connection
108
+ ? `Check the provider connection "${check.connection}".`
109
+ : `Check the profile's provider connection.`)
110
+ );
111
+ case "transient":
112
+ return `A test request through this profile could not complete: ${check.detail} This may be temporary.`;
113
+ default:
114
+ return `A test request through this profile failed: ${check.detail}`;
115
+ }
116
+ }
117
+
118
+ /**
119
+ * Probe a saved profile with one minimal request. Returns `null` when there
120
+ * is no verdict to give: the profile does not exist or is disabled, it rides
121
+ * a managed or routing-identity route (probing would bill managed credits),
122
+ * or the probe timed out without an answer.
123
+ */
124
+ export async function probeInferenceProfile(
125
+ name: string,
126
+ ): Promise<ProfileCheck | null> {
127
+ const { llm } = getConfigReadOnly();
128
+ const entry = llm.profiles?.[name] as Record<string, unknown> | undefined;
129
+ if (!entry || entry.status === "disabled" || entry.source === "managed") {
130
+ return null;
131
+ }
132
+
133
+ const selectionSeed = crypto.randomUUID();
134
+
135
+ // If the resolver would fall back past this profile, the probe must not run:
136
+ // it would silently exercise (and bill) a different profile. The fallback
137
+ // reason is itself the verdict.
138
+ let fallbackReason: ResolutionFallbackReason | undefined;
139
+ const winner = selectWinningProfile("inference", llm, {
140
+ overrideProfile: name,
141
+ selectionSeed,
142
+ // The same resolvability predicate dispatch applies: without it this
143
+ // pre-check can report the requested profile as winner while the actual
144
+ // dispatch below falls back to (and bills) a different profile.
145
+ isResolvableProvider: dispatchProviderResolvable,
146
+ onResolutionFallback: (info) => {
147
+ if (info.requested === name) {
148
+ fallbackReason = info.reason;
149
+ }
150
+ },
151
+ });
152
+ if (winner.profileName !== name) {
153
+ const detail = `The profile is ${fallbackReason ?? "unusable"} and requests would silently run on ${winner.profileName ?? "the default"} instead.`;
154
+ return {
155
+ ok: false,
156
+ blame: "profile",
157
+ reason: "resolution",
158
+ detail,
159
+ message: checkMessage({ blame: "profile", detail }),
160
+ };
161
+ }
162
+
163
+ // Billing guards run on the fully resolved call-site config — the same
164
+ // composition dispatch consumes (mix arm expanded, call-site overrides
165
+ // applied) — so neither a mix nor an `llm.callSites.inference` provider
166
+ // tweak can route the probe onto a managed or platform-billed path the
167
+ // stored entry hid. Same seed as the dispatch below, so the judged arm is
168
+ // the dispatched arm.
169
+ const resolved = resolveCallSiteConfig("inference", llm, {
170
+ overrideProfile: name,
171
+ selectionSeed,
172
+ });
173
+ const provider = resolved.provider ?? "";
174
+ if (ROUTING_IDENTITY_PROVIDERS.has(provider)) {
175
+ return null;
176
+ }
177
+ const connection = resolved.provider_connection ?? (provider || undefined);
178
+ // Canonical managed names always dispatch platform-billed: routing ignores
179
+ // a user-owned row that merely claims such a name
180
+ // (tryResolveProviderForConnectionName), so the name gates the probe
181
+ // regardless of the row's stored auth.
182
+ if (connection && MANAGED_CONNECTION_NAMES.has(connection)) {
183
+ return null;
184
+ }
185
+ // A legacy profile can declare a concrete provider while staying bound to
186
+ // a platform-billed connection row; the row's auth is the billing fact, so
187
+ // it gates the probe too.
188
+ const boundRow = connection ? getConnection(getDb(), connection) : null;
189
+ if (boundRow?.auth.type === "platform") {
190
+ return null;
191
+ }
192
+
193
+ const timeout = createTimeout(PROBE_TIMEOUT_MS);
194
+ try {
195
+ const providerInstance = await getConfiguredProvider("inference", {
196
+ overrideProfile: name,
197
+ selectionSeed,
198
+ });
199
+ if (!providerInstance) {
200
+ const detail = "No dispatchable provider is configured for this profile.";
201
+ return {
202
+ ok: false,
203
+ blame: "provider",
204
+ reason: "not_configured",
205
+ detail,
206
+ connection,
207
+ message: checkMessage({ blame: "provider", detail, connection }),
208
+ };
209
+ }
210
+ // Final billing gate on the route dispatch ACTUALLY resolved: a profile
211
+ // without a binding gets its row auto-selected inside
212
+ // getConfiguredProvider, so only the stamped attribution names the row
213
+ // this request would sign with.
214
+ const route = providerInstance.routeAttribution;
215
+ if (route?.isManagedRoute) {
216
+ return null;
217
+ }
218
+ const routedRow = route?.connectionName
219
+ ? getConnection(getDb(), route.connectionName)
220
+ : null;
221
+ if (routedRow?.auth.type === "platform") {
222
+ return null;
223
+ }
224
+ await providerInstance.sendMessage(
225
+ [userMessage("Reply with only the word OK.")],
226
+ {
227
+ signal: timeout.signal,
228
+ config: {
229
+ callSite: "inference",
230
+ overrideProfile: name,
231
+ selectionSeed,
232
+ },
233
+ },
234
+ );
235
+ return { ok: true };
236
+ } catch (err) {
237
+ if (timeout.signal.aborted) {
238
+ // No verdict: a slow model and a dead endpoint look identical here,
239
+ // and the connection-level probe already covers unreachable hosts.
240
+ log.info({ profile: name }, "Profile probe timed out without a verdict");
241
+ return null;
242
+ }
243
+ const classified = classifyProbeFailure(err);
244
+ return {
245
+ ok: false,
246
+ ...classified,
247
+ ...(classified.blame === "provider" ? { connection } : {}),
248
+ message: checkMessage({
249
+ ...classified,
250
+ ...(classified.blame === "provider" ? { connection } : {}),
251
+ }),
252
+ };
253
+ } finally {
254
+ timeout.cleanup();
255
+ }
256
+ }
@@ -17,7 +17,7 @@ import {
17
17
  } from "../platform-proxy/context.js";
18
18
  import {
19
19
  type Auth,
20
- PROVIDERS_REQUIRING_BASE_URL_AND_MODELS,
20
+ PROVIDERS_ALLOWING_CUSTOM_BASE_URL,
21
21
  type ResolvedAuth,
22
22
  } from "./auth.js";
23
23
 
@@ -36,14 +36,14 @@ export async function resolveAuth(
36
36
  { ok: true; resolved: ResolvedAuth } | { ok: false; error: ResolveAuthError }
37
37
  > {
38
38
  // Defense-in-depth: strip baseUrl for providers that should not accept one.
39
- // The route layer rejects base_url for non-openai-compatible providers, but
40
- // this guard catches any code path that bypasses route validation (e.g.
41
- // corrupted DB rows, direct calls from internal code).
39
+ // The route layer rejects base_url for those providers, but this guard
40
+ // catches any code path that bypasses route validation (e.g. corrupted DB
41
+ // rows, direct calls from internal code).
42
42
  let safeBaseUrl = opts.baseUrl;
43
- if (safeBaseUrl && !PROVIDERS_REQUIRING_BASE_URL_AND_MODELS.has(provider)) {
43
+ if (safeBaseUrl && !PROVIDERS_ALLOWING_CUSTOM_BASE_URL.has(provider)) {
44
44
  log.warn(
45
45
  { provider, baseUrl: safeBaseUrl },
46
- `Stripping baseUrl for provider "${provider}" — base_url is only valid for openai-compatible providers.`,
46
+ `Stripping baseUrl for provider "${provider}": base_url is only valid for openai-compatible and ollama providers.`,
47
47
  );
48
48
  safeBaseUrl = null;
49
49
  }
@@ -85,7 +85,7 @@ export async function resolveAuth(
85
85
 
86
86
  case "none":
87
87
  // Keyless endpoints still need their custom baseUrl threaded through
88
- // (LM Studio / vLLM openai-compatible servers).
88
+ // (Ollama, LM Studio / vLLM openai-compatible servers).
89
89
  return {
90
90
  ok: true,
91
91
  resolved: {
@@ -12,13 +12,21 @@
12
12
  * can keep reading `block.source.data` exactly as before.
13
13
  *
14
14
  * Blocks that already carry base64 (a live, in-flight turn) pass through
15
- * untouched — no disk read — so only reloaded history pays the resolution cost,
15
+ * untouched (no disk read) so only reloaded history pays the resolution cost,
16
16
  * and only on its first send after reload. The walk is pure: it returns fresh
17
17
  * block/message objects and never mutates the caller's in-memory history.
18
+ *
19
+ * The walk is also the last place an image block can be checked before a
20
+ * provider serializes it, so it is where an image the provider cannot decode
21
+ * is replaced by a text note: see {@link UNSENDABLE_IMAGE_FORMAT_NOTE}.
18
22
  */
19
23
 
20
24
  import { optimizeImageForTransport } from "../agent/image-optimize.js";
21
25
  import { getAttachmentContent } from "../persistence/attachments-store.js";
26
+ import {
27
+ heifImageMimeType,
28
+ sniffImageMimeType,
29
+ } from "../util/image-conversion.js";
22
30
  import { getLogger } from "../util/logger.js";
23
31
  import type {
24
32
  Base64MediaSource,
@@ -43,6 +51,65 @@ export function mediaSourceBytes(source: MediaSource): Buffer | null {
43
51
  return getAttachmentContent(source.attachmentId);
44
52
  }
45
53
 
54
+ /**
55
+ * Whether an image source's bytes are not one of the formats providers accept,
56
+ * so {@link resolveMediaReferences} replaces the block with
57
+ * {@link UNSENDABLE_IMAGE_FORMAT_NOTE} instead of sending it. Applies the same
58
+ * rule as {@link resolveImageBlock}, including its treatment of a reference
59
+ * whose payload is gone: that is an unavailable attachment, not an unreadable
60
+ * format. Lets a caller holding a persisted block ask whether the model will
61
+ * see the image.
62
+ *
63
+ * Answers for the formats every provider reads, so HEIF counts as unsendable
64
+ * even though a Gemini turn ({@link MediaResolveOptions}) would carry it.
65
+ */
66
+ export function isUnsendableImageSource(source: MediaSource): boolean {
67
+ const bytes =
68
+ source.type === "base64"
69
+ ? imageHeadBytes(source.data)
70
+ : getAttachmentContent(source.attachmentId);
71
+ return bytes != null && sendableImageMimeType(bytes, {}) === null;
72
+ }
73
+
74
+ /**
75
+ * Media types a provider accepts for an image block beyond the four every
76
+ * provider reads (PNG, JPEG, GIF, WebP).
77
+ *
78
+ * HEIF is the one format where providers genuinely disagree: Gemini decodes
79
+ * `image/heic` and `image/heif` directly
80
+ * (https://ai.google.dev/gemini-api/docs/image-understanding), while
81
+ * OpenAI-compatible and Anthropic endpoints answer HTTP 400 for the whole
82
+ * request. HEIF bytes reach a provider whenever transcoding was unavailable
83
+ * (`convertImageToJpeg` is backed by macOS `sips`), so a Linux-hosted assistant
84
+ * on Gemini depends on them passing through.
85
+ */
86
+ export interface MediaResolveOptions {
87
+ acceptsHeif?: boolean;
88
+ }
89
+
90
+ // 16 base64 chars decode to the 12 bytes the longest signature needs.
91
+ function imageHeadBytes(dataBase64: string): Buffer {
92
+ return Buffer.from(dataBase64.slice(0, 16), "base64");
93
+ }
94
+
95
+ /**
96
+ * Media type the provider should be told an image is, or null when it cannot
97
+ * read the bytes at all.
98
+ */
99
+ function sendableImageMimeType(
100
+ bytes: Uint8Array,
101
+ options: MediaResolveOptions,
102
+ ): string | null {
103
+ const sniffed = sniffImageMimeType(bytes);
104
+ if (sniffed) {
105
+ return sniffed;
106
+ }
107
+ if (options.acceptsHeif) {
108
+ return heifImageMimeType(bytes);
109
+ }
110
+ return null;
111
+ }
112
+
46
113
  /**
47
114
  * Narrow a media source to its base64 arm. After {@link resolveMediaReferences}
48
115
  * runs, provider transforms only ever see base64 sources; this asserts that
@@ -96,9 +163,57 @@ export function resolveMediaSourceData(
96
163
  return { data: bytes.toString("base64"), media_type: source.media_type };
97
164
  }
98
165
 
99
- async function resolveImageBlock(block: ImageContent): Promise<ContentBlock> {
166
+ /**
167
+ * Note that replaces an image block whose bytes are not one of the formats
168
+ * providers accept (PNG, JPEG, GIF, WebP). Sending the block instead costs the
169
+ * user the whole turn: an OpenAI-compatible endpoint answers HTTP 400 ("The
170
+ * image data you provided does not represent a valid image") for the entire
171
+ * request, so one unreadable image in a batch of eight blocks the reply to all
172
+ * of them. The note keeps the turn alive and tells the model what it is
173
+ * missing, and the daemon posts a system card naming the file so the user
174
+ * knows too.
175
+ */
176
+ export const UNSENDABLE_IMAGE_FORMAT_NOTE =
177
+ "[Image omitted: its format is not one the model can read (PNG, JPEG, GIF, and WebP are)]";
178
+
179
+ /**
180
+ * Resolve an image block into one the provider can accept, keyed on the bytes
181
+ * rather than on the declared media type.
182
+ *
183
+ * The declaration is only a claim: web clients derive it from the filename
184
+ * extension, so a HEIC photo saved as `.png` arrives declared `image/png`. One
185
+ * sniff covers both failure modes that claim produces. Bytes that sniff as
186
+ * nothing the target provider reads are unsendable (a corrupt head, or a format
187
+ * outside both the universal four and the provider's own extras: AVIF, BMP,
188
+ * TIFF, SVG) and become {@link UNSENDABLE_IMAGE_FORMAT_NOTE}. Bytes that sniff
189
+ * as something other than the declaration are relabeled, since a mismatch is
190
+ * fatal on its own (Anthropic answers "image does not match the provided media
191
+ * type").
192
+ *
193
+ * The sniff runs ahead of the transport optimization so sendability cannot
194
+ * depend on an unrelated size threshold: optimization rewrites only the images
195
+ * it decides to rescale.
196
+ */
197
+ async function resolveImageBlock(
198
+ block: ImageContent,
199
+ options: MediaResolveOptions,
200
+ ): Promise<ContentBlock> {
100
201
  if (block.source.type === "base64") {
101
- return block;
202
+ const sniffed = sendableImageMimeType(
203
+ imageHeadBytes(block.source.data),
204
+ options,
205
+ );
206
+ if (!sniffed) {
207
+ log.warn(
208
+ { declaredMediaType: block.source.media_type },
209
+ "Inline image is not in a provider-readable format; substituting a text note",
210
+ );
211
+ return { type: "text", text: UNSENDABLE_IMAGE_FORMAT_NOTE };
212
+ }
213
+ return {
214
+ type: "image",
215
+ source: { type: "base64", media_type: sniffed, data: block.source.data },
216
+ };
102
217
  }
103
218
  const bytes = getAttachmentContent(block.source.attachmentId);
104
219
  if (!bytes) {
@@ -111,11 +226,22 @@ async function resolveImageBlock(block: ImageContent): Promise<ContentBlock> {
111
226
  text: "[Attachment unavailable: image could not be loaded]",
112
227
  };
113
228
  }
229
+ const sniffed = sendableImageMimeType(bytes, options);
230
+ if (!sniffed) {
231
+ log.warn(
232
+ {
233
+ attachmentId: block.source.attachmentId,
234
+ declaredMediaType: block.source.media_type,
235
+ },
236
+ "Referenced image is not in a provider-readable format; substituting a text note",
237
+ );
238
+ return { type: "text", text: UNSENDABLE_IMAGE_FORMAT_NOTE };
239
+ }
114
240
  // Apply the same transport optimization the inline-base64 path used, so a
115
241
  // reloaded (reference) turn sends the model the same bytes a live turn would.
116
242
  const { data, mediaType } = await optimizeImageForTransport(
117
243
  bytes.toString("base64"),
118
- block.source.media_type,
244
+ sniffed,
119
245
  );
120
246
  return {
121
247
  type: "image",
@@ -161,10 +287,13 @@ function resolveFileBlock(block: FileContent): ContentBlock {
161
287
  };
162
288
  }
163
289
 
164
- async function resolveBlock(block: ContentBlock): Promise<ContentBlock> {
290
+ async function resolveBlock(
291
+ block: ContentBlock,
292
+ options: MediaResolveOptions,
293
+ ): Promise<ContentBlock> {
165
294
  switch (block.type) {
166
295
  case "image":
167
- return resolveImageBlock(block);
296
+ return resolveImageBlock(block, options);
168
297
  case "file":
169
298
  return resolveFileBlock(block);
170
299
  case "tool_result": {
@@ -174,7 +303,9 @@ async function resolveBlock(block: ContentBlock): Promise<ContentBlock> {
174
303
  }
175
304
  return {
176
305
  ...block,
177
- contentBlocks: await Promise.all(block.contentBlocks.map(resolveBlock)),
306
+ contentBlocks: await Promise.all(
307
+ block.contentBlocks.map((nested) => resolveBlock(nested, options)),
308
+ ),
178
309
  };
179
310
  }
180
311
  default:
@@ -182,13 +313,40 @@ async function resolveBlock(block: ContentBlock): Promise<ContentBlock> {
182
313
  }
183
314
  }
184
315
 
185
- function contentHasReference(content: ContentBlock[]): boolean {
316
+ /**
317
+ * Whether an inline image needs rebuilding before it can be sent: its payload
318
+ * is unreadable, or its declared media type is not exactly what the bytes are.
319
+ * The comparison is exact so a non-canonical spelling of a real format is
320
+ * rewritten too (`image/jpg` is not in any provider's accepted set, and clients
321
+ * derive it from a `.jpg` extension). Sniffing here, rather than rebuilding
322
+ * every inline block unconditionally, keeps the clean live turn on the identity
323
+ * fast path below at the cost of decoding a 12-byte head per image.
324
+ */
325
+ function base64ImageNeedsRewrite(
326
+ source: Base64MediaSource,
327
+ options: MediaResolveOptions,
328
+ ): boolean {
329
+ return (
330
+ sendableImageMimeType(imageHeadBytes(source.data), options) !==
331
+ source.media_type
332
+ );
333
+ }
334
+
335
+ function contentNeedsResolution(
336
+ content: ContentBlock[],
337
+ options: MediaResolveOptions,
338
+ ): boolean {
186
339
  return content.some((block) => {
187
- if (block.type === "image" || block.type === "file") {
340
+ if (block.type === "image") {
341
+ return block.source.type === "workspace_ref"
342
+ ? true
343
+ : base64ImageNeedsRewrite(block.source, options);
344
+ }
345
+ if (block.type === "file") {
188
346
  return block.source.type === "workspace_ref";
189
347
  }
190
348
  if (block.type === "tool_result" && block.contentBlocks?.length) {
191
- return contentHasReference(block.contentBlocks);
349
+ return contentNeedsResolution(block.contentBlocks, options);
192
350
  }
193
351
  return false;
194
352
  });
@@ -196,21 +354,24 @@ function contentHasReference(content: ContentBlock[]): boolean {
196
354
 
197
355
  /**
198
356
  * Return a copy of `messages` with every {@link WorkspaceRefMediaSource}
199
- * resolved to inline base64. Messages with no references are returned unchanged
200
- * (same object reference) so the common all-base64 live turn does no allocation
201
- * or disk I/O.
357
+ * resolved to inline base64 and every image block reconciled with its own
358
+ * bytes. Messages that need neither are returned unchanged (same object
359
+ * reference) so the common live turn does no allocation or disk I/O.
202
360
  */
203
361
  export function resolveMediaReferences(
204
362
  messages: Message[],
363
+ options: MediaResolveOptions = {},
205
364
  ): Promise<Message[]> {
206
365
  return Promise.all(
207
366
  messages.map(async (message) => {
208
- if (!contentHasReference(message.content)) {
367
+ if (!contentNeedsResolution(message.content, options)) {
209
368
  return message;
210
369
  }
211
370
  return {
212
371
  ...message,
213
- content: await Promise.all(message.content.map(resolveBlock)),
372
+ content: await Promise.all(
373
+ message.content.map((block) => resolveBlock(block, options)),
374
+ ),
214
375
  };
215
376
  }),
216
377
  );
@@ -2190,6 +2190,16 @@ export function catalogMaxOutputTokens(
2190
2190
  )?.maxOutputTokens;
2191
2191
  }
2192
2192
 
2193
+ /** The `contextWindowTokens` declared for a (provider, model) catalog entry, if any. */
2194
+ export function catalogContextWindowTokens(
2195
+ provider: string,
2196
+ modelId: string,
2197
+ ): number | undefined {
2198
+ return PROVIDER_CATALOG.find((p) => p.id === provider)?.models.find(
2199
+ (m) => m.id === modelId,
2200
+ )?.contextWindowTokens;
2201
+ }
2202
+
2193
2203
  /**
2194
2204
  * Model IDs (across all catalog providers) flagged
2195
2205
  * `supportsPromptCacheBreakpoints`. Consumed by the OpenAI Responses
@@ -1,6 +1,6 @@
1
1
  import { describe, expect, test } from "bun:test";
2
2
 
3
- import type OpenAI from "openai";
3
+ import OpenAI from "openai";
4
4
 
5
5
  import type { NormalizedOpenAIAPIError } from "../api-error-normalization.js";
6
6
  import {
@@ -44,6 +44,22 @@ describe("normalizeOpenAIAPIError", () => {
44
44
  );
45
45
  });
46
46
 
47
+ test("stamps network_error for SDK connection failures", () => {
48
+ const cause = Object.assign(new Error("connect ECONNREFUSED 127.0.0.1:9"), {
49
+ code: "ECONNREFUSED",
50
+ });
51
+ const n = normalizeOpenAIAPIError(new OpenAI.APIConnectionError({ cause }));
52
+ expect(n.reason).toBe("network_error");
53
+ });
54
+
55
+ test("does not stamp network_error for user aborts", () => {
56
+ // APIUserAbortError covers caller cancellation and inner stream
57
+ // deadlines; classifying it as transient would make retry re-run
58
+ // 30-minute deadline failures.
59
+ const n = normalizeOpenAIAPIError(new OpenAI.APIUserAbortError());
60
+ expect(n.reason).not.toBe("network_error");
61
+ });
62
+
47
63
  test("extracts OpenAI-shaped error metadata", () => {
48
64
  const n = normalizeOpenAIAPIError(
49
65
  apiError(401),