@vellumai/assistant 0.8.10 → 0.8.11-staging.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (400) hide show
  1. package/bun.lock +62 -1
  2. package/docs/workspace-tools.md +196 -0
  3. package/examples/plugins/echo/README.md +3 -3
  4. package/knip.json +1 -0
  5. package/openapi.yaml +460 -128
  6. package/package.json +2 -1
  7. package/scripts/build-plugin-api.ts +299 -0
  8. package/src/__tests__/agent-loop-callsite-precedence.test.ts +7 -0
  9. package/src/__tests__/agent-loop-compaction-events.test.ts +197 -0
  10. package/src/__tests__/agent-loop-exit-reason.test.ts +93 -96
  11. package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +2 -0
  12. package/src/__tests__/agent-loop-output-hooks.test.ts +274 -1
  13. package/src/__tests__/agent-loop-override-profile.test.ts +3 -0
  14. package/src/__tests__/agent-loop-provider-error-recording.test.ts +4 -0
  15. package/src/__tests__/agent-loop-thinking.test.ts +4 -0
  16. package/src/__tests__/agent-loop.test.ts +578 -5
  17. package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +0 -1
  18. package/src/__tests__/approval-cascade.test.ts +1 -0
  19. package/src/__tests__/background-workers-disk-pressure.test.ts +0 -2
  20. package/src/__tests__/btw-routes.test.ts +0 -1
  21. package/src/__tests__/build-persisted-content.test.ts +75 -1
  22. package/src/__tests__/catalog-install-normalize.test.ts +141 -0
  23. package/src/__tests__/ces-startup-timeout.test.ts +60 -0
  24. package/src/__tests__/compaction-events.test.ts +1 -0
  25. package/src/__tests__/config-managed-gemini-defaults.test.ts +2 -46
  26. package/src/__tests__/context-overflow-reducer.test.ts +264 -124
  27. package/src/__tests__/context-window-manager-overflow-rung.test.ts +351 -0
  28. package/src/__tests__/conversation-abort-tool-results.test.ts +1 -1
  29. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +13 -5
  30. package/src/__tests__/conversation-agent-loop-overflow.test.ts +284 -455
  31. package/src/__tests__/conversation-agent-loop.test.ts +131 -551
  32. package/src/__tests__/conversation-app-control-instantiation.test.ts +13 -0
  33. package/src/__tests__/conversation-confirmation-signals.test.ts +1 -0
  34. package/src/__tests__/conversation-fork-crud.test.ts +259 -0
  35. package/src/__tests__/conversation-history-web-search.test.ts +1 -1
  36. package/src/__tests__/conversation-lifecycle.test.ts +257 -1
  37. package/src/__tests__/conversation-process-callsite.test.ts +1 -0
  38. package/src/__tests__/conversation-provider-retry-repair.test.ts +38 -377
  39. package/src/__tests__/conversation-queue.test.ts +1 -39
  40. package/src/__tests__/conversation-runtime-assembly.test.ts +119 -8
  41. package/src/__tests__/conversation-skill-tools.test.ts +491 -5
  42. package/src/__tests__/conversation-slash-queue.test.ts +1 -1
  43. package/src/__tests__/conversation-slash-unknown.test.ts +1 -0
  44. package/src/__tests__/conversation-speed-override.test.ts +1 -0
  45. package/src/__tests__/conversation-store.test.ts +74 -0
  46. package/src/__tests__/conversation-surfaces-app-control.test.ts +4 -1
  47. package/src/__tests__/conversation-tool-setup-attribution.test.ts +323 -0
  48. package/src/__tests__/conversation-tool-setup-tools-disabled.test.ts +34 -0
  49. package/src/__tests__/conversation-workspace-cache-state.test.ts +1 -0
  50. package/src/__tests__/conversation-workspace-injection.test.ts +1 -1
  51. package/src/__tests__/conversation-workspace-tool-tracking.test.ts +1 -0
  52. package/src/__tests__/corrected-target.test.ts +93 -0
  53. package/src/__tests__/credential-execution-feature-gates.test.ts +3 -5
  54. package/src/__tests__/credential-execution-tools.test.ts +23 -11
  55. package/src/__tests__/credential-security-invariants.test.ts +6 -1
  56. package/src/__tests__/db-schedule-syntax-migration.test.ts +80 -0
  57. package/src/__tests__/device-id.test.ts +70 -1
  58. package/src/__tests__/embedding-managed-proxy-selection.test.ts +6 -40
  59. package/src/__tests__/empty-response-hook.test.ts +242 -66
  60. package/src/__tests__/external-plugin-loader.test.ts +0 -31
  61. package/src/__tests__/get-skill-detail-audit.test.ts +43 -1
  62. package/src/__tests__/guardian-routing-invariants.test.ts +91 -0
  63. package/src/__tests__/history-repair-hook.test.ts +228 -3
  64. package/src/__tests__/host-app-control-proxy.test.ts +45 -0
  65. package/src/__tests__/host-browser-proxy.test.ts +254 -9
  66. package/src/__tests__/identity-routes.test.ts +1 -0
  67. package/src/__tests__/image-recovery-hook.test.ts +387 -0
  68. package/src/__tests__/injector-chain.test.ts +5 -4
  69. package/src/__tests__/injector-v3-suppression.test.ts +373 -47
  70. package/src/__tests__/intent-routing.test.ts +7 -0
  71. package/src/__tests__/memory-retrieval-hook.test.ts +117 -15
  72. package/src/__tests__/notification-decision-strategy.test.ts +3 -3
  73. package/src/__tests__/oauth-store.test.ts +0 -85
  74. package/src/__tests__/{context-overflow-policy.test.ts → overflow-policy.test.ts} +1 -1
  75. package/src/__tests__/parallel-tool.benchmark.test.ts +4 -0
  76. package/src/__tests__/persist-unsendable-image-downscale.test.ts +29 -9
  77. package/src/__tests__/persist-unsendable-image.test.ts +4 -4
  78. package/src/__tests__/persistence-secret-redaction.test.ts +78 -0
  79. package/src/__tests__/plugin-bootstrap.test.ts +82 -73
  80. package/src/__tests__/plugin-tool-contribution.test.ts +7 -4
  81. package/src/__tests__/plugin-types.test.ts +0 -8
  82. package/src/__tests__/provider-catalog-visibility.test.ts +1 -9
  83. package/src/__tests__/prune-old-conversations-job.test.ts +99 -0
  84. package/src/__tests__/registry.test.ts +240 -1
  85. package/src/__tests__/require-fresh-approval.test.ts +3 -0
  86. package/src/__tests__/schedule-routes.test.ts +116 -1
  87. package/src/__tests__/schedule-store.test.ts +28 -0
  88. package/src/__tests__/schedule-tools.test.ts +94 -1
  89. package/src/__tests__/server-history-render.test.ts +39 -0
  90. package/src/__tests__/skill-projection-feature-flag.test.ts +13 -0
  91. package/src/__tests__/skill-projection.benchmark.test.ts +25 -7
  92. package/src/__tests__/skills.test.ts +202 -0
  93. package/src/__tests__/slim-skill-category.test.ts +195 -0
  94. package/src/__tests__/strip-memory-injections.test.ts +33 -39
  95. package/src/__tests__/test-support/tool-invocation-seed.ts +79 -0
  96. package/src/__tests__/title-generate-hook.test.ts +9 -7
  97. package/src/__tests__/tool-audit-listener.test.ts +264 -1
  98. package/src/__tests__/tool-error-hook.test.ts +4 -3
  99. package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +1 -0
  100. package/src/__tests__/tool-executor-lifecycle-events.test.ts +273 -0
  101. package/src/__tests__/tool-result-truncate-hook.test.ts +1 -0
  102. package/src/__tests__/tool-start-timestamp.test.ts +218 -0
  103. package/src/__tests__/tools-get-route.test.ts +202 -0
  104. package/src/__tests__/workspace-tool-loader.test.ts +319 -0
  105. package/src/__tests__/workspace-tools-watcher-flag.test.ts +70 -0
  106. package/src/agent/loop.ts +569 -319
  107. package/src/api/events/tool-result.ts +9 -0
  108. package/src/api/events/tool-use-start.ts +7 -0
  109. package/src/api/index.ts +10 -0
  110. package/src/api/responses/conversation-message.ts +135 -27
  111. package/src/api/responses/memory-v3-selection-log.ts +4 -4
  112. package/src/approvals/guardian-request-resolvers.ts +26 -0
  113. package/src/browser-session/backends/host-bridge.ts +29 -0
  114. package/src/browser-session/index.ts +1 -0
  115. package/src/browser-session/types.ts +5 -1
  116. package/src/cli/commands/__tests__/schedules.test.ts +62 -4
  117. package/src/cli/commands/__tests__/skills.test.ts +53 -0
  118. package/src/cli/commands/channel-verification-sessions.ts +6 -6
  119. package/src/cli/commands/inference-providers.ts +0 -8
  120. package/src/cli/commands/plugins.ts +2 -2
  121. package/src/cli/commands/schedules.ts +27 -4
  122. package/src/cli/commands/skills.ts +187 -146
  123. package/src/cli/commands/tools.ts +106 -0
  124. package/src/cli/lib/__tests__/install-from-github.test.ts +256 -328
  125. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +6 -2
  126. package/src/cli/lib/__tests__/plugin-details.test.ts +10 -16
  127. package/src/cli/lib/__tests__/plugin-marketplace.test.ts +2 -2
  128. package/src/cli/lib/__tests__/search-plugins.test.ts +145 -240
  129. package/src/cli/lib/install-from-github.ts +187 -117
  130. package/src/cli/lib/plugin-catalog-cache.ts +9 -9
  131. package/src/cli/lib/plugin-details.ts +38 -68
  132. package/src/cli/lib/plugin-marketplace.ts +42 -14
  133. package/src/cli/lib/search-plugins.ts +29 -129
  134. package/src/cli/program.ts +2 -0
  135. package/src/config/bundled-skills/acp/SKILL.md +1 -0
  136. package/src/config/bundled-skills/app-builder/SKILL.md +1 -0
  137. package/src/config/bundled-skills/app-control/SKILL.md +1 -0
  138. package/src/config/bundled-skills/computer-use/SKILL.md +1 -0
  139. package/src/config/bundled-skills/contacts/SKILL.md +1 -0
  140. package/src/config/bundled-skills/document-editor/SKILL.md +1 -0
  141. package/src/config/bundled-skills/followups/SKILL.md +1 -0
  142. package/src/config/bundled-skills/image-studio/SKILL.md +1 -0
  143. package/src/config/bundled-skills/media-processing/SKILL.md +1 -0
  144. package/src/config/bundled-skills/messaging/SKILL.md +1 -0
  145. package/src/config/bundled-skills/phone-calls/SKILL.md +1 -0
  146. package/src/config/bundled-skills/playbooks/SKILL.md +1 -0
  147. package/src/config/bundled-skills/schedule/SKILL.md +1 -0
  148. package/src/config/bundled-skills/schedule/TOOLS.json +11 -3
  149. package/src/config/bundled-skills/sequences/SKILL.md +1 -0
  150. package/src/config/bundled-skills/settings/SKILL.md +1 -0
  151. package/src/config/bundled-skills/skill-management/SKILL.md +101 -1
  152. package/src/config/bundled-skills/subagent/SKILL.md +1 -0
  153. package/src/config/bundled-skills/transcribe/SKILL.md +1 -0
  154. package/src/config/env-registry.ts +23 -0
  155. package/src/config/feature-flag-registry.json +17 -81
  156. package/src/config/loader.ts +5 -22
  157. package/src/config/schema.ts +2 -0
  158. package/src/config/schemas/__tests__/compaction-logs.test.ts +56 -0
  159. package/src/config/schemas/__tests__/memory-v2.test.ts +0 -1
  160. package/src/config/schemas/__tests__/memory-v3.test.ts +61 -1
  161. package/src/config/schemas/compaction-logs.ts +79 -0
  162. package/src/config/schemas/memory-v2.ts +0 -8
  163. package/src/config/schemas/memory-v3.ts +104 -33
  164. package/src/config/seed-inference-profiles.ts +1 -1
  165. package/src/config/skills.ts +117 -47
  166. package/src/context/compactor.ts +11 -0
  167. package/src/context/strip-injections.ts +38 -4
  168. package/src/credential-execution/feature-gates.ts +0 -21
  169. package/src/credential-execution/startup-timeout.ts +32 -4
  170. package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +18 -0
  171. package/src/daemon/conversation-agent-loop-handlers.ts +140 -91
  172. package/src/daemon/conversation-agent-loop.ts +123 -658
  173. package/src/daemon/conversation-error.ts +6 -33
  174. package/src/daemon/conversation-lifecycle.ts +1 -1
  175. package/src/daemon/conversation-runtime-assembly.ts +183 -22
  176. package/src/daemon/conversation-skill-tools.ts +137 -8
  177. package/src/daemon/conversation-slash.ts +0 -14
  178. package/src/daemon/conversation-store.ts +2 -19
  179. package/src/daemon/conversation-tool-setup.ts +87 -1
  180. package/src/daemon/conversation.ts +119 -50
  181. package/src/daemon/external-plugins-bootstrap.ts +36 -95
  182. package/src/daemon/handlers/config-channels.ts +11 -2
  183. package/src/daemon/handlers/shared.ts +9 -1
  184. package/src/daemon/handlers/skills.ts +10 -4
  185. package/src/daemon/host-app-control-proxy.ts +72 -57
  186. package/src/daemon/host-browser-proxy.ts +117 -22
  187. package/src/daemon/lifecycle.ts +34 -5
  188. package/src/daemon/message-protocol.ts +0 -7
  189. package/src/daemon/message-types/schedules.ts +1 -0
  190. package/src/daemon/message-types/skills.ts +17 -0
  191. package/src/daemon/providers-setup.ts +3 -0
  192. package/src/daemon/server.ts +3 -3
  193. package/src/daemon/tool-setup-types.ts +9 -3
  194. package/src/daemon/trust-context.ts +23 -0
  195. package/src/daemon/workspace-tools-watcher.ts +324 -0
  196. package/src/events/tool-audit-listener.ts +78 -16
  197. package/src/events/tool-metrics-listener.ts +2 -5
  198. package/src/memory/__tests__/compaction-log-writer-clickhouse.test.ts +227 -0
  199. package/src/memory/__tests__/conversation-queries.test.ts +176 -0
  200. package/src/memory/__tests__/jobs-worker-v2-schedule.test.ts +20 -32
  201. package/src/memory/compaction-log-writer-clickhouse.ts +418 -0
  202. package/src/memory/conversation-crud.ts +134 -3
  203. package/src/memory/conversation-queries.ts +64 -7
  204. package/src/memory/db-init.ts +14 -0
  205. package/src/memory/embedding-backend.test.ts +130 -1
  206. package/src/memory/embedding-backend.ts +79 -106
  207. package/src/memory/embedding-gemini.ts +5 -0
  208. package/src/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +12 -0
  209. package/src/memory/graph/__tests__/handle-remember-v2.test.ts +19 -0
  210. package/src/memory/graph/conversation-graph-memory.ts +36 -25
  211. package/src/memory/graph/tool-handlers.ts +3 -0
  212. package/src/memory/job-handlers/cleanup.ts +3 -1
  213. package/src/memory/jobs-store.ts +0 -28
  214. package/src/memory/jobs-worker.ts +10 -22
  215. package/src/memory/memory-marker.ts +29 -0
  216. package/src/memory/memory-retrospective-startup-cleanup.ts +1 -1
  217. package/src/memory/migrations/268-add-memory-v3-selections.ts +6 -0
  218. package/src/memory/migrations/270-schedule-description.ts +36 -0
  219. package/src/memory/migrations/275-tool-invocations-add-skill-id.test.ts +81 -0
  220. package/src/memory/migrations/275-tool-invocations-add-skill-id.ts +20 -0
  221. package/src/memory/migrations/276-tool-invocations-created-at-id-index.test.ts +68 -0
  222. package/src/memory/migrations/276-tool-invocations-created-at-id-index.ts +20 -0
  223. package/src/memory/migrations/277-add-memory-v3-ever-injected.ts +29 -0
  224. package/src/memory/migrations/278-tool-invocations-telemetry-columns.test.ts +96 -0
  225. package/src/memory/migrations/278-tool-invocations-telemetry-columns.ts +39 -0
  226. package/src/memory/migrations/279-create-skill-loaded-events.test.ts +84 -0
  227. package/src/memory/migrations/279-create-skill-loaded-events.ts +26 -0
  228. package/src/memory/migrations/280-conversations-surfaced-at.test.ts +88 -0
  229. package/src/memory/migrations/280-conversations-surfaced-at.ts +24 -0
  230. package/src/memory/migrations/index.ts +10 -0
  231. package/src/memory/migrations/registry.ts +8 -0
  232. package/src/memory/schema/conversations.ts +16 -0
  233. package/src/memory/schema/infrastructure.ts +26 -0
  234. package/src/memory/skill-loaded-events-store.test.ts +160 -0
  235. package/src/memory/skill-loaded-events-store.ts +95 -0
  236. package/src/memory/tool-executed-events-store.test.ts +219 -0
  237. package/src/memory/tool-executed-events-store.ts +102 -0
  238. package/src/memory/tool-usage-store.ts +15 -3
  239. package/src/memory/v2/__tests__/consolidation-job.test.ts +117 -12
  240. package/src/memory/v2/__tests__/consolidation-prompt-flag-gating-guard.test.ts +189 -0
  241. package/src/memory/v2/__tests__/injected-block-slugs.test.ts +90 -0
  242. package/src/memory/v2/__tests__/page-store.test.ts +33 -0
  243. package/src/memory/v2/__tests__/prompts-consolidation.test.ts +88 -15
  244. package/src/memory/v2/activation-store.ts +50 -1
  245. package/src/memory/v2/consolidation-job.ts +113 -29
  246. package/src/memory/v2/injected-block-slugs.ts +79 -0
  247. package/src/memory/v2/injection.ts +6 -1
  248. package/src/memory/v2/prompts/consolidation.ts +414 -13
  249. package/src/memory/v2/router.ts +2 -28
  250. package/src/memory/v2/static-context.ts +1 -1
  251. package/src/memory/v2/types.ts +16 -0
  252. package/src/notifications/__tests__/copy-composer.test.ts +244 -0
  253. package/src/notifications/access-request-copy.ts +298 -0
  254. package/src/notifications/adapters/slack.ts +3 -3
  255. package/src/notifications/adapters/telegram.ts +2 -1
  256. package/src/notifications/copy-composer.ts +49 -267
  257. package/src/notifications/decision-engine.ts +16 -35
  258. package/src/notifications/home-feed-side-effect.ts +1 -6
  259. package/src/oauth/oauth-store.ts +0 -9
  260. package/src/permissions/checker.test.ts +83 -1
  261. package/src/permissions/checker.ts +25 -2
  262. package/src/permissions/gateway-threshold-reader.test.ts +182 -0
  263. package/src/permissions/gateway-threshold-reader.ts +80 -0
  264. package/src/platform/client.ts +1 -3
  265. package/src/platform/feature-gate.ts +3 -12
  266. package/src/plugin-api/constants.ts +4 -2
  267. package/src/plugin-api/index.ts +63 -11
  268. package/src/plugin-api/types.ts +236 -71
  269. package/src/plugins/defaults/compaction/compact.ts +66 -2
  270. package/src/plugins/defaults/compaction/context-overflow-reducer.ts +240 -32
  271. package/src/plugins/defaults/compaction/corrected-target.ts +53 -0
  272. package/src/{daemon/context-overflow-policy.ts → plugins/defaults/compaction/overflow-policy.ts} +1 -1
  273. package/src/plugins/defaults/compaction/window-manager.ts +303 -1
  274. package/src/plugins/defaults/empty-response/hooks/post-model-call.ts +173 -0
  275. package/src/plugins/defaults/empty-response/hooks/stop.ts +11 -115
  276. package/src/plugins/defaults/empty-response/nudge-state-store.ts +46 -0
  277. package/src/plugins/defaults/history-repair/hooks/post-model-call.ts +50 -0
  278. package/src/plugins/defaults/history-repair/hooks/stop.ts +22 -0
  279. package/src/plugins/defaults/history-repair/repair-state-store.ts +51 -0
  280. package/src/plugins/defaults/history-repair/terminal.ts +39 -2
  281. package/src/plugins/defaults/image-recovery/detect.ts +25 -0
  282. package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +73 -0
  283. package/src/plugins/defaults/image-recovery/hooks/stop.ts +22 -0
  284. package/src/plugins/defaults/image-recovery/image-recovery-state-store.ts +48 -0
  285. package/src/plugins/defaults/image-recovery/package.json +14 -0
  286. package/src/{daemon/persist-unsendable-image.ts → plugins/defaults/image-recovery/recover.ts} +67 -14
  287. package/src/plugins/defaults/index.ts +71 -5
  288. package/src/plugins/defaults/memory-retrieval/hooks/post-compact.ts +76 -112
  289. package/src/plugins/defaults/memory-retrieval/hooks/{user-prompt-submit-temp.ts → user-prompt-submit.ts} +83 -74
  290. package/src/plugins/defaults/memory-retrieval/injector-chain.ts +14 -8
  291. package/src/plugins/defaults/memory-retrieval/injectors.ts +2 -18
  292. package/src/plugins/defaults/memory-retrieval/package.json +14 -0
  293. package/src/plugins/defaults/memory-v3-shadow/__tests__/carry-integration.test.ts +1157 -0
  294. package/src/plugins/defaults/memory-v3-shadow/__tests__/injection.test.ts +683 -0
  295. package/src/plugins/defaults/memory-v3-shadow/__tests__/live-integration.test.ts +161 -140
  296. package/src/plugins/defaults/memory-v3-shadow/__tests__/maintain-job.test.ts +160 -0
  297. package/src/plugins/defaults/memory-v3-shadow/__tests__/orchestrate.test.ts +335 -316
  298. package/src/plugins/defaults/memory-v3-shadow/__tests__/pool-select.test.ts +145 -53
  299. package/src/plugins/defaults/memory-v3-shadow/__tests__/render-injection.test.ts +38 -1
  300. package/src/plugins/defaults/memory-v3-shadow/__tests__/selection-log-store.test.ts +18 -8
  301. package/src/plugins/defaults/memory-v3-shadow/__tests__/shadow-integration.test.ts +112 -71
  302. package/src/plugins/defaults/memory-v3-shadow/__tests__/shadow-plugin.test.ts +192 -51
  303. package/src/plugins/defaults/memory-v3-shadow/__tests__/types.test.ts +4 -16
  304. package/src/plugins/defaults/memory-v3-shadow/card.test.ts +173 -0
  305. package/src/plugins/defaults/memory-v3-shadow/card.ts +116 -0
  306. package/src/plugins/defaults/memory-v3-shadow/core-set.test.ts +104 -0
  307. package/src/plugins/defaults/memory-v3-shadow/core-set.ts +59 -0
  308. package/src/plugins/defaults/memory-v3-shadow/ever-injected-store.test.ts +305 -0
  309. package/src/plugins/defaults/memory-v3-shadow/ever-injected-store.ts +278 -0
  310. package/src/plugins/defaults/memory-v3-shadow/hot-set.test.ts +138 -0
  311. package/src/plugins/defaults/memory-v3-shadow/hot-set.ts +85 -0
  312. package/src/plugins/defaults/memory-v3-shadow/injector.ts +331 -24
  313. package/src/plugins/defaults/memory-v3-shadow/maintain-job.ts +119 -13
  314. package/src/plugins/defaults/memory-v3-shadow/orchestrate.ts +169 -114
  315. package/src/plugins/defaults/memory-v3-shadow/page-content.ts +47 -16
  316. package/src/plugins/defaults/memory-v3-shadow/pool-select.ts +144 -66
  317. package/src/plugins/defaults/memory-v3-shadow/prune.test.ts +758 -0
  318. package/src/plugins/defaults/memory-v3-shadow/prune.ts +471 -0
  319. package/src/plugins/defaults/memory-v3-shadow/render-injection.ts +68 -16
  320. package/src/plugins/defaults/memory-v3-shadow/selection-log-store.ts +19 -12
  321. package/src/plugins/defaults/memory-v3-shadow/shadow-plugin.ts +96 -43
  322. package/src/plugins/defaults/memory-v3-shadow/types.ts +34 -17
  323. package/src/plugins/defaults/title-generate/hooks/stop.ts +9 -11
  324. package/src/plugins/pipeline.ts +8 -5
  325. package/src/plugins/types.ts +5 -51
  326. package/src/providers/cache-control.ts +26 -0
  327. package/src/providers/inference/__tests__/base-url-route-validation.test.ts +1 -2
  328. package/src/providers/model-catalog.ts +13 -1
  329. package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +147 -0
  330. package/src/providers/openai/chat-completions-provider.ts +46 -0
  331. package/src/providers/openai/responses-provider.ts +45 -0
  332. package/src/providers/registry.ts +0 -8
  333. package/src/runtime/__tests__/agent-wake.test.ts +0 -1
  334. package/src/runtime/agent-wake.ts +13 -0
  335. package/src/runtime/routes/__tests__/acp-routes.test.ts +151 -0
  336. package/src/runtime/routes/__tests__/consolidation-routes.test.ts +12 -50
  337. package/src/runtime/routes/__tests__/conversation-surface-routes.test.ts +322 -0
  338. package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +0 -62
  339. package/src/runtime/routes/__tests__/plugins-routes.test.ts +31 -11
  340. package/src/runtime/routes/acp-routes.test.ts +106 -0
  341. package/src/runtime/routes/acp-routes.ts +248 -2
  342. package/src/runtime/routes/browser-tabs-routes.ts +1 -1
  343. package/src/runtime/routes/channel-verification-routes.ts +14 -5
  344. package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +0 -14
  345. package/src/runtime/routes/consolidation-routes.ts +6 -82
  346. package/src/runtime/routes/conversation-list-routes.ts +6 -0
  347. package/src/runtime/routes/conversation-management-routes.ts +71 -0
  348. package/src/runtime/routes/identity-routes.ts +8 -0
  349. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +33 -34
  350. package/src/runtime/routes/inference-provider-connection-routes.ts +0 -45
  351. package/src/runtime/routes/plugins-routes.ts +34 -45
  352. package/src/runtime/routes/schedule-routes.ts +43 -5
  353. package/src/runtime/routes/settings-routes.ts +140 -15
  354. package/src/runtime/routes/skills-routes.ts +18 -6
  355. package/src/runtime/services/__tests__/conversation-serializer.test.ts +140 -0
  356. package/src/runtime/services/conversation-serializer.ts +38 -1
  357. package/src/runtime/verification-outbound-actions.ts +147 -2
  358. package/src/runtime/verification-templates.ts +29 -3
  359. package/src/schedule/schedule-store.ts +19 -0
  360. package/src/skills/catalog-install.ts +77 -13
  361. package/src/tasks/task-scheduler.ts +1 -0
  362. package/src/telemetry/types.ts +66 -1
  363. package/src/telemetry/usage-telemetry-reporter.test.ts +542 -13
  364. package/src/telemetry/usage-telemetry-reporter.ts +213 -20
  365. package/src/tools/browser/__tests__/browser-execution-acquire.test.ts +49 -2
  366. package/src/tools/browser/__tests__/browser-status.test.ts +29 -5
  367. package/src/tools/browser/browser-execution.ts +27 -9
  368. package/src/tools/browser/cdp-client/__tests__/factory.test.ts +380 -4
  369. package/src/tools/browser/cdp-client/__tests__/host-bridge-cdp-client.test.ts +107 -0
  370. package/src/tools/browser/cdp-client/__tests__/types.test.ts +6 -1
  371. package/src/tools/browser/cdp-client/factory.ts +217 -17
  372. package/src/tools/browser/cdp-client/host-bridge-cdp-client.ts +67 -0
  373. package/src/tools/browser/cdp-client/types.ts +22 -2
  374. package/src/tools/credential-execution/make-authenticated-request.ts +2 -1
  375. package/src/tools/credential-execution/manage-secure-command-tool.ts +169 -164
  376. package/src/tools/credential-execution/run-authenticated-command.ts +2 -1
  377. package/src/tools/executor.ts +39 -7
  378. package/src/tools/registry.ts +387 -5
  379. package/src/tools/schedule/create.ts +16 -0
  380. package/src/tools/schedule/list.ts +12 -4
  381. package/src/tools/schedule/update.ts +12 -0
  382. package/src/tools/skills/load.ts +11 -6
  383. package/src/tools/terminal/safe-env.ts +2 -0
  384. package/src/tools/types.ts +65 -9
  385. package/src/tools/workspace-tools/loader.ts +673 -0
  386. package/src/usage/attribution.ts +28 -0
  387. package/src/util/device-id.ts +17 -3
  388. package/src/util/platform.ts +16 -0
  389. package/tsconfig.plugin-api.json +13 -0
  390. package/src/__tests__/plugin-external-api.test.ts +0 -68
  391. package/src/__tests__/plugin-skill-contribution.test.ts +0 -355
  392. package/src/daemon/message-types/browser.ts +0 -10
  393. package/src/notifications/__tests__/emit-signal-home-feed.test.ts +0 -187
  394. package/src/plugins/defaults/memory-v3-shadow/__tests__/fixtures/eval-turns.json +0 -36
  395. package/src/plugins/defaults/memory-v3-shadow/__tests__/fixtures/live-turns.json +0 -37
  396. package/src/plugins/defaults/memory-v3-shadow/__tests__/working-set-eviction.test.ts +0 -106
  397. package/src/plugins/defaults/memory-v3-shadow/__tests__/working-set-skeleton.test.ts +0 -44
  398. package/src/plugins/defaults/memory-v3-shadow/working-set.ts +0 -91
  399. package/src/plugins/external-api.ts +0 -114
  400. package/src/plugins/plugin-skill-contributions.ts +0 -292
@@ -0,0 +1,46 @@
1
+ /**
2
+ * Per-conversation empty-response nudge state for the default empty-response
3
+ * module.
4
+ *
5
+ * The `post-model-call` hook recovers from an empty turn after tool use by
6
+ * appending a nudge and asking the loop to re-query. That recovery is bounded
7
+ * to one pass per run: a second empty turn means the nudge could not coax a
8
+ * real answer, so the hook lets the turn end rather than looping.
9
+ *
10
+ * The two hooks split this state's lifecycle: `post-model-call` marks a
11
+ * conversation when it issues a nudge-retry, and the sibling `stop` hook clears
12
+ * the mark when the turn terminates. A conversation therefore only holds an
13
+ * entry while a nudge is in flight, and the next run always nudges afresh.
14
+ *
15
+ * This module is side-effect free: importing it only initializes an empty store
16
+ * and registers no plugin.
17
+ */
18
+
19
+ /** Conversations with an empty-response nudge in flight for the current run. */
20
+ const nudgeInFlight = new Set<string>();
21
+
22
+ /** Whether the conversation already issued its one nudge this run. */
23
+ export function isEmptyResponseNudged(conversationId: string): boolean {
24
+ return nudgeInFlight.has(conversationId);
25
+ }
26
+
27
+ /** Record that the conversation issued its one nudge-retry this run. */
28
+ export function markEmptyResponseNudged(conversationId: string): void {
29
+ nudgeInFlight.add(conversationId);
30
+ }
31
+
32
+ /**
33
+ * Clear the conversation's nudge mark so the next run nudges afresh. The
34
+ * sibling `stop` hook calls this when the turn terminates.
35
+ */
36
+ export function clearEmptyResponseNudged(conversationId: string): void {
37
+ nudgeInFlight.delete(conversationId);
38
+ }
39
+
40
+ /**
41
+ * Test-only: drop every conversation's nudge state so a suite that drives the
42
+ * hook directly starts each case from an empty store.
43
+ */
44
+ export function resetEmptyResponseNudgeStoreForTests(): void {
45
+ nudgeInFlight.clear();
46
+ }
@@ -0,0 +1,50 @@
1
+ /**
2
+ * Default `post-model-call` hook: recovers from a provider ordering rejection.
3
+ *
4
+ * A provider rejects the call when the history violates tool-use/tool-result
5
+ * pairing or role-alternation rules (orphan `tool_use`, orphan
6
+ * `web_search_tool_result`, leading assistant turn, consecutive same-role
7
+ * turns). That rejection is a model-call outcome — the loop runs the
8
+ * `post-model-call` chain with the rejection attached. This hook recognizes the
9
+ * ordering class and runs {@link deepRepairHistory} directly — deliberately not
10
+ * through the `user-prompt-submit` hook chain, whose user/plugin hooks may have
11
+ * caused the drift — to re-normalize the history, then asks the loop to retry
12
+ * the call.
13
+ *
14
+ * Bounded to one pass per turn via the per-conversation repair state: a second
15
+ * consecutive ordering rejection means the repair could not recover the
16
+ * history, so the hook leaves the error to surface rather than looping. The
17
+ * loop's per-run backstop caps these retries globally; this one-shot mark keeps
18
+ * a single recovery attempt per turn. This hook only ever marks the
19
+ * conversation; the sibling `stop` hook (see `./stop.ts`) clears the mark when
20
+ * the turn terminates, so the next turn always repairs afresh.
21
+ *
22
+ * A finalized reply (the model returned content) is left untouched for the
23
+ * empty-response hook; only a provider rejection is this hook's to act on.
24
+ */
25
+
26
+ import type { PluginHookFn, PostModelCallContext } from "@vellumai/plugin-api";
27
+
28
+ import {
29
+ isOrderingRepairAttempted,
30
+ markOrderingRepairAttempted,
31
+ } from "../repair-state-store.js";
32
+ import { deepRepairHistory, isRepairableOrderingError } from "../terminal.js";
33
+
34
+ const postModelCall: PluginHookFn<PostModelCallContext> = async (ctx) => {
35
+ if (
36
+ ctx.error &&
37
+ isRepairableOrderingError(ctx.error.message) &&
38
+ !isOrderingRepairAttempted(ctx.conversationId)
39
+ ) {
40
+ markOrderingRepairAttempted(ctx.conversationId);
41
+ ctx.messages = deepRepairHistory(ctx.messages).messages;
42
+ ctx.decision = "continue";
43
+ ctx.logger.warn(
44
+ { plugin: "history-repair", messageCount: ctx.messages.length },
45
+ "Provider ordering error — recovering via history deep-repair",
46
+ );
47
+ }
48
+ };
49
+
50
+ export default postModelCall;
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Default `stop` hook: clears the per-conversation ordering-repair bound when a
3
+ * turn terminates.
4
+ *
5
+ * The `post-model-call` hook (see `./post-model-call.ts`) marks the bound when
6
+ * it deep-repairs the history and asks the loop to retry. `stop` is the
7
+ * definitive terminal hook — it fires exactly once when the turn is truly
8
+ * ending, after every retry decision has been made — so clearing the bound here
9
+ * unconditionally guarantees the next turn always repairs afresh, no matter how
10
+ * the turn ended (a finalized reply, an unrecovered rejection, an abort, or a
11
+ * retry the loop's per-run backstop refused).
12
+ */
13
+
14
+ import type { PluginHookFn, StopContext } from "@vellumai/plugin-api";
15
+
16
+ import { clearOrderingRepairAttempted } from "../repair-state-store.js";
17
+
18
+ const stop: PluginHookFn<StopContext> = async (ctx) => {
19
+ clearOrderingRepairAttempted(ctx.conversationId);
20
+ };
21
+
22
+ export default stop;
@@ -0,0 +1,51 @@
1
+ /**
2
+ * Per-conversation ordering-repair state for the default history-repair module.
3
+ *
4
+ * The `post-model-call` hook recovers from a provider ordering rejection by
5
+ * running a deep repair and asking the loop to retry. That recovery is bounded
6
+ * to one pass per turn: a second consecutive ordering rejection means the
7
+ * repair could not recover the history, so the hook lets the error surface
8
+ * rather than looping.
9
+ *
10
+ * The `post-model-call` hook marks a conversation when it issues a repair-retry
11
+ * and clears the mark on any outcome it resolves — a finalized reply, a
12
+ * non-ordering rejection, or an exhausted second ordering rejection. The
13
+ * `stop` hook clears it on the one terminal that hook does not resolve: a retry
14
+ * the loop's per-run backstop overrides, surfacing the rejection without
15
+ * re-running `post-model-call`. A conversation therefore only holds an entry
16
+ * while a repair is in flight, so the store stays bounded without a separate
17
+ * teardown step.
18
+ *
19
+ * This module is side-effect free: importing it only initializes an empty store
20
+ * and registers no plugin.
21
+ */
22
+
23
+ /** Conversations with a deep-repair pass in flight for the current turn. */
24
+ const repairInFlight = new Set<string>();
25
+
26
+ /** Whether the conversation already attempted a deep repair this turn. */
27
+ export function isOrderingRepairAttempted(conversationId: string): boolean {
28
+ return repairInFlight.has(conversationId);
29
+ }
30
+
31
+ /** Record that the conversation issued its one deep-repair retry this turn. */
32
+ export function markOrderingRepairAttempted(conversationId: string): void {
33
+ repairInFlight.add(conversationId);
34
+ }
35
+
36
+ /**
37
+ * Clear the conversation's repair mark so the next turn (or run) repairs afresh.
38
+ * The `post-model-call` hook calls this on the outcomes it resolves; the `stop`
39
+ * hook calls it on a terminal stop to cover a backstop-overridden retry.
40
+ */
41
+ export function clearOrderingRepairAttempted(conversationId: string): void {
42
+ repairInFlight.delete(conversationId);
43
+ }
44
+
45
+ /**
46
+ * Test-only: drop every conversation's repair state so a suite that drives the
47
+ * hook directly starts each case from an empty store.
48
+ */
49
+ export function resetRepairStateStoreForTests(): void {
50
+ repairInFlight.clear();
51
+ }
@@ -7,8 +7,9 @@
7
7
  * `repairHistory` is the canonical repair pass, run per turn via the plugin's
8
8
  * `user-prompt-submit` hook (`./hooks/user-prompt-submit.ts`) and exported so
9
9
  * daemon call sites and tests can reach it directly. `deepRepairHistory` is an
10
- * aggressive one-shot fallback the orchestrator invokes only after a provider
11
- * ordering error.
10
+ * aggressive one-shot fallback applied after a provider ordering rejection, and
11
+ * `isRepairableOrderingError` recognizes the rejections it can recover — so
12
+ * detection and repair of ordering drift live together as one unit.
12
13
  */
13
14
 
14
15
  import type {
@@ -333,6 +334,42 @@ function formatWebSearchContent(content: unknown): string {
333
334
  return "results unavailable";
334
335
  }
335
336
 
337
+ /**
338
+ * Provider rejection messages indicating a tool_use/tool_result pairing or
339
+ * message-ordering violation — the structural drift `deepRepairHistory`
340
+ * re-normalizes.
341
+ */
342
+ export const ORDERING_ERROR_PATTERNS: readonly RegExp[] = [
343
+ /tool_result.*not immediately after.*tool_use/i,
344
+ /tool_use.*must have.*tool_result/i,
345
+ /tool_use_id.*without.*tool_result/i,
346
+ /tool_result.*tool_use_id.*not found/i,
347
+ /messages.*invalid.*order/i,
348
+ ];
349
+
350
+ /**
351
+ * Provider rejection messages for the server-side web-search tool's own
352
+ * tool_use/result pairing violations, which `repairHistory` resolves by
353
+ * downgrading the orphaned block to text.
354
+ */
355
+ export const WEB_SEARCH_ORDERING_PATTERNS: readonly RegExp[] = [
356
+ /web_search.*tool_use.*without/i,
357
+ /web_search.*tool_result/i,
358
+ ];
359
+
360
+ /**
361
+ * Whether a provider error message denotes an ordering/pairing violation that
362
+ * `deepRepairHistory` can recover by re-normalizing the history. Detection and
363
+ * repair are co-located so callers recognize and fix ordering drift through a
364
+ * single source of truth.
365
+ */
366
+ export function isRepairableOrderingError(message: string): boolean {
367
+ return (
368
+ ORDERING_ERROR_PATTERNS.some((pattern) => pattern.test(message)) ||
369
+ WEB_SEARCH_ORDERING_PATTERNS.some((pattern) => pattern.test(message))
370
+ );
371
+ }
372
+
336
373
  /**
337
374
  * Aggressive repair pass that handles edge cases beyond repairHistory:
338
375
  * - Removes empty messages
@@ -0,0 +1,25 @@
1
+ /**
2
+ * Detection for the image-too-large recovery path.
3
+ *
4
+ * Anthropic returns a 400 with one of these messages when an image block
5
+ * violates a hard limit. The first matches the per-side pixel cap ("image
6
+ * dimensions exceed max allowed size"); the second matches the base64 payload
7
+ * cap ("image exceeds 5 MB maximum: 7465044 bytes > 5242880 bytes"). Distinct
8
+ * classification matters because retrying with the same oversized image is
9
+ * futile — the recovery path must downscale or note it instead.
10
+ *
11
+ * Exported as the single source of truth: the image-recovery `stop` hook reads
12
+ * it to recognize the rejection it can recover, and `daemon/conversation-error`
13
+ * imports it so the user-facing classification stays in lockstep with what the
14
+ * hook actually recovers.
15
+ */
16
+
17
+ export const IMAGE_DIMENSIONS_TOO_LARGE_PATTERNS: readonly RegExp[] = [
18
+ /image dimensions? exceeds? max allowed size/i,
19
+ /image exceeds \d+\s*MB maximum/i,
20
+ ];
21
+
22
+ /** Whether an error message indicates an image-input dimension/payload failure. */
23
+ export function isImageDimensionsTooLargeError(message: string): boolean {
24
+ return IMAGE_DIMENSIONS_TOO_LARGE_PATTERNS.some((p) => p.test(message));
25
+ }
@@ -0,0 +1,73 @@
1
+ /**
2
+ * Default `post-model-call` hook: recovers from a provider image-too-large
3
+ * rejection.
4
+ *
5
+ * A provider rejects the call when an attached image violates a hard limit —
6
+ * its longest side exceeds the per-side pixel cap, or its base64 payload
7
+ * exceeds the size cap. That rejection is a model-call outcome — the loop runs
8
+ * the `post-model-call` chain with the rejection attached. This hook recognizes
9
+ * the image-too-large class, downscales the oversized image blocks in the
10
+ * working history via {@link recoverOversizedImages}, and asks the loop to
11
+ * retry the call. It also persists the same downgrade durably via {@link
12
+ * persistUnsendableImageDowngrades} so the rejected image cannot rehydrate from
13
+ * the stored row and re-reject on every later turn.
14
+ *
15
+ * Bounded to one pass per turn via the per-conversation recovery state: a
16
+ * second consecutive image-too-large rejection means the downscale could not
17
+ * bring the image under the cap, so the hook leaves the error to surface rather
18
+ * than looping. The loop's per-run backstop caps these retries globally; this
19
+ * one-shot mark keeps a single recovery attempt per turn. This hook only ever
20
+ * marks the conversation; the sibling `stop` hook (see `./stop.ts`) clears the
21
+ * mark when the turn terminates, so the next turn recovers afresh.
22
+ *
23
+ * A finalized reply (the model returned content) is left untouched for the
24
+ * empty-response hook; only a provider rejection is this hook's to act on.
25
+ */
26
+
27
+ import type { PluginHookFn, PostModelCallContext } from "@vellumai/plugin-api";
28
+
29
+ import { isImageDimensionsTooLargeError } from "../detect.js";
30
+ import {
31
+ isImageRecoveryAttempted,
32
+ markImageRecoveryAttempted,
33
+ } from "../image-recovery-state-store.js";
34
+ import {
35
+ persistUnsendableImageDowngrades,
36
+ recoverOversizedImages,
37
+ } from "../recover.js";
38
+
39
+ const postModelCall: PluginHookFn<PostModelCallContext> = async (ctx) => {
40
+ if (
41
+ ctx.error &&
42
+ isImageDimensionsTooLargeError(ctx.error.message) &&
43
+ !isImageRecoveryAttempted(ctx.conversationId)
44
+ ) {
45
+ markImageRecoveryAttempted(ctx.conversationId);
46
+ ctx.messages = recoverOversizedImages(ctx.messages);
47
+ // Make the downgrade durable so the rejected image can't rehydrate from the
48
+ // stored row and re-reject on later turns. This is cleanup for future
49
+ // turns, so a persistence failure must never abort the retry that is about
50
+ // to run — log it and continue with the in-memory recovery.
51
+ try {
52
+ const rewritten = persistUnsendableImageDowngrades(ctx.conversationId);
53
+ if (rewritten > 0) {
54
+ ctx.logger.info(
55
+ { plugin: "image-recovery", rewritten },
56
+ "Persisted unsendable-image downgrades so they cannot resurface",
57
+ );
58
+ }
59
+ } catch (err) {
60
+ ctx.logger.warn(
61
+ { plugin: "image-recovery", err },
62
+ "Failed to persist unsendable-image downgrade; continuing with in-memory recovery",
63
+ );
64
+ }
65
+ ctx.decision = "continue";
66
+ ctx.logger.warn(
67
+ { plugin: "image-recovery", messageCount: ctx.messages.length },
68
+ "Provider image-too-large error — recovering oversized image blocks and retrying",
69
+ );
70
+ }
71
+ };
72
+
73
+ export default postModelCall;
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Default `stop` hook: clears the per-conversation image-recovery bound when a
3
+ * turn terminates.
4
+ *
5
+ * The `post-model-call` hook (see `./post-model-call.ts`) marks the bound when
6
+ * it downscales an oversized image and asks the loop to retry. `stop` is the
7
+ * definitive terminal hook — it fires exactly once when the turn is truly
8
+ * ending, after every retry decision has been made — so clearing the bound here
9
+ * unconditionally guarantees the next turn always recovers afresh, no matter how
10
+ * the turn ended (a finalized reply, an unrecovered rejection, an abort, or a
11
+ * retry the loop's per-run backstop refused).
12
+ */
13
+
14
+ import type { PluginHookFn, StopContext } from "@vellumai/plugin-api";
15
+
16
+ import { clearImageRecoveryAttempted } from "../image-recovery-state-store.js";
17
+
18
+ const stop: PluginHookFn<StopContext> = async (ctx) => {
19
+ clearImageRecoveryAttempted(ctx.conversationId);
20
+ };
21
+
22
+ export default stop;
@@ -0,0 +1,48 @@
1
+ /**
2
+ * Per-conversation image-recovery state for the default image-recovery module.
3
+ *
4
+ * The `stop` hook recovers from a provider image-too-large rejection by
5
+ * downscaling the oversized image blocks and asking the loop to retry. That
6
+ * recovery is bounded to one pass per turn: a second consecutive image-too-large
7
+ * rejection means the downscale could not get the image under the provider's
8
+ * cap, so the hook lets the error surface rather than looping.
9
+ *
10
+ * The `stop` hook owns this state end-to-end. It marks a conversation when it
11
+ * issues a recovery-retry and clears the mark on any terminal stop — a
12
+ * successful response, a non-image rejection, or an exhausted second
13
+ * image-too-large rejection — i.e. whenever the loop is leaving rather than
14
+ * retrying. A conversation therefore only holds an entry while a recovery is in
15
+ * flight, so the store stays bounded without a separate teardown step.
16
+ *
17
+ * This module is side-effect free: importing it only initializes an empty store
18
+ * and registers no plugin.
19
+ */
20
+
21
+ /** Conversations with an image-recovery pass in flight for the current turn. */
22
+ const recoveryInFlight = new Set<string>();
23
+
24
+ /** Whether the conversation already attempted an image recovery this turn. */
25
+ export function isImageRecoveryAttempted(conversationId: string): boolean {
26
+ return recoveryInFlight.has(conversationId);
27
+ }
28
+
29
+ /** Record that the conversation issued its one image-recovery retry this turn. */
30
+ export function markImageRecoveryAttempted(conversationId: string): void {
31
+ recoveryInFlight.add(conversationId);
32
+ }
33
+
34
+ /**
35
+ * Clear the conversation's recovery mark so the next turn (or run) recovers
36
+ * afresh. The `stop` hook calls this on every terminal stop.
37
+ */
38
+ export function clearImageRecoveryAttempted(conversationId: string): void {
39
+ recoveryInFlight.delete(conversationId);
40
+ }
41
+
42
+ /**
43
+ * Test-only: drop every conversation's recovery state so a suite that drives the
44
+ * hook directly starts each case from an empty store.
45
+ */
46
+ export function resetImageRecoveryStoreForTests(): void {
47
+ recoveryInFlight.clear();
48
+ }
@@ -0,0 +1,14 @@
1
+ {
2
+ "name": "default-image-recovery",
3
+ "version": "1.0.0",
4
+ "description": "First-party default plugin that recovers from provider image-too-large rejections by downscaling oversized image blocks and retrying.",
5
+ "private": true,
6
+ "license": "MIT",
7
+ "type": "module",
8
+ "engines": {
9
+ "node": ">=20.12.0"
10
+ },
11
+ "peerDependencies": {
12
+ "@vellumai/plugin-api": "^0.8.0"
13
+ }
14
+ }
@@ -1,26 +1,28 @@
1
1
  /**
2
- * Persistence for the image-too-large recovery path.
2
+ * Image-too-large recovery transforms for the image-recovery plugin.
3
3
  *
4
4
  * When the provider rejects a turn because an attached image exceeds its
5
- * limits, the agent loop downgrades the offending image blocks in memory and
6
- * retries. That transformation is transient — the stored message row keeps the
7
- * full-size image block, so the rejected image is rehydrated on every later
8
- * turn and keeps re-entering the model's context. This module makes the
9
- * downgrade durable: the oversized block is rewritten to its downscaled form,
10
- * or to a text note when it cannot be shrunk on this host, so a rejected image
11
- * cannot resurface and re-reject on every later turn.
5
+ * limits, the `stop` hook downgrades the offending image blocks in the working
6
+ * history and asks the loop to retry. {@link recoverOversizedImages} performs
7
+ * that in-memory transform for the immediate retry; {@link
8
+ * persistUnsendableImageDowngrades} makes the same downgrade durable, because
9
+ * the stored message row otherwise keeps the full-size image block and the
10
+ * rejected image rehydrates on every later turn and keeps re-entering the
11
+ * model's context. The durable rewrite replaces the oversized block with its
12
+ * downscaled form, or with a text note when it cannot be shrunk on this host,
13
+ * so a rejected image cannot resurface and re-reject on every later turn.
12
14
  */
13
15
 
14
- import { optimizeImageForTransport } from "../agent/image-optimize.js";
15
- import { parseImageDimensions } from "../context/image-dimensions.js";
16
+ import { optimizeImageForTransport } from "../../../agent/image-optimize.js";
17
+ import { parseImageDimensions } from "../../../context/image-dimensions.js";
16
18
  import {
17
19
  getMessages,
18
20
  updateMessageContent,
19
- } from "../memory/conversation-crud.js";
20
- import type { ContentBlock } from "../providers/types.js";
21
- import { getLogger } from "../util/logger.js";
21
+ } from "../../../memory/conversation-crud.js";
22
+ import type { ContentBlock, Message } from "../../../providers/types.js";
23
+ import { getLogger } from "../../../util/logger.js";
22
24
 
23
- const log = getLogger("persist-unsendable-image");
25
+ const log = getLogger("image-recovery");
24
26
 
25
27
  // Anthropic rejects any image whose longest side exceeds this many pixels,
26
28
  // regardless of payload size. Mirrors the user-facing message surfaced by
@@ -152,3 +154,54 @@ export function persistUnsendableImageDowngrades(
152
154
  }
153
155
  return rewritten;
154
156
  }
157
+
158
+ /**
159
+ * True when a message's content holds an image the provider may have rejected
160
+ * for being oversized — either a top-level image block (user upload) or one
161
+ * nested inside a tool_result's contentBlocks (e.g. a browser screenshot).
162
+ */
163
+ function messageHasImageBlock(content: ReadonlyArray<ContentBlock>): boolean {
164
+ return content.some(
165
+ (b) =>
166
+ b.type === "image" ||
167
+ (b.type === "tool_result" &&
168
+ (b.contentBlocks?.some((cb) => cb.type === "image") ?? false)),
169
+ );
170
+ }
171
+
172
+ /**
173
+ * Downscale every oversized image in the working history for an immediate
174
+ * retry, leaving still-sendable images untouched. Recovers both top-level image
175
+ * blocks (user uploads) and images nested inside a tool_result's contentBlocks
176
+ * (e.g. a browser screenshot) in place, so the tool_use/tool_result pairing
177
+ * stays intact rather than dropping the whole tool_result. Applies the same
178
+ * provider-cap gate as {@link persistUnsendableImageDowngrades} so the in-memory
179
+ * retry and the durable rewrite agree on which images are unsendable.
180
+ */
181
+ export function recoverOversizedImages(
182
+ messages: ReadonlyArray<Message>,
183
+ ): Message[] {
184
+ return messages.map((msg) => {
185
+ if (!Array.isArray(msg.content)) return msg;
186
+ if (!messageHasImageBlock(msg.content)) return msg;
187
+ return {
188
+ ...msg,
189
+ content: msg.content.flatMap((b): ContentBlock[] => {
190
+ if (b.type === "image") {
191
+ return [oversizedImageReplacement(b) ?? b];
192
+ }
193
+ if (b.type === "tool_result" && b.contentBlocks?.length) {
194
+ return [
195
+ {
196
+ ...b,
197
+ contentBlocks: b.contentBlocks.map((cb) =>
198
+ cb.type === "image" ? (oversizedImageReplacement(cb) ?? cb) : cb,
199
+ ),
200
+ },
201
+ ];
202
+ }
203
+ return [b];
204
+ }),
205
+ };
206
+ });
207
+ }
@@ -27,10 +27,22 @@
27
27
  import { registerPlugin, resetPluginRegistryForTests } from "../registry.js";
28
28
  import { type Plugin, PluginExecutionError } from "../types.js";
29
29
  import compactionPkg from "./compaction/package.json" with { type: "json" };
30
+ import emptyResponsePostModelCall from "./empty-response/hooks/post-model-call.js";
30
31
  import emptyResponseStop from "./empty-response/hooks/stop.js";
32
+ import { resetEmptyResponseNudgeStoreForTests } from "./empty-response/nudge-state-store.js";
31
33
  import emptyResponsePkg from "./empty-response/package.json" with { type: "json" };
34
+ import historyRepairPostModelCall from "./history-repair/hooks/post-model-call.js";
35
+ import historyRepairStop from "./history-repair/hooks/stop.js";
32
36
  import historyRepairUserPromptSubmit from "./history-repair/hooks/user-prompt-submit.js";
33
37
  import historyRepairPkg from "./history-repair/package.json" with { type: "json" };
38
+ import { resetRepairStateStoreForTests } from "./history-repair/repair-state-store.js";
39
+ import imageRecoveryPostModelCall from "./image-recovery/hooks/post-model-call.js";
40
+ import imageRecoveryStop from "./image-recovery/hooks/stop.js";
41
+ import { resetImageRecoveryStoreForTests } from "./image-recovery/image-recovery-state-store.js";
42
+ import imageRecoveryPkg from "./image-recovery/package.json" with { type: "json" };
43
+ import memoryRetrievalPostCompact from "./memory-retrieval/hooks/post-compact.js";
44
+ import memoryRetrievalUserPromptSubmit from "./memory-retrieval/hooks/user-prompt-submit.js";
45
+ import memoryRetrievalPkg from "./memory-retrieval/package.json" with { type: "json" };
34
46
  import memoryV3PostCompact from "./memory-v3-shadow/hooks/post-compact.js";
35
47
  import memoryV3UserPromptSubmit from "./memory-v3-shadow/hooks/user-prompt-submit.js";
36
48
  import memoryV3Pkg from "./memory-v3-shadow/package.json" with { type: "json" };
@@ -57,8 +69,10 @@ export const defaultCompactionPlugin: Plugin = {
57
69
  };
58
70
 
59
71
  /**
60
- * `empty-response` — a `stop` hook that re-queries the model when a turn
61
- * yields with no tool calls but came back empty (or as a provider refusal).
72
+ * `empty-response` — a `post-model-call` hook that re-queries the model when a
73
+ * turn yields with no tool calls but came back empty (or as a provider
74
+ * refusal); the `stop` hook clears the one-shot nudge bound on a terminal stop
75
+ * so the next run nudges afresh.
62
76
  */
63
77
  export const defaultEmptyResponsePlugin: Plugin = {
64
78
  manifest: {
@@ -66,14 +80,39 @@ export const defaultEmptyResponsePlugin: Plugin = {
66
80
  version: emptyResponsePkg.version,
67
81
  },
68
82
  hooks: {
83
+ "post-model-call": emptyResponsePostModelCall,
69
84
  stop: emptyResponseStop,
70
85
  },
71
86
  };
72
87
 
73
88
  /**
74
- * `history-repair` — a `user-prompt-submit` hook that normalizes the working
75
- * message history (tool-use/tool-result pairing, role alternation) before the
76
- * agent loop hands it to the provider.
89
+ * `memory-retrieval` — assembles the turn's runtime injections (the unified
90
+ * `<turn_context>` block, Slack chronological transcript, NOW.md / PKB /
91
+ * memory-v2 / workspace blocks) via two hooks: `user-prompt-submit` runs
92
+ * memory-graph retrieval and the initial injection, and `post-compact`
93
+ * re-applies the injections onto the compacted history after a mid-turn
94
+ * compaction. Registered first in the chain so later `user-prompt-submit`
95
+ * hooks (history repair, title) see the fully memory-injected history.
96
+ */
97
+ export const defaultMemoryRetrievalPlugin: Plugin = {
98
+ manifest: {
99
+ name: memoryRetrievalPkg.name,
100
+ version: memoryRetrievalPkg.version,
101
+ },
102
+ hooks: {
103
+ "user-prompt-submit": memoryRetrievalUserPromptSubmit,
104
+ "post-compact": memoryRetrievalPostCompact,
105
+ },
106
+ };
107
+
108
+ /**
109
+ * `history-repair` — normalizes the working message history (tool-use/tool-result
110
+ * pairing, role alternation). The `user-prompt-submit` hook normalizes the
111
+ * history before each provider call; the `post-model-call` hook handles the
112
+ * provider rejection where the call failed on an ordering violation,
113
+ * deep-repairing the history and asking the loop to retry; the `stop` hook
114
+ * clears the one-shot repair bound on a terminal stop so the next turn repairs
115
+ * afresh.
77
116
  */
78
117
  export const defaultHistoryRepairPlugin: Plugin = {
79
118
  manifest: {
@@ -82,6 +121,28 @@ export const defaultHistoryRepairPlugin: Plugin = {
82
121
  },
83
122
  hooks: {
84
123
  "user-prompt-submit": historyRepairUserPromptSubmit,
124
+ "post-model-call": historyRepairPostModelCall,
125
+ stop: historyRepairStop,
126
+ },
127
+ };
128
+
129
+ /**
130
+ * `image-recovery` — recovers from a provider image-too-large rejection. The
131
+ * `post-model-call` hook handles the rejection, downscaling the oversized image
132
+ * blocks in the working history and asking the loop to retry, and persisting
133
+ * the same downgrade durably so the rejected image cannot rehydrate from the
134
+ * stored row and re-reject on later turns; the `stop` hook clears the one-shot
135
+ * recovery bound on a terminal stop so the next turn recovers afresh. Bounded
136
+ * to one pass per turn.
137
+ */
138
+ export const defaultImageRecoveryPlugin: Plugin = {
139
+ manifest: {
140
+ name: imageRecoveryPkg.name,
141
+ version: imageRecoveryPkg.version,
142
+ },
143
+ hooks: {
144
+ "post-model-call": imageRecoveryPostModelCall,
145
+ stop: imageRecoveryStop,
85
146
  },
86
147
  };
87
148
 
@@ -156,10 +217,12 @@ export const defaultToolResultTruncatePlugin: Plugin = {
156
217
  */
157
218
  function getAllDefaultPlugins(): readonly Plugin[] {
158
219
  return [
220
+ defaultMemoryRetrievalPlugin,
159
221
  defaultToolResultTruncatePlugin,
160
222
  defaultEmptyResponsePlugin,
161
223
  defaultToolErrorPlugin,
162
224
  defaultHistoryRepairPlugin,
225
+ defaultImageRecoveryPlugin,
163
226
  defaultCompactionPlugin,
164
227
  defaultTitleGeneratePlugin,
165
228
  memoryV3ShadowPlugin,
@@ -202,5 +265,8 @@ export function registerDefaultPlugins(): void {
202
265
  */
203
266
  export function resetPluginRegistryAndRegisterDefaults(): void {
204
267
  resetPluginRegistryForTests();
268
+ resetEmptyResponseNudgeStoreForTests();
269
+ resetRepairStateStoreForTests();
270
+ resetImageRecoveryStoreForTests();
205
271
  registerDefaultPlugins();
206
272
  }