@vellumai/assistant 0.4.56 → 0.4.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (450) hide show
  1. package/ARCHITECTURE.md +10 -10
  2. package/Dockerfile +3 -0
  3. package/README.md +11 -11
  4. package/docs/architecture/integrations.md +2 -2
  5. package/docs/architecture/memory.md +3 -4
  6. package/docs/credential-execution-service.md +13 -20
  7. package/node_modules/@vellumai/ces-contracts/src/error.ts +5 -4
  8. package/package.json +1 -1
  9. package/src/__tests__/actor-token-service.test.ts +7 -7
  10. package/src/__tests__/anthropic-provider.test.ts +172 -0
  11. package/src/__tests__/app-builder-tool-scripts.test.ts +15 -1
  12. package/src/__tests__/approval-cascade.test.ts +2 -2
  13. package/src/__tests__/approval-routes-http.test.ts +3 -4
  14. package/src/__tests__/asset-materialize-tool.test.ts +5 -5
  15. package/src/__tests__/asset-search-tool.test.ts +1 -1
  16. package/src/__tests__/assistant-attachments.test.ts +5 -5
  17. package/src/__tests__/assistant-events-sse-hardening.test.ts +1 -1
  18. package/src/__tests__/assistant-feature-flags-integration.test.ts +50 -38
  19. package/src/__tests__/attachments-store.test.ts +2 -2
  20. package/src/__tests__/avatar-e2e.test.ts +5 -3
  21. package/src/__tests__/browser-skill-endstate.test.ts +0 -1
  22. package/src/__tests__/call-routes-http.test.ts +2 -2
  23. package/src/__tests__/callback-handoff-copy.test.ts +1 -1
  24. package/src/__tests__/cancel-resolves-conversation-key.test.ts +158 -0
  25. package/src/__tests__/channel-readiness-routes.test.ts +0 -1
  26. package/src/__tests__/channel-readiness-service.test.ts +0 -1
  27. package/src/__tests__/checker.test.ts +31 -32
  28. package/src/__tests__/chrome-cdp.test.ts +47 -18
  29. package/src/__tests__/claude-code-skill-regression.test.ts +2 -2
  30. package/src/__tests__/config-schema-cmd.test.ts +2 -2
  31. package/src/__tests__/config-schema.test.ts +9 -18
  32. package/src/__tests__/confirmation-request-guardian-bridge.test.ts +1 -1
  33. package/src/__tests__/conversation-abort-tool-results.test.ts +4 -4
  34. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -2
  35. package/src/__tests__/conversation-agent-loop.test.ts +11 -4
  36. package/src/__tests__/conversation-attachments.test.ts +1 -1
  37. package/src/__tests__/conversation-confirmation-signals.test.ts +2 -2
  38. package/src/__tests__/conversation-error.test.ts +33 -0
  39. package/src/__tests__/conversation-init.benchmark.test.ts +0 -1
  40. package/src/__tests__/conversation-load-history-repair.test.ts +1 -1
  41. package/src/__tests__/conversation-pairing.test.ts +1 -1
  42. package/src/__tests__/conversation-pre-run-repair.test.ts +4 -4
  43. package/src/__tests__/conversation-provider-retry-repair.test.ts +4 -4
  44. package/src/__tests__/conversation-queue.test.ts +23 -14
  45. package/src/__tests__/conversation-routes-slash-commands.test.ts +3 -3
  46. package/src/__tests__/conversation-runtime-assembly.test.ts +185 -173
  47. package/src/__tests__/conversation-seed-composer.test.ts +1 -1
  48. package/src/__tests__/conversation-slash-queue.test.ts +4 -4
  49. package/src/__tests__/conversation-slash-unknown.test.ts +4 -4
  50. package/src/__tests__/conversation-starter-routes.test.ts +291 -0
  51. package/src/__tests__/conversation-wipe.test.ts +438 -0
  52. package/src/__tests__/conversation-workspace-cache-state.test.ts +2 -3
  53. package/src/__tests__/conversation-workspace-injection.test.ts +4 -5
  54. package/src/__tests__/conversation-workspace-tool-tracking.test.ts +4 -5
  55. package/src/__tests__/credential-security-e2e.test.ts +20 -0
  56. package/src/__tests__/credential-security-invariants.test.ts +1 -0
  57. package/src/__tests__/credential-vault-unit.test.ts +227 -0
  58. package/src/__tests__/credentials-cli.test.ts +3 -0
  59. package/src/__tests__/date-context.test.ts +59 -377
  60. package/src/__tests__/drop-capability-card-state-migration.test.ts +169 -0
  61. package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +11 -45
  62. package/src/__tests__/emit-signal-routing-intent.test.ts +3 -3
  63. package/src/__tests__/encrypted-store.test.ts +237 -15
  64. package/src/__tests__/ephemeral-permissions.test.ts +4 -5
  65. package/src/__tests__/event-bus.test.ts +3 -3
  66. package/src/__tests__/gateway-only-enforcement.test.ts +2 -2
  67. package/src/__tests__/gateway-only-guard.test.ts +1 -0
  68. package/src/__tests__/gemini-image-service.test.ts +4 -4
  69. package/src/__tests__/gemini-provider.test.ts +6 -9
  70. package/src/__tests__/guardian-binding-drift-heal.test.ts +128 -0
  71. package/src/__tests__/guardian-dispatch.test.ts +0 -1
  72. package/src/__tests__/host-shell-tool.test.ts +6 -6
  73. package/src/__tests__/http-user-message-parity.test.ts +2 -2
  74. package/src/__tests__/intent-routing.test.ts +51 -99
  75. package/src/__tests__/invite-routes-http.test.ts +5 -0
  76. package/src/__tests__/list-messages-attachments.test.ts +1 -1
  77. package/src/__tests__/managed-proxy-context.test.ts +2 -5
  78. package/src/__tests__/managed-skill-lifecycle.test.ts +8 -8
  79. package/src/__tests__/media-generate-image.test.ts +32 -15
  80. package/src/__tests__/media-reuse-story.e2e.test.ts +1 -1
  81. package/src/__tests__/memory-context-benchmark.benchmark.test.ts +1 -1
  82. package/src/__tests__/memory-lifecycle-e2e.test.ts +24 -18
  83. package/src/__tests__/memory-recall-quality.test.ts +4 -3
  84. package/src/__tests__/memory-regressions.test.ts +86 -90
  85. package/src/__tests__/migration-cross-version-compatibility.test.ts +32 -32
  86. package/src/__tests__/migration-export-http.test.ts +26 -27
  87. package/src/__tests__/migration-import-commit-http.test.ts +165 -37
  88. package/src/__tests__/migration-import-preflight-http.test.ts +81 -20
  89. package/src/__tests__/migration-validate-http.test.ts +16 -16
  90. package/src/__tests__/model-intents.test.ts +1 -1
  91. package/src/__tests__/no-domain-routing-in-prompt-guard.test.ts +1 -1
  92. package/src/__tests__/notification-broadcaster.test.ts +1 -1
  93. package/src/__tests__/notification-decision-fallback.test.ts +2 -2
  94. package/src/__tests__/notification-decision-identity.test.ts +8 -9
  95. package/src/__tests__/notification-decision-strategy.test.ts +1 -1
  96. package/src/__tests__/notification-deep-link.test.ts +1 -1
  97. package/src/__tests__/notification-guardian-path.test.ts +0 -1
  98. package/src/__tests__/notification-schedule-dedup.test.ts +7 -7
  99. package/src/__tests__/oauth-store.test.ts +1 -3
  100. package/src/__tests__/oauth2-gateway-transport.test.ts +6 -1
  101. package/src/__tests__/onboarding-template-contract.test.ts +23 -59
  102. package/src/__tests__/provider-error-scenarios.test.ts +154 -0
  103. package/src/__tests__/provider-fail-open-selection.test.ts +2 -2
  104. package/src/__tests__/provider-managed-proxy-integration.test.ts +8 -9
  105. package/src/__tests__/provider-registry-ollama.test.ts +5 -2
  106. package/src/__tests__/qdrant-manager.test.ts +7 -7
  107. package/src/__tests__/ratelimit.test.ts +0 -74
  108. package/src/__tests__/recording-handler.test.ts +0 -1
  109. package/src/__tests__/require-fresh-approval.test.ts +1 -1
  110. package/src/__tests__/runtime-attachment-metadata.test.ts +1 -1
  111. package/src/__tests__/runtime-events-sse-parity.test.ts +1 -1
  112. package/src/__tests__/runtime-events-sse.test.ts +1 -1
  113. package/src/__tests__/scheduler-recurrence.test.ts +46 -2
  114. package/src/__tests__/schema-transforms.test.ts +114 -54
  115. package/src/__tests__/secret-onetime-send.test.ts +20 -0
  116. package/src/__tests__/secret-routes-managed-proxy.test.ts +5 -2
  117. package/src/__tests__/secret-scanner-executor.test.ts +1 -2
  118. package/src/__tests__/send-endpoint-busy.test.ts +63 -4
  119. package/src/__tests__/send-notification-tool.test.ts +2 -2
  120. package/src/__tests__/shell-credential-ref.test.ts +0 -1
  121. package/src/__tests__/shell-tool-proxy-mode.test.ts +1 -2
  122. package/src/__tests__/skill-memory.test.ts +547 -0
  123. package/src/__tests__/skill-script-runner-sandbox.test.ts +1 -2
  124. package/src/__tests__/slack-app-setup-skill-regression.test.ts +37 -0
  125. package/src/__tests__/slack-channel-config.test.ts +109 -94
  126. package/src/__tests__/swarm-conversation-integration.test.ts +2 -2
  127. package/src/__tests__/swarm-recursion.test.ts +2 -2
  128. package/src/__tests__/swarm-tool.test.ts +2 -2
  129. package/src/__tests__/system-prompt.test.ts +19 -66
  130. package/src/__tests__/telegram-config.test.ts +121 -0
  131. package/src/__tests__/terminal-tools.test.ts +1 -1
  132. package/src/__tests__/tool-execution-abort-cleanup.test.ts +1 -2
  133. package/src/__tests__/tool-executor-lifecycle-events.test.ts +1 -1
  134. package/src/__tests__/tool-executor-shell-integration.test.ts +1 -1
  135. package/src/__tests__/tool-executor.test.ts +1 -1
  136. package/src/__tests__/trace-emitter.test.ts +8 -1
  137. package/src/__tests__/trust-store.test.ts +7 -8
  138. package/src/__tests__/twilio-routes.test.ts +1 -18
  139. package/src/__tests__/user-reference.test.ts +82 -2
  140. package/src/__tests__/vbundle-pax-and-symlink.test.ts +196 -0
  141. package/src/__tests__/verification-control-plane-policy.test.ts +1 -1
  142. package/src/approvals/guardian-request-resolvers.ts +3 -3
  143. package/src/avatar/ascii-renderer.ts +2 -2
  144. package/src/avatar/png-renderer.ts +2 -2
  145. package/src/avatar/resvg-lazy.ts +21 -0
  146. package/src/calls/guardian-dispatch.ts +1 -1
  147. package/src/calls/relay-access-wait.ts +2 -2
  148. package/src/calls/twilio-rest.ts +0 -248
  149. package/src/cli/AGENTS.md +5 -8
  150. package/src/cli/__tests__/notifications.test.ts +5 -5
  151. package/src/cli/commands/avatar.ts +64 -2
  152. package/src/cli/commands/conversations.ts +131 -1
  153. package/src/cli/commands/credentials.ts +2 -0
  154. package/src/cli/commands/notifications.ts +3 -3
  155. package/src/cli.ts +10 -0
  156. package/src/config/bundled-skills/acp/SKILL.md +5 -5
  157. package/src/config/bundled-skills/acp/TOOLS.json +6 -6
  158. package/src/config/bundled-skills/app-builder/SKILL.md +42 -42
  159. package/src/config/bundled-skills/app-builder/TOOLS.json +10 -10
  160. package/src/config/bundled-skills/browser/SKILL.md +15 -15
  161. package/src/config/bundled-skills/browser/TOOLS.json +14 -14
  162. package/src/config/bundled-skills/chatgpt-import/SKILL.md +2 -2
  163. package/src/config/bundled-skills/chatgpt-import/TOOLS.json +1 -1
  164. package/src/config/bundled-skills/chatgpt-import/tools/chatgpt-import.ts +1 -1
  165. package/src/config/bundled-skills/claude-code/SKILL.md +5 -5
  166. package/src/config/bundled-skills/computer-use/SKILL.md +2 -2
  167. package/src/config/bundled-skills/computer-use/TOOLS.json +15 -15
  168. package/src/config/bundled-skills/contacts/SKILL.md +3 -3
  169. package/src/config/bundled-skills/contacts/TOOLS.json +4 -4
  170. package/src/config/bundled-skills/document/SKILL.md +4 -4
  171. package/src/config/bundled-skills/document/TOOLS.json +2 -2
  172. package/src/config/bundled-skills/followups/TOOLS.json +3 -3
  173. package/src/config/bundled-skills/gmail/SKILL.md +32 -32
  174. package/src/config/bundled-skills/gmail/TOOLS.json +16 -16
  175. package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +1 -1
  176. package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +1 -1
  177. package/src/config/bundled-skills/google-calendar/SKILL.md +1 -1
  178. package/src/config/bundled-skills/google-calendar/TOOLS.json +5 -5
  179. package/src/config/bundled-skills/google-calendar/types.ts +1 -1
  180. package/src/config/bundled-skills/heartbeat/SKILL.md +43 -0
  181. package/src/config/bundled-skills/image-studio/SKILL.md +3 -3
  182. package/src/config/bundled-skills/image-studio/TOOLS.json +2 -3
  183. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +16 -12
  184. package/src/config/bundled-skills/media-processing/SKILL.md +40 -40
  185. package/src/config/bundled-skills/media-processing/TOOLS.json +8 -8
  186. package/src/config/bundled-skills/media-processing/__tests__/concurrency-pool.test.ts +2 -2
  187. package/src/config/bundled-skills/media-processing/__tests__/preprocess.test.ts +1 -1
  188. package/src/config/bundled-skills/media-processing/services/gemini-map.ts +5 -5
  189. package/src/config/bundled-skills/media-processing/services/gemini-video.ts +2 -2
  190. package/src/config/bundled-skills/media-processing/services/preprocess.ts +2 -2
  191. package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +2 -2
  192. package/src/config/bundled-skills/media-processing/services/reduce.ts +3 -3
  193. package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +2 -2
  194. package/src/config/bundled-skills/media-processing/tools/query-media-events.ts +1 -1
  195. package/src/config/bundled-skills/messaging/SKILL.md +29 -25
  196. package/src/config/bundled-skills/messaging/TOOLS.json +11 -11
  197. package/src/config/bundled-skills/messaging/tools/messaging-send.ts +1 -1
  198. package/src/config/bundled-skills/messaging/tools/shared.ts +1 -1
  199. package/src/config/bundled-skills/notifications/SKILL.md +3 -3
  200. package/src/config/bundled-skills/notifications/TOOLS.json +2 -2
  201. package/src/config/bundled-skills/notifications/tools/send-notification.ts +3 -3
  202. package/src/config/bundled-skills/orchestration/SKILL.md +1 -1
  203. package/src/config/bundled-skills/orchestration/TOOLS.json +1 -1
  204. package/src/config/bundled-skills/phone-calls/SKILL.md +18 -14
  205. package/src/config/bundled-skills/phone-calls/TOOLS.json +3 -3
  206. package/src/config/bundled-skills/phone-calls/references/CONFIG.md +2 -2
  207. package/src/config/bundled-skills/phone-calls/references/TRANSCRIPTS.md +2 -2
  208. package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +1 -1
  209. package/src/config/bundled-skills/playbooks/TOOLS.json +4 -4
  210. package/src/config/bundled-skills/schedule/SKILL.md +26 -26
  211. package/src/config/bundled-skills/schedule/TOOLS.json +5 -5
  212. package/src/config/bundled-skills/screen-watch/SKILL.md +3 -3
  213. package/src/config/bundled-skills/screen-watch/TOOLS.json +1 -1
  214. package/src/config/bundled-skills/sequences/SKILL.md +2 -2
  215. package/src/config/bundled-skills/sequences/TOOLS.json +10 -10
  216. package/src/config/bundled-skills/sequences/tools/sequence-analytics.ts +2 -2
  217. package/src/config/bundled-skills/sequences/tools/sequence-enroll.ts +2 -2
  218. package/src/config/bundled-skills/sequences/tools/sequence-enrollment-list.ts +1 -1
  219. package/src/config/bundled-skills/sequences/tools/sequence-get.ts +1 -1
  220. package/src/config/bundled-skills/sequences/tools/sequence-import.ts +3 -3
  221. package/src/config/bundled-skills/sequences/tools/sequence-list.ts +1 -1
  222. package/src/config/bundled-skills/sequences/tools/sequence-update.ts +1 -1
  223. package/src/config/bundled-skills/settings/TOOLS.json +3 -3
  224. package/src/config/bundled-skills/settings/tools/open-system-settings.ts +1 -1
  225. package/src/config/bundled-skills/skill-management/TOOLS.json +5 -5
  226. package/src/config/bundled-skills/skills-catalog/SKILL.md +84 -0
  227. package/src/config/bundled-skills/slack/SKILL.md +2 -2
  228. package/src/config/bundled-skills/slack/TOOLS.json +8 -8
  229. package/src/config/bundled-skills/slack/tools/slack-scan-digest.ts +3 -3
  230. package/src/config/bundled-skills/subagent/TOOLS.json +5 -5
  231. package/src/config/bundled-skills/tasks/SKILL.md +1 -1
  232. package/src/config/bundled-skills/tasks/TOOLS.json +9 -9
  233. package/src/config/bundled-skills/transcribe/SKILL.md +5 -5
  234. package/src/config/bundled-skills/transcribe/TOOLS.json +1 -1
  235. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +10 -10
  236. package/src/config/bundled-skills/watcher/SKILL.md +4 -4
  237. package/src/config/bundled-skills/watcher/TOOLS.json +5 -5
  238. package/src/config/feature-flag-registry.json +33 -17
  239. package/src/config/schemas/sandbox.ts +1 -1
  240. package/src/config/schemas/services.ts +13 -3
  241. package/src/config/schemas/timeouts.ts +0 -10
  242. package/src/contacts/contact-store.ts +63 -0
  243. package/src/contacts/contacts-write.ts +1 -1
  244. package/src/daemon/assistant-attachments.ts +2 -2
  245. package/src/daemon/conversation-agent-loop-handlers.ts +2 -2
  246. package/src/daemon/conversation-agent-loop.ts +7 -30
  247. package/src/daemon/conversation-error.ts +24 -0
  248. package/src/daemon/conversation-memory.ts +8 -7
  249. package/src/daemon/conversation-runtime-assembly.ts +139 -274
  250. package/src/daemon/conversation-slash.ts +7 -26
  251. package/src/daemon/conversation-surfaces.ts +14 -0
  252. package/src/daemon/conversation-tool-setup.ts +9 -8
  253. package/src/daemon/conversation.ts +2 -0
  254. package/src/daemon/daemon-control.ts +1 -1
  255. package/src/daemon/date-context.ts +10 -83
  256. package/src/daemon/handlers/config-channels.ts +12 -2
  257. package/src/daemon/handlers/config-slack-channel.ts +7 -1
  258. package/src/daemon/handlers/config-telegram.ts +6 -1
  259. package/src/daemon/handlers/conversations.ts +2 -2
  260. package/src/daemon/handlers/skills.ts +4 -0
  261. package/src/daemon/lifecycle.ts +28 -4
  262. package/src/daemon/providers-setup.ts +1 -1
  263. package/src/daemon/server.ts +1 -5
  264. package/src/daemon/shutdown-handlers.ts +9 -3
  265. package/src/daemon/tool-side-effects.ts +40 -0
  266. package/src/daemon/trace-emitter.ts +25 -2
  267. package/src/events/domain-events.ts +1 -1
  268. package/src/events/tool-permission-telemetry-listener.ts +46 -0
  269. package/src/inbound/platform-callback-registration.ts +0 -18
  270. package/src/media/app-icon-generator.ts +15 -8
  271. package/src/media/avatar-router.ts +15 -8
  272. package/src/media/gemini-image-service.ts +125 -21
  273. package/src/memory/attachments-store.ts +3 -3
  274. package/src/memory/channel-verification-sessions.ts +6 -6
  275. package/src/memory/conversation-crud.ts +196 -1
  276. package/src/memory/{thread-starters-cadence.ts → conversation-starters-cadence.ts} +9 -42
  277. package/src/memory/conversation-title-service.ts +2 -3
  278. package/src/memory/db-init.ts +25 -1
  279. package/src/memory/invite-store.ts +4 -4
  280. package/src/memory/items-extractor.ts +4 -4
  281. package/src/memory/job-handlers/{thread-starters.ts → conversation-starters.ts} +123 -38
  282. package/src/memory/jobs-store.ts +3 -2
  283. package/src/memory/jobs-worker.ts +7 -5
  284. package/src/memory/lifecycle-events-store.ts +63 -0
  285. package/src/memory/migrations/172-rename-created-by-session-id.ts +27 -0
  286. package/src/memory/migrations/173-rename-source-session-id.ts +16 -0
  287. package/src/memory/migrations/174-rename-thread-starters-table.ts +52 -0
  288. package/src/memory/migrations/175-create-lifecycle-events.ts +15 -0
  289. package/src/memory/migrations/176-drop-capability-card-state.ts +36 -0
  290. package/src/memory/migrations/177-create-trace-events-table.ts +40 -0
  291. package/src/memory/migrations/index.ts +6 -0
  292. package/src/memory/migrations/registry.ts +13 -0
  293. package/src/memory/retriever.test.ts +223 -96
  294. package/src/memory/retriever.ts +115 -138
  295. package/src/memory/schema/calls.ts +1 -1
  296. package/src/memory/schema/contacts.ts +1 -1
  297. package/src/memory/schema/infrastructure.ts +29 -0
  298. package/src/memory/schema/memory-core.ts +7 -17
  299. package/src/memory/schema/notifications.ts +1 -1
  300. package/src/memory/search/formatting.ts +23 -6
  301. package/src/memory/search/lexical.ts +2 -0
  302. package/src/memory/search/semantic.ts +2 -0
  303. package/src/memory/search/staleness.ts +1 -0
  304. package/src/memory/search/types.ts +4 -0
  305. package/src/memory/task-memory-cleanup.ts +96 -6
  306. package/src/memory/trace-event-store.ts +148 -0
  307. package/src/notifications/README.md +1 -1
  308. package/src/notifications/decision-engine.ts +2 -2
  309. package/src/notifications/emit-signal.ts +4 -4
  310. package/src/notifications/events-store.ts +4 -4
  311. package/src/notifications/signal.ts +1 -1
  312. package/src/oauth/manual-token-connection.ts +49 -25
  313. package/src/permissions/checker.ts +6 -5
  314. package/src/permissions/defaults.ts +4 -4
  315. package/src/prompts/__tests__/build-cli-reference-section.test.ts +9 -90
  316. package/src/prompts/cache-boundary.ts +8 -0
  317. package/src/prompts/system-prompt.ts +105 -634
  318. package/src/prompts/templates/BOOTSTRAP.md +166 -33
  319. package/src/prompts/templates/IDENTITY.md +8 -23
  320. package/src/prompts/templates/SOUL.md +20 -41
  321. package/src/prompts/templates/USER.md +3 -19
  322. package/src/prompts/user-reference.ts +14 -16
  323. package/src/providers/anthropic/client.ts +46 -2
  324. package/src/providers/gemini/client.ts +6 -9
  325. package/src/providers/managed-proxy/constants.ts +1 -7
  326. package/src/providers/managed-proxy/context.ts +0 -1
  327. package/src/providers/model-intents.ts +5 -5
  328. package/src/providers/openai/client.ts +10 -1
  329. package/src/providers/openrouter/client.ts +1 -0
  330. package/src/providers/ratelimit.ts +0 -35
  331. package/src/providers/registry.ts +3 -5
  332. package/src/providers/retry.ts +18 -1
  333. package/src/runtime/access-request-helper.ts +1 -1
  334. package/src/runtime/auth/route-policy.ts +7 -0
  335. package/src/runtime/channel-verification-service.ts +1 -1
  336. package/src/runtime/confirmation-request-guardian-bridge.ts +1 -1
  337. package/src/runtime/guardian-vellum-migration.ts +63 -1
  338. package/src/runtime/http-server.ts +8 -4
  339. package/src/runtime/migrations/vbundle-builder.ts +212 -32
  340. package/src/runtime/migrations/vbundle-import-analyzer.ts +74 -8
  341. package/src/runtime/migrations/vbundle-importer.ts +66 -1
  342. package/src/runtime/migrations/vbundle-validator.ts +17 -3
  343. package/src/runtime/routes/approval-strategies/guardian-callback-strategy.ts +4 -4
  344. package/src/runtime/routes/attachment-routes.ts +2 -2
  345. package/src/runtime/routes/btw-routes.ts +9 -0
  346. package/src/runtime/routes/channel-verification-routes.ts +19 -2
  347. package/src/runtime/routes/conversation-management-routes.ts +55 -1
  348. package/src/runtime/routes/conversation-query-routes.ts +1 -1
  349. package/src/runtime/routes/conversation-routes.ts +49 -5
  350. package/src/runtime/routes/conversation-starter-routes.ts +207 -0
  351. package/src/runtime/routes/guardian-bootstrap-routes.ts +13 -9
  352. package/src/runtime/routes/inbound-stages/escalation-intercept.ts +1 -1
  353. package/src/runtime/routes/inbound-stages/verification-intercept.ts +1 -1
  354. package/src/runtime/routes/migration-routes.ts +25 -13
  355. package/src/runtime/routes/secret-routes.ts +18 -0
  356. package/src/runtime/routes/settings-routes.ts +8 -8
  357. package/src/runtime/routes/telemetry-routes.ts +53 -0
  358. package/src/runtime/routes/trace-event-routes.ts +62 -0
  359. package/src/runtime/tool-grant-request-helper.ts +1 -1
  360. package/src/runtime/verification-outbound-actions.ts +47 -31
  361. package/src/security/encrypted-store.ts +263 -78
  362. package/src/skills/catalog-install.ts +10 -0
  363. package/src/skills/managed-store.ts +2 -0
  364. package/src/skills/skill-memory.ts +220 -0
  365. package/src/subagent/manager.ts +1 -4
  366. package/src/telemetry/types.ts +10 -1
  367. package/src/telemetry/usage-telemetry-reporter.test.ts +1 -1
  368. package/src/telemetry/usage-telemetry-reporter.ts +51 -4
  369. package/src/tools/AGENTS.md +11 -11
  370. package/src/tools/acp/spawn.ts +1 -1
  371. package/src/tools/apps/executors.ts +8 -8
  372. package/src/tools/apps/registry.ts +1 -1
  373. package/src/tools/assets/materialize.ts +6 -6
  374. package/src/tools/assets/search.ts +10 -10
  375. package/src/tools/browser/__tests__/auth-cache.test.ts +2 -2
  376. package/src/tools/browser/__tests__/auth-detector.test.ts +4 -4
  377. package/src/tools/browser/auth-detector.ts +6 -6
  378. package/src/tools/browser/browser-execution.ts +13 -13
  379. package/src/tools/browser/browser-manager.ts +3 -3
  380. package/src/tools/browser/chrome-cdp.ts +5 -5
  381. package/src/tools/browser/jit-auth.ts +2 -2
  382. package/src/tools/browser/network-recorder.test.ts +2 -2
  383. package/src/tools/browser/network-recorder.ts +3 -3
  384. package/src/tools/browser/runtime-check.ts +3 -3
  385. package/src/tools/claude-code/claude-code.ts +2 -2
  386. package/src/tools/computer-use/definitions.ts +18 -18
  387. package/src/tools/credential-execution/make-authenticated-request.ts +4 -4
  388. package/src/tools/credential-execution/manage-secure-command-tool.ts +3 -3
  389. package/src/tools/credential-execution/run-authenticated-command.ts +4 -4
  390. package/src/tools/credentials/broker-types.ts +5 -5
  391. package/src/tools/credentials/broker.ts +15 -15
  392. package/src/tools/credentials/metadata-store.ts +2 -2
  393. package/src/tools/credentials/resolve.ts +1 -1
  394. package/src/tools/credentials/selection.ts +1 -1
  395. package/src/tools/credentials/tool-policy.ts +1 -1
  396. package/src/tools/credentials/vault.ts +115 -25
  397. package/src/tools/execution-target.ts +2 -2
  398. package/src/tools/executor.ts +7 -7
  399. package/src/tools/filesystem/edit.ts +2 -2
  400. package/src/tools/filesystem/read.ts +1 -1
  401. package/src/tools/filesystem/write.ts +1 -1
  402. package/src/tools/host-filesystem/edit.ts +2 -1
  403. package/src/tools/host-filesystem/read.ts +2 -1
  404. package/src/tools/host-filesystem/write.ts +1 -1
  405. package/src/tools/host-terminal/host-shell.ts +9 -8
  406. package/src/tools/mcp/mcp-tool-factory.ts +7 -6
  407. package/src/tools/memory/definitions.ts +6 -5
  408. package/src/tools/memory/handlers.test.ts +1 -1
  409. package/src/tools/network/__tests__/web-search.test.ts +3 -3
  410. package/src/tools/network/domain-normalize.ts +2 -2
  411. package/src/tools/network/script-proxy/session-manager.ts +10 -10
  412. package/src/tools/network/web-fetch.ts +1 -1
  413. package/src/tools/network/web-search.ts +3 -3
  414. package/src/tools/permission-checker.ts +8 -8
  415. package/src/tools/registry.ts +7 -7
  416. package/src/tools/schedule/list.ts +2 -2
  417. package/src/tools/schema-transforms.ts +31 -21
  418. package/src/tools/secret-detection-handler.ts +1 -1
  419. package/src/tools/sensitive-output-placeholders.ts +1 -1
  420. package/src/tools/shared/filesystem/edit-engine.ts +1 -1
  421. package/src/tools/shared/filesystem/file-ops-service.ts +3 -3
  422. package/src/tools/shared/filesystem/image-read.ts +25 -5
  423. package/src/tools/shared/filesystem/path-policy.ts +2 -2
  424. package/src/tools/shared/shell-output.ts +1 -1
  425. package/src/tools/side-effects.ts +1 -1
  426. package/src/tools/skills/execute.ts +1 -1
  427. package/src/tools/skills/load.ts +3 -3
  428. package/src/tools/skills/sandbox-runner.ts +3 -3
  429. package/src/tools/subagent/read.ts +1 -1
  430. package/src/tools/subagent/spawn.ts +2 -2
  431. package/src/tools/swarm/delegate.ts +3 -3
  432. package/src/tools/system/request-permission.ts +5 -4
  433. package/src/tools/terminal/backends/native.ts +4 -4
  434. package/src/tools/terminal/parser.ts +6 -6
  435. package/src/tools/terminal/sandbox-diagnostics.ts +1 -1
  436. package/src/tools/terminal/shell.ts +16 -16
  437. package/src/tools/tool-approval-handler.ts +21 -12
  438. package/src/tools/tool-manifest.ts +4 -4
  439. package/src/tools/types.ts +3 -3
  440. package/src/tools/ui-surface/definitions.ts +9 -37
  441. package/src/tools/watcher/list.ts +1 -1
  442. package/src/util/logger.ts +7 -2
  443. package/src/util/retry.ts +29 -1
  444. package/src/workspace/migrations/007-web-search-provider-rename.ts +37 -0
  445. package/src/workspace/migrations/registry.ts +2 -0
  446. package/src/__tests__/cli-help-reference-sync.test.ts +0 -26
  447. package/src/__tests__/onboarding-starter-tasks.test.ts +0 -190
  448. package/src/cli/reference.ts +0 -38
  449. package/src/memory/job-handlers/capability-cards.ts +0 -420
  450. package/src/runtime/routes/thread-starter-routes.ts +0 -294
@@ -38,7 +38,7 @@
38
38
  "enum": ["startTime", "updated"],
39
39
  "description": "Sort order (default \"startTime\"). Only valid when single_events is true."
40
40
  },
41
- "reason": {
41
+ "activity": {
42
42
  "type": "string",
43
43
  "description": "Brief non-technical explanation of why this tool is being called"
44
44
  },
@@ -67,7 +67,7 @@
67
67
  "type": "string",
68
68
  "description": "Calendar ID (default \"primary\")"
69
69
  },
70
- "reason": {
70
+ "activity": {
71
71
  "type": "string",
72
72
  "description": "Brief non-technical explanation of why this tool is being called"
73
73
  },
@@ -128,7 +128,7 @@
128
128
  "type": "number",
129
129
  "description": "Confidence score (0-1) for this action"
130
130
  },
131
- "reason": {
131
+ "activity": {
132
132
  "type": "string",
133
133
  "description": "Brief non-technical explanation of why this tool is being called"
134
134
  },
@@ -169,7 +169,7 @@
169
169
  "type": "string",
170
170
  "description": "IANA timezone for interpreting results (e.g. \"America/New_York\")"
171
171
  },
172
- "reason": {
172
+ "activity": {
173
173
  "type": "string",
174
174
  "description": "Brief non-technical explanation of why this tool is being called"
175
175
  },
@@ -208,7 +208,7 @@
208
208
  "type": "number",
209
209
  "description": "Confidence score (0-1) for this action"
210
210
  },
211
- "reason": {
211
+ "activity": {
212
212
  "type": "string",
213
213
  "description": "Brief non-technical explanation of why this tool is being called"
214
214
  },
@@ -1,4 +1,4 @@
1
- /** Event time either a dateTime with timezone or a date for all-day events. */
1
+ /** Event time - either a dateTime with timezone or a date for all-day events. */
2
2
  export interface EventDateTime {
3
3
  dateTime?: string;
4
4
  date?: string;
@@ -0,0 +1,43 @@
1
+ ---
2
+ name: heartbeat
3
+ description: Configure periodic background checklist runs
4
+ compatibility: "Designed for Vellum personal assistants"
5
+ metadata:
6
+ emoji: "\U0001F493"
7
+ vellum:
8
+ display-name: "Heartbeat"
9
+ activation-hints:
10
+ - "Set up a heartbeat, periodic checklist, background health check, or recurring background task"
11
+ avoid-when:
12
+ - "One-off or recurring schedules with specific payloads - use the schedule skill instead"
13
+ ---
14
+
15
+ The heartbeat feature runs your `HEARTBEAT.md` checklist periodically in a background conversation. Each run, the assistant works through the checklist and flags anything that needs attention.
16
+
17
+ ## Setup
18
+
19
+ Edit `config.json` using `file_edit`:
20
+
21
+ 1. **Enable heartbeat**: Set `heartbeat.enabled` to `true`.
22
+ 2. **Set interval**: Set `heartbeat.intervalMs` (milliseconds between runs, default: 3600000 = 1 hour).
23
+ 3. **Optional active hours**: Set `heartbeat.activeHoursStart` and `heartbeat.activeHoursEnd` (0-23) to restrict runs to certain hours. Both must be set together.
24
+
25
+ Example config.json heartbeat section:
26
+ ```json
27
+ {
28
+ "heartbeat": {
29
+ "enabled": true,
30
+ "intervalMs": 1800000,
31
+ "activeHoursStart": 8,
32
+ "activeHoursEnd": 22
33
+ }
34
+ }
35
+ ```
36
+
37
+ Then edit `HEARTBEAT.md` with the checklist items. The assistant will work through this file each heartbeat run.
38
+
39
+ ## Notes
40
+
41
+ - Toggling `heartbeat.enabled` requires an assistant restart to take effect.
42
+ - Changes to `HEARTBEAT.md` take effect on the next heartbeat run (no restart needed).
43
+ - The heartbeat runs in a separate background conversation, not the user's active chat.
@@ -23,12 +23,12 @@ You are an image generation assistant. When the user asks you to create or edit
23
23
 
24
24
  ## Models
25
25
 
26
- - `gemini-2.5-flash-image` (default) fast, good quality
27
- - `gemini-3-pro-image` higher quality, slower
26
+ - `gemini-3.1-flash-image-preview` (default) - Nano Banana 2, fast, good quality
27
+ - `gemini-3-pro-image-preview` - Nano Banana Pro, higher quality, slower
28
28
 
29
29
  ## Tips
30
30
 
31
31
  - Be descriptive in your prompts for better results. Include details about style, composition, lighting, and mood.
32
32
  - When editing images, clearly describe what changes you want made to the source image.
33
33
  - Use the `variants` parameter (1-4) to generate multiple options and pick the best one.
34
- - If no Gemini API key is configured, the tool will return an error ask the user to set one up.
34
+ - If no Gemini API key is configured, the tool will return an error - ask the user to set one up.
@@ -28,8 +28,7 @@
28
28
  "model": {
29
29
  "type": "string",
30
30
  "enum": [
31
- "gemini-2.5-flash-image",
32
- "gemini-3-pro-image",
31
+ "gemini-3.1-flash-image-preview",
33
32
  "gemini-3-pro-image-preview"
34
33
  ],
35
34
  "description": "Which model to use for generation. If omitted, uses the user's configured preference."
@@ -38,7 +37,7 @@
38
37
  "type": "number",
39
38
  "description": "Number of image variants to generate (1-4, default: 1)"
40
39
  },
41
- "reason": {
40
+ "activity": {
42
41
  "type": "string",
43
42
  "description": "Brief non-technical explanation of why this tool is being called"
44
43
  }
@@ -41,7 +41,7 @@ function isAttachmentAccessible(
41
41
  if (hasStandard) {
42
42
  return true;
43
43
  }
44
- // All sources are private visible only if the caller is in one of those conversations
44
+ // All sources are private - visible only if the caller is in one of those conversations
45
45
  return sources.some((s) =>
46
46
  isAttachmentVisible(
47
47
  { conversationId: s.conversationId, isPrivate: true },
@@ -55,14 +55,13 @@ export async function run(
55
55
  context: ToolContext,
56
56
  ): Promise<ToolExecutionResult> {
57
57
  const config = getConfig();
58
- const apiKey = await getProviderKeyAsync("gemini");
58
+ const imageGenMode = config.services["image-generation"].mode;
59
59
 
60
- // Resolve credentials: prefer direct API key, fall back to managed proxy
60
+ // Resolve credentials strictly based on mode no cross-mode fallbacks
61
61
  let credentials: ImageGenCredentials | undefined;
62
- if (apiKey) {
63
- credentials = { type: "direct", apiKey };
64
- } else {
65
- const managedBaseUrl = await buildManagedBaseUrl("vertex");
62
+
63
+ if (imageGenMode === "managed") {
64
+ const managedBaseUrl = await buildManagedBaseUrl("gemini");
66
65
  if (managedBaseUrl) {
67
66
  const ctx = await resolveManagedProxyContext();
68
67
  credentials = {
@@ -71,14 +70,19 @@ export async function run(
71
70
  baseUrl: managedBaseUrl,
72
71
  };
73
72
  }
73
+ } else {
74
+ const apiKey = await getProviderKeyAsync("gemini");
75
+ if (apiKey) {
76
+ credentials = { type: "direct", apiKey };
77
+ }
74
78
  }
75
79
 
76
80
  if (!credentials) {
77
- return {
78
- content:
79
- "No Gemini API key configured. Please set your Gemini API key to use image generation.",
80
- isError: true,
81
- };
81
+ const hint =
82
+ imageGenMode === "managed"
83
+ ? "Managed proxy is not available. Please log in to Vellum or switch to Your Own mode."
84
+ : "No Gemini API key configured. Please set your Gemini API key in Settings > Models & Services.";
85
+ return { content: hint, isError: true };
82
86
  }
83
87
 
84
88
  const prompt = input.prompt as string;
@@ -1,6 +1,6 @@
1
1
  ---
2
2
  name: media-processing
3
- description: "Ingest and process media files (video, audio, image) through a 3-phase pipeline: preprocess, map (Gemini), and reduce (Claude)"
3
+ description: "Ingest and process media files (video, audio, image)"
4
4
  compatibility: "Designed for Vellum personal assistants"
5
5
  metadata:
6
6
  emoji: "🎬"
@@ -14,11 +14,11 @@ Ingest and track processing of media files (video, audio, images) through a conf
14
14
 
15
15
  The processing pipeline follows a sequential 3-phase flow:
16
16
 
17
- 1. **Ingest** (`ingest_media`) Register a media file, detect MIME type, extract duration, deduplicate by content hash.
18
- 2. **Preprocess** (`extract_keyframes`) Detect dead time, segment the video into windows, extract downscaled keyframes, build a subject registry, and write a pipeline manifest.
19
- 3. **Map** (`analyze_keyframes`) Send each segment's frames to Gemini 2.5 Flash with assistant-provided extraction instructions and a JSON Schema for guaranteed structured output. Supports concurrency pooling, cost tracking, resumability, and automatic retries.
20
- 4. **Reduce / Query** (`query_media`) Send all map output to Claude for intelligent analysis and Q&A. Supports arbitrary natural language queries about video content.
21
- 5. **Clip** (`generate_clip`) Extract video clips around specific moments.
17
+ 1. **Ingest** (`ingest_media`) - Register a media file, detect MIME type, extract duration, deduplicate by content hash.
18
+ 2. **Preprocess** (`extract_keyframes`) - Detect dead time, segment the video into windows, extract downscaled keyframes, build a subject registry, and write a pipeline manifest.
19
+ 3. **Map** (`analyze_keyframes`) - Send each segment's frames to Gemini 2.5 Flash with assistant-provided extraction instructions and a JSON Schema for guaranteed structured output. Supports concurrency pooling, cost tracking, resumability, and automatic retries.
20
+ 4. **Reduce / Query** (`query_media`) - Send all map output to Claude for intelligent analysis and Q&A. Supports arbitrary natural language queries about video content.
21
+ 5. **Clip** (`generate_clip`) - Extract video clips around specific moments.
22
22
 
23
23
  The processing pipeline service (`services/processing-pipeline.ts`) orchestrates phases 2-4 automatically with retries, resumability, and cancellation support.
24
24
 
@@ -38,35 +38,35 @@ Preprocess a video asset: detect dead time via mpdecimate, segment the video int
38
38
 
39
39
  Parameters:
40
40
 
41
- - `asset_id` (required) ID of the media asset.
42
- - `interval_seconds` Interval between keyframes (default: 1s). Use 0.5s for sports/action content where frame density matters.
43
- - `segment_duration` Duration of each segment window (default: 15s).
44
- - `dead_time_threshold` Sensitivity for dead-time detection (default: 0.02).
45
- - `section_config` Path to a JSON file with manual section boundaries.
46
- - `detect_dead_time` Whether to detect and skip dead time (default: false). Dead-time detection can be too aggressive for continuous action video like sports it may incorrectly skip live play. Enable only for content with clear idle periods (e.g., lectures, surveillance footage).
47
- - `short_edge` Short edge resolution for downscaled frames in pixels (default: 480).
48
- - `include_audio` Whether to extract and transcribe audio for each segment (default: false). When enabled, each segment's audio is transcribed and stored alongside visual frames.
49
- - `transcription_mode` Transcription backend: `'api'` (OpenAI Whisper cloud) or `'local'` (whisper.cpp on-device). Default: `'local'`. The `'api'` mode requires an OpenAI API key configured in settings.
41
+ - `asset_id` (required) - ID of the media asset.
42
+ - `interval_seconds` - Interval between keyframes (default: 1s). Use 0.5s for sports/action content where frame density matters.
43
+ - `segment_duration` - Duration of each segment window (default: 15s).
44
+ - `dead_time_threshold` - Sensitivity for dead-time detection (default: 0.02).
45
+ - `section_config` - Path to a JSON file with manual section boundaries.
46
+ - `detect_dead_time` - Whether to detect and skip dead time (default: false). Dead-time detection can be too aggressive for continuous action video like sports - it may incorrectly skip live play. Enable only for content with clear idle periods (e.g., lectures, surveillance footage).
47
+ - `short_edge` - Short edge resolution for downscaled frames in pixels (default: 480).
48
+ - `include_audio` - Whether to extract and transcribe audio for each segment (default: false). When enabled, each segment's audio is transcribed and stored alongside visual frames.
49
+ - `transcription_mode` - Transcription backend: `'api'` (OpenAI Whisper cloud) or `'local'` (whisper.cpp on-device). Default: `'local'`. The `'api'` mode requires an OpenAI API key configured in settings.
50
50
 
51
51
  ### analyze_keyframes
52
52
 
53
53
  Map video segments through Gemini's structured output API. Supports two modes:
54
54
 
55
- - **`keyframes`** (default) Reads frames from the preprocess manifest, sends each segment's images to Gemini. Requires `extract_keyframes` to be run first. Best for longer videos (> 1 hour) or when you need fine-grained control over frame selection (interval, segment duration, dead-time skipping).
56
- - **`direct_video`** Uploads the video file directly to Gemini's Files API. Gemini sees actual motion and temporal context instead of static frames. Best for shorter videos (< 1 hour) where temporal context matters (detecting actions, transitions, motion patterns). Has a 2 GB file size limit. Does not require `extract_keyframes` preprocessing.
55
+ - **`keyframes`** (default) - Reads frames from the preprocess manifest, sends each segment's images to Gemini. Requires `extract_keyframes` to be run first. Best for longer videos (> 1 hour) or when you need fine-grained control over frame selection (interval, segment duration, dead-time skipping).
56
+ - **`direct_video`** - Uploads the video file directly to Gemini's Files API. Gemini sees actual motion and temporal context instead of static frames. Best for shorter videos (< 1 hour) where temporal context matters (detecting actions, transitions, motion patterns). Has a 2 GB file size limit. Does not require `extract_keyframes` preprocessing.
57
57
 
58
58
  Both modes produce the same `MapOutput` format, so `query_media` works identically regardless of which mode was used.
59
59
 
60
60
  Parameters:
61
61
 
62
- - `asset_id` (required) ID of the media asset.
63
- - `system_prompt` (required) Extraction instructions for Gemini.
64
- - `output_schema` (required) JSON Schema for structured output.
65
- - `mode` Analysis mode: `'keyframes'` (default) or `'direct_video'`.
66
- - `context` Additional context to include in the prompt.
67
- - `model` Gemini model to use (default: `gemini-2.5-flash`).
68
- - `concurrency` Maximum concurrent API requests (default: 10, keyframes mode only).
69
- - `max_retries` Retry attempts per segment on failure (default: 3).
62
+ - `asset_id` (required) - ID of the media asset.
63
+ - `system_prompt` (required) - Extraction instructions for Gemini.
64
+ - `output_schema` (required) - JSON Schema for structured output.
65
+ - `mode` - Analysis mode: `'keyframes'` (default) or `'direct_video'`.
66
+ - `context` - Additional context to include in the prompt.
67
+ - `model` - Gemini model to use (default: `gemini-2.5-flash`).
68
+ - `concurrency` - Maximum concurrent API requests (default: 10, keyframes mode only).
69
+ - `max_retries` - Retry attempts per segment on failure (default: 3).
70
70
 
71
71
  ### query_media
72
72
 
@@ -74,10 +74,10 @@ Query video analysis data using natural language. Sends map output (from analyze
74
74
 
75
75
  Parameters:
76
76
 
77
- - `asset_id` (required) ID of the media asset.
78
- - `query` (required) Natural language query about the video data.
79
- - `system_prompt` Optional system prompt for Claude.
80
- - `model` LLM model to use (default: `claude-sonnet-4-6`).
77
+ - `asset_id` (required) - ID of the media asset.
78
+ - `query` (required) - Natural language query about the video data.
79
+ - `system_prompt` - Optional system prompt for Claude.
80
+ - `model` - LLM model to use (default: `claude-sonnet-4-6`).
81
81
 
82
82
  ### generate_clip
83
83
 
@@ -135,7 +135,7 @@ Transcription modes:
135
135
  - **`local`** (default): Uses whisper.cpp for on-device transcription. Requires `whisper-cpp` to be installed (`brew install whisper-cpp`). No API costs, but slower.
136
136
  - **`api`**: Uses the OpenAI Whisper API for cloud-based transcription. Faster and more accurate, but requires an OpenAI API key and incurs per-minute costs.
137
137
 
138
- The audio transcription degrades gracefully if transcription fails for a segment (missing tools, no audio track, API errors), the segment proceeds with visual-only analysis.
138
+ The audio transcription degrades gracefully - if transcription fails for a segment (missing tools, no audio track, API errors), the segment proceeds with visual-only analysis.
139
139
 
140
140
  ## Best Practices
141
141
 
@@ -195,13 +195,13 @@ Be specific and factual. Describe what you see, not what you infer happened betw
195
195
 
196
196
  ### Clip Delivery
197
197
 
198
- The `generate_clip` tool automatically opens clips in the user's default video player after extraction (handled internally do **not** run `open` via `host_bash`). Clips are saved persistently in the asset's pipeline directory (`pipeline/<assetId>/clips/`), falling back to a temp directory when the source location is read-only. Each clip gets a unique filename so concurrent or repeated extractions at the same range never collide. The `clipPath` field in the tool response contains the absolute file path.
198
+ The `generate_clip` tool automatically opens clips in the user's default video player after extraction (handled internally - do **not** run `open` via `host_bash`). Clips are saved persistently in the asset's pipeline directory (`pipeline/<assetId>/clips/`), falling back to a temp directory when the source location is read-only. Each clip gets a unique filename so concurrent or repeated extractions at the same range never collide. The `clipPath` field in the tool response contains the absolute file path.
199
199
 
200
- The tool handles high-bitrate and incompatible codec sources automatically it tries stream copy first for speed, then falls back to H.264 re-encoding if needed. **Always use `generate_clip` rather than manual ffmpeg commands.**
200
+ The tool handles high-bitrate and incompatible codec sources automatically - it tries stream copy first for speed, then falls back to H.264 re-encoding if needed. **Always use `generate_clip` rather than manual ffmpeg commands.**
201
201
 
202
202
  Always provide a descriptive `title` parameter (e.g. `"snow-dive-closeup"`, `"goal-celebration"`) so clips get meaningful filenames instead of timestamp-based names.
203
203
 
204
- ## Known Limitations Vision Analysis
204
+ ## Known Limitations - Vision Analysis
205
205
 
206
206
  Gemini performs well at **spatial/descriptive analysis** from static keyframes:
207
207
 
@@ -217,7 +217,7 @@ Gemini **hallucinates when asked to detect fast temporal events** from static fr
217
217
  - Fast transitions and split-second actions
218
218
  - Causality between frames (what "happened" vs. what's visible)
219
219
 
220
- The model is good at describing **what is there** but bad at detecting **what happened** from static frames. For content where temporal context matters, consider using `mode: 'direct_video'` which lets Gemini see actual motion. For keyframes mode, structure your map prompts and queries accordingly ask the model to describe scenes, then use `query_media` (Claude) to reason about patterns and events across the descriptive data.
220
+ The model is good at describing **what is there** but bad at detecting **what happened** from static frames. For content where temporal context matters, consider using `mode: 'direct_video'` which lets Gemini see actual motion. For keyframes mode, structure your map prompts and queries accordingly - ask the model to describe scenes, then use `query_media` (Claude) to reason about patterns and events across the descriptive data.
221
221
 
222
222
  ## Operator Runbook
223
223
 
@@ -225,10 +225,10 @@ The model is good at describing **what is there** but bad at detecting **what ha
225
225
 
226
226
  Use `media_status` to check the current state of any asset:
227
227
 
228
- - **registered** Ingested but not yet processed.
229
- - **processing** Pipeline is running.
230
- - **indexed** All stages completed successfully.
231
- - **failed** A stage failed. Check stage details for the error.
228
+ - **registered** - Ingested but not yet processed.
229
+ - **processing** - Pipeline is running.
230
+ - **indexed** - All stages completed successfully.
231
+ - **failed** - A stage failed. Check stage details for the error.
232
232
 
233
233
  The response includes per-stage progress (0-100%) so you can see exactly where processing stands.
234
234
 
@@ -244,11 +244,11 @@ Use `media_status` to check processing stages:
244
244
  - **map**: Gemini API key not configured, API rate limits, network errors.
245
245
  - **reduce**: No LLM provider configured, no map output exists.
246
246
 
247
- After fixing the root cause, re-run the failed stage. The pipeline is resumable it picks up from where it left off.
247
+ After fixing the root cause, re-run the failed stage. The pipeline is resumable - it picks up from where it left off.
248
248
 
249
249
  ### Cost Expectations
250
250
 
251
- The Map phase (Gemini) is the primary cost driver it scales with video duration and keyframe interval. The Q&A phase (Claude) is negligible per query.
251
+ The Map phase (Gemini) is the primary cost driver - it scales with video duration and keyframe interval. The Q&A phase (Claude) is negligible per query.
252
252
 
253
253
  ### Known Limitations
254
254
 
@@ -21,7 +21,7 @@
21
21
  "type": "object",
22
22
  "description": "Optional JSON metadata to attach to the asset (e.g., pipeline config, source info)"
23
23
  },
24
- "reason": {
24
+ "activity": {
25
25
  "type": "string",
26
26
  "description": "Brief non-technical explanation of why this tool is being called"
27
27
  }
@@ -52,7 +52,7 @@
52
52
  "enum": ["registered", "processing", "indexed", "failed"],
53
53
  "description": "Filter assets by processing status"
54
54
  },
55
- "reason": {
55
+ "activity": {
56
56
  "type": "string",
57
57
  "description": "Brief non-technical explanation of why this tool is being called"
58
58
  }
@@ -106,7 +106,7 @@
106
106
  "enum": ["api", "local"],
107
107
  "description": "Transcription backend: 'api' (OpenAI Whisper cloud) or 'local' (whisper.cpp). Default: 'local'."
108
108
  },
109
- "reason": {
109
+ "activity": {
110
110
  "type": "string",
111
111
  "description": "Brief non-technical explanation of why this tool is being called"
112
112
  }
@@ -139,7 +139,7 @@
139
139
  },
140
140
  "output_schema": {
141
141
  "type": "object",
142
- "description": "JSON Schema for structured output Gemini will enforce this schema on the response"
142
+ "description": "JSON Schema for structured output - Gemini will enforce this schema on the response"
143
143
  },
144
144
  "context": {
145
145
  "type": "object",
@@ -159,7 +159,7 @@
159
159
  "minimum": 0,
160
160
  "description": "Maximum retry attempts per segment on failure. Default: 3"
161
161
  },
162
- "reason": {
162
+ "activity": {
163
163
  "type": "string",
164
164
  "description": "Brief non-technical explanation of why this tool is being called"
165
165
  }
@@ -171,7 +171,7 @@
171
171
  },
172
172
  {
173
173
  "name": "query_media",
174
- "description": "Query video analysis data using natural language. Sends map output (from analyze_keyframes) to Claude for intelligent analysis and Q&A. Supports arbitrary questions about video content Claude reads the full structured analysis and answers based on the data.",
174
+ "description": "Query video analysis data using natural language. Sends map output (from analyze_keyframes) to Claude for intelligent analysis and Q&A. Supports arbitrary questions about video content - Claude reads the full structured analysis and answers based on the data.",
175
175
  "category": "media",
176
176
  "risk": "low",
177
177
  "input_schema": {
@@ -193,7 +193,7 @@
193
193
  "type": "string",
194
194
  "description": "LLM model to use for analysis. Default: 'claude-sonnet-4-6'"
195
195
  },
196
- "reason": {
196
+ "activity": {
197
197
  "type": "string",
198
198
  "description": "Brief non-technical explanation of why this tool is being called"
199
199
  }
@@ -240,7 +240,7 @@
240
240
  "type": "string",
241
241
  "description": "Short descriptive title for the clip (e.g. 'snow-dive-closeup', 'goal-celebration'). Used as the filename. If omitted, falls back to timestamp-based naming."
242
242
  },
243
- "reason": {
243
+ "activity": {
244
244
  "type": "string",
245
245
  "description": "Brief non-technical explanation of why this tool is being called"
246
246
  }
@@ -23,7 +23,7 @@ describe("ConcurrencyPool", () => {
23
23
  expect(thirdResolved).toBe(false);
24
24
  expect(pool.waitingCount).toBe(1);
25
25
 
26
- // Release one slot third should now resolve
26
+ // Release one slot - third should now resolve
27
27
  pool.release();
28
28
  await thirdPromise;
29
29
  expect(thirdResolved).toBe(true);
@@ -62,7 +62,7 @@ describe("ConcurrencyPool", () => {
62
62
  it("defaults to maxConcurrency of 10", async () => {
63
63
  const pool = new ConcurrencyPool();
64
64
 
65
- // Acquire 10 slots all should resolve immediately
65
+ // Acquire 10 slots - all should resolve immediately
66
66
  for (let i = 0; i < 10; i++) {
67
67
  await pool.acquire();
68
68
  }
@@ -80,7 +80,7 @@ describe("buildDeadTimeRanges", () => {
80
80
  });
81
81
 
82
82
  it("filters out short ranges below minDuration", () => {
83
- // Only 3 seconds of drops below the 5s minimum
83
+ // Only 3 seconds of drops - below the 5s minimum
84
84
  const timestamps = [10, 10.5, 11, 11.5, 12, 12.5, 13];
85
85
  const ranges = buildDeadTimeRanges(timestamps, 1.0, 5.0);
86
86
  expect(ranges).toEqual([]);
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Gemini Map service processes video segments through Gemini's structured
2
+ * Gemini Map service - processes video segments through Gemini's structured
3
3
  * output API for vision-based analysis with guaranteed valid JSON responses.
4
4
  *
5
5
  * Uses @google/genai SDK directly (not the GeminiProvider wrapper) to leverage
@@ -194,7 +194,7 @@ async function processSegmentWithRetry(
194
194
  },
195
195
  };
196
196
  } catch (err) {
197
- // Handle Gemini safety blocks not retryable
197
+ // Handle Gemini safety blocks - not retryable
198
198
  if (err instanceof ApiError) {
199
199
  const message = err.message ?? "";
200
200
  if (message.includes("SAFETY") || message.includes("safety")) {
@@ -208,7 +208,7 @@ async function processSegmentWithRetry(
208
208
  };
209
209
  }
210
210
 
211
- // Non-retryable client errors (400, 401, 403, etc.) fail immediately
211
+ // Non-retryable client errors (400, 401, 403, etc.) - fail immediately
212
212
  if (
213
213
  err.status !== undefined &&
214
214
  err.status < 500 &&
@@ -225,7 +225,7 @@ async function processSegmentWithRetry(
225
225
  };
226
226
  }
227
227
 
228
- // 429 rate limits retryable with backoff
228
+ // 429 rate limits - retryable with backoff
229
229
  if (err.status === 429 && attempt < maxRetries) {
230
230
  const delay = computeRetryDelay(attempt);
231
231
  onProgress?.(
@@ -332,7 +332,7 @@ export async function mapSegments(
332
332
  onProgress?.(` Segment ${segment.id}: loaded from cache.\n`);
333
333
  return;
334
334
  } catch {
335
- // Corrupted cache file reprocess
335
+ // Corrupted cache file - reprocess
336
336
  }
337
337
  }
338
338
 
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Gemini Video service uploads video directly to Gemini's Files API for
2
+ * Gemini Video service - uploads video directly to Gemini's Files API for
3
3
  * analysis, letting Gemini see actual motion and temporal context instead
4
4
  * of static keyframes.
5
5
  *
@@ -266,7 +266,7 @@ export async function analyzeVideoDirectly(
266
266
  await client.files.delete({ name: uploadedFileName });
267
267
  onProgress?.(`Cleaned up uploaded file from Gemini.\n`);
268
268
  } catch {
269
- // Best-effort cleanup don't fail the operation
269
+ // Best-effort cleanup - don't fail the operation
270
270
  }
271
271
  }
272
272
  }
@@ -319,7 +319,7 @@ async function extractDominantColors(framePath: string): Promise<string[]> {
319
319
  // Fallback: return empty if analysis fails
320
320
  if (result.exitCode !== 0) return [];
321
321
 
322
- // Parse palette info from stderr look for color hex values
322
+ // Parse palette info from stderr - look for color hex values
323
323
  const colors: string[] = [];
324
324
  const colorMatches = result.stderr.matchAll(/0x([0-9a-fA-F]{6})/g);
325
325
  for (const m of colorMatches) {
@@ -549,7 +549,7 @@ export async function preprocessForAsset(
549
549
  );
550
550
  if (rawSegments.length > 0 && totalFrames === 0) {
551
551
  throw new Error(
552
- `All ${rawSegments.length} segment(s) failed frame extraction zero usable frames produced.`,
552
+ `All ${rawSegments.length} segment(s) failed frame extraction - zero usable frames produced.`,
553
553
  );
554
554
  }
555
555
  onProgress?.(
@@ -151,7 +151,7 @@ export async function runPipeline(
151
151
  resumedFrom = STAGE_ORDER[startIndex];
152
152
  onProgress?.(`Resuming pipeline from stage: ${resumedFrom}`);
153
153
  } else if (startIndex >= STAGE_ORDER.length) {
154
- // All stages already completed idempotent no-op
154
+ // All stages already completed - idempotent no-op
155
155
  onProgress?.("All pipeline stages already completed.");
156
156
  updateMediaAssetStatus(assetId, "indexed");
157
157
  return {
@@ -225,7 +225,7 @@ export async function runPipeline(
225
225
  `Stage ${stageName} failed (attempt ${attempt + 1}/${maxRetries + 1}): ${errorMsg}`,
226
226
  );
227
227
 
228
- // Save partial progress the stage handler should have already
228
+ // Save partial progress - the stage handler should have already
229
229
  // persisted any partial results before throwing
230
230
  updateProcessingStage(stageRecord.id, {
231
231
  status: attempt >= maxRetries ? "failed" : "running",
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Reduce service sends Map output to Claude as text-only for analysis.
2
+ * Reduce service - sends Map output to Claude as text-only for analysis.
3
3
  *
4
4
  * Two modes:
5
5
  * - One-shot merge: assembles all Map results into a single document,
@@ -75,7 +75,7 @@ async function loadMapOutput(assetId: string): Promise<MapOutput> {
75
75
 
76
76
  /**
77
77
  * Format map output segments into a text document for Claude.
78
- * Strips image data text only.
78
+ * Strips image data - text only.
79
79
  */
80
80
  function formatMapOutputAsText(mapOutput: MapOutput): string {
81
81
  const lines: string[] = [];
@@ -151,7 +151,7 @@ async function persistReduceCost(
151
151
  const raw = await readFile(costPath, "utf-8");
152
152
  existing = JSON.parse(raw) as ReduceCostData;
153
153
  } catch {
154
- // First query start fresh
154
+ // First query - start fresh
155
155
  }
156
156
 
157
157
  existing.entries.push({
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Clip generation tool extract a video segment from a media asset.
2
+ * Clip generation tool - extract a video segment from a media asset.
3
3
  *
4
4
  * Uses ffmpeg to cut a segment with configurable pre/post-roll padding,
5
5
  * then registers the resulting clip as an attachment for in-chat delivery.
@@ -199,7 +199,7 @@ export async function run(
199
199
  const result = await spawnWithTimeout(ffmpegArgs, FFMPEG_CLIP_TIMEOUT_MS);
200
200
 
201
201
  if (result.exitCode !== 0) {
202
- // Stream copy failed fall back to re-encoding (handles high-bitrate
202
+ // Stream copy failed - fall back to re-encoding (handles high-bitrate
203
203
  // sources, incompatible codecs, and missing keyframes at cut points)
204
204
  context.onOutput?.("Stream copy failed, re-encoding clip...\n");
205
205
  // Select codecs based on output format (WebM needs VP9/Opus, MP4/MOV use H.264/AAC)
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Query media tool sends natural language queries against video
2
+ * Query media tool - sends natural language queries against video
3
3
  * analysis data (map output) via Claude for intelligent answers.
4
4
  *
5
5
  * Replaces the old keyword-matching approach with an LLM-powered