@poolzin/pool-bot 2026.1.39 → 2026.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (511) hide show
  1. package/assets/chrome-extension/README.md +3 -3
  2. package/assets/chrome-extension/background.js +5 -5
  3. package/assets/chrome-extension/manifest.json +3 -3
  4. package/assets/chrome-extension/options.html +4 -4
  5. package/assets/chrome-extension/options.js +1 -1
  6. package/dist/acp/client.js +3 -3
  7. package/dist/acp/types.js +1 -1
  8. package/dist/agents/agent-paths.js +3 -3
  9. package/dist/agents/auth-profiles/paths.js +3 -3
  10. package/dist/agents/bash-tools.exec.js +76 -25
  11. package/dist/agents/cli-runner/helpers.js +10 -12
  12. package/dist/agents/cli-runner.js +2 -2
  13. package/dist/agents/cloudflare-ai-gateway.js +31 -0
  14. package/dist/agents/compaction.js +16 -2
  15. package/dist/agents/context-window-guard.js +13 -10
  16. package/dist/agents/context.js +4 -4
  17. package/dist/agents/docs-path.js +1 -1
  18. package/dist/agents/identity.js +47 -7
  19. package/dist/agents/memory-search.js +25 -8
  20. package/dist/agents/minimax-vlm.js +1 -1
  21. package/dist/agents/model-auth.js +12 -1
  22. package/dist/agents/model-catalog.js +4 -4
  23. package/dist/agents/model-selection.js +31 -4
  24. package/dist/agents/models-config.js +3 -3
  25. package/dist/agents/models-config.providers.js +147 -39
  26. package/dist/agents/pi-embedded-block-chunker.js +117 -42
  27. package/dist/agents/pi-embedded-helpers/errors.js +183 -78
  28. package/dist/agents/pi-embedded-helpers/openai.js +1 -1
  29. package/dist/agents/pi-embedded-helpers.js +1 -1
  30. package/dist/agents/pi-embedded-runner/compact.js +9 -8
  31. package/dist/agents/pi-embedded-runner/model.js +63 -4
  32. package/dist/agents/pi-embedded-runner/run/attempt.js +27 -17
  33. package/dist/agents/pi-embedded-runner/run.js +203 -50
  34. package/dist/agents/pi-embedded-runner/system-prompt.js +10 -2
  35. package/dist/agents/pi-embedded-runner/tool-result-truncation.js +275 -0
  36. package/dist/agents/pi-embedded-runner/utils.js +1 -1
  37. package/dist/agents/pi-embedded-subscribe.js +118 -29
  38. package/dist/agents/pi-model-discovery.js +10 -0
  39. package/dist/agents/pi-tool-definition-adapter.js +50 -9
  40. package/dist/agents/pi-tools.before-tool-call.js +67 -0
  41. package/dist/agents/pi-tools.js +20 -10
  42. package/dist/agents/pi-tools.read.js +2 -2
  43. package/dist/agents/poolbot-tools.js +15 -10
  44. package/dist/agents/sandbox-paths.js +31 -0
  45. package/dist/agents/session-file-repair.js +83 -0
  46. package/dist/agents/session-tool-result-guard.js +94 -15
  47. package/dist/agents/session-transcript-repair.js +68 -0
  48. package/dist/agents/shell-utils.js +51 -0
  49. package/dist/agents/skills/bundled-context.js +23 -0
  50. package/dist/agents/skills/bundled-dir.js +41 -7
  51. package/dist/agents/skills/frontmatter.js +1 -1
  52. package/dist/agents/skills/workspace.js +2 -2
  53. package/dist/agents/skills-install.js +60 -23
  54. package/dist/agents/subagent-announce.js +79 -34
  55. package/dist/agents/system-prompt.js +28 -4
  56. package/dist/agents/together-models.js +127 -0
  57. package/dist/agents/tool-images.js +1 -1
  58. package/dist/agents/tool-policy.conformance.js +14 -0
  59. package/dist/agents/tool-policy.js +25 -1
  60. package/dist/agents/tools/browser-tool.js +3 -3
  61. package/dist/agents/tools/cron-tool.js +166 -19
  62. package/dist/agents/tools/discord-actions-presence.js +78 -0
  63. package/dist/agents/tools/image-tool.js +2 -2
  64. package/dist/agents/tools/memory-tool.js +93 -5
  65. package/dist/agents/tools/message-tool.js +56 -2
  66. package/dist/agents/tools/sessions-history-tool.js +69 -1
  67. package/dist/agents/tools/web-search.js +211 -42
  68. package/dist/agents/usage.js +23 -1
  69. package/dist/agents/workspace-run.js +67 -0
  70. package/dist/agents/workspace-templates.js +44 -0
  71. package/dist/auto-reply/command-auth.js +121 -6
  72. package/dist/auto-reply/commands-registry.data.js +1 -1
  73. package/dist/auto-reply/envelope.js +50 -72
  74. package/dist/auto-reply/reply/commands-compact.js +1 -0
  75. package/dist/auto-reply/reply/commands-context-report.js +3 -2
  76. package/dist/auto-reply/reply/commands-context.js +1 -0
  77. package/dist/auto-reply/reply/commands-models.js +107 -60
  78. package/dist/auto-reply/reply/commands-ptt.js +171 -0
  79. package/dist/auto-reply/reply/commands-session.js +2 -2
  80. package/dist/auto-reply/reply/get-reply-run.js +16 -5
  81. package/dist/auto-reply/reply/groups.js +1 -1
  82. package/dist/auto-reply/reply/inbound-context.js +9 -1
  83. package/dist/auto-reply/reply/inbound-meta.js +130 -0
  84. package/dist/auto-reply/reply/model-selection.js +3 -3
  85. package/dist/auto-reply/reply/untrusted-context.js +15 -0
  86. package/dist/auto-reply/status.js +1 -1
  87. package/dist/auto-reply/thinking.js +88 -43
  88. package/dist/browser/bridge-server.js +13 -0
  89. package/dist/browser/cdp.helpers.js +38 -24
  90. package/dist/browser/client-fetch.js +51 -8
  91. package/dist/browser/config.js +2 -11
  92. package/dist/browser/extension-relay.js +104 -43
  93. package/dist/browser/pw-ai.js +1 -1
  94. package/dist/browser/pw-session.js +143 -8
  95. package/dist/browser/pw-tools-core.interactions.js +125 -27
  96. package/dist/browser/pw-tools-core.responses.js +1 -1
  97. package/dist/browser/pw-tools-core.state.js +1 -1
  98. package/dist/browser/routes/agent.act.js +86 -41
  99. package/dist/browser/routes/dispatcher.js +4 -4
  100. package/dist/browser/screenshot.js +1 -1
  101. package/dist/browser/server-context.js +2 -2
  102. package/dist/browser/server.js +13 -0
  103. package/dist/build-info.json +3 -3
  104. package/dist/canvas-host/a2ui.js +3 -3
  105. package/dist/channels/plugins/catalog.js +2 -2
  106. package/dist/channels/plugins/onboarding/imessage.js +1 -1
  107. package/dist/channels/plugins/onboarding/signal.js +1 -1
  108. package/dist/channels/plugins/onboarding/slack.js +4 -4
  109. package/dist/channels/plugins/onboarding/whatsapp.js +3 -3
  110. package/dist/channels/plugins/pairing-message.js +1 -1
  111. package/dist/channels/reply-prefix.js +8 -1
  112. package/dist/cli/browser-cli-extension.js +2 -2
  113. package/dist/cli/cron-cli/register.cron-add.js +61 -40
  114. package/dist/cli/cron-cli/register.cron-edit.js +60 -34
  115. package/dist/cli/cron-cli/shared.js +56 -41
  116. package/dist/cli/dns-cli.js +26 -14
  117. package/dist/cli/docs-cli.js +1 -1
  118. package/dist/cli/gateway-cli/dev.js +1 -1
  119. package/dist/cli/gateway-cli/register.js +37 -19
  120. package/dist/cli/memory-cli.js +30 -20
  121. package/dist/cli/nodes-cli/register.canvas.js +1 -1
  122. package/dist/cli/parse-bytes.js +37 -0
  123. package/dist/cli/plugins-cli.js +1 -1
  124. package/dist/cli/run-main.js +2 -2
  125. package/dist/cli/security-cli.js +1 -1
  126. package/dist/cli/tagline.js +1 -1
  127. package/dist/cli/update-cli.js +173 -52
  128. package/dist/cli/webhooks-cli.js +5 -5
  129. package/dist/commands/agent.js +1 -0
  130. package/dist/commands/agents.commands.add.js +1 -1
  131. package/dist/commands/auth-choice.apply.api-providers.js +305 -17
  132. package/dist/commands/auth-choice.apply.js +4 -1
  133. package/dist/commands/auth-choice.apply.plugin-provider.js +2 -2
  134. package/dist/commands/auth-choice.apply.xai.js +63 -0
  135. package/dist/commands/auth-choice.preferred-provider.js +7 -1
  136. package/dist/commands/configure.wizard.js +1 -1
  137. package/dist/commands/dashboard.js +1 -1
  138. package/dist/commands/docs.js +1 -1
  139. package/dist/commands/doctor-config-flow.js +61 -5
  140. package/dist/commands/doctor-gateway-services.js +3 -3
  141. package/dist/commands/doctor-state-migrations.js +1 -1
  142. package/dist/commands/doctor-update.js +3 -3
  143. package/dist/commands/doctor.js +1 -1
  144. package/dist/commands/health.js +1 -1
  145. package/dist/commands/model-allowlist.js +29 -0
  146. package/dist/commands/model-picker.js +2 -1
  147. package/dist/commands/models/list.probe.js +2 -2
  148. package/dist/commands/models/list.registry.js +4 -4
  149. package/dist/commands/models/list.status-command.js +44 -24
  150. package/dist/commands/models/shared.js +15 -0
  151. package/dist/commands/onboard-auth.config-core.js +366 -28
  152. package/dist/commands/onboard-auth.credentials.js +71 -9
  153. package/dist/commands/onboard-auth.js +3 -3
  154. package/dist/commands/onboard-auth.models.js +26 -24
  155. package/dist/commands/onboard-custom.js +384 -0
  156. package/dist/commands/onboard-non-interactive/local/auth-choice-inference.js +35 -0
  157. package/dist/commands/onboard-non-interactive/local/auth-choice.js +146 -9
  158. package/dist/commands/onboard-skills.js +63 -38
  159. package/dist/commands/openai-model-default.js +41 -0
  160. package/dist/commands/status-all/report-lines.js +1 -1
  161. package/dist/commands/status.command.js +1 -1
  162. package/dist/commands/uninstall.js +3 -3
  163. package/dist/compat/legacy-names.js +1 -1
  164. package/dist/config/defaults.js +3 -2
  165. package/dist/config/io.js +3 -3
  166. package/dist/config/paths.js +136 -35
  167. package/dist/config/plugin-auto-enable.js +21 -5
  168. package/dist/config/redact-snapshot.js +153 -0
  169. package/dist/config/schema.field-metadata.js +590 -0
  170. package/dist/config/schema.js +3 -3
  171. package/dist/config/sessions/store.js +291 -23
  172. package/dist/config/types.memory.js +1 -0
  173. package/dist/config/version.js +4 -4
  174. package/dist/config/zod-schema.agent-defaults.js +3 -0
  175. package/dist/config/zod-schema.agent-runtime.js +13 -2
  176. package/dist/config/zod-schema.providers-core.js +142 -0
  177. package/dist/config/zod-schema.session.js +3 -0
  178. package/dist/cron/delivery.js +57 -0
  179. package/dist/cron/isolated-agent/delivery-target.js +18 -3
  180. package/dist/cron/isolated-agent/helpers.js +22 -5
  181. package/dist/cron/isolated-agent/run.js +171 -63
  182. package/dist/cron/isolated-agent/session.js +2 -0
  183. package/dist/cron/normalize.js +356 -28
  184. package/dist/cron/parse.js +10 -5
  185. package/dist/cron/run-log.js +35 -10
  186. package/dist/cron/schedule.js +41 -6
  187. package/dist/cron/service/jobs.js +208 -35
  188. package/dist/cron/service/ops.js +72 -16
  189. package/dist/cron/service/state.js +2 -0
  190. package/dist/cron/service/store.js +386 -14
  191. package/dist/cron/service/timer.js +390 -147
  192. package/dist/cron/session-reaper.js +86 -0
  193. package/dist/cron/store.js +23 -8
  194. package/dist/cron/validate-timestamp.js +43 -0
  195. package/dist/daemon/constants.js +7 -7
  196. package/dist/daemon/inspect.js +6 -6
  197. package/dist/daemon/systemd-unit.js +1 -1
  198. package/dist/discord/monitor/agent-components.js +438 -0
  199. package/dist/discord/monitor/allow-list.js +28 -5
  200. package/dist/discord/monitor/gateway-registry.js +29 -0
  201. package/dist/discord/monitor/native-command.js +44 -23
  202. package/dist/discord/monitor/sender-identity.js +45 -0
  203. package/dist/discord/pluralkit.js +27 -0
  204. package/dist/discord/send.outbound.js +92 -5
  205. package/dist/discord/send.shared.js +60 -23
  206. package/dist/discord/targets.js +84 -1
  207. package/dist/entry.js +15 -9
  208. package/dist/extensionAPI.js +8 -0
  209. package/dist/gateway/control-ui.js +8 -1
  210. package/dist/gateway/hooks-mapping.js +3 -0
  211. package/dist/gateway/hooks.js +65 -0
  212. package/dist/gateway/live-image-probe.js +1 -66
  213. package/dist/gateway/net.js +96 -31
  214. package/dist/gateway/node-command-policy.js +50 -15
  215. package/dist/gateway/openai-http.js +2 -2
  216. package/dist/gateway/openresponses-http.js +4 -4
  217. package/dist/gateway/origin-check.js +56 -0
  218. package/dist/gateway/protocol/client-info.js +9 -0
  219. package/dist/gateway/protocol/index.js +9 -2
  220. package/dist/gateway/protocol/schema/agents-models-skills.js +71 -1
  221. package/dist/gateway/protocol/schema/cron.js +22 -10
  222. package/dist/gateway/protocol/schema/protocol-schemas.js +16 -2
  223. package/dist/gateway/protocol/schema/sessions.js +12 -0
  224. package/dist/gateway/server/hooks.js +1 -1
  225. package/dist/gateway/server-broadcast.js +26 -9
  226. package/dist/gateway/server-chat.js +112 -23
  227. package/dist/gateway/server-discovery-runtime.js +10 -2
  228. package/dist/gateway/server-discovery.js +2 -2
  229. package/dist/gateway/server-http.js +110 -12
  230. package/dist/gateway/server-methods/agent-timestamp.js +60 -0
  231. package/dist/gateway/server-methods/agents.js +321 -2
  232. package/dist/gateway/server-methods/usage.js +559 -16
  233. package/dist/gateway/server-runtime-state.js +22 -8
  234. package/dist/gateway/server-startup-memory.js +16 -0
  235. package/dist/gateway/server.impl.js +7 -3
  236. package/dist/gateway/session-utils.fs.js +23 -25
  237. package/dist/gateway/session-utils.js +20 -10
  238. package/dist/gateway/sessions-patch.js +7 -22
  239. package/dist/gateway/test-helpers.server.js +35 -2
  240. package/dist/hooks/frontmatter.js +1 -1
  241. package/dist/hooks/hooks-status.js +1 -1
  242. package/dist/hooks/install.js +2 -2
  243. package/dist/hooks/loader.js +1 -1
  244. package/dist/hooks/workspace.js +3 -3
  245. package/dist/imessage/constants.js +2 -0
  246. package/dist/imessage/monitor/deliver.js +4 -1
  247. package/dist/imessage/monitor/monitor-provider.js +51 -1
  248. package/dist/index.js +2 -2
  249. package/dist/infra/bonjour-discovery.js +131 -70
  250. package/dist/infra/bonjour.js +3 -3
  251. package/dist/infra/control-ui-assets.js +134 -12
  252. package/dist/infra/errors.js +12 -0
  253. package/dist/infra/exec-approvals.js +266 -57
  254. package/dist/infra/format-time/format-datetime.js +79 -0
  255. package/dist/infra/format-time/format-duration.js +81 -0
  256. package/dist/infra/format-time/format-relative.js +80 -0
  257. package/dist/infra/heartbeat-runner.js +140 -49
  258. package/dist/infra/home-dir.js +54 -0
  259. package/dist/infra/net/fetch-guard.js +122 -0
  260. package/dist/infra/net/ssrf.js +65 -29
  261. package/dist/infra/outbound/abort.js +14 -0
  262. package/dist/infra/outbound/message-action-runner.js +77 -13
  263. package/dist/infra/outbound/outbound-session.js +143 -37
  264. package/dist/infra/path-env.js +3 -3
  265. package/dist/infra/poolbot-root.js +43 -1
  266. package/dist/infra/provider-usage.fetch.minimax.js +1 -1
  267. package/dist/infra/restart.js +1 -1
  268. package/dist/infra/session-cost-usage.js +631 -41
  269. package/dist/infra/state-migrations.js +317 -47
  270. package/dist/infra/tailscale.js +1 -1
  271. package/dist/infra/update-global.js +35 -0
  272. package/dist/infra/update-runner.js +149 -43
  273. package/dist/infra/warning-filter.js +65 -0
  274. package/dist/infra/widearea-dns.js +30 -9
  275. package/dist/logging/redact-identifier.js +12 -0
  276. package/dist/macos/relay.js +2 -2
  277. package/dist/media/fetch.js +81 -58
  278. package/dist/media/input-files.js +1 -1
  279. package/dist/media/mime.js +4 -0
  280. package/dist/media/png-encode.js +74 -0
  281. package/dist/media-understanding/apply.js +403 -3
  282. package/dist/media-understanding/attachments.js +38 -27
  283. package/dist/media-understanding/defaults.js +16 -0
  284. package/dist/media-understanding/providers/deepgram/audio.js +22 -14
  285. package/dist/media-understanding/providers/google/audio.js +24 -17
  286. package/dist/media-understanding/providers/google/video.js +24 -17
  287. package/dist/media-understanding/providers/image.js +4 -4
  288. package/dist/media-understanding/providers/index.js +4 -1
  289. package/dist/media-understanding/providers/openai/audio.js +22 -14
  290. package/dist/media-understanding/providers/shared.js +16 -11
  291. package/dist/media-understanding/providers/zai/index.js +6 -0
  292. package/dist/media-understanding/runner.js +158 -90
  293. package/dist/memory/backend-config.js +207 -0
  294. package/dist/memory/batch-voyage.js +277 -0
  295. package/dist/memory/embeddings-voyage.js +75 -0
  296. package/dist/memory/embeddings.js +29 -17
  297. package/dist/memory/internal.js +101 -18
  298. package/dist/memory/manager.js +155 -48
  299. package/dist/memory/search-manager.js +173 -0
  300. package/dist/memory/session-files.js +9 -3
  301. package/dist/memory/types.js +1 -0
  302. package/dist/node-host/runner.js +36 -26
  303. package/dist/node-host/with-timeout.js +27 -0
  304. package/dist/pairing/pairing-messages.js +1 -1
  305. package/dist/plugins/commands.js +5 -1
  306. package/dist/plugins/config-state.js +86 -7
  307. package/dist/plugins/discovery.js +1 -1
  308. package/dist/plugins/install.js +2 -2
  309. package/dist/plugins/source-display.js +51 -0
  310. package/dist/plugins/update.js +1 -1
  311. package/dist/process/exec.js +20 -2
  312. package/dist/routing/resolve-route.js +12 -0
  313. package/dist/routing/session-key.js +15 -0
  314. package/dist/runtime.js +2 -0
  315. package/dist/security/audit-extra.async.js +601 -0
  316. package/dist/security/audit-extra.js +2 -830
  317. package/dist/security/audit-extra.sync.js +505 -0
  318. package/dist/security/audit.js +2 -2
  319. package/dist/security/channel-metadata.js +34 -0
  320. package/dist/security/external-content.js +88 -6
  321. package/dist/security/skill-scanner.js +330 -0
  322. package/dist/sessions/session-key-utils.js +7 -0
  323. package/dist/shared/text/reasoning-tags.js +52 -7
  324. package/dist/signal/monitor/event-handler.js +80 -1
  325. package/dist/slack/monitor/media.js +85 -15
  326. package/dist/tailscale/detect.js +145 -0
  327. package/dist/telegram/bot/helpers.js +109 -28
  328. package/dist/telegram/bot-handlers.js +144 -3
  329. package/dist/telegram/bot-message-context.js +38 -11
  330. package/dist/telegram/bot-message-dispatch.js +48 -15
  331. package/dist/telegram/bot-native-commands.js +86 -29
  332. package/dist/telegram/bot.js +30 -29
  333. package/dist/telegram/model-buttons.js +163 -0
  334. package/dist/telegram/monitor.js +110 -85
  335. package/dist/telegram/send.js +129 -47
  336. package/dist/terminal/restore.js +45 -0
  337. package/dist/test-helpers/state-dir-env.js +16 -0
  338. package/dist/test-helpers/workspace.js +11 -0
  339. package/dist/test-utils/channel-plugins.js +82 -0
  340. package/dist/test-utils/ports.js +73 -0
  341. package/dist/tts/tts.js +12 -6
  342. package/dist/tui/tui-session-actions.js +166 -54
  343. package/dist/utils/fetch-timeout.js +20 -0
  344. package/dist/utils/normalize-secret-input.js +19 -0
  345. package/dist/utils/shell-argv.js +61 -0
  346. package/dist/utils/transcript-tools.js +58 -0
  347. package/dist/utils.js +55 -14
  348. package/dist/version.js +42 -5
  349. package/dist/web/qr-image.js +1 -61
  350. package/dist/wizard/onboarding.finalize.js +7 -7
  351. package/dist/wizard/onboarding.js +3 -3
  352. package/docs/RELEASE_WORKFOTS_COMPARISON.md +3 -3
  353. package/docs/_config.yml +2 -2
  354. package/docs/_layouts/default.html +9 -9
  355. package/docs/concepts/typebox.md +1 -1
  356. package/docs/docs.json +1 -1
  357. package/docs/northflank.mdx +7 -7
  358. package/docs/railway.mdx +3 -3
  359. package/docs/render.mdx +5 -5
  360. package/docs/start/lore.md +2 -2
  361. package/extensions/bluebubbles/index.ts +2 -2
  362. package/extensions/bluebubbles/package.json +1 -1
  363. package/extensions/bluebubbles/src/accounts.ts +8 -8
  364. package/extensions/bluebubbles/src/actions.test.ts +22 -22
  365. package/extensions/bluebubbles/src/actions.ts +5 -5
  366. package/extensions/bluebubbles/src/attachments.ts +2 -2
  367. package/extensions/bluebubbles/src/channel.ts +16 -16
  368. package/extensions/bluebubbles/src/chat.ts +2 -2
  369. package/extensions/bluebubbles/src/media-send.ts +2 -2
  370. package/extensions/bluebubbles/src/monitor.test.ts +46 -46
  371. package/extensions/bluebubbles/src/monitor.ts +5 -5
  372. package/extensions/bluebubbles/src/onboarding.ts +7 -7
  373. package/extensions/bluebubbles/src/reactions.ts +2 -2
  374. package/extensions/bluebubbles/src/send.ts +2 -2
  375. package/extensions/copilot-proxy/README.md +1 -1
  376. package/extensions/copilot-proxy/package.json +1 -1
  377. package/extensions/diagnostics-otel/index.ts +2 -2
  378. package/extensions/diagnostics-otel/package.json +1 -1
  379. package/extensions/diagnostics-otel/src/service.ts +3 -3
  380. package/extensions/discord/index.ts +2 -2
  381. package/extensions/discord/package.json +1 -1
  382. package/extensions/google-antigravity-auth/README.md +1 -1
  383. package/extensions/google-antigravity-auth/index.ts +1 -1
  384. package/extensions/google-antigravity-auth/package.json +1 -1
  385. package/extensions/google-gemini-cli-auth/README.md +1 -1
  386. package/extensions/google-gemini-cli-auth/oauth.ts +1 -1
  387. package/extensions/google-gemini-cli-auth/package.json +1 -1
  388. package/extensions/googlechat/index.ts +3 -3
  389. package/extensions/googlechat/package.json +1 -1
  390. package/extensions/googlechat/src/accounts.ts +8 -8
  391. package/extensions/googlechat/src/actions.ts +6 -6
  392. package/extensions/googlechat/src/channel.ts +21 -21
  393. package/extensions/googlechat/src/monitor.ts +8 -8
  394. package/extensions/googlechat/src/onboarding.ts +10 -10
  395. package/extensions/imessage/index.ts +2 -2
  396. package/extensions/imessage/package.json +1 -1
  397. package/extensions/line/index.ts +2 -2
  398. package/extensions/line/package.json +1 -1
  399. package/extensions/line/src/card-command.ts +2 -2
  400. package/extensions/line/src/channel.logout.test.ts +4 -4
  401. package/extensions/line/src/channel.sendPayload.test.ts +8 -8
  402. package/extensions/line/src/channel.ts +3 -3
  403. package/extensions/llm-task/README.md +3 -3
  404. package/extensions/llm-task/index.ts +2 -2
  405. package/extensions/llm-task/package.json +1 -1
  406. package/extensions/llm-task/src/llm-task-tool.ts +4 -4
  407. package/extensions/lobster/README.md +6 -6
  408. package/extensions/lobster/index.ts +2 -2
  409. package/extensions/lobster/src/lobster-tool.test.ts +4 -4
  410. package/extensions/lobster/src/lobster-tool.ts +2 -2
  411. package/extensions/matrix/index.ts +2 -2
  412. package/extensions/matrix/package.json +1 -1
  413. package/extensions/matrix/src/matrix/client/config.ts +1 -1
  414. package/extensions/matrix/src/matrix/monitor/handler.ts +1 -1
  415. package/extensions/matrix/src/onboarding.ts +1 -1
  416. package/extensions/mattermost/index.ts +2 -2
  417. package/extensions/mattermost/package.json +1 -1
  418. package/extensions/mattermost/src/mattermost/accounts.ts +8 -8
  419. package/extensions/mattermost/src/mattermost/monitor-helpers.ts +5 -5
  420. package/extensions/mattermost/src/mattermost/monitor.ts +2 -2
  421. package/extensions/mattermost/src/onboarding-helpers.ts +3 -3
  422. package/extensions/mattermost/src/onboarding.ts +2 -2
  423. package/extensions/memory-core/index.ts +2 -2
  424. package/extensions/memory-core/package.json +1 -1
  425. package/extensions/memory-lancedb/index.ts +3 -3
  426. package/extensions/memory-lancedb/package.json +1 -1
  427. package/extensions/msteams/index.ts +2 -2
  428. package/extensions/msteams/package.json +1 -1
  429. package/extensions/msteams/src/channel.directory.test.ts +2 -2
  430. package/extensions/msteams/src/channel.ts +2 -2
  431. package/extensions/msteams/src/graph-upload.ts +4 -4
  432. package/extensions/msteams/src/monitor-handler.ts +2 -2
  433. package/extensions/msteams/src/monitor.ts +2 -2
  434. package/extensions/msteams/src/onboarding.ts +9 -9
  435. package/extensions/msteams/src/reply-dispatcher.ts +2 -2
  436. package/extensions/msteams/src/send-context.ts +2 -2
  437. package/extensions/msteams/src/send.ts +4 -4
  438. package/extensions/nextcloud-talk/index.ts +2 -2
  439. package/extensions/nextcloud-talk/package.json +1 -1
  440. package/extensions/nextcloud-talk/src/channel.ts +7 -7
  441. package/extensions/nextcloud-talk/src/inbound.ts +7 -7
  442. package/extensions/nextcloud-talk/src/onboarding.ts +1 -1
  443. package/extensions/nostr/README.md +2 -2
  444. package/extensions/nostr/index.ts +5 -5
  445. package/extensions/nostr/package.json +1 -1
  446. package/extensions/nostr/src/types.ts +4 -4
  447. package/extensions/open-prose/index.ts +2 -2
  448. package/extensions/qwen-portal-auth/README.md +1 -1
  449. package/extensions/signal/index.ts +2 -2
  450. package/extensions/signal/package.json +1 -1
  451. package/extensions/slack/index.ts +2 -2
  452. package/extensions/slack/package.json +1 -1
  453. package/extensions/telegram/index.ts +2 -2
  454. package/extensions/telegram/package.json +1 -1
  455. package/extensions/telegram/src/channel.ts +2 -2
  456. package/extensions/tlon/README.md +2 -2
  457. package/extensions/tlon/index.ts +2 -2
  458. package/extensions/tlon/package.json +1 -1
  459. package/extensions/tlon/src/channel.ts +13 -13
  460. package/extensions/tlon/src/monitor/index.ts +3 -3
  461. package/extensions/tlon/src/onboarding.ts +3 -3
  462. package/extensions/tlon/src/types.ts +3 -3
  463. package/extensions/twitch/README.md +1 -1
  464. package/extensions/twitch/index.ts +2 -2
  465. package/extensions/twitch/package.json +1 -1
  466. package/extensions/twitch/src/config.ts +3 -3
  467. package/extensions/twitch/src/monitor.ts +3 -3
  468. package/extensions/twitch/src/onboarding.ts +9 -9
  469. package/extensions/twitch/src/outbound.test.ts +2 -2
  470. package/extensions/twitch/src/plugin.test.ts +2 -2
  471. package/extensions/twitch/src/plugin.ts +8 -8
  472. package/extensions/twitch/src/send.test.ts +2 -2
  473. package/extensions/twitch/src/send.ts +4 -4
  474. package/extensions/twitch/src/token.test.ts +8 -8
  475. package/extensions/twitch/src/token.ts +3 -3
  476. package/extensions/twitch/src/twitch-client.ts +3 -3
  477. package/extensions/twitch/src/types.ts +3 -3
  478. package/extensions/twitch/src/utils/markdown.ts +1 -1
  479. package/extensions/voice-call/README.md +3 -3
  480. package/extensions/voice-call/package.json +1 -1
  481. package/extensions/voice-call/src/core-bridge.ts +2 -2
  482. package/extensions/voice-call/src/response-generator.ts +1 -1
  483. package/extensions/whatsapp/index.ts +2 -2
  484. package/extensions/whatsapp/package.json +1 -1
  485. package/extensions/zalo/README.md +1 -1
  486. package/extensions/zalo/index.ts +2 -2
  487. package/extensions/zalo/package.json +1 -1
  488. package/extensions/zalo/src/accounts.ts +8 -8
  489. package/extensions/zalo/src/actions.ts +4 -4
  490. package/extensions/zalo/src/channel.directory.test.ts +2 -2
  491. package/extensions/zalo/src/channel.ts +18 -18
  492. package/extensions/zalo/src/monitor.ts +9 -9
  493. package/extensions/zalo/src/monitor.webhook.test.ts +2 -2
  494. package/extensions/zalo/src/onboarding.ts +24 -24
  495. package/extensions/zalo/src/send.ts +2 -2
  496. package/extensions/zalouser/README.md +2 -2
  497. package/extensions/zalouser/index.ts +2 -2
  498. package/extensions/zalouser/package.json +1 -1
  499. package/extensions/zalouser/src/accounts.ts +9 -9
  500. package/extensions/zalouser/src/channel.ts +24 -24
  501. package/extensions/zalouser/src/monitor.ts +4 -4
  502. package/extensions/zalouser/src/onboarding.ts +28 -28
  503. package/package.json +13 -251
  504. package/skills/nano-banana-pro/scripts/generate_image.py +1 -1
  505. package/skills/tmux/scripts/find-sessions.sh +1 -1
  506. package/CHANGELOG.md +0 -102
  507. package/README-header.png +0 -0
  508. package/git-hooks/pre-commit +0 -4
  509. package/scripts/format-staged.js +0 -148
  510. package/scripts/postinstall.js +0 -300
  511. package/scripts/setup-git-hooks.js +0 -96
@@ -1,9 +1,390 @@
1
+ import path from "node:path";
1
2
  import { finalizeInboundContext } from "../auto-reply/reply/inbound-context.js";
2
- import { extractMediaUserText, formatAudioTranscripts, formatMediaUnderstandingBody, } from "./format.js";
3
+ import { logVerbose, shouldLogVerbose } from "../globals.js";
4
+ import { DEFAULT_INPUT_FILE_MAX_BYTES, DEFAULT_INPUT_FILE_MAX_CHARS, DEFAULT_INPUT_FILE_MIMES, DEFAULT_INPUT_MAX_REDIRECTS, DEFAULT_INPUT_PDF_MAX_PAGES, DEFAULT_INPUT_PDF_MAX_PIXELS, DEFAULT_INPUT_PDF_MIN_TEXT_CHARS, DEFAULT_INPUT_TIMEOUT_MS, extractFileContentFromSource, normalizeMimeList, normalizeMimeType, } from "../media/input-files.js";
5
+ import { resolveAttachmentKind } from "./attachments.js";
3
6
  import { runWithConcurrency } from "./concurrency.js";
7
+ import { extractMediaUserText, formatAudioTranscripts, formatMediaUnderstandingBody, } from "./format.js";
4
8
  import { resolveConcurrency } from "./resolve.js";
5
9
  import { buildProviderRegistry, createMediaAttachmentCache, normalizeMediaAttachments, runCapability, } from "./runner.js";
6
10
  const CAPABILITY_ORDER = ["image", "audio", "video"];
11
+ const EXTRA_TEXT_MIMES = [
12
+ "application/xml",
13
+ "text/xml",
14
+ "application/x-yaml",
15
+ "text/yaml",
16
+ "application/yaml",
17
+ "application/javascript",
18
+ "text/javascript",
19
+ "text/tab-separated-values",
20
+ ];
21
+ const TEXT_EXT_MIME = new Map([
22
+ [".csv", "text/csv"],
23
+ [".tsv", "text/tab-separated-values"],
24
+ [".txt", "text/plain"],
25
+ [".md", "text/markdown"],
26
+ [".log", "text/plain"],
27
+ [".ini", "text/plain"],
28
+ [".cfg", "text/plain"],
29
+ [".conf", "text/plain"],
30
+ [".env", "text/plain"],
31
+ [".json", "application/json"],
32
+ [".yaml", "text/yaml"],
33
+ [".yml", "text/yaml"],
34
+ [".xml", "application/xml"],
35
+ ]);
36
+ const XML_ESCAPE_MAP = {
37
+ "<": "&lt;",
38
+ ">": "&gt;",
39
+ "&": "&amp;",
40
+ '"': "&quot;",
41
+ "'": "&apos;",
42
+ };
43
+ /**
44
+ * Escapes special XML characters in attribute values to prevent injection.
45
+ */
46
+ function xmlEscapeAttr(value) {
47
+ return value.replace(/[<>&"']/g, (char) => XML_ESCAPE_MAP[char] ?? char);
48
+ }
49
+ function escapeFileBlockContent(value) {
50
+ return value.replace(/<\s*\/\s*file\s*>/gi, "&lt;/file&gt;").replace(/<\s*file\b/gi, "&lt;file");
51
+ }
52
+ function sanitizeMimeType(value) {
53
+ if (!value) {
54
+ return undefined;
55
+ }
56
+ const trimmed = value.trim().toLowerCase();
57
+ if (!trimmed) {
58
+ return undefined;
59
+ }
60
+ const match = trimmed.match(/^([a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+)/);
61
+ return match?.[1];
62
+ }
63
+ function resolveFileLimits(cfg) {
64
+ const files = cfg.gateway?.http?.endpoints?.responses?.files;
65
+ const allowedMimesConfigured = Boolean(files?.allowedMimes && files.allowedMimes.length > 0);
66
+ return {
67
+ allowUrl: files?.allowUrl ?? true,
68
+ allowedMimes: normalizeMimeList(files?.allowedMimes, DEFAULT_INPUT_FILE_MIMES),
69
+ allowedMimesConfigured,
70
+ maxBytes: files?.maxBytes ?? DEFAULT_INPUT_FILE_MAX_BYTES,
71
+ maxChars: files?.maxChars ?? DEFAULT_INPUT_FILE_MAX_CHARS,
72
+ maxRedirects: files?.maxRedirects ?? DEFAULT_INPUT_MAX_REDIRECTS,
73
+ timeoutMs: files?.timeoutMs ?? DEFAULT_INPUT_TIMEOUT_MS,
74
+ pdf: {
75
+ maxPages: files?.pdf?.maxPages ?? DEFAULT_INPUT_PDF_MAX_PAGES,
76
+ maxPixels: files?.pdf?.maxPixels ?? DEFAULT_INPUT_PDF_MAX_PIXELS,
77
+ minTextChars: files?.pdf?.minTextChars ?? DEFAULT_INPUT_PDF_MIN_TEXT_CHARS,
78
+ },
79
+ };
80
+ }
81
+ function appendFileBlocks(body, blocks) {
82
+ if (!blocks || blocks.length === 0) {
83
+ return body ?? "";
84
+ }
85
+ const base = typeof body === "string" ? body.trim() : "";
86
+ const suffix = blocks.join("\n\n").trim();
87
+ if (!base) {
88
+ return suffix;
89
+ }
90
+ return `${base}\n\n${suffix}`.trim();
91
+ }
92
+ function resolveUtf16Charset(buffer) {
93
+ if (!buffer || buffer.length < 2) {
94
+ return undefined;
95
+ }
96
+ const b0 = buffer[0];
97
+ const b1 = buffer[1];
98
+ if (b0 === 0xff && b1 === 0xfe) {
99
+ return "utf-16le";
100
+ }
101
+ if (b0 === 0xfe && b1 === 0xff) {
102
+ return "utf-16be";
103
+ }
104
+ const sampleLen = Math.min(buffer.length, 2048);
105
+ let zeroEven = 0;
106
+ let zeroOdd = 0;
107
+ for (let i = 0; i < sampleLen; i += 1) {
108
+ if (buffer[i] !== 0) {
109
+ continue;
110
+ }
111
+ if (i % 2 === 0) {
112
+ zeroEven += 1;
113
+ }
114
+ else {
115
+ zeroOdd += 1;
116
+ }
117
+ }
118
+ const zeroCount = zeroEven + zeroOdd;
119
+ if (zeroCount / sampleLen > 0.2) {
120
+ return zeroOdd >= zeroEven ? "utf-16le" : "utf-16be";
121
+ }
122
+ return undefined;
123
+ }
124
+ const WORDISH_CHAR = /[\p{L}\p{N}]/u;
125
+ const CP1252_MAP = [
126
+ "\u20ac",
127
+ undefined,
128
+ "\u201a",
129
+ "\u0192",
130
+ "\u201e",
131
+ "\u2026",
132
+ "\u2020",
133
+ "\u2021",
134
+ "\u02c6",
135
+ "\u2030",
136
+ "\u0160",
137
+ "\u2039",
138
+ "\u0152",
139
+ undefined,
140
+ "\u017d",
141
+ undefined,
142
+ undefined,
143
+ "\u2018",
144
+ "\u2019",
145
+ "\u201c",
146
+ "\u201d",
147
+ "\u2022",
148
+ "\u2013",
149
+ "\u2014",
150
+ "\u02dc",
151
+ "\u2122",
152
+ "\u0161",
153
+ "\u203a",
154
+ "\u0153",
155
+ undefined,
156
+ "\u017e",
157
+ "\u0178",
158
+ ];
159
+ function decodeLegacyText(buffer) {
160
+ let output = "";
161
+ for (const byte of buffer) {
162
+ if (byte >= 0x80 && byte <= 0x9f) {
163
+ const mapped = CP1252_MAP[byte - 0x80];
164
+ output += mapped ?? String.fromCharCode(byte);
165
+ continue;
166
+ }
167
+ output += String.fromCharCode(byte);
168
+ }
169
+ return output;
170
+ }
171
+ function getTextStats(text) {
172
+ if (!text) {
173
+ return { printableRatio: 0, wordishRatio: 0 };
174
+ }
175
+ let printable = 0;
176
+ let control = 0;
177
+ let wordish = 0;
178
+ for (const char of text) {
179
+ const code = char.codePointAt(0) ?? 0;
180
+ if (code === 9 || code === 10 || code === 13 || code === 32) {
181
+ printable += 1;
182
+ wordish += 1;
183
+ continue;
184
+ }
185
+ if (code < 32 || (code >= 0x7f && code <= 0x9f)) {
186
+ control += 1;
187
+ continue;
188
+ }
189
+ printable += 1;
190
+ if (WORDISH_CHAR.test(char)) {
191
+ wordish += 1;
192
+ }
193
+ }
194
+ const total = printable + control;
195
+ if (total === 0) {
196
+ return { printableRatio: 0, wordishRatio: 0 };
197
+ }
198
+ return { printableRatio: printable / total, wordishRatio: wordish / total };
199
+ }
200
+ function isMostlyPrintable(text) {
201
+ return getTextStats(text).printableRatio > 0.85;
202
+ }
203
+ function looksLikeLegacyTextBytes(buffer) {
204
+ if (buffer.length === 0) {
205
+ return false;
206
+ }
207
+ const text = decodeLegacyText(buffer);
208
+ const { printableRatio, wordishRatio } = getTextStats(text);
209
+ return printableRatio > 0.95 && wordishRatio > 0.3;
210
+ }
211
+ function looksLikeUtf8Text(buffer) {
212
+ if (!buffer || buffer.length === 0) {
213
+ return false;
214
+ }
215
+ const sample = buffer.subarray(0, Math.min(buffer.length, 4096));
216
+ try {
217
+ const text = new TextDecoder("utf-8", { fatal: true }).decode(sample);
218
+ return isMostlyPrintable(text);
219
+ }
220
+ catch {
221
+ return looksLikeLegacyTextBytes(sample);
222
+ }
223
+ }
224
+ function decodeTextSample(buffer) {
225
+ if (!buffer || buffer.length === 0) {
226
+ return "";
227
+ }
228
+ const sample = buffer.subarray(0, Math.min(buffer.length, 8192));
229
+ const utf16Charset = resolveUtf16Charset(sample);
230
+ if (utf16Charset === "utf-16be") {
231
+ const swapped = Buffer.alloc(sample.length);
232
+ for (let i = 0; i + 1 < sample.length; i += 2) {
233
+ swapped[i] = sample[i + 1];
234
+ swapped[i + 1] = sample[i];
235
+ }
236
+ return new TextDecoder("utf-16le").decode(swapped);
237
+ }
238
+ if (utf16Charset === "utf-16le") {
239
+ return new TextDecoder("utf-16le").decode(sample);
240
+ }
241
+ return new TextDecoder("utf-8").decode(sample);
242
+ }
243
+ function guessDelimitedMime(text) {
244
+ if (!text) {
245
+ return undefined;
246
+ }
247
+ const line = text.split(/\r?\n/)[0] ?? "";
248
+ const tabs = (line.match(/\t/g) ?? []).length;
249
+ const commas = (line.match(/,/g) ?? []).length;
250
+ if (commas > 0) {
251
+ return "text/csv";
252
+ }
253
+ if (tabs > 0) {
254
+ return "text/tab-separated-values";
255
+ }
256
+ return undefined;
257
+ }
258
+ function resolveTextMimeFromName(name) {
259
+ if (!name) {
260
+ return undefined;
261
+ }
262
+ const ext = path.extname(name).toLowerCase();
263
+ return TEXT_EXT_MIME.get(ext);
264
+ }
265
+ function isBinaryMediaMime(mime) {
266
+ if (!mime) {
267
+ return false;
268
+ }
269
+ return mime.startsWith("image/") || mime.startsWith("audio/") || mime.startsWith("video/");
270
+ }
271
+ async function extractFileBlocks(params) {
272
+ const { attachments, cache, limits, skipAttachmentIndexes } = params;
273
+ if (!attachments || attachments.length === 0) {
274
+ return [];
275
+ }
276
+ const blocks = [];
277
+ for (const attachment of attachments) {
278
+ if (!attachment) {
279
+ continue;
280
+ }
281
+ if (skipAttachmentIndexes?.has(attachment.index)) {
282
+ continue;
283
+ }
284
+ const forcedTextMime = resolveTextMimeFromName(attachment.path ?? attachment.url ?? "");
285
+ const kind = forcedTextMime ? "document" : resolveAttachmentKind(attachment);
286
+ if (!forcedTextMime && (kind === "image" || kind === "video" || kind === "audio")) {
287
+ continue;
288
+ }
289
+ if (!limits.allowUrl && attachment.url && !attachment.path) {
290
+ if (shouldLogVerbose()) {
291
+ logVerbose(`media: file attachment skipped (url disabled) index=${attachment.index}`);
292
+ }
293
+ continue;
294
+ }
295
+ let bufferResult;
296
+ try {
297
+ bufferResult = await cache.getBuffer({
298
+ attachmentIndex: attachment.index,
299
+ maxBytes: limits.maxBytes,
300
+ timeoutMs: limits.timeoutMs,
301
+ });
302
+ }
303
+ catch (err) {
304
+ if (shouldLogVerbose()) {
305
+ logVerbose(`media: file attachment skipped (buffer): ${String(err)}`);
306
+ }
307
+ continue;
308
+ }
309
+ const nameHint = bufferResult?.fileName ?? attachment.path ?? attachment.url;
310
+ const forcedTextMimeResolved = forcedTextMime ?? resolveTextMimeFromName(nameHint ?? "");
311
+ const rawMime = bufferResult?.mime ?? attachment.mime;
312
+ const normalizedRawMime = normalizeMimeType(rawMime);
313
+ if (!forcedTextMimeResolved && isBinaryMediaMime(normalizedRawMime)) {
314
+ continue;
315
+ }
316
+ const utf16Charset = resolveUtf16Charset(bufferResult?.buffer);
317
+ const textSample = decodeTextSample(bufferResult?.buffer);
318
+ const textLike = Boolean(utf16Charset) || looksLikeUtf8Text(bufferResult?.buffer);
319
+ const guessedDelimited = textLike ? guessDelimitedMime(textSample) : undefined;
320
+ const textHint = forcedTextMimeResolved ?? guessedDelimited ?? (textLike ? "text/plain" : undefined);
321
+ const mimeType = sanitizeMimeType(textHint ?? normalizeMimeType(rawMime));
322
+ // Log when MIME type is overridden from non-text to text for auditability
323
+ if (textHint && rawMime && !rawMime.startsWith("text/")) {
324
+ logVerbose(`media: MIME override from "${rawMime}" to "${textHint}" for index=${attachment.index}`);
325
+ }
326
+ if (!mimeType) {
327
+ if (shouldLogVerbose()) {
328
+ logVerbose(`media: file attachment skipped (unknown mime) index=${attachment.index}`);
329
+ }
330
+ continue;
331
+ }
332
+ const allowedMimes = new Set(limits.allowedMimes);
333
+ if (!limits.allowedMimesConfigured) {
334
+ for (const extra of EXTRA_TEXT_MIMES) {
335
+ allowedMimes.add(extra);
336
+ }
337
+ if (mimeType.startsWith("text/")) {
338
+ allowedMimes.add(mimeType);
339
+ }
340
+ }
341
+ if (!allowedMimes.has(mimeType)) {
342
+ if (shouldLogVerbose()) {
343
+ logVerbose(`media: file attachment skipped (unsupported mime ${mimeType}) index=${attachment.index}`);
344
+ }
345
+ continue;
346
+ }
347
+ let extracted;
348
+ try {
349
+ const mediaType = utf16Charset ? `${mimeType}; charset=${utf16Charset}` : mimeType;
350
+ const { allowedMimesConfigured: _allowedMimesConfigured, ...baseLimits } = limits;
351
+ extracted = await extractFileContentFromSource({
352
+ source: {
353
+ type: "base64",
354
+ data: bufferResult.buffer.toString("base64"),
355
+ mediaType,
356
+ filename: bufferResult.fileName,
357
+ },
358
+ limits: {
359
+ ...baseLimits,
360
+ allowedMimes,
361
+ },
362
+ });
363
+ }
364
+ catch (err) {
365
+ if (shouldLogVerbose()) {
366
+ logVerbose(`media: file attachment skipped (extract): ${String(err)}`);
367
+ }
368
+ continue;
369
+ }
370
+ const text = extracted?.text?.trim() ?? "";
371
+ let blockText = text;
372
+ if (!blockText) {
373
+ if (extracted?.images && extracted.images.length > 0) {
374
+ blockText = "[PDF content rendered to images; images not forwarded to model]";
375
+ }
376
+ else {
377
+ blockText = "[No extractable text]";
378
+ }
379
+ }
380
+ const safeName = (bufferResult.fileName ?? `file-${attachment.index + 1}`)
381
+ .replace(/[\r\n\t]+/g, " ")
382
+ .trim();
383
+ // Escape XML special characters in attributes to prevent injection
384
+ blocks.push(`<file name="${xmlEscapeAttr(safeName)}" mime="${xmlEscapeAttr(mimeType)}">\n${escapeFileBlockContent(blockText)}\n</file>`);
385
+ }
386
+ return blocks;
387
+ }
7
388
  export async function applyMediaUnderstanding(params) {
8
389
  const { ctx, cfg } = params;
9
390
  const commandCandidates = [ctx.CommandBody, ctx.RawBody, ctx.Body];
@@ -32,8 +413,9 @@ export async function applyMediaUnderstanding(params) {
32
413
  const outputs = [];
33
414
  const decisions = [];
34
415
  for (const entry of results) {
35
- if (!entry)
416
+ if (!entry) {
36
417
  continue;
418
+ }
37
419
  for (const output of entry.outputs) {
38
420
  outputs.push(output);
39
421
  }
@@ -62,7 +444,24 @@ export async function applyMediaUnderstanding(params) {
62
444
  ctx.RawBody = originalUserText;
63
445
  }
64
446
  ctx.MediaUnderstanding = [...(ctx.MediaUnderstanding ?? []), ...outputs];
65
- finalizeInboundContext(ctx, { forceBodyForAgent: true, forceBodyForCommands: true });
447
+ }
448
+ const audioAttachmentIndexes = new Set(outputs
449
+ .filter((output) => output.kind === "audio.transcription")
450
+ .map((output) => output.attachmentIndex));
451
+ const fileBlocks = await extractFileBlocks({
452
+ attachments,
453
+ cache,
454
+ limits: resolveFileLimits(cfg),
455
+ skipAttachmentIndexes: audioAttachmentIndexes.size > 0 ? audioAttachmentIndexes : undefined,
456
+ });
457
+ if (fileBlocks.length > 0) {
458
+ ctx.Body = appendFileBlocks(ctx.Body, fileBlocks);
459
+ }
460
+ if (outputs.length > 0 || fileBlocks.length > 0) {
461
+ finalizeInboundContext(ctx, {
462
+ forceBodyForAgent: true,
463
+ forceBodyForCommands: outputs.length > 0 || fileBlocks.length > 0,
464
+ });
66
465
  }
67
466
  return {
68
467
  outputs,
@@ -70,6 +469,7 @@ export async function applyMediaUnderstanding(params) {
70
469
  appliedImage: outputs.some((output) => output.kind === "image.description"),
71
470
  appliedAudio: outputs.some((output) => output.kind === "audio.transcription"),
72
471
  appliedVideo: outputs.some((output) => output.kind === "video.description"),
472
+ appliedFile: fileBlocks.length > 0,
73
473
  };
74
474
  }
75
475
  finally {
@@ -3,16 +3,18 @@ import fs from "node:fs/promises";
3
3
  import os from "node:os";
4
4
  import path from "node:path";
5
5
  import { fileURLToPath } from "node:url";
6
+ import { logVerbose, shouldLogVerbose } from "../globals.js";
7
+ import { isAbortError } from "../infra/unhandled-rejections.js";
6
8
  import { fetchRemoteMedia, MediaFetchError } from "../media/fetch.js";
7
9
  import { detectMime, getFileExtension, isAudioFileName, kindFromMime } from "../media/mime.js";
8
- import { logVerbose, shouldLogVerbose } from "../globals.js";
9
- import { fetchWithTimeout } from "./providers/shared.js";
10
10
  import { MediaUnderstandingSkipError } from "./errors.js";
11
+ import { fetchWithTimeout } from "./providers/shared.js";
11
12
  const DEFAULT_MAX_ATTACHMENTS = 1;
12
13
  function normalizeAttachmentPath(raw) {
13
14
  const value = raw?.trim();
14
- if (!value)
15
+ if (!value) {
15
16
  return undefined;
17
+ }
16
18
  if (value.startsWith("file://")) {
17
19
  try {
18
20
  return fileURLToPath(value);
@@ -30,8 +32,9 @@ export function normalizeAttachments(ctx) {
30
32
  const resolveMime = (count, index) => {
31
33
  const typeHint = typesFromArray?.[index];
32
34
  const trimmed = typeof typeHint === "string" ? typeHint.trim() : "";
33
- if (trimmed)
35
+ if (trimmed) {
34
36
  return trimmed;
37
+ }
35
38
  return count === 1 ? ctx.MediaType : undefined;
36
39
  };
37
40
  if (pathsFromArray && pathsFromArray.length > 0) {
@@ -59,8 +62,9 @@ export function normalizeAttachments(ctx) {
59
62
  }
60
63
  const pathValue = ctx.MediaPath?.trim();
61
64
  const url = ctx.MediaUrl?.trim();
62
- if (!pathValue && !url)
65
+ if (!pathValue && !url) {
63
66
  return [];
67
+ }
64
68
  return [
65
69
  {
66
70
  path: pathValue || undefined,
@@ -72,15 +76,19 @@ export function normalizeAttachments(ctx) {
72
76
  }
73
77
  export function resolveAttachmentKind(attachment) {
74
78
  const kind = kindFromMime(attachment.mime);
75
- if (kind === "image" || kind === "audio" || kind === "video")
79
+ if (kind === "image" || kind === "audio" || kind === "video") {
76
80
  return kind;
81
+ }
77
82
  const ext = getFileExtension(attachment.path ?? attachment.url);
78
- if (!ext)
83
+ if (!ext) {
79
84
  return "unknown";
80
- if ([".mp4", ".mov", ".mkv", ".webm", ".avi", ".m4v"].includes(ext))
85
+ }
86
+ if ([".mp4", ".mov", ".mkv", ".webm", ".avi", ".m4v"].includes(ext)) {
81
87
  return "video";
82
- if (isAudioFileName(attachment.path ?? attachment.url))
88
+ }
89
+ if (isAudioFileName(attachment.path ?? attachment.url)) {
83
90
  return "audio";
91
+ }
84
92
  if ([".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff", ".tif"].includes(ext)) {
85
93
  return "image";
86
94
  }
@@ -95,25 +103,22 @@ export function isAudioAttachment(attachment) {
95
103
  export function isImageAttachment(attachment) {
96
104
  return resolveAttachmentKind(attachment) === "image";
97
105
  }
98
- function isAbortError(err) {
99
- if (!err)
100
- return false;
101
- if (err instanceof Error && err.name === "AbortError")
102
- return true;
103
- return false;
104
- }
105
106
  function resolveRequestUrl(input) {
106
- if (typeof input === "string")
107
+ if (typeof input === "string") {
107
108
  return input;
108
- if (input instanceof URL)
109
+ }
110
+ if (input instanceof URL) {
109
111
  return input.toString();
112
+ }
110
113
  return input.url;
111
114
  }
112
115
  function orderAttachments(attachments, prefer) {
113
- if (!prefer || prefer === "first")
116
+ if (!prefer || prefer === "first") {
114
117
  return attachments;
115
- if (prefer === "last")
116
- return [...attachments].reverse();
118
+ }
119
+ if (prefer === "last") {
120
+ return [...attachments].toReversed();
121
+ }
117
122
  if (prefer === "path") {
118
123
  const withPath = attachments.filter((item) => item.path);
119
124
  const withoutPath = attachments.filter((item) => !item.path);
@@ -129,14 +134,17 @@ function orderAttachments(attachments, prefer) {
129
134
  export function selectAttachments(params) {
130
135
  const { capability, attachments, policy } = params;
131
136
  const matches = attachments.filter((item) => {
132
- if (capability === "image")
137
+ if (capability === "image") {
133
138
  return isImageAttachment(item);
134
- if (capability === "audio")
139
+ }
140
+ if (capability === "audio") {
135
141
  return isAudioAttachment(item);
142
+ }
136
143
  return isVideoAttachment(item);
137
144
  });
138
- if (matches.length === 0)
145
+ if (matches.length === 0) {
139
146
  return [];
147
+ }
140
148
  const ordered = orderAttachments(matches, policy?.prefer);
141
149
  const mode = policy?.mode ?? "first";
142
150
  const maxAttachments = policy?.maxAttachments ?? DEFAULT_MAX_ATTACHMENTS;
@@ -291,15 +299,18 @@ export class MediaAttachmentCache {
291
299
  }
292
300
  resolveLocalPath(attachment) {
293
301
  const rawPath = normalizeAttachmentPath(attachment.path);
294
- if (!rawPath)
302
+ if (!rawPath) {
295
303
  return undefined;
304
+ }
296
305
  return path.isAbsolute(rawPath) ? rawPath : path.resolve(rawPath);
297
306
  }
298
307
  async ensureLocalStat(entry) {
299
- if (!entry.resolvedPath)
308
+ if (!entry.resolvedPath) {
300
309
  return undefined;
301
- if (entry.statSize !== undefined)
310
+ }
311
+ if (entry.statSize !== undefined) {
302
312
  return entry.statSize;
313
+ }
303
314
  try {
304
315
  const stat = await fs.stat(entry.resolvedPath);
305
316
  if (!stat.isFile()) {
@@ -26,5 +26,21 @@ export const DEFAULT_AUDIO_MODELS = {
26
26
  openai: "gpt-4o-mini-transcribe",
27
27
  deepgram: "nova-3",
28
28
  };
29
+ export const AUTO_AUDIO_KEY_PROVIDERS = ["openai", "groq", "deepgram", "google"];
30
+ export const AUTO_IMAGE_KEY_PROVIDERS = [
31
+ "openai",
32
+ "anthropic",
33
+ "google",
34
+ "minimax",
35
+ "zai",
36
+ ];
37
+ export const AUTO_VIDEO_KEY_PROVIDERS = ["google"];
38
+ export const DEFAULT_IMAGE_MODELS = {
39
+ openai: "gpt-5-mini",
40
+ anthropic: "claude-opus-4-6",
41
+ google: "gemini-3-flash-preview",
42
+ minimax: "MiniMax-VL-01",
43
+ zai: "glm-4.6v",
44
+ };
29
45
  export const CLI_OUTPUT_MAX_BUFFER = 5 * MB;
30
46
  export const DEFAULT_MEDIA_CONCURRENCY = 2;
@@ -1,4 +1,4 @@
1
- import { fetchWithTimeout, normalizeBaseUrl, readErrorResponse } from "../shared.js";
1
+ import { fetchWithTimeoutGuarded, normalizeBaseUrl, readErrorResponse } from "../shared.js";
2
2
  export const DEFAULT_DEEPGRAM_AUDIO_BASE_URL = "https://api.deepgram.com/v1";
3
3
  export const DEFAULT_DEEPGRAM_AUDIO_MODEL = "nova-3";
4
4
  function resolveModel(model) {
@@ -8,15 +8,18 @@ function resolveModel(model) {
8
8
  export async function transcribeDeepgramAudio(params) {
9
9
  const fetchFn = params.fetchFn ?? fetch;
10
10
  const baseUrl = normalizeBaseUrl(params.baseUrl, DEFAULT_DEEPGRAM_AUDIO_BASE_URL);
11
+ const allowPrivate = Boolean(params.baseUrl?.trim());
11
12
  const model = resolveModel(params.model);
12
13
  const url = new URL(`${baseUrl}/listen`);
13
14
  url.searchParams.set("model", model);
14
- if (params.language?.trim())
15
+ if (params.language?.trim()) {
15
16
  url.searchParams.set("language", params.language.trim());
17
+ }
16
18
  if (params.query) {
17
19
  for (const [key, value] of Object.entries(params.query)) {
18
- if (value === undefined)
20
+ if (value === undefined) {
19
21
  continue;
22
+ }
20
23
  url.searchParams.set(key, String(value));
21
24
  }
22
25
  }
@@ -28,20 +31,25 @@ export async function transcribeDeepgramAudio(params) {
28
31
  headers.set("content-type", params.mime ?? "application/octet-stream");
29
32
  }
30
33
  const body = new Uint8Array(params.buffer);
31
- const res = await fetchWithTimeout(url.toString(), {
34
+ const { response: res, release } = await fetchWithTimeoutGuarded(url.toString(), {
32
35
  method: "POST",
33
36
  headers,
34
37
  body,
35
- }, params.timeoutMs, fetchFn);
36
- if (!res.ok) {
37
- const detail = await readErrorResponse(res);
38
- const suffix = detail ? `: ${detail}` : "";
39
- throw new Error(`Audio transcription failed (HTTP ${res.status})${suffix}`);
38
+ }, params.timeoutMs, fetchFn, allowPrivate ? { ssrfPolicy: { allowPrivateNetwork: true } } : undefined);
39
+ try {
40
+ if (!res.ok) {
41
+ const detail = await readErrorResponse(res);
42
+ const suffix = detail ? `: ${detail}` : "";
43
+ throw new Error(`Audio transcription failed (HTTP ${res.status})${suffix}`);
44
+ }
45
+ const payload = (await res.json());
46
+ const transcript = payload.results?.channels?.[0]?.alternatives?.[0]?.transcript?.trim();
47
+ if (!transcript) {
48
+ throw new Error("Audio transcription response missing transcript");
49
+ }
50
+ return { text: transcript, model };
40
51
  }
41
- const payload = (await res.json());
42
- const transcript = payload.results?.channels?.[0]?.alternatives?.[0]?.transcript?.trim();
43
- if (!transcript) {
44
- throw new Error("Audio transcription response missing transcript");
52
+ finally {
53
+ await release();
45
54
  }
46
- return { text: transcript, model };
47
55
  }