@poolzin/pool-bot 2026.1.39 → 2026.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/chrome-extension/README.md +3 -3
- package/assets/chrome-extension/background.js +5 -5
- package/assets/chrome-extension/manifest.json +3 -3
- package/assets/chrome-extension/options.html +4 -4
- package/assets/chrome-extension/options.js +1 -1
- package/dist/acp/client.js +3 -3
- package/dist/acp/types.js +1 -1
- package/dist/agents/agent-paths.js +3 -3
- package/dist/agents/auth-profiles/paths.js +3 -3
- package/dist/agents/bash-tools.exec.js +76 -25
- package/dist/agents/cli-runner/helpers.js +10 -12
- package/dist/agents/cli-runner.js +2 -2
- package/dist/agents/cloudflare-ai-gateway.js +31 -0
- package/dist/agents/compaction.js +16 -2
- package/dist/agents/context-window-guard.js +13 -10
- package/dist/agents/context.js +4 -4
- package/dist/agents/docs-path.js +1 -1
- package/dist/agents/identity.js +47 -7
- package/dist/agents/memory-search.js +25 -8
- package/dist/agents/minimax-vlm.js +1 -1
- package/dist/agents/model-auth.js +12 -1
- package/dist/agents/model-catalog.js +4 -4
- package/dist/agents/model-selection.js +31 -4
- package/dist/agents/models-config.js +3 -3
- package/dist/agents/models-config.providers.js +147 -39
- package/dist/agents/pi-embedded-block-chunker.js +117 -42
- package/dist/agents/pi-embedded-helpers/errors.js +183 -78
- package/dist/agents/pi-embedded-helpers/openai.js +1 -1
- package/dist/agents/pi-embedded-helpers.js +1 -1
- package/dist/agents/pi-embedded-runner/compact.js +9 -8
- package/dist/agents/pi-embedded-runner/model.js +63 -4
- package/dist/agents/pi-embedded-runner/run/attempt.js +27 -17
- package/dist/agents/pi-embedded-runner/run.js +203 -50
- package/dist/agents/pi-embedded-runner/system-prompt.js +10 -2
- package/dist/agents/pi-embedded-runner/tool-result-truncation.js +275 -0
- package/dist/agents/pi-embedded-runner/utils.js +1 -1
- package/dist/agents/pi-embedded-subscribe.js +118 -29
- package/dist/agents/pi-model-discovery.js +10 -0
- package/dist/agents/pi-tool-definition-adapter.js +50 -9
- package/dist/agents/pi-tools.before-tool-call.js +67 -0
- package/dist/agents/pi-tools.js +20 -10
- package/dist/agents/pi-tools.read.js +2 -2
- package/dist/agents/poolbot-tools.js +15 -10
- package/dist/agents/sandbox-paths.js +31 -0
- package/dist/agents/session-file-repair.js +83 -0
- package/dist/agents/session-tool-result-guard.js +94 -15
- package/dist/agents/session-transcript-repair.js +68 -0
- package/dist/agents/shell-utils.js +51 -0
- package/dist/agents/skills/bundled-context.js +23 -0
- package/dist/agents/skills/bundled-dir.js +41 -7
- package/dist/agents/skills/frontmatter.js +1 -1
- package/dist/agents/skills/workspace.js +2 -2
- package/dist/agents/skills-install.js +60 -23
- package/dist/agents/subagent-announce.js +79 -34
- package/dist/agents/system-prompt.js +28 -4
- package/dist/agents/together-models.js +127 -0
- package/dist/agents/tool-images.js +1 -1
- package/dist/agents/tool-policy.conformance.js +14 -0
- package/dist/agents/tool-policy.js +25 -1
- package/dist/agents/tools/browser-tool.js +3 -3
- package/dist/agents/tools/cron-tool.js +166 -19
- package/dist/agents/tools/discord-actions-presence.js +78 -0
- package/dist/agents/tools/image-tool.js +2 -2
- package/dist/agents/tools/memory-tool.js +93 -5
- package/dist/agents/tools/message-tool.js +56 -2
- package/dist/agents/tools/sessions-history-tool.js +69 -1
- package/dist/agents/tools/web-search.js +211 -42
- package/dist/agents/usage.js +23 -1
- package/dist/agents/workspace-run.js +67 -0
- package/dist/agents/workspace-templates.js +44 -0
- package/dist/auto-reply/command-auth.js +121 -6
- package/dist/auto-reply/commands-registry.data.js +1 -1
- package/dist/auto-reply/envelope.js +50 -72
- package/dist/auto-reply/reply/commands-compact.js +1 -0
- package/dist/auto-reply/reply/commands-context-report.js +3 -2
- package/dist/auto-reply/reply/commands-context.js +1 -0
- package/dist/auto-reply/reply/commands-models.js +107 -60
- package/dist/auto-reply/reply/commands-ptt.js +171 -0
- package/dist/auto-reply/reply/commands-session.js +2 -2
- package/dist/auto-reply/reply/get-reply-run.js +16 -5
- package/dist/auto-reply/reply/groups.js +1 -1
- package/dist/auto-reply/reply/inbound-context.js +9 -1
- package/dist/auto-reply/reply/inbound-meta.js +130 -0
- package/dist/auto-reply/reply/model-selection.js +3 -3
- package/dist/auto-reply/reply/untrusted-context.js +15 -0
- package/dist/auto-reply/status.js +1 -1
- package/dist/auto-reply/thinking.js +88 -43
- package/dist/browser/bridge-server.js +13 -0
- package/dist/browser/cdp.helpers.js +38 -24
- package/dist/browser/client-fetch.js +51 -8
- package/dist/browser/config.js +2 -11
- package/dist/browser/extension-relay.js +104 -43
- package/dist/browser/pw-ai.js +1 -1
- package/dist/browser/pw-session.js +143 -8
- package/dist/browser/pw-tools-core.interactions.js +125 -27
- package/dist/browser/pw-tools-core.responses.js +1 -1
- package/dist/browser/pw-tools-core.state.js +1 -1
- package/dist/browser/routes/agent.act.js +86 -41
- package/dist/browser/routes/dispatcher.js +4 -4
- package/dist/browser/screenshot.js +1 -1
- package/dist/browser/server-context.js +2 -2
- package/dist/browser/server.js +13 -0
- package/dist/build-info.json +3 -3
- package/dist/canvas-host/a2ui.js +3 -3
- package/dist/channels/plugins/catalog.js +2 -2
- package/dist/channels/plugins/onboarding/imessage.js +1 -1
- package/dist/channels/plugins/onboarding/signal.js +1 -1
- package/dist/channels/plugins/onboarding/slack.js +4 -4
- package/dist/channels/plugins/onboarding/whatsapp.js +3 -3
- package/dist/channels/plugins/pairing-message.js +1 -1
- package/dist/channels/reply-prefix.js +8 -1
- package/dist/cli/browser-cli-extension.js +2 -2
- package/dist/cli/cron-cli/register.cron-add.js +61 -40
- package/dist/cli/cron-cli/register.cron-edit.js +60 -34
- package/dist/cli/cron-cli/shared.js +56 -41
- package/dist/cli/dns-cli.js +26 -14
- package/dist/cli/docs-cli.js +1 -1
- package/dist/cli/gateway-cli/dev.js +1 -1
- package/dist/cli/gateway-cli/register.js +37 -19
- package/dist/cli/memory-cli.js +30 -20
- package/dist/cli/nodes-cli/register.canvas.js +1 -1
- package/dist/cli/parse-bytes.js +37 -0
- package/dist/cli/plugins-cli.js +1 -1
- package/dist/cli/run-main.js +2 -2
- package/dist/cli/security-cli.js +1 -1
- package/dist/cli/tagline.js +1 -1
- package/dist/cli/update-cli.js +173 -52
- package/dist/cli/webhooks-cli.js +5 -5
- package/dist/commands/agent.js +1 -0
- package/dist/commands/agents.commands.add.js +1 -1
- package/dist/commands/auth-choice.apply.api-providers.js +305 -17
- package/dist/commands/auth-choice.apply.js +4 -1
- package/dist/commands/auth-choice.apply.plugin-provider.js +2 -2
- package/dist/commands/auth-choice.apply.xai.js +63 -0
- package/dist/commands/auth-choice.preferred-provider.js +7 -1
- package/dist/commands/configure.wizard.js +1 -1
- package/dist/commands/dashboard.js +1 -1
- package/dist/commands/docs.js +1 -1
- package/dist/commands/doctor-config-flow.js +61 -5
- package/dist/commands/doctor-gateway-services.js +3 -3
- package/dist/commands/doctor-state-migrations.js +1 -1
- package/dist/commands/doctor-update.js +3 -3
- package/dist/commands/doctor.js +1 -1
- package/dist/commands/health.js +1 -1
- package/dist/commands/model-allowlist.js +29 -0
- package/dist/commands/model-picker.js +2 -1
- package/dist/commands/models/list.probe.js +2 -2
- package/dist/commands/models/list.registry.js +4 -4
- package/dist/commands/models/list.status-command.js +44 -24
- package/dist/commands/models/shared.js +15 -0
- package/dist/commands/onboard-auth.config-core.js +366 -28
- package/dist/commands/onboard-auth.credentials.js +71 -9
- package/dist/commands/onboard-auth.js +3 -3
- package/dist/commands/onboard-auth.models.js +26 -24
- package/dist/commands/onboard-custom.js +384 -0
- package/dist/commands/onboard-non-interactive/local/auth-choice-inference.js +35 -0
- package/dist/commands/onboard-non-interactive/local/auth-choice.js +146 -9
- package/dist/commands/onboard-skills.js +63 -38
- package/dist/commands/openai-model-default.js +41 -0
- package/dist/commands/status-all/report-lines.js +1 -1
- package/dist/commands/status.command.js +1 -1
- package/dist/commands/uninstall.js +3 -3
- package/dist/compat/legacy-names.js +1 -1
- package/dist/config/defaults.js +3 -2
- package/dist/config/io.js +3 -3
- package/dist/config/paths.js +136 -35
- package/dist/config/plugin-auto-enable.js +21 -5
- package/dist/config/redact-snapshot.js +153 -0
- package/dist/config/schema.field-metadata.js +590 -0
- package/dist/config/schema.js +3 -3
- package/dist/config/sessions/store.js +291 -23
- package/dist/config/types.memory.js +1 -0
- package/dist/config/version.js +4 -4
- package/dist/config/zod-schema.agent-defaults.js +3 -0
- package/dist/config/zod-schema.agent-runtime.js +13 -2
- package/dist/config/zod-schema.providers-core.js +142 -0
- package/dist/config/zod-schema.session.js +3 -0
- package/dist/cron/delivery.js +57 -0
- package/dist/cron/isolated-agent/delivery-target.js +18 -3
- package/dist/cron/isolated-agent/helpers.js +22 -5
- package/dist/cron/isolated-agent/run.js +171 -63
- package/dist/cron/isolated-agent/session.js +2 -0
- package/dist/cron/normalize.js +356 -28
- package/dist/cron/parse.js +10 -5
- package/dist/cron/run-log.js +35 -10
- package/dist/cron/schedule.js +41 -6
- package/dist/cron/service/jobs.js +208 -35
- package/dist/cron/service/ops.js +72 -16
- package/dist/cron/service/state.js +2 -0
- package/dist/cron/service/store.js +386 -14
- package/dist/cron/service/timer.js +390 -147
- package/dist/cron/session-reaper.js +86 -0
- package/dist/cron/store.js +23 -8
- package/dist/cron/validate-timestamp.js +43 -0
- package/dist/daemon/constants.js +7 -7
- package/dist/daemon/inspect.js +6 -6
- package/dist/daemon/systemd-unit.js +1 -1
- package/dist/discord/monitor/agent-components.js +438 -0
- package/dist/discord/monitor/allow-list.js +28 -5
- package/dist/discord/monitor/gateway-registry.js +29 -0
- package/dist/discord/monitor/native-command.js +44 -23
- package/dist/discord/monitor/sender-identity.js +45 -0
- package/dist/discord/pluralkit.js +27 -0
- package/dist/discord/send.outbound.js +92 -5
- package/dist/discord/send.shared.js +60 -23
- package/dist/discord/targets.js +84 -1
- package/dist/entry.js +15 -9
- package/dist/extensionAPI.js +8 -0
- package/dist/gateway/control-ui.js +8 -1
- package/dist/gateway/hooks-mapping.js +3 -0
- package/dist/gateway/hooks.js +65 -0
- package/dist/gateway/live-image-probe.js +1 -66
- package/dist/gateway/net.js +96 -31
- package/dist/gateway/node-command-policy.js +50 -15
- package/dist/gateway/openai-http.js +2 -2
- package/dist/gateway/openresponses-http.js +4 -4
- package/dist/gateway/origin-check.js +56 -0
- package/dist/gateway/protocol/client-info.js +9 -0
- package/dist/gateway/protocol/index.js +9 -2
- package/dist/gateway/protocol/schema/agents-models-skills.js +71 -1
- package/dist/gateway/protocol/schema/cron.js +22 -10
- package/dist/gateway/protocol/schema/protocol-schemas.js +16 -2
- package/dist/gateway/protocol/schema/sessions.js +12 -0
- package/dist/gateway/server/hooks.js +1 -1
- package/dist/gateway/server-broadcast.js +26 -9
- package/dist/gateway/server-chat.js +112 -23
- package/dist/gateway/server-discovery-runtime.js +10 -2
- package/dist/gateway/server-discovery.js +2 -2
- package/dist/gateway/server-http.js +110 -12
- package/dist/gateway/server-methods/agent-timestamp.js +60 -0
- package/dist/gateway/server-methods/agents.js +321 -2
- package/dist/gateway/server-methods/usage.js +559 -16
- package/dist/gateway/server-runtime-state.js +22 -8
- package/dist/gateway/server-startup-memory.js +16 -0
- package/dist/gateway/server.impl.js +7 -3
- package/dist/gateway/session-utils.fs.js +23 -25
- package/dist/gateway/session-utils.js +20 -10
- package/dist/gateway/sessions-patch.js +7 -22
- package/dist/gateway/test-helpers.server.js +35 -2
- package/dist/hooks/frontmatter.js +1 -1
- package/dist/hooks/hooks-status.js +1 -1
- package/dist/hooks/install.js +2 -2
- package/dist/hooks/loader.js +1 -1
- package/dist/hooks/workspace.js +3 -3
- package/dist/imessage/constants.js +2 -0
- package/dist/imessage/monitor/deliver.js +4 -1
- package/dist/imessage/monitor/monitor-provider.js +51 -1
- package/dist/index.js +2 -2
- package/dist/infra/bonjour-discovery.js +131 -70
- package/dist/infra/bonjour.js +3 -3
- package/dist/infra/control-ui-assets.js +134 -12
- package/dist/infra/errors.js +12 -0
- package/dist/infra/exec-approvals.js +266 -57
- package/dist/infra/format-time/format-datetime.js +79 -0
- package/dist/infra/format-time/format-duration.js +81 -0
- package/dist/infra/format-time/format-relative.js +80 -0
- package/dist/infra/heartbeat-runner.js +140 -49
- package/dist/infra/home-dir.js +54 -0
- package/dist/infra/net/fetch-guard.js +122 -0
- package/dist/infra/net/ssrf.js +65 -29
- package/dist/infra/outbound/abort.js +14 -0
- package/dist/infra/outbound/message-action-runner.js +77 -13
- package/dist/infra/outbound/outbound-session.js +143 -37
- package/dist/infra/path-env.js +3 -3
- package/dist/infra/poolbot-root.js +43 -1
- package/dist/infra/provider-usage.fetch.minimax.js +1 -1
- package/dist/infra/restart.js +1 -1
- package/dist/infra/session-cost-usage.js +631 -41
- package/dist/infra/state-migrations.js +317 -47
- package/dist/infra/tailscale.js +1 -1
- package/dist/infra/update-global.js +35 -0
- package/dist/infra/update-runner.js +149 -43
- package/dist/infra/warning-filter.js +65 -0
- package/dist/infra/widearea-dns.js +30 -9
- package/dist/logging/redact-identifier.js +12 -0
- package/dist/macos/relay.js +2 -2
- package/dist/media/fetch.js +81 -58
- package/dist/media/input-files.js +1 -1
- package/dist/media/mime.js +4 -0
- package/dist/media/png-encode.js +74 -0
- package/dist/media-understanding/apply.js +403 -3
- package/dist/media-understanding/attachments.js +38 -27
- package/dist/media-understanding/defaults.js +16 -0
- package/dist/media-understanding/providers/deepgram/audio.js +22 -14
- package/dist/media-understanding/providers/google/audio.js +24 -17
- package/dist/media-understanding/providers/google/video.js +24 -17
- package/dist/media-understanding/providers/image.js +4 -4
- package/dist/media-understanding/providers/index.js +4 -1
- package/dist/media-understanding/providers/openai/audio.js +22 -14
- package/dist/media-understanding/providers/shared.js +16 -11
- package/dist/media-understanding/providers/zai/index.js +6 -0
- package/dist/media-understanding/runner.js +158 -90
- package/dist/memory/backend-config.js +207 -0
- package/dist/memory/batch-voyage.js +277 -0
- package/dist/memory/embeddings-voyage.js +75 -0
- package/dist/memory/embeddings.js +29 -17
- package/dist/memory/internal.js +101 -18
- package/dist/memory/manager.js +155 -48
- package/dist/memory/search-manager.js +173 -0
- package/dist/memory/session-files.js +9 -3
- package/dist/memory/types.js +1 -0
- package/dist/node-host/runner.js +36 -26
- package/dist/node-host/with-timeout.js +27 -0
- package/dist/pairing/pairing-messages.js +1 -1
- package/dist/plugins/commands.js +5 -1
- package/dist/plugins/config-state.js +86 -7
- package/dist/plugins/discovery.js +1 -1
- package/dist/plugins/install.js +2 -2
- package/dist/plugins/source-display.js +51 -0
- package/dist/plugins/update.js +1 -1
- package/dist/process/exec.js +20 -2
- package/dist/routing/resolve-route.js +12 -0
- package/dist/routing/session-key.js +15 -0
- package/dist/runtime.js +2 -0
- package/dist/security/audit-extra.async.js +601 -0
- package/dist/security/audit-extra.js +2 -830
- package/dist/security/audit-extra.sync.js +505 -0
- package/dist/security/audit.js +2 -2
- package/dist/security/channel-metadata.js +34 -0
- package/dist/security/external-content.js +88 -6
- package/dist/security/skill-scanner.js +330 -0
- package/dist/sessions/session-key-utils.js +7 -0
- package/dist/shared/text/reasoning-tags.js +52 -7
- package/dist/signal/monitor/event-handler.js +80 -1
- package/dist/slack/monitor/media.js +85 -15
- package/dist/tailscale/detect.js +145 -0
- package/dist/telegram/bot/helpers.js +109 -28
- package/dist/telegram/bot-handlers.js +144 -3
- package/dist/telegram/bot-message-context.js +38 -11
- package/dist/telegram/bot-message-dispatch.js +48 -15
- package/dist/telegram/bot-native-commands.js +86 -29
- package/dist/telegram/bot.js +30 -29
- package/dist/telegram/model-buttons.js +163 -0
- package/dist/telegram/monitor.js +110 -85
- package/dist/telegram/send.js +129 -47
- package/dist/terminal/restore.js +45 -0
- package/dist/test-helpers/state-dir-env.js +16 -0
- package/dist/test-helpers/workspace.js +11 -0
- package/dist/test-utils/channel-plugins.js +82 -0
- package/dist/test-utils/ports.js +73 -0
- package/dist/tts/tts.js +12 -6
- package/dist/tui/tui-session-actions.js +166 -54
- package/dist/utils/fetch-timeout.js +20 -0
- package/dist/utils/normalize-secret-input.js +19 -0
- package/dist/utils/shell-argv.js +61 -0
- package/dist/utils/transcript-tools.js +58 -0
- package/dist/utils.js +55 -14
- package/dist/version.js +42 -5
- package/dist/web/qr-image.js +1 -61
- package/dist/wizard/onboarding.finalize.js +7 -7
- package/dist/wizard/onboarding.js +3 -3
- package/docs/RELEASE_WORKFOTS_COMPARISON.md +3 -3
- package/docs/_config.yml +2 -2
- package/docs/_layouts/default.html +9 -9
- package/docs/concepts/typebox.md +1 -1
- package/docs/docs.json +1 -1
- package/docs/northflank.mdx +7 -7
- package/docs/railway.mdx +3 -3
- package/docs/render.mdx +5 -5
- package/docs/start/lore.md +2 -2
- package/extensions/bluebubbles/index.ts +2 -2
- package/extensions/bluebubbles/package.json +1 -1
- package/extensions/bluebubbles/src/accounts.ts +8 -8
- package/extensions/bluebubbles/src/actions.test.ts +22 -22
- package/extensions/bluebubbles/src/actions.ts +5 -5
- package/extensions/bluebubbles/src/attachments.ts +2 -2
- package/extensions/bluebubbles/src/channel.ts +16 -16
- package/extensions/bluebubbles/src/chat.ts +2 -2
- package/extensions/bluebubbles/src/media-send.ts +2 -2
- package/extensions/bluebubbles/src/monitor.test.ts +46 -46
- package/extensions/bluebubbles/src/monitor.ts +5 -5
- package/extensions/bluebubbles/src/onboarding.ts +7 -7
- package/extensions/bluebubbles/src/reactions.ts +2 -2
- package/extensions/bluebubbles/src/send.ts +2 -2
- package/extensions/copilot-proxy/README.md +1 -1
- package/extensions/copilot-proxy/package.json +1 -1
- package/extensions/diagnostics-otel/index.ts +2 -2
- package/extensions/diagnostics-otel/package.json +1 -1
- package/extensions/diagnostics-otel/src/service.ts +3 -3
- package/extensions/discord/index.ts +2 -2
- package/extensions/discord/package.json +1 -1
- package/extensions/google-antigravity-auth/README.md +1 -1
- package/extensions/google-antigravity-auth/index.ts +1 -1
- package/extensions/google-antigravity-auth/package.json +1 -1
- package/extensions/google-gemini-cli-auth/README.md +1 -1
- package/extensions/google-gemini-cli-auth/oauth.ts +1 -1
- package/extensions/google-gemini-cli-auth/package.json +1 -1
- package/extensions/googlechat/index.ts +3 -3
- package/extensions/googlechat/package.json +1 -1
- package/extensions/googlechat/src/accounts.ts +8 -8
- package/extensions/googlechat/src/actions.ts +6 -6
- package/extensions/googlechat/src/channel.ts +21 -21
- package/extensions/googlechat/src/monitor.ts +8 -8
- package/extensions/googlechat/src/onboarding.ts +10 -10
- package/extensions/imessage/index.ts +2 -2
- package/extensions/imessage/package.json +1 -1
- package/extensions/line/index.ts +2 -2
- package/extensions/line/package.json +1 -1
- package/extensions/line/src/card-command.ts +2 -2
- package/extensions/line/src/channel.logout.test.ts +4 -4
- package/extensions/line/src/channel.sendPayload.test.ts +8 -8
- package/extensions/line/src/channel.ts +3 -3
- package/extensions/llm-task/README.md +3 -3
- package/extensions/llm-task/index.ts +2 -2
- package/extensions/llm-task/package.json +1 -1
- package/extensions/llm-task/src/llm-task-tool.ts +4 -4
- package/extensions/lobster/README.md +6 -6
- package/extensions/lobster/index.ts +2 -2
- package/extensions/lobster/src/lobster-tool.test.ts +4 -4
- package/extensions/lobster/src/lobster-tool.ts +2 -2
- package/extensions/matrix/index.ts +2 -2
- package/extensions/matrix/package.json +1 -1
- package/extensions/matrix/src/matrix/client/config.ts +1 -1
- package/extensions/matrix/src/matrix/monitor/handler.ts +1 -1
- package/extensions/matrix/src/onboarding.ts +1 -1
- package/extensions/mattermost/index.ts +2 -2
- package/extensions/mattermost/package.json +1 -1
- package/extensions/mattermost/src/mattermost/accounts.ts +8 -8
- package/extensions/mattermost/src/mattermost/monitor-helpers.ts +5 -5
- package/extensions/mattermost/src/mattermost/monitor.ts +2 -2
- package/extensions/mattermost/src/onboarding-helpers.ts +3 -3
- package/extensions/mattermost/src/onboarding.ts +2 -2
- package/extensions/memory-core/index.ts +2 -2
- package/extensions/memory-core/package.json +1 -1
- package/extensions/memory-lancedb/index.ts +3 -3
- package/extensions/memory-lancedb/package.json +1 -1
- package/extensions/msteams/index.ts +2 -2
- package/extensions/msteams/package.json +1 -1
- package/extensions/msteams/src/channel.directory.test.ts +2 -2
- package/extensions/msteams/src/channel.ts +2 -2
- package/extensions/msteams/src/graph-upload.ts +4 -4
- package/extensions/msteams/src/monitor-handler.ts +2 -2
- package/extensions/msteams/src/monitor.ts +2 -2
- package/extensions/msteams/src/onboarding.ts +9 -9
- package/extensions/msteams/src/reply-dispatcher.ts +2 -2
- package/extensions/msteams/src/send-context.ts +2 -2
- package/extensions/msteams/src/send.ts +4 -4
- package/extensions/nextcloud-talk/index.ts +2 -2
- package/extensions/nextcloud-talk/package.json +1 -1
- package/extensions/nextcloud-talk/src/channel.ts +7 -7
- package/extensions/nextcloud-talk/src/inbound.ts +7 -7
- package/extensions/nextcloud-talk/src/onboarding.ts +1 -1
- package/extensions/nostr/README.md +2 -2
- package/extensions/nostr/index.ts +5 -5
- package/extensions/nostr/package.json +1 -1
- package/extensions/nostr/src/types.ts +4 -4
- package/extensions/open-prose/index.ts +2 -2
- package/extensions/qwen-portal-auth/README.md +1 -1
- package/extensions/signal/index.ts +2 -2
- package/extensions/signal/package.json +1 -1
- package/extensions/slack/index.ts +2 -2
- package/extensions/slack/package.json +1 -1
- package/extensions/telegram/index.ts +2 -2
- package/extensions/telegram/package.json +1 -1
- package/extensions/telegram/src/channel.ts +2 -2
- package/extensions/tlon/README.md +2 -2
- package/extensions/tlon/index.ts +2 -2
- package/extensions/tlon/package.json +1 -1
- package/extensions/tlon/src/channel.ts +13 -13
- package/extensions/tlon/src/monitor/index.ts +3 -3
- package/extensions/tlon/src/onboarding.ts +3 -3
- package/extensions/tlon/src/types.ts +3 -3
- package/extensions/twitch/README.md +1 -1
- package/extensions/twitch/index.ts +2 -2
- package/extensions/twitch/package.json +1 -1
- package/extensions/twitch/src/config.ts +3 -3
- package/extensions/twitch/src/monitor.ts +3 -3
- package/extensions/twitch/src/onboarding.ts +9 -9
- package/extensions/twitch/src/outbound.test.ts +2 -2
- package/extensions/twitch/src/plugin.test.ts +2 -2
- package/extensions/twitch/src/plugin.ts +8 -8
- package/extensions/twitch/src/send.test.ts +2 -2
- package/extensions/twitch/src/send.ts +4 -4
- package/extensions/twitch/src/token.test.ts +8 -8
- package/extensions/twitch/src/token.ts +3 -3
- package/extensions/twitch/src/twitch-client.ts +3 -3
- package/extensions/twitch/src/types.ts +3 -3
- package/extensions/twitch/src/utils/markdown.ts +1 -1
- package/extensions/voice-call/README.md +3 -3
- package/extensions/voice-call/package.json +1 -1
- package/extensions/voice-call/src/core-bridge.ts +2 -2
- package/extensions/voice-call/src/response-generator.ts +1 -1
- package/extensions/whatsapp/index.ts +2 -2
- package/extensions/whatsapp/package.json +1 -1
- package/extensions/zalo/README.md +1 -1
- package/extensions/zalo/index.ts +2 -2
- package/extensions/zalo/package.json +1 -1
- package/extensions/zalo/src/accounts.ts +8 -8
- package/extensions/zalo/src/actions.ts +4 -4
- package/extensions/zalo/src/channel.directory.test.ts +2 -2
- package/extensions/zalo/src/channel.ts +18 -18
- package/extensions/zalo/src/monitor.ts +9 -9
- package/extensions/zalo/src/monitor.webhook.test.ts +2 -2
- package/extensions/zalo/src/onboarding.ts +24 -24
- package/extensions/zalo/src/send.ts +2 -2
- package/extensions/zalouser/README.md +2 -2
- package/extensions/zalouser/index.ts +2 -2
- package/extensions/zalouser/package.json +1 -1
- package/extensions/zalouser/src/accounts.ts +9 -9
- package/extensions/zalouser/src/channel.ts +24 -24
- package/extensions/zalouser/src/monitor.ts +4 -4
- package/extensions/zalouser/src/onboarding.ts +28 -28
- package/package.json +13 -251
- package/skills/nano-banana-pro/scripts/generate_image.py +1 -1
- package/skills/tmux/scripts/find-sessions.sh +1 -1
- package/CHANGELOG.md +0 -102
- package/README-header.png +0 -0
- package/git-hooks/pre-commit +0 -4
- package/scripts/format-staged.js +0 -148
- package/scripts/postinstall.js +0 -300
- package/scripts/setup-git-hooks.js +0 -96
|
@@ -1,9 +1,390 @@
|
|
|
1
|
+
import path from "node:path";
|
|
1
2
|
import { finalizeInboundContext } from "../auto-reply/reply/inbound-context.js";
|
|
2
|
-
import {
|
|
3
|
+
import { logVerbose, shouldLogVerbose } from "../globals.js";
|
|
4
|
+
import { DEFAULT_INPUT_FILE_MAX_BYTES, DEFAULT_INPUT_FILE_MAX_CHARS, DEFAULT_INPUT_FILE_MIMES, DEFAULT_INPUT_MAX_REDIRECTS, DEFAULT_INPUT_PDF_MAX_PAGES, DEFAULT_INPUT_PDF_MAX_PIXELS, DEFAULT_INPUT_PDF_MIN_TEXT_CHARS, DEFAULT_INPUT_TIMEOUT_MS, extractFileContentFromSource, normalizeMimeList, normalizeMimeType, } from "../media/input-files.js";
|
|
5
|
+
import { resolveAttachmentKind } from "./attachments.js";
|
|
3
6
|
import { runWithConcurrency } from "./concurrency.js";
|
|
7
|
+
import { extractMediaUserText, formatAudioTranscripts, formatMediaUnderstandingBody, } from "./format.js";
|
|
4
8
|
import { resolveConcurrency } from "./resolve.js";
|
|
5
9
|
import { buildProviderRegistry, createMediaAttachmentCache, normalizeMediaAttachments, runCapability, } from "./runner.js";
|
|
6
10
|
const CAPABILITY_ORDER = ["image", "audio", "video"];
|
|
11
|
+
const EXTRA_TEXT_MIMES = [
|
|
12
|
+
"application/xml",
|
|
13
|
+
"text/xml",
|
|
14
|
+
"application/x-yaml",
|
|
15
|
+
"text/yaml",
|
|
16
|
+
"application/yaml",
|
|
17
|
+
"application/javascript",
|
|
18
|
+
"text/javascript",
|
|
19
|
+
"text/tab-separated-values",
|
|
20
|
+
];
|
|
21
|
+
const TEXT_EXT_MIME = new Map([
|
|
22
|
+
[".csv", "text/csv"],
|
|
23
|
+
[".tsv", "text/tab-separated-values"],
|
|
24
|
+
[".txt", "text/plain"],
|
|
25
|
+
[".md", "text/markdown"],
|
|
26
|
+
[".log", "text/plain"],
|
|
27
|
+
[".ini", "text/plain"],
|
|
28
|
+
[".cfg", "text/plain"],
|
|
29
|
+
[".conf", "text/plain"],
|
|
30
|
+
[".env", "text/plain"],
|
|
31
|
+
[".json", "application/json"],
|
|
32
|
+
[".yaml", "text/yaml"],
|
|
33
|
+
[".yml", "text/yaml"],
|
|
34
|
+
[".xml", "application/xml"],
|
|
35
|
+
]);
|
|
36
|
+
const XML_ESCAPE_MAP = {
|
|
37
|
+
"<": "<",
|
|
38
|
+
">": ">",
|
|
39
|
+
"&": "&",
|
|
40
|
+
'"': """,
|
|
41
|
+
"'": "'",
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* Escapes special XML characters in attribute values to prevent injection.
|
|
45
|
+
*/
|
|
46
|
+
function xmlEscapeAttr(value) {
|
|
47
|
+
return value.replace(/[<>&"']/g, (char) => XML_ESCAPE_MAP[char] ?? char);
|
|
48
|
+
}
|
|
49
|
+
function escapeFileBlockContent(value) {
|
|
50
|
+
return value.replace(/<\s*\/\s*file\s*>/gi, "</file>").replace(/<\s*file\b/gi, "<file");
|
|
51
|
+
}
|
|
52
|
+
function sanitizeMimeType(value) {
|
|
53
|
+
if (!value) {
|
|
54
|
+
return undefined;
|
|
55
|
+
}
|
|
56
|
+
const trimmed = value.trim().toLowerCase();
|
|
57
|
+
if (!trimmed) {
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
const match = trimmed.match(/^([a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+)/);
|
|
61
|
+
return match?.[1];
|
|
62
|
+
}
|
|
63
|
+
function resolveFileLimits(cfg) {
|
|
64
|
+
const files = cfg.gateway?.http?.endpoints?.responses?.files;
|
|
65
|
+
const allowedMimesConfigured = Boolean(files?.allowedMimes && files.allowedMimes.length > 0);
|
|
66
|
+
return {
|
|
67
|
+
allowUrl: files?.allowUrl ?? true,
|
|
68
|
+
allowedMimes: normalizeMimeList(files?.allowedMimes, DEFAULT_INPUT_FILE_MIMES),
|
|
69
|
+
allowedMimesConfigured,
|
|
70
|
+
maxBytes: files?.maxBytes ?? DEFAULT_INPUT_FILE_MAX_BYTES,
|
|
71
|
+
maxChars: files?.maxChars ?? DEFAULT_INPUT_FILE_MAX_CHARS,
|
|
72
|
+
maxRedirects: files?.maxRedirects ?? DEFAULT_INPUT_MAX_REDIRECTS,
|
|
73
|
+
timeoutMs: files?.timeoutMs ?? DEFAULT_INPUT_TIMEOUT_MS,
|
|
74
|
+
pdf: {
|
|
75
|
+
maxPages: files?.pdf?.maxPages ?? DEFAULT_INPUT_PDF_MAX_PAGES,
|
|
76
|
+
maxPixels: files?.pdf?.maxPixels ?? DEFAULT_INPUT_PDF_MAX_PIXELS,
|
|
77
|
+
minTextChars: files?.pdf?.minTextChars ?? DEFAULT_INPUT_PDF_MIN_TEXT_CHARS,
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
function appendFileBlocks(body, blocks) {
|
|
82
|
+
if (!blocks || blocks.length === 0) {
|
|
83
|
+
return body ?? "";
|
|
84
|
+
}
|
|
85
|
+
const base = typeof body === "string" ? body.trim() : "";
|
|
86
|
+
const suffix = blocks.join("\n\n").trim();
|
|
87
|
+
if (!base) {
|
|
88
|
+
return suffix;
|
|
89
|
+
}
|
|
90
|
+
return `${base}\n\n${suffix}`.trim();
|
|
91
|
+
}
|
|
92
|
+
function resolveUtf16Charset(buffer) {
|
|
93
|
+
if (!buffer || buffer.length < 2) {
|
|
94
|
+
return undefined;
|
|
95
|
+
}
|
|
96
|
+
const b0 = buffer[0];
|
|
97
|
+
const b1 = buffer[1];
|
|
98
|
+
if (b0 === 0xff && b1 === 0xfe) {
|
|
99
|
+
return "utf-16le";
|
|
100
|
+
}
|
|
101
|
+
if (b0 === 0xfe && b1 === 0xff) {
|
|
102
|
+
return "utf-16be";
|
|
103
|
+
}
|
|
104
|
+
const sampleLen = Math.min(buffer.length, 2048);
|
|
105
|
+
let zeroEven = 0;
|
|
106
|
+
let zeroOdd = 0;
|
|
107
|
+
for (let i = 0; i < sampleLen; i += 1) {
|
|
108
|
+
if (buffer[i] !== 0) {
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
if (i % 2 === 0) {
|
|
112
|
+
zeroEven += 1;
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
zeroOdd += 1;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
const zeroCount = zeroEven + zeroOdd;
|
|
119
|
+
if (zeroCount / sampleLen > 0.2) {
|
|
120
|
+
return zeroOdd >= zeroEven ? "utf-16le" : "utf-16be";
|
|
121
|
+
}
|
|
122
|
+
return undefined;
|
|
123
|
+
}
|
|
124
|
+
const WORDISH_CHAR = /[\p{L}\p{N}]/u;
|
|
125
|
+
const CP1252_MAP = [
|
|
126
|
+
"\u20ac",
|
|
127
|
+
undefined,
|
|
128
|
+
"\u201a",
|
|
129
|
+
"\u0192",
|
|
130
|
+
"\u201e",
|
|
131
|
+
"\u2026",
|
|
132
|
+
"\u2020",
|
|
133
|
+
"\u2021",
|
|
134
|
+
"\u02c6",
|
|
135
|
+
"\u2030",
|
|
136
|
+
"\u0160",
|
|
137
|
+
"\u2039",
|
|
138
|
+
"\u0152",
|
|
139
|
+
undefined,
|
|
140
|
+
"\u017d",
|
|
141
|
+
undefined,
|
|
142
|
+
undefined,
|
|
143
|
+
"\u2018",
|
|
144
|
+
"\u2019",
|
|
145
|
+
"\u201c",
|
|
146
|
+
"\u201d",
|
|
147
|
+
"\u2022",
|
|
148
|
+
"\u2013",
|
|
149
|
+
"\u2014",
|
|
150
|
+
"\u02dc",
|
|
151
|
+
"\u2122",
|
|
152
|
+
"\u0161",
|
|
153
|
+
"\u203a",
|
|
154
|
+
"\u0153",
|
|
155
|
+
undefined,
|
|
156
|
+
"\u017e",
|
|
157
|
+
"\u0178",
|
|
158
|
+
];
|
|
159
|
+
function decodeLegacyText(buffer) {
|
|
160
|
+
let output = "";
|
|
161
|
+
for (const byte of buffer) {
|
|
162
|
+
if (byte >= 0x80 && byte <= 0x9f) {
|
|
163
|
+
const mapped = CP1252_MAP[byte - 0x80];
|
|
164
|
+
output += mapped ?? String.fromCharCode(byte);
|
|
165
|
+
continue;
|
|
166
|
+
}
|
|
167
|
+
output += String.fromCharCode(byte);
|
|
168
|
+
}
|
|
169
|
+
return output;
|
|
170
|
+
}
|
|
171
|
+
function getTextStats(text) {
|
|
172
|
+
if (!text) {
|
|
173
|
+
return { printableRatio: 0, wordishRatio: 0 };
|
|
174
|
+
}
|
|
175
|
+
let printable = 0;
|
|
176
|
+
let control = 0;
|
|
177
|
+
let wordish = 0;
|
|
178
|
+
for (const char of text) {
|
|
179
|
+
const code = char.codePointAt(0) ?? 0;
|
|
180
|
+
if (code === 9 || code === 10 || code === 13 || code === 32) {
|
|
181
|
+
printable += 1;
|
|
182
|
+
wordish += 1;
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
if (code < 32 || (code >= 0x7f && code <= 0x9f)) {
|
|
186
|
+
control += 1;
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
printable += 1;
|
|
190
|
+
if (WORDISH_CHAR.test(char)) {
|
|
191
|
+
wordish += 1;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
const total = printable + control;
|
|
195
|
+
if (total === 0) {
|
|
196
|
+
return { printableRatio: 0, wordishRatio: 0 };
|
|
197
|
+
}
|
|
198
|
+
return { printableRatio: printable / total, wordishRatio: wordish / total };
|
|
199
|
+
}
|
|
200
|
+
function isMostlyPrintable(text) {
|
|
201
|
+
return getTextStats(text).printableRatio > 0.85;
|
|
202
|
+
}
|
|
203
|
+
function looksLikeLegacyTextBytes(buffer) {
|
|
204
|
+
if (buffer.length === 0) {
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
207
|
+
const text = decodeLegacyText(buffer);
|
|
208
|
+
const { printableRatio, wordishRatio } = getTextStats(text);
|
|
209
|
+
return printableRatio > 0.95 && wordishRatio > 0.3;
|
|
210
|
+
}
|
|
211
|
+
function looksLikeUtf8Text(buffer) {
|
|
212
|
+
if (!buffer || buffer.length === 0) {
|
|
213
|
+
return false;
|
|
214
|
+
}
|
|
215
|
+
const sample = buffer.subarray(0, Math.min(buffer.length, 4096));
|
|
216
|
+
try {
|
|
217
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(sample);
|
|
218
|
+
return isMostlyPrintable(text);
|
|
219
|
+
}
|
|
220
|
+
catch {
|
|
221
|
+
return looksLikeLegacyTextBytes(sample);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
function decodeTextSample(buffer) {
|
|
225
|
+
if (!buffer || buffer.length === 0) {
|
|
226
|
+
return "";
|
|
227
|
+
}
|
|
228
|
+
const sample = buffer.subarray(0, Math.min(buffer.length, 8192));
|
|
229
|
+
const utf16Charset = resolveUtf16Charset(sample);
|
|
230
|
+
if (utf16Charset === "utf-16be") {
|
|
231
|
+
const swapped = Buffer.alloc(sample.length);
|
|
232
|
+
for (let i = 0; i + 1 < sample.length; i += 2) {
|
|
233
|
+
swapped[i] = sample[i + 1];
|
|
234
|
+
swapped[i + 1] = sample[i];
|
|
235
|
+
}
|
|
236
|
+
return new TextDecoder("utf-16le").decode(swapped);
|
|
237
|
+
}
|
|
238
|
+
if (utf16Charset === "utf-16le") {
|
|
239
|
+
return new TextDecoder("utf-16le").decode(sample);
|
|
240
|
+
}
|
|
241
|
+
return new TextDecoder("utf-8").decode(sample);
|
|
242
|
+
}
|
|
243
|
+
function guessDelimitedMime(text) {
|
|
244
|
+
if (!text) {
|
|
245
|
+
return undefined;
|
|
246
|
+
}
|
|
247
|
+
const line = text.split(/\r?\n/)[0] ?? "";
|
|
248
|
+
const tabs = (line.match(/\t/g) ?? []).length;
|
|
249
|
+
const commas = (line.match(/,/g) ?? []).length;
|
|
250
|
+
if (commas > 0) {
|
|
251
|
+
return "text/csv";
|
|
252
|
+
}
|
|
253
|
+
if (tabs > 0) {
|
|
254
|
+
return "text/tab-separated-values";
|
|
255
|
+
}
|
|
256
|
+
return undefined;
|
|
257
|
+
}
|
|
258
|
+
function resolveTextMimeFromName(name) {
|
|
259
|
+
if (!name) {
|
|
260
|
+
return undefined;
|
|
261
|
+
}
|
|
262
|
+
const ext = path.extname(name).toLowerCase();
|
|
263
|
+
return TEXT_EXT_MIME.get(ext);
|
|
264
|
+
}
|
|
265
|
+
function isBinaryMediaMime(mime) {
|
|
266
|
+
if (!mime) {
|
|
267
|
+
return false;
|
|
268
|
+
}
|
|
269
|
+
return mime.startsWith("image/") || mime.startsWith("audio/") || mime.startsWith("video/");
|
|
270
|
+
}
|
|
271
|
+
async function extractFileBlocks(params) {
|
|
272
|
+
const { attachments, cache, limits, skipAttachmentIndexes } = params;
|
|
273
|
+
if (!attachments || attachments.length === 0) {
|
|
274
|
+
return [];
|
|
275
|
+
}
|
|
276
|
+
const blocks = [];
|
|
277
|
+
for (const attachment of attachments) {
|
|
278
|
+
if (!attachment) {
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
if (skipAttachmentIndexes?.has(attachment.index)) {
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
const forcedTextMime = resolveTextMimeFromName(attachment.path ?? attachment.url ?? "");
|
|
285
|
+
const kind = forcedTextMime ? "document" : resolveAttachmentKind(attachment);
|
|
286
|
+
if (!forcedTextMime && (kind === "image" || kind === "video" || kind === "audio")) {
|
|
287
|
+
continue;
|
|
288
|
+
}
|
|
289
|
+
if (!limits.allowUrl && attachment.url && !attachment.path) {
|
|
290
|
+
if (shouldLogVerbose()) {
|
|
291
|
+
logVerbose(`media: file attachment skipped (url disabled) index=${attachment.index}`);
|
|
292
|
+
}
|
|
293
|
+
continue;
|
|
294
|
+
}
|
|
295
|
+
let bufferResult;
|
|
296
|
+
try {
|
|
297
|
+
bufferResult = await cache.getBuffer({
|
|
298
|
+
attachmentIndex: attachment.index,
|
|
299
|
+
maxBytes: limits.maxBytes,
|
|
300
|
+
timeoutMs: limits.timeoutMs,
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
catch (err) {
|
|
304
|
+
if (shouldLogVerbose()) {
|
|
305
|
+
logVerbose(`media: file attachment skipped (buffer): ${String(err)}`);
|
|
306
|
+
}
|
|
307
|
+
continue;
|
|
308
|
+
}
|
|
309
|
+
const nameHint = bufferResult?.fileName ?? attachment.path ?? attachment.url;
|
|
310
|
+
const forcedTextMimeResolved = forcedTextMime ?? resolveTextMimeFromName(nameHint ?? "");
|
|
311
|
+
const rawMime = bufferResult?.mime ?? attachment.mime;
|
|
312
|
+
const normalizedRawMime = normalizeMimeType(rawMime);
|
|
313
|
+
if (!forcedTextMimeResolved && isBinaryMediaMime(normalizedRawMime)) {
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
const utf16Charset = resolveUtf16Charset(bufferResult?.buffer);
|
|
317
|
+
const textSample = decodeTextSample(bufferResult?.buffer);
|
|
318
|
+
const textLike = Boolean(utf16Charset) || looksLikeUtf8Text(bufferResult?.buffer);
|
|
319
|
+
const guessedDelimited = textLike ? guessDelimitedMime(textSample) : undefined;
|
|
320
|
+
const textHint = forcedTextMimeResolved ?? guessedDelimited ?? (textLike ? "text/plain" : undefined);
|
|
321
|
+
const mimeType = sanitizeMimeType(textHint ?? normalizeMimeType(rawMime));
|
|
322
|
+
// Log when MIME type is overridden from non-text to text for auditability
|
|
323
|
+
if (textHint && rawMime && !rawMime.startsWith("text/")) {
|
|
324
|
+
logVerbose(`media: MIME override from "${rawMime}" to "${textHint}" for index=${attachment.index}`);
|
|
325
|
+
}
|
|
326
|
+
if (!mimeType) {
|
|
327
|
+
if (shouldLogVerbose()) {
|
|
328
|
+
logVerbose(`media: file attachment skipped (unknown mime) index=${attachment.index}`);
|
|
329
|
+
}
|
|
330
|
+
continue;
|
|
331
|
+
}
|
|
332
|
+
const allowedMimes = new Set(limits.allowedMimes);
|
|
333
|
+
if (!limits.allowedMimesConfigured) {
|
|
334
|
+
for (const extra of EXTRA_TEXT_MIMES) {
|
|
335
|
+
allowedMimes.add(extra);
|
|
336
|
+
}
|
|
337
|
+
if (mimeType.startsWith("text/")) {
|
|
338
|
+
allowedMimes.add(mimeType);
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
if (!allowedMimes.has(mimeType)) {
|
|
342
|
+
if (shouldLogVerbose()) {
|
|
343
|
+
logVerbose(`media: file attachment skipped (unsupported mime ${mimeType}) index=${attachment.index}`);
|
|
344
|
+
}
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
let extracted;
|
|
348
|
+
try {
|
|
349
|
+
const mediaType = utf16Charset ? `${mimeType}; charset=${utf16Charset}` : mimeType;
|
|
350
|
+
const { allowedMimesConfigured: _allowedMimesConfigured, ...baseLimits } = limits;
|
|
351
|
+
extracted = await extractFileContentFromSource({
|
|
352
|
+
source: {
|
|
353
|
+
type: "base64",
|
|
354
|
+
data: bufferResult.buffer.toString("base64"),
|
|
355
|
+
mediaType,
|
|
356
|
+
filename: bufferResult.fileName,
|
|
357
|
+
},
|
|
358
|
+
limits: {
|
|
359
|
+
...baseLimits,
|
|
360
|
+
allowedMimes,
|
|
361
|
+
},
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
catch (err) {
|
|
365
|
+
if (shouldLogVerbose()) {
|
|
366
|
+
logVerbose(`media: file attachment skipped (extract): ${String(err)}`);
|
|
367
|
+
}
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
const text = extracted?.text?.trim() ?? "";
|
|
371
|
+
let blockText = text;
|
|
372
|
+
if (!blockText) {
|
|
373
|
+
if (extracted?.images && extracted.images.length > 0) {
|
|
374
|
+
blockText = "[PDF content rendered to images; images not forwarded to model]";
|
|
375
|
+
}
|
|
376
|
+
else {
|
|
377
|
+
blockText = "[No extractable text]";
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
const safeName = (bufferResult.fileName ?? `file-${attachment.index + 1}`)
|
|
381
|
+
.replace(/[\r\n\t]+/g, " ")
|
|
382
|
+
.trim();
|
|
383
|
+
// Escape XML special characters in attributes to prevent injection
|
|
384
|
+
blocks.push(`<file name="${xmlEscapeAttr(safeName)}" mime="${xmlEscapeAttr(mimeType)}">\n${escapeFileBlockContent(blockText)}\n</file>`);
|
|
385
|
+
}
|
|
386
|
+
return blocks;
|
|
387
|
+
}
|
|
7
388
|
export async function applyMediaUnderstanding(params) {
|
|
8
389
|
const { ctx, cfg } = params;
|
|
9
390
|
const commandCandidates = [ctx.CommandBody, ctx.RawBody, ctx.Body];
|
|
@@ -32,8 +413,9 @@ export async function applyMediaUnderstanding(params) {
|
|
|
32
413
|
const outputs = [];
|
|
33
414
|
const decisions = [];
|
|
34
415
|
for (const entry of results) {
|
|
35
|
-
if (!entry)
|
|
416
|
+
if (!entry) {
|
|
36
417
|
continue;
|
|
418
|
+
}
|
|
37
419
|
for (const output of entry.outputs) {
|
|
38
420
|
outputs.push(output);
|
|
39
421
|
}
|
|
@@ -62,7 +444,24 @@ export async function applyMediaUnderstanding(params) {
|
|
|
62
444
|
ctx.RawBody = originalUserText;
|
|
63
445
|
}
|
|
64
446
|
ctx.MediaUnderstanding = [...(ctx.MediaUnderstanding ?? []), ...outputs];
|
|
65
|
-
|
|
447
|
+
}
|
|
448
|
+
const audioAttachmentIndexes = new Set(outputs
|
|
449
|
+
.filter((output) => output.kind === "audio.transcription")
|
|
450
|
+
.map((output) => output.attachmentIndex));
|
|
451
|
+
const fileBlocks = await extractFileBlocks({
|
|
452
|
+
attachments,
|
|
453
|
+
cache,
|
|
454
|
+
limits: resolveFileLimits(cfg),
|
|
455
|
+
skipAttachmentIndexes: audioAttachmentIndexes.size > 0 ? audioAttachmentIndexes : undefined,
|
|
456
|
+
});
|
|
457
|
+
if (fileBlocks.length > 0) {
|
|
458
|
+
ctx.Body = appendFileBlocks(ctx.Body, fileBlocks);
|
|
459
|
+
}
|
|
460
|
+
if (outputs.length > 0 || fileBlocks.length > 0) {
|
|
461
|
+
finalizeInboundContext(ctx, {
|
|
462
|
+
forceBodyForAgent: true,
|
|
463
|
+
forceBodyForCommands: outputs.length > 0 || fileBlocks.length > 0,
|
|
464
|
+
});
|
|
66
465
|
}
|
|
67
466
|
return {
|
|
68
467
|
outputs,
|
|
@@ -70,6 +469,7 @@ export async function applyMediaUnderstanding(params) {
|
|
|
70
469
|
appliedImage: outputs.some((output) => output.kind === "image.description"),
|
|
71
470
|
appliedAudio: outputs.some((output) => output.kind === "audio.transcription"),
|
|
72
471
|
appliedVideo: outputs.some((output) => output.kind === "video.description"),
|
|
472
|
+
appliedFile: fileBlocks.length > 0,
|
|
73
473
|
};
|
|
74
474
|
}
|
|
75
475
|
finally {
|
|
@@ -3,16 +3,18 @@ import fs from "node:fs/promises";
|
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
|
+
import { logVerbose, shouldLogVerbose } from "../globals.js";
|
|
7
|
+
import { isAbortError } from "../infra/unhandled-rejections.js";
|
|
6
8
|
import { fetchRemoteMedia, MediaFetchError } from "../media/fetch.js";
|
|
7
9
|
import { detectMime, getFileExtension, isAudioFileName, kindFromMime } from "../media/mime.js";
|
|
8
|
-
import { logVerbose, shouldLogVerbose } from "../globals.js";
|
|
9
|
-
import { fetchWithTimeout } from "./providers/shared.js";
|
|
10
10
|
import { MediaUnderstandingSkipError } from "./errors.js";
|
|
11
|
+
import { fetchWithTimeout } from "./providers/shared.js";
|
|
11
12
|
const DEFAULT_MAX_ATTACHMENTS = 1;
|
|
12
13
|
function normalizeAttachmentPath(raw) {
|
|
13
14
|
const value = raw?.trim();
|
|
14
|
-
if (!value)
|
|
15
|
+
if (!value) {
|
|
15
16
|
return undefined;
|
|
17
|
+
}
|
|
16
18
|
if (value.startsWith("file://")) {
|
|
17
19
|
try {
|
|
18
20
|
return fileURLToPath(value);
|
|
@@ -30,8 +32,9 @@ export function normalizeAttachments(ctx) {
|
|
|
30
32
|
const resolveMime = (count, index) => {
|
|
31
33
|
const typeHint = typesFromArray?.[index];
|
|
32
34
|
const trimmed = typeof typeHint === "string" ? typeHint.trim() : "";
|
|
33
|
-
if (trimmed)
|
|
35
|
+
if (trimmed) {
|
|
34
36
|
return trimmed;
|
|
37
|
+
}
|
|
35
38
|
return count === 1 ? ctx.MediaType : undefined;
|
|
36
39
|
};
|
|
37
40
|
if (pathsFromArray && pathsFromArray.length > 0) {
|
|
@@ -59,8 +62,9 @@ export function normalizeAttachments(ctx) {
|
|
|
59
62
|
}
|
|
60
63
|
const pathValue = ctx.MediaPath?.trim();
|
|
61
64
|
const url = ctx.MediaUrl?.trim();
|
|
62
|
-
if (!pathValue && !url)
|
|
65
|
+
if (!pathValue && !url) {
|
|
63
66
|
return [];
|
|
67
|
+
}
|
|
64
68
|
return [
|
|
65
69
|
{
|
|
66
70
|
path: pathValue || undefined,
|
|
@@ -72,15 +76,19 @@ export function normalizeAttachments(ctx) {
|
|
|
72
76
|
}
|
|
73
77
|
export function resolveAttachmentKind(attachment) {
|
|
74
78
|
const kind = kindFromMime(attachment.mime);
|
|
75
|
-
if (kind === "image" || kind === "audio" || kind === "video")
|
|
79
|
+
if (kind === "image" || kind === "audio" || kind === "video") {
|
|
76
80
|
return kind;
|
|
81
|
+
}
|
|
77
82
|
const ext = getFileExtension(attachment.path ?? attachment.url);
|
|
78
|
-
if (!ext)
|
|
83
|
+
if (!ext) {
|
|
79
84
|
return "unknown";
|
|
80
|
-
|
|
85
|
+
}
|
|
86
|
+
if ([".mp4", ".mov", ".mkv", ".webm", ".avi", ".m4v"].includes(ext)) {
|
|
81
87
|
return "video";
|
|
82
|
-
|
|
88
|
+
}
|
|
89
|
+
if (isAudioFileName(attachment.path ?? attachment.url)) {
|
|
83
90
|
return "audio";
|
|
91
|
+
}
|
|
84
92
|
if ([".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tiff", ".tif"].includes(ext)) {
|
|
85
93
|
return "image";
|
|
86
94
|
}
|
|
@@ -95,25 +103,22 @@ export function isAudioAttachment(attachment) {
|
|
|
95
103
|
export function isImageAttachment(attachment) {
|
|
96
104
|
return resolveAttachmentKind(attachment) === "image";
|
|
97
105
|
}
|
|
98
|
-
function isAbortError(err) {
|
|
99
|
-
if (!err)
|
|
100
|
-
return false;
|
|
101
|
-
if (err instanceof Error && err.name === "AbortError")
|
|
102
|
-
return true;
|
|
103
|
-
return false;
|
|
104
|
-
}
|
|
105
106
|
function resolveRequestUrl(input) {
|
|
106
|
-
if (typeof input === "string")
|
|
107
|
+
if (typeof input === "string") {
|
|
107
108
|
return input;
|
|
108
|
-
|
|
109
|
+
}
|
|
110
|
+
if (input instanceof URL) {
|
|
109
111
|
return input.toString();
|
|
112
|
+
}
|
|
110
113
|
return input.url;
|
|
111
114
|
}
|
|
112
115
|
function orderAttachments(attachments, prefer) {
|
|
113
|
-
if (!prefer || prefer === "first")
|
|
116
|
+
if (!prefer || prefer === "first") {
|
|
114
117
|
return attachments;
|
|
115
|
-
|
|
116
|
-
|
|
118
|
+
}
|
|
119
|
+
if (prefer === "last") {
|
|
120
|
+
return [...attachments].toReversed();
|
|
121
|
+
}
|
|
117
122
|
if (prefer === "path") {
|
|
118
123
|
const withPath = attachments.filter((item) => item.path);
|
|
119
124
|
const withoutPath = attachments.filter((item) => !item.path);
|
|
@@ -129,14 +134,17 @@ function orderAttachments(attachments, prefer) {
|
|
|
129
134
|
export function selectAttachments(params) {
|
|
130
135
|
const { capability, attachments, policy } = params;
|
|
131
136
|
const matches = attachments.filter((item) => {
|
|
132
|
-
if (capability === "image")
|
|
137
|
+
if (capability === "image") {
|
|
133
138
|
return isImageAttachment(item);
|
|
134
|
-
|
|
139
|
+
}
|
|
140
|
+
if (capability === "audio") {
|
|
135
141
|
return isAudioAttachment(item);
|
|
142
|
+
}
|
|
136
143
|
return isVideoAttachment(item);
|
|
137
144
|
});
|
|
138
|
-
if (matches.length === 0)
|
|
145
|
+
if (matches.length === 0) {
|
|
139
146
|
return [];
|
|
147
|
+
}
|
|
140
148
|
const ordered = orderAttachments(matches, policy?.prefer);
|
|
141
149
|
const mode = policy?.mode ?? "first";
|
|
142
150
|
const maxAttachments = policy?.maxAttachments ?? DEFAULT_MAX_ATTACHMENTS;
|
|
@@ -291,15 +299,18 @@ export class MediaAttachmentCache {
|
|
|
291
299
|
}
|
|
292
300
|
resolveLocalPath(attachment) {
|
|
293
301
|
const rawPath = normalizeAttachmentPath(attachment.path);
|
|
294
|
-
if (!rawPath)
|
|
302
|
+
if (!rawPath) {
|
|
295
303
|
return undefined;
|
|
304
|
+
}
|
|
296
305
|
return path.isAbsolute(rawPath) ? rawPath : path.resolve(rawPath);
|
|
297
306
|
}
|
|
298
307
|
async ensureLocalStat(entry) {
|
|
299
|
-
if (!entry.resolvedPath)
|
|
308
|
+
if (!entry.resolvedPath) {
|
|
300
309
|
return undefined;
|
|
301
|
-
|
|
310
|
+
}
|
|
311
|
+
if (entry.statSize !== undefined) {
|
|
302
312
|
return entry.statSize;
|
|
313
|
+
}
|
|
303
314
|
try {
|
|
304
315
|
const stat = await fs.stat(entry.resolvedPath);
|
|
305
316
|
if (!stat.isFile()) {
|
|
@@ -26,5 +26,21 @@ export const DEFAULT_AUDIO_MODELS = {
|
|
|
26
26
|
openai: "gpt-4o-mini-transcribe",
|
|
27
27
|
deepgram: "nova-3",
|
|
28
28
|
};
|
|
29
|
+
export const AUTO_AUDIO_KEY_PROVIDERS = ["openai", "groq", "deepgram", "google"];
|
|
30
|
+
export const AUTO_IMAGE_KEY_PROVIDERS = [
|
|
31
|
+
"openai",
|
|
32
|
+
"anthropic",
|
|
33
|
+
"google",
|
|
34
|
+
"minimax",
|
|
35
|
+
"zai",
|
|
36
|
+
];
|
|
37
|
+
export const AUTO_VIDEO_KEY_PROVIDERS = ["google"];
|
|
38
|
+
export const DEFAULT_IMAGE_MODELS = {
|
|
39
|
+
openai: "gpt-5-mini",
|
|
40
|
+
anthropic: "claude-opus-4-6",
|
|
41
|
+
google: "gemini-3-flash-preview",
|
|
42
|
+
minimax: "MiniMax-VL-01",
|
|
43
|
+
zai: "glm-4.6v",
|
|
44
|
+
};
|
|
29
45
|
export const CLI_OUTPUT_MAX_BUFFER = 5 * MB;
|
|
30
46
|
export const DEFAULT_MEDIA_CONCURRENCY = 2;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { fetchWithTimeoutGuarded, normalizeBaseUrl, readErrorResponse } from "../shared.js";
|
|
2
2
|
export const DEFAULT_DEEPGRAM_AUDIO_BASE_URL = "https://api.deepgram.com/v1";
|
|
3
3
|
export const DEFAULT_DEEPGRAM_AUDIO_MODEL = "nova-3";
|
|
4
4
|
function resolveModel(model) {
|
|
@@ -8,15 +8,18 @@ function resolveModel(model) {
|
|
|
8
8
|
export async function transcribeDeepgramAudio(params) {
|
|
9
9
|
const fetchFn = params.fetchFn ?? fetch;
|
|
10
10
|
const baseUrl = normalizeBaseUrl(params.baseUrl, DEFAULT_DEEPGRAM_AUDIO_BASE_URL);
|
|
11
|
+
const allowPrivate = Boolean(params.baseUrl?.trim());
|
|
11
12
|
const model = resolveModel(params.model);
|
|
12
13
|
const url = new URL(`${baseUrl}/listen`);
|
|
13
14
|
url.searchParams.set("model", model);
|
|
14
|
-
if (params.language?.trim())
|
|
15
|
+
if (params.language?.trim()) {
|
|
15
16
|
url.searchParams.set("language", params.language.trim());
|
|
17
|
+
}
|
|
16
18
|
if (params.query) {
|
|
17
19
|
for (const [key, value] of Object.entries(params.query)) {
|
|
18
|
-
if (value === undefined)
|
|
20
|
+
if (value === undefined) {
|
|
19
21
|
continue;
|
|
22
|
+
}
|
|
20
23
|
url.searchParams.set(key, String(value));
|
|
21
24
|
}
|
|
22
25
|
}
|
|
@@ -28,20 +31,25 @@ export async function transcribeDeepgramAudio(params) {
|
|
|
28
31
|
headers.set("content-type", params.mime ?? "application/octet-stream");
|
|
29
32
|
}
|
|
30
33
|
const body = new Uint8Array(params.buffer);
|
|
31
|
-
const res = await
|
|
34
|
+
const { response: res, release } = await fetchWithTimeoutGuarded(url.toString(), {
|
|
32
35
|
method: "POST",
|
|
33
36
|
headers,
|
|
34
37
|
body,
|
|
35
|
-
}, params.timeoutMs, fetchFn);
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
38
|
+
}, params.timeoutMs, fetchFn, allowPrivate ? { ssrfPolicy: { allowPrivateNetwork: true } } : undefined);
|
|
39
|
+
try {
|
|
40
|
+
if (!res.ok) {
|
|
41
|
+
const detail = await readErrorResponse(res);
|
|
42
|
+
const suffix = detail ? `: ${detail}` : "";
|
|
43
|
+
throw new Error(`Audio transcription failed (HTTP ${res.status})${suffix}`);
|
|
44
|
+
}
|
|
45
|
+
const payload = (await res.json());
|
|
46
|
+
const transcript = payload.results?.channels?.[0]?.alternatives?.[0]?.transcript?.trim();
|
|
47
|
+
if (!transcript) {
|
|
48
|
+
throw new Error("Audio transcription response missing transcript");
|
|
49
|
+
}
|
|
50
|
+
return { text: transcript, model };
|
|
40
51
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
if (!transcript) {
|
|
44
|
-
throw new Error("Audio transcription response missing transcript");
|
|
52
|
+
finally {
|
|
53
|
+
await release();
|
|
45
54
|
}
|
|
46
|
-
return { text: transcript, model };
|
|
47
55
|
}
|