@vellumai/assistant 0.11.4 → 0.11.5-dev.202608220019.a778cf0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -2
- package/ARCHITECTURE.md +29 -4
- package/README.md +1 -1
- package/docs/architecture/memory.md +17 -5
- package/docs/architecture/security.md +96 -152
- package/docs/guardian-request-flow.md +13 -2
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/url-normalization.test.ts +135 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/url-normalization.ts +107 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/url-normalization.test.ts +135 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/url-normalization.ts +107 -0
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +166 -1
- package/node_modules/@vellumai/gateway-client/src/http-delivery.ts +7 -14
- package/node_modules/@vellumai/gateway-client/src/index.ts +2 -0
- package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +30 -32
- package/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/url-normalization.test.ts +135 -0
- package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
- package/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
- package/node_modules/@vellumai/service-contracts/src/url-normalization.ts +107 -0
- package/openapi.yaml +487 -11
- package/package.json +1 -1
- package/scripts/sync-web-search-catalog.ts +6 -0
- package/src/__tests__/anthropic-provider.test.ts +10 -3
- package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
- package/src/__tests__/background-workers-disk-pressure.test.ts +1 -1
- package/src/__tests__/channel-approval-routes.test.ts +4 -0
- package/src/__tests__/channel-inbound-disk-pressure.test.ts +5 -6
- package/src/__tests__/channel-readiness-service.test.ts +126 -0
- package/src/__tests__/channel-reply-delivery.test.ts +30 -17
- package/src/__tests__/checker.test.ts +20 -314
- package/src/__tests__/cli-memory-v2-reembed-skills.test.ts +6 -2
- package/src/__tests__/client-os-metadata-persistence.test.ts +16 -0
- package/src/__tests__/compactor-retained-image-validation.test.ts +142 -0
- package/src/__tests__/config-loader-backfill.test.ts +19 -10
- package/src/__tests__/container-cpu-sampler.test.ts +153 -0
- package/src/__tests__/context-search-memory-v2-source.test.ts +0 -1
- package/src/__tests__/conversation-attachments.test.ts +0 -1
- package/src/__tests__/conversation-delete-watch-timeline.test.ts +231 -0
- package/src/__tests__/conversation-error.test.ts +17 -0
- package/src/__tests__/conversation-lifecycle.test.ts +20 -26
- package/src/__tests__/conversation-pairing.test.ts +186 -0
- package/src/__tests__/conversation-runtime-assembly.test.ts +21 -2
- package/src/__tests__/credential-broker-server-use.test.ts +24 -5
- package/src/__tests__/credential-routes.test.ts +22 -3
- package/src/__tests__/credential-security-invariants.test.ts +2 -1
- package/src/__tests__/daemon-credential-client.test.ts +88 -0
- package/src/__tests__/default-plugin-names-guard.test.ts +12 -1
- package/src/__tests__/dm-backfill.test.ts +63 -0
- package/src/__tests__/dynamic-skill-background-guard.test.ts +0 -2
- package/src/__tests__/edit-propagation.test.ts +107 -4
- package/src/__tests__/events-client-registration.test.ts +28 -0
- package/src/__tests__/file-write-tool.test.ts +4 -2
- package/src/__tests__/guardian-prompt-notice-privacy.test.ts +133 -0
- package/src/__tests__/guardian-verify-setup-skill-regression.test.ts +143 -97
- package/src/__tests__/helpers/gateway-classify-mock.ts +27 -6
- package/src/__tests__/host-proxy-interface.test.ts +11 -1
- package/src/__tests__/host-shell-tool.test.ts +23 -4
- package/src/__tests__/image-conversion.test.ts +143 -1
- package/src/__tests__/inline-skill-load-permissions.test.ts +47 -36
- package/src/__tests__/input-repairs.test.ts +207 -0
- package/src/__tests__/live-workspace-guard.test.ts +75 -0
- package/src/__tests__/managed-profile-guard.test.ts +4 -2
- package/src/__tests__/mcp-abort-signal.test.ts +1 -1
- package/src/__tests__/mcp-client-auth.test.ts +1 -1
- package/src/__tests__/mcp-tool-annotations-risk.test.ts +1 -1
- package/src/__tests__/media-resolve-image-validation.test.ts +310 -0
- package/src/__tests__/mtime-cache.test.ts +2 -0
- package/src/__tests__/notification-vellum-adapter.test.ts +124 -1
- package/src/__tests__/openai-provider.test.ts +40 -0
- package/src/__tests__/openai-responses-provider.test.ts +18 -0
- package/src/__tests__/personal-memory-auth-bypass.test.ts +22 -6
- package/src/__tests__/platform-bash-auto-approve.test.ts +0 -4
- package/src/__tests__/platform.test.ts +16 -1
- package/src/__tests__/plugin-api-resolve-credential.test.ts +22 -13
- package/src/__tests__/plugin-api-shim.test.ts +5 -0
- package/src/__tests__/plugin-api-store-credential.test.ts +268 -0
- package/src/__tests__/plugin-config-data-migration.test.ts +8 -1
- package/src/__tests__/plugin-disabled-state.test.ts +2 -0
- package/src/__tests__/plugin-effective-enabled-set.test.ts +55 -3
- package/src/__tests__/plugin-execution-context.test.ts +73 -0
- package/src/__tests__/plugin-import-boundary-guard.test.ts +0 -1
- package/src/__tests__/plugin-import-boundary-reverse-guard.test.ts +0 -1
- package/src/__tests__/reaction-persistence.test.ts +200 -7
- package/src/__tests__/require-fresh-approval.test.ts +0 -4
- package/src/__tests__/resource-pressure-guard.test.ts +428 -0
- package/src/__tests__/resource-pressure-routes.test.ts +113 -0
- package/src/__tests__/risk-classification-boundary-guard.test.ts +115 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +27 -1
- package/src/__tests__/sanitize-config-for-transfer.test.ts +28 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +7 -0
- package/src/__tests__/settings-routes.test.ts +85 -3
- package/src/__tests__/skill-tool-factory.test.ts +161 -0
- package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +1 -1
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +132 -5
- package/src/__tests__/tool-executor.test.ts +63 -38
- package/src/__tests__/tool-policy.test.ts +97 -1
- package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +7 -4
- package/src/__tests__/user-plugin-loader.test.ts +2 -0
- package/src/__tests__/validate-input.test.ts +243 -7
- package/src/__tests__/verification-control-plane-policy.test.ts +0 -2
- package/src/__tests__/voice-scoped-grant-consumer.test.ts +2 -0
- package/src/__tests__/voice-session-bridge.test.ts +42 -0
- package/src/__tests__/web-search-catalog-parity.test.ts +8 -0
- package/src/acp/__tests__/acp-claude-oauth.test.ts +97 -29
- package/src/acp/__tests__/prepare-agent-env.test.ts +376 -153
- package/src/acp/acp-claude-oauth.ts +38 -21
- package/src/acp/prepare-agent-env.ts +146 -55
- package/src/agent/loop-tool-dedup.test.ts +161 -0
- package/src/agent/loop.ts +68 -22
- package/src/api/constants/app-tools.ts +34 -0
- package/src/api/constants/profile-config-validation.ts +60 -0
- package/src/api/events/resource-pressure-status-changed.ts +53 -0
- package/src/api/events/tool-result.ts +6 -1
- package/src/api/events/watch-retro-completed.ts +52 -0
- package/src/api/index.ts +30 -0
- package/src/api/responses/resource-pressure-status.ts +25 -0
- package/src/approvals/guardian-channel-delivery.ts +70 -6
- package/src/approvals/guardian-request-resolvers.ts +32 -66
- package/src/calls/__tests__/voice-session-bridge.test.ts +114 -0
- package/src/calls/voice-session-bridge.ts +60 -6
- package/src/channels/__tests__/message-audience.test.ts +49 -0
- package/src/channels/__tests__/types.test.ts +17 -3
- package/src/channels/message-audience.ts +40 -0
- package/src/channels/types.ts +16 -12
- package/src/cli/__tests__/catalog-search-help.test.ts +50 -0
- package/src/cli/commands/__tests__/channel-verification-sessions.test.ts +31 -0
- package/src/cli/commands/__tests__/conversations-search.test.ts +289 -0
- package/src/cli/commands/__tests__/conversations-slack.test.ts +2 -1
- package/src/cli/commands/__tests__/inference-profiles.test.ts +16 -4
- package/src/cli/commands/__tests__/inference-providers.test.ts +135 -2
- package/src/cli/commands/__tests__/keys.test.ts +89 -8
- package/src/cli/commands/__tests__/plugins.test.ts +57 -1
- package/src/cli/commands/channel-verification-sessions.help.ts +11 -9
- package/src/cli/commands/channel-verification-sessions.ts +11 -21
- package/src/cli/commands/conversations.help.ts +34 -0
- package/src/cli/commands/conversations.ts +125 -0
- package/src/cli/commands/inference-profiles.ts +56 -3
- package/src/cli/commands/inference-providers.ts +37 -6
- package/src/cli/commands/inference.help.ts +9 -4
- package/src/cli/commands/keys.help.ts +13 -1
- package/src/cli/commands/keys.ts +26 -15
- package/src/cli/commands/memory/__tests__/memory-v2.test.ts +57 -7
- package/src/cli/commands/memory/index.help.ts +41 -27
- package/src/cli/commands/memory/index.ts +2 -0
- package/src/cli/commands/memory/memory-v2.ts +58 -54
- package/src/cli/commands/memory/memory-validate.ts +18 -0
- package/src/cli/commands/monitoring.ts +1 -0
- package/src/cli/commands/oauth/request.test.ts +93 -0
- package/src/cli/commands/oauth/request.ts +23 -9
- package/src/cli/commands/plugins.help.ts +28 -31
- package/src/cli/commands/plugins.ts +10 -0
- package/src/cli/commands/skills.help.ts +13 -16
- package/src/cli/commands/trust.ts +3 -14
- package/src/cli/lib/__tests__/install-from-github.test.ts +38 -0
- package/src/cli/lib/__tests__/merge-plugin-tree.test.ts +34 -0
- package/src/cli/lib/__tests__/plugin-surfaces.test.ts +32 -1
- package/src/cli/lib/__tests__/toggle-plugin.test.ts +2 -1
- package/src/cli/lib/__tests__/upgrade-plugin.test.ts +64 -0
- package/src/cli/lib/bundled-marketplace.json +26 -0
- package/src/cli/lib/daemon-credential-client.ts +25 -3
- package/src/cli/lib/install-from-github.ts +35 -24
- package/src/cli/lib/merge-plugin-tree.ts +22 -4
- package/src/cli/lib/open-browser.test.ts +67 -0
- package/src/cli/lib/open-browser.ts +24 -5
- package/src/cli/lib/plugin-surfaces.ts +27 -0
- package/src/config/__tests__/balanced-model-experiment.test.ts +7 -7
- package/src/config/__tests__/default-profile-catalog.test.ts +3 -3
- package/src/config/__tests__/profile-materialization.test.ts +26 -0
- package/src/config/default-profile-catalog.ts +5 -6
- package/src/config/feature-flag-registry.json +36 -28
- package/src/config/profile-materialization.ts +29 -0
- package/src/config/sanitize-for-transfer.ts +16 -0
- package/src/config/schemas/llm.ts +7 -0
- package/src/config/schemas/services.ts +6 -0
- package/src/config/webhook-routing.ts +8 -0
- package/src/context/compactor.ts +31 -2
- package/src/context/outbound-sanitize.ts +6 -0
- package/src/daemon/__tests__/lifecycle-watch-timeline-sweep.test.ts +98 -0
- package/src/daemon/__tests__/provider-rejection-log-fields.test.ts +272 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +4 -1
- package/src/daemon/conversation-error.ts +19 -2
- package/src/daemon/conversation-messaging.ts +22 -2
- package/src/daemon/conversation-process.ts +11 -0
- package/src/daemon/conversation-store.ts +10 -0
- package/src/daemon/conversation-surfaces.ts +21 -8
- package/src/daemon/conversation.ts +12 -10
- package/src/daemon/daemon-control.ts +1 -0
- package/src/daemon/handlers/__tests__/config-ingress-tunnel-records.test.ts +208 -0
- package/src/daemon/handlers/config-ingress.ts +115 -5
- package/src/daemon/lifecycle.ts +30 -2
- package/src/daemon/message-types/conversations.ts +5 -7
- package/src/daemon/message-types/web-activity.ts +3 -2
- package/src/daemon/provider-rejection-log-fields.ts +124 -0
- package/src/daemon/resource-pressure-guard-lifecycle.ts +66 -0
- package/src/daemon/resource-pressure-guard.ts +387 -0
- package/src/daemon/shutdown-handlers.ts +2 -0
- package/src/daemon/startup-error.ts +16 -0
- package/src/daemon/trust-context.ts +14 -10
- package/src/daemon/unsendable-image-notice.ts +76 -0
- package/src/hooks/registry.ts +65 -51
- package/src/inbound/__tests__/tunnel-probe.test.ts +448 -0
- package/src/inbound/tunnel-probe.ts +261 -0
- package/src/ipc/__tests__/socket-path.test.ts +15 -7
- package/src/ipc/gateway-client.test.ts +1 -1
- package/src/ipc/gateway-client.ts +14 -21
- package/src/ipc/socket-cleanup.ts +2 -11
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +1 -1
- package/src/messaging/provider-message-metadata.ts +128 -0
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +138 -73
- package/src/messaging/providers/channel-transport.ts +100 -16
- package/src/messaging/providers/discord/send.ts +16 -0
- package/src/messaging/providers/discord/transport.ts +23 -1
- package/src/messaging/providers/index.ts +84 -16
- package/src/messaging/providers/slack/message-metadata.ts +45 -0
- package/src/messaging/providers/slack/render-transcript.ts +1 -4
- package/src/messaging/providers/slack/send.test.ts +88 -34
- package/src/messaging/providers/slack/send.ts +99 -50
- package/src/messaging/providers/slack/transport.ts +35 -37
- package/src/messaging/providers/telegram-bot/send.test.ts +88 -18
- package/src/messaging/providers/telegram-bot/send.ts +36 -0
- package/src/messaging/providers/telegram-bot/transport.ts +22 -8
- package/src/messaging/read-provider-metadata.test.ts +157 -0
- package/src/messaging/read-provider-metadata.ts +51 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +94 -0
- package/src/notifications/__tests__/edit-notification.test.ts +174 -2
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +536 -4
- package/src/notifications/adapters/macos.ts +55 -0
- package/src/notifications/adapters/slack.ts +10 -5
- package/src/notifications/assistant-reply-producer.ts +37 -4
- package/src/notifications/conversation-pairing.ts +111 -21
- package/src/notifications/edit-notification.ts +38 -8
- package/src/notifications/emit-signal.ts +12 -14
- package/src/notifications/home-feed-side-effect.ts +218 -15
- package/src/notifications/types.ts +5 -0
- package/src/permissions/AGENTS.md +16 -0
- package/src/permissions/checker.test.ts +75 -122
- package/src/permissions/checker.ts +44 -560
- package/src/permissions/confirmation-guardian-request.test.ts +28 -3
- package/src/permissions/confirmation-guardian-request.ts +5 -1
- package/src/persistence/conversation-crud.ts +69 -20
- package/src/persistence/conversation-queries.ts +14 -1
- package/src/persistence/conversation-types.ts +22 -0
- package/src/persistence/db-async-query.ts +2 -1
- package/src/persistence/db-maintenance.ts +15 -0
- package/src/persistence/delivery-crud.ts +122 -15
- package/src/persistence/embeddings/qdrant-manager.ts +1 -0
- package/src/persistence/migrations/367-create-watch-timeline-entries.ts +46 -0
- package/src/persistence/migrations/368-watch-timeline-screenshot-blob.ts +33 -0
- package/src/persistence/migrations/__tests__/367-create-watch-timeline-entries.test.ts +98 -0
- package/src/persistence/migrations/__tests__/368-watch-timeline-screenshot-blob.test.ts +98 -0
- package/src/persistence/schema/index.ts +1 -0
- package/src/persistence/schema/watch.ts +29 -0
- package/src/persistence/steps.ts +4 -0
- package/src/plugin-api/__tests__/import-graph-partial-mock.test.ts +67 -0
- package/src/plugin-api/conversation-turn.ts +27 -0
- package/src/plugin-api/credential-scope.test.ts +15 -0
- package/src/plugin-api/credential-scope.ts +14 -0
- package/src/plugin-api/index.ts +19 -1
- package/src/plugin-api/resolve-credential.ts +15 -10
- package/src/plugin-api/store-credential.ts +148 -0
- package/src/plugin-api/system-card.ts +36 -0
- package/src/plugin-api/vision-support.test.ts +82 -0
- package/src/plugin-api/vision-support.ts +14 -2
- package/src/plugins/defaults/image-fallback/__tests__/image-fallback.test.ts +361 -110
- package/src/plugins/defaults/image-fallback/__tests__/vision-recovery.test.ts +4 -0
- package/src/plugins/defaults/image-fallback/hooks/user-prompt-submit.ts +58 -0
- package/src/plugins/defaults/image-fallback/src/caption-blocks.ts +64 -23
- package/src/plugins/defaults/image-recovery/detect.ts +24 -1
- package/src/plugins/defaults/main.ts +15 -0
- package/src/plugins/defaults/memory/AGENTS.md +11 -3
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +0 -1
- package/src/plugins/defaults/memory/graph-topology/pending-buffer.ts +9 -12
- package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +1 -1
- package/src/plugins/defaults/memory/src/memory-v2-routes.ts +12 -10
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +114 -0
- package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +8 -0
- package/src/plugins/defaults/memory/substrate/__tests__/edge-index.test.ts +0 -30
- package/src/plugins/defaults/memory/substrate/__tests__/ingest.test.ts +73 -1
- package/src/plugins/defaults/memory/substrate/__tests__/page-index.test.ts +37 -0
- package/src/plugins/defaults/memory/substrate/__tests__/page-links.test.ts +208 -0
- package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +158 -1
- package/src/plugins/defaults/memory/substrate/__tests__/static-context.test.ts +1 -61
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +68 -6
- package/src/plugins/defaults/memory/substrate/edge-index.ts +0 -31
- package/src/plugins/defaults/memory/substrate/ingest.ts +59 -0
- package/src/plugins/defaults/memory/substrate/page-index.ts +35 -8
- package/src/plugins/defaults/memory/substrate/page-links.ts +133 -0
- package/src/plugins/defaults/memory/substrate/page-store.ts +10 -0
- package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +124 -34
- package/src/plugins/defaults/memory/substrate/static-context.ts +0 -29
- package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +3 -1
- package/src/plugins/defaults/memory/v3/card.ts +9 -9
- package/src/plugins/defaults/memory/v3/core-set.test.ts +7 -0
- package/src/plugins/defaults/memory/v3/core-set.ts +5 -1
- package/src/plugins/defaults/memory/v3/edge.ts +13 -57
- package/src/plugins/mtime-cache.ts +1 -1
- package/src/plugins/pipeline.ts +14 -7
- package/src/plugins/plugin-execution-context.ts +35 -11
- package/src/plugins/plugin-tree-walk.ts +63 -4
- package/src/providers/__tests__/retry-network-error.test.ts +84 -0
- package/src/providers/anthropic/__tests__/pause-turn-continuation.test.ts +170 -0
- package/src/providers/anthropic/client.ts +332 -241
- package/src/providers/connection-resolution.ts +25 -2
- package/src/providers/content-blocks.ts +22 -0
- package/src/providers/fetch-provider-catalog.ts +19 -0
- package/src/providers/gemini/client.ts +5 -2
- package/src/providers/inference/__tests__/adapter-factory-litellm.test.ts +47 -1
- package/src/providers/inference/__tests__/adapter-factory-ollama.test.ts +83 -0
- package/src/providers/inference/__tests__/adapter-factory-openai-compatible.test.ts +98 -1
- package/src/providers/inference/__tests__/base-url-route-validation.test.ts +38 -0
- package/src/providers/inference/__tests__/base-url-security.test.ts +10 -0
- package/src/providers/inference/__tests__/connections-ollama.test.ts +90 -0
- package/src/providers/inference/__tests__/endpoint-probe.test.ts +92 -0
- package/src/providers/inference/__tests__/missing-credential-guard.test.ts +117 -0
- package/src/providers/inference/__tests__/profile-config-validation.test.ts +39 -0
- package/src/providers/inference/__tests__/profile-probe-classify.test.ts +66 -0
- package/src/providers/inference/adapter-factory.ts +33 -2
- package/src/providers/inference/auth.ts +13 -2
- package/src/providers/inference/credential-usage.ts +37 -0
- package/src/providers/inference/endpoint-probe.ts +115 -0
- package/src/providers/inference/missing-credential-guard.ts +110 -0
- package/src/providers/inference/profile-probe.ts +256 -0
- package/src/providers/inference/resolve-auth.ts +7 -7
- package/src/providers/media-resolve.ts +176 -15
- package/src/providers/model-catalog.ts +10 -0
- package/src/providers/openai/__tests__/api-error-normalization.test.ts +17 -1
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +576 -46
- package/src/providers/openai/__tests__/connection-error-wrap.test.ts +44 -0
- package/src/providers/openai/__tests__/tool-choice-mapping.test.ts +125 -2
- package/src/providers/openai/api-error-normalization.ts +16 -2
- package/src/providers/openai/chat-completions-provider.ts +326 -44
- package/src/providers/openai/responses-provider.ts +5 -2
- package/src/providers/provider-send-message.ts +11 -0
- package/src/providers/retry.ts +6 -0
- package/src/providers/search-provider-catalog.ts +20 -0
- package/src/routes/route-host-protocol.ts +7 -0
- package/src/routes/worker.ts +31 -10
- package/src/runtime/AGENTS.md +36 -0
- package/src/runtime/__tests__/host-observe.test.ts +302 -0
- package/src/runtime/__tests__/runtime-http-port-collision.test.ts +131 -0
- package/src/runtime/__tests__/web-presence.test.ts +234 -0
- package/src/runtime/assistant-event-hub.ts +38 -0
- package/src/runtime/channel-readiness-service.ts +39 -13
- package/src/runtime/channel-readiness-types.ts +49 -2
- package/src/runtime/channel-reply-delivery.ts +55 -32
- package/src/runtime/channel-retry-sweep.ts +6 -4
- package/src/runtime/effective-capabilities.test.ts +19 -0
- package/src/runtime/effective-capabilities.ts +25 -0
- package/src/runtime/guardian-reply-router.ts +6 -2
- package/src/runtime/host-observe.ts +246 -0
- package/src/runtime/http-errors.ts +1 -0
- package/src/runtime/http-server.ts +185 -6
- package/src/runtime/routes/__tests__/client-routes.test.ts +150 -0
- package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +133 -0
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +80 -0
- package/src/runtime/routes/__tests__/credential-delete-in-use.test.ts +155 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +118 -0
- package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +72 -1
- package/src/runtime/routes/__tests__/ingress-status-routes.test.ts +508 -0
- package/src/runtime/routes/__tests__/monitoring-routes.test.ts +1 -0
- package/src/runtime/routes/__tests__/pending-interactions-route.test.ts +175 -0
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +103 -71
- package/src/runtime/routes/__tests__/surface-action-routes.test.ts +55 -6
- package/src/runtime/routes/__tests__/system-card-turn-termination.test.ts +107 -0
- package/src/runtime/routes/__tests__/user-route-dispatcher-host.test.ts +52 -1
- package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +75 -4
- package/src/runtime/routes/__tests__/watch-routes-guardian-cache.test.ts +139 -0
- package/src/runtime/routes/__tests__/watch-routes.test.ts +598 -0
- package/src/runtime/routes/approval-routes.ts +37 -4
- package/src/runtime/routes/approval-strategies/guardian-text-engine-strategy.ts +16 -22
- package/src/runtime/routes/canned-message-complete.ts +68 -29
- package/src/runtime/routes/channel-readiness-routes.ts +14 -2
- package/src/runtime/routes/client-routes.ts +113 -0
- package/src/runtime/routes/conversation-list-routes.ts +22 -7
- package/src/runtime/routes/conversation-management-routes.ts +15 -9
- package/src/runtime/routes/conversation-query-routes.ts +10 -0
- package/src/runtime/routes/conversation-routes.ts +36 -22
- package/src/runtime/routes/credential-in-use.ts +85 -0
- package/src/runtime/routes/credential-routes.ts +55 -85
- package/src/runtime/routes/guardian-approval-interception.ts +44 -53
- package/src/runtime/routes/guardian-approval-reply-helpers.ts +14 -12
- package/src/runtime/routes/host-cu-routes.ts +18 -0
- package/src/runtime/routes/identity-routes.ts +5 -121
- package/src/runtime/routes/inbound-message-handler.ts +74 -30
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +16 -17
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +182 -297
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +173 -386
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +101 -77
- package/src/runtime/routes/inbound-stages/guardian-reply-intercept.ts +17 -8
- package/src/runtime/routes/inbound-stages/reaction-intercept.test.ts +91 -7
- package/src/runtime/routes/inbound-stages/reaction-intercept.ts +72 -70
- package/src/runtime/routes/index.ts +4 -0
- package/src/runtime/routes/inference-profiles-routes.ts +232 -31
- package/src/runtime/routes/inference-provider-connection-routes.ts +33 -11
- package/src/runtime/routes/ingress-status-routes.ts +180 -0
- package/src/runtime/routes/monitoring-routes.ts +9 -1
- package/src/runtime/routes/oauth-commands-routes.ts +1 -1
- package/src/runtime/routes/plugins-routes.ts +28 -32
- package/src/runtime/routes/resource-pressure-routes.ts +22 -0
- package/src/runtime/routes/secret-routes.ts +39 -8
- package/src/runtime/routes/settings-routes.ts +15 -12
- package/src/runtime/routes/surface-action-routes.ts +20 -19
- package/src/runtime/routes/user-route-dispatcher.ts +31 -2
- package/src/runtime/routes/user-route-resolution.ts +18 -1
- package/src/runtime/routes/watch-routes.ts +807 -0
- package/src/runtime/slack-reply-session.test.ts +23 -13
- package/src/runtime/slack-reply-session.ts +38 -55
- package/src/runtime/web-presence.ts +90 -0
- package/src/security/__tests__/untrusted-content.test.ts +42 -0
- package/src/security/untrusted-content.ts +28 -9
- package/src/skills/validate-input.ts +177 -40
- package/src/subagent/__tests__/consult-context-gating.test.ts +6 -10
- package/src/subagent/consult-context.ts +7 -23
- package/src/tools/credentials/broker.ts +24 -79
- package/src/tools/credentials/ref-parse.ts +35 -0
- package/src/tools/credentials/resolve.ts +4 -10
- package/src/tools/credentials/store.ts +168 -0
- package/src/tools/credentials/tool-policy.ts +67 -0
- package/src/tools/executor.ts +53 -15
- package/src/tools/network/__tests__/firecrawl-compat.test.ts +77 -0
- package/src/tools/network/__tests__/web-fetch-fastcrw.test.ts +169 -0
- package/src/tools/network/__tests__/web-search.test.ts +138 -3
- package/src/tools/network/firecrawl-compat.ts +90 -0
- package/src/tools/network/url-safety.ts +9 -16
- package/src/tools/network/web-fetch.ts +142 -62
- package/src/tools/network/web-search.ts +162 -58
- package/src/tools/permission-checker.ts +43 -52
- package/src/tools/schema-transforms.ts +40 -5
- package/src/tools/shared/input-repairs.ts +160 -0
- package/src/tools/skills/skill-tool-factory.ts +18 -4
- package/src/tools/subagent/spawn.ts +0 -1
- package/src/tools/tool-approval-handler.ts +16 -10
- package/src/tools/tool-types.ts +26 -13
- package/src/tools/types.ts +6 -6
- package/src/util/__tests__/cgroup-memory.test.ts +3 -0
- package/src/util/cgroup-memory.ts +3 -0
- package/src/util/container-cpu-sampler.ts +250 -0
- package/src/util/image-conversion.ts +178 -14
- package/src/util/platform.ts +93 -10
- package/src/util/worker-process.ts +1 -0
- package/src/watch/__tests__/watch-retro.test.ts +665 -0
- package/src/watch/__tests__/watch-session-manager.test.ts +566 -0
- package/src/watch/__tests__/watch-timeline.test.ts +670 -0
- package/src/watch/watch-retro.ts +480 -0
- package/src/watch/watch-session-manager.ts +575 -0
- package/src/watch/watch-timeline.ts +848 -0
- package/src/permissions/ipc-risk-types.ts +0 -143
- package/src/permissions/risk-types.ts +0 -76
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Live save-time probe for a user-authored inference profile: one minimal
|
|
3
|
+
* request dispatched through the profile's own resolved model and connection,
|
|
4
|
+
* so a wrong model name, a rejected key, or an impossible token budget
|
|
5
|
+
* surfaces when the profile is saved instead of on the first chat message.
|
|
6
|
+
*
|
|
7
|
+
* The verdict is advisory and classified by which object the user should fix
|
|
8
|
+
* (this profile vs its provider connection), using the semantic
|
|
9
|
+
* `ProviderErrorReason` the provider adapters stamp at their throw sites.
|
|
10
|
+
* Managed and routing-identity profiles are never probed: their pins are
|
|
11
|
+
* hand-validated, and a probe there would spend managed credits on every
|
|
12
|
+
* save.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import type { ResolutionFallbackReason } from "../../config/llm-resolver.js";
|
|
16
|
+
import {
|
|
17
|
+
resolveCallSiteConfig,
|
|
18
|
+
selectWinningProfile,
|
|
19
|
+
} from "../../config/llm-resolver.js";
|
|
20
|
+
import { getConfigReadOnly } from "../../config/loader.js";
|
|
21
|
+
import { getDb } from "../../persistence/db-connection.js";
|
|
22
|
+
import { ProviderError, type ProviderErrorReason } from "../../util/errors.js";
|
|
23
|
+
import { getLogger } from "../../util/logger.js";
|
|
24
|
+
import { dispatchProviderResolvable } from "../connection-resolution.js";
|
|
25
|
+
import {
|
|
26
|
+
createTimeout,
|
|
27
|
+
getConfiguredProvider,
|
|
28
|
+
userMessage,
|
|
29
|
+
} from "../provider-send-message.js";
|
|
30
|
+
import { ROUTING_IDENTITY_PROVIDERS } from "./auth.js";
|
|
31
|
+
import { getConnection, MANAGED_CONNECTION_NAMES } from "./connections.js";
|
|
32
|
+
import { PROBE_TIMEOUT_MS } from "./endpoint-probe.js";
|
|
33
|
+
|
|
34
|
+
const log = getLogger("inference-profile-probe");
|
|
35
|
+
|
|
36
|
+
/** Which object the user should fix when the probe fails. */
|
|
37
|
+
export type ProfileCheckBlame =
|
|
38
|
+
| "profile"
|
|
39
|
+
| "provider"
|
|
40
|
+
| "transient"
|
|
41
|
+
| "unknown";
|
|
42
|
+
|
|
43
|
+
export interface ProfileCheck {
|
|
44
|
+
ok: boolean;
|
|
45
|
+
blame?: ProfileCheckBlame;
|
|
46
|
+
/** Semantic reason (ProviderErrorReason, or "resolution"/"not_configured"). */
|
|
47
|
+
reason?: string;
|
|
48
|
+
/** Upstream or resolver error text, verbatim. */
|
|
49
|
+
detail?: string;
|
|
50
|
+
/** The connection to inspect when `blame` is "provider". */
|
|
51
|
+
connection?: string;
|
|
52
|
+
/** English summary for non-localized surfaces (the CLI). */
|
|
53
|
+
message?: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const PROVIDER_BLAME_REASONS: ReadonlySet<ProviderErrorReason> = new Set([
|
|
57
|
+
"invalid_credentials",
|
|
58
|
+
"insufficient_credits",
|
|
59
|
+
"daily_limit_reached",
|
|
60
|
+
"network_error",
|
|
61
|
+
"server_error",
|
|
62
|
+
]);
|
|
63
|
+
|
|
64
|
+
const TRANSIENT_BLAME_REASONS: ReadonlySet<ProviderErrorReason> = new Set([
|
|
65
|
+
"rate_limited",
|
|
66
|
+
"overloaded",
|
|
67
|
+
]);
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Map a failed probe dispatch onto the object the user should fix. Reasons
|
|
71
|
+
* the adapters stamp about the request itself (model unknown or restricted,
|
|
72
|
+
* parameter and token-budget rejections) blame the profile; credential,
|
|
73
|
+
* billing, and reachability reasons blame the provider connection.
|
|
74
|
+
*/
|
|
75
|
+
export function classifyProbeFailure(err: unknown): {
|
|
76
|
+
blame: ProfileCheckBlame;
|
|
77
|
+
reason?: string;
|
|
78
|
+
detail: string;
|
|
79
|
+
} {
|
|
80
|
+
const detail = err instanceof Error ? err.message : String(err);
|
|
81
|
+
if (!(err instanceof ProviderError) || err.reason === undefined) {
|
|
82
|
+
return { blame: "unknown", detail };
|
|
83
|
+
}
|
|
84
|
+
if (PROVIDER_BLAME_REASONS.has(err.reason)) {
|
|
85
|
+
return { blame: "provider", reason: err.reason, detail };
|
|
86
|
+
}
|
|
87
|
+
if (TRANSIENT_BLAME_REASONS.has(err.reason)) {
|
|
88
|
+
return { blame: "transient", reason: err.reason, detail };
|
|
89
|
+
}
|
|
90
|
+
if (err.reason === "unknown") {
|
|
91
|
+
return { blame: "unknown", reason: err.reason, detail };
|
|
92
|
+
}
|
|
93
|
+
return { blame: "profile", reason: err.reason, detail };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function checkMessage(check: {
|
|
97
|
+
blame: ProfileCheckBlame;
|
|
98
|
+
detail: string;
|
|
99
|
+
connection?: string;
|
|
100
|
+
}): string {
|
|
101
|
+
switch (check.blame) {
|
|
102
|
+
case "profile":
|
|
103
|
+
return `A test request through this profile failed: ${check.detail} Check the profile's model and parameters.`;
|
|
104
|
+
case "provider":
|
|
105
|
+
return (
|
|
106
|
+
`A test request through this profile failed: ${check.detail} ` +
|
|
107
|
+
(check.connection
|
|
108
|
+
? `Check the provider connection "${check.connection}".`
|
|
109
|
+
: `Check the profile's provider connection.`)
|
|
110
|
+
);
|
|
111
|
+
case "transient":
|
|
112
|
+
return `A test request through this profile could not complete: ${check.detail} This may be temporary.`;
|
|
113
|
+
default:
|
|
114
|
+
return `A test request through this profile failed: ${check.detail}`;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Probe a saved profile with one minimal request. Returns `null` when there
|
|
120
|
+
* is no verdict to give: the profile does not exist or is disabled, it rides
|
|
121
|
+
* a managed or routing-identity route (probing would bill managed credits),
|
|
122
|
+
* or the probe timed out without an answer.
|
|
123
|
+
*/
|
|
124
|
+
export async function probeInferenceProfile(
|
|
125
|
+
name: string,
|
|
126
|
+
): Promise<ProfileCheck | null> {
|
|
127
|
+
const { llm } = getConfigReadOnly();
|
|
128
|
+
const entry = llm.profiles?.[name] as Record<string, unknown> | undefined;
|
|
129
|
+
if (!entry || entry.status === "disabled" || entry.source === "managed") {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const selectionSeed = crypto.randomUUID();
|
|
134
|
+
|
|
135
|
+
// If the resolver would fall back past this profile, the probe must not run:
|
|
136
|
+
// it would silently exercise (and bill) a different profile. The fallback
|
|
137
|
+
// reason is itself the verdict.
|
|
138
|
+
let fallbackReason: ResolutionFallbackReason | undefined;
|
|
139
|
+
const winner = selectWinningProfile("inference", llm, {
|
|
140
|
+
overrideProfile: name,
|
|
141
|
+
selectionSeed,
|
|
142
|
+
// The same resolvability predicate dispatch applies: without it this
|
|
143
|
+
// pre-check can report the requested profile as winner while the actual
|
|
144
|
+
// dispatch below falls back to (and bills) a different profile.
|
|
145
|
+
isResolvableProvider: dispatchProviderResolvable,
|
|
146
|
+
onResolutionFallback: (info) => {
|
|
147
|
+
if (info.requested === name) {
|
|
148
|
+
fallbackReason = info.reason;
|
|
149
|
+
}
|
|
150
|
+
},
|
|
151
|
+
});
|
|
152
|
+
if (winner.profileName !== name) {
|
|
153
|
+
const detail = `The profile is ${fallbackReason ?? "unusable"} and requests would silently run on ${winner.profileName ?? "the default"} instead.`;
|
|
154
|
+
return {
|
|
155
|
+
ok: false,
|
|
156
|
+
blame: "profile",
|
|
157
|
+
reason: "resolution",
|
|
158
|
+
detail,
|
|
159
|
+
message: checkMessage({ blame: "profile", detail }),
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// Billing guards run on the fully resolved call-site config — the same
|
|
164
|
+
// composition dispatch consumes (mix arm expanded, call-site overrides
|
|
165
|
+
// applied) — so neither a mix nor an `llm.callSites.inference` provider
|
|
166
|
+
// tweak can route the probe onto a managed or platform-billed path the
|
|
167
|
+
// stored entry hid. Same seed as the dispatch below, so the judged arm is
|
|
168
|
+
// the dispatched arm.
|
|
169
|
+
const resolved = resolveCallSiteConfig("inference", llm, {
|
|
170
|
+
overrideProfile: name,
|
|
171
|
+
selectionSeed,
|
|
172
|
+
});
|
|
173
|
+
const provider = resolved.provider ?? "";
|
|
174
|
+
if (ROUTING_IDENTITY_PROVIDERS.has(provider)) {
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
const connection = resolved.provider_connection ?? (provider || undefined);
|
|
178
|
+
// Canonical managed names always dispatch platform-billed: routing ignores
|
|
179
|
+
// a user-owned row that merely claims such a name
|
|
180
|
+
// (tryResolveProviderForConnectionName), so the name gates the probe
|
|
181
|
+
// regardless of the row's stored auth.
|
|
182
|
+
if (connection && MANAGED_CONNECTION_NAMES.has(connection)) {
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
// A legacy profile can declare a concrete provider while staying bound to
|
|
186
|
+
// a platform-billed connection row; the row's auth is the billing fact, so
|
|
187
|
+
// it gates the probe too.
|
|
188
|
+
const boundRow = connection ? getConnection(getDb(), connection) : null;
|
|
189
|
+
if (boundRow?.auth.type === "platform") {
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const timeout = createTimeout(PROBE_TIMEOUT_MS);
|
|
194
|
+
try {
|
|
195
|
+
const providerInstance = await getConfiguredProvider("inference", {
|
|
196
|
+
overrideProfile: name,
|
|
197
|
+
selectionSeed,
|
|
198
|
+
});
|
|
199
|
+
if (!providerInstance) {
|
|
200
|
+
const detail = "No dispatchable provider is configured for this profile.";
|
|
201
|
+
return {
|
|
202
|
+
ok: false,
|
|
203
|
+
blame: "provider",
|
|
204
|
+
reason: "not_configured",
|
|
205
|
+
detail,
|
|
206
|
+
connection,
|
|
207
|
+
message: checkMessage({ blame: "provider", detail, connection }),
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
// Final billing gate on the route dispatch ACTUALLY resolved: a profile
|
|
211
|
+
// without a binding gets its row auto-selected inside
|
|
212
|
+
// getConfiguredProvider, so only the stamped attribution names the row
|
|
213
|
+
// this request would sign with.
|
|
214
|
+
const route = providerInstance.routeAttribution;
|
|
215
|
+
if (route?.isManagedRoute) {
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
218
|
+
const routedRow = route?.connectionName
|
|
219
|
+
? getConnection(getDb(), route.connectionName)
|
|
220
|
+
: null;
|
|
221
|
+
if (routedRow?.auth.type === "platform") {
|
|
222
|
+
return null;
|
|
223
|
+
}
|
|
224
|
+
await providerInstance.sendMessage(
|
|
225
|
+
[userMessage("Reply with only the word OK.")],
|
|
226
|
+
{
|
|
227
|
+
signal: timeout.signal,
|
|
228
|
+
config: {
|
|
229
|
+
callSite: "inference",
|
|
230
|
+
overrideProfile: name,
|
|
231
|
+
selectionSeed,
|
|
232
|
+
},
|
|
233
|
+
},
|
|
234
|
+
);
|
|
235
|
+
return { ok: true };
|
|
236
|
+
} catch (err) {
|
|
237
|
+
if (timeout.signal.aborted) {
|
|
238
|
+
// No verdict: a slow model and a dead endpoint look identical here,
|
|
239
|
+
// and the connection-level probe already covers unreachable hosts.
|
|
240
|
+
log.info({ profile: name }, "Profile probe timed out without a verdict");
|
|
241
|
+
return null;
|
|
242
|
+
}
|
|
243
|
+
const classified = classifyProbeFailure(err);
|
|
244
|
+
return {
|
|
245
|
+
ok: false,
|
|
246
|
+
...classified,
|
|
247
|
+
...(classified.blame === "provider" ? { connection } : {}),
|
|
248
|
+
message: checkMessage({
|
|
249
|
+
...classified,
|
|
250
|
+
...(classified.blame === "provider" ? { connection } : {}),
|
|
251
|
+
}),
|
|
252
|
+
};
|
|
253
|
+
} finally {
|
|
254
|
+
timeout.cleanup();
|
|
255
|
+
}
|
|
256
|
+
}
|
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
} from "../platform-proxy/context.js";
|
|
18
18
|
import {
|
|
19
19
|
type Auth,
|
|
20
|
-
|
|
20
|
+
PROVIDERS_ALLOWING_CUSTOM_BASE_URL,
|
|
21
21
|
type ResolvedAuth,
|
|
22
22
|
} from "./auth.js";
|
|
23
23
|
|
|
@@ -36,14 +36,14 @@ export async function resolveAuth(
|
|
|
36
36
|
{ ok: true; resolved: ResolvedAuth } | { ok: false; error: ResolveAuthError }
|
|
37
37
|
> {
|
|
38
38
|
// Defense-in-depth: strip baseUrl for providers that should not accept one.
|
|
39
|
-
// The route layer rejects base_url for
|
|
40
|
-
//
|
|
41
|
-
//
|
|
39
|
+
// The route layer rejects base_url for those providers, but this guard
|
|
40
|
+
// catches any code path that bypasses route validation (e.g. corrupted DB
|
|
41
|
+
// rows, direct calls from internal code).
|
|
42
42
|
let safeBaseUrl = opts.baseUrl;
|
|
43
|
-
if (safeBaseUrl && !
|
|
43
|
+
if (safeBaseUrl && !PROVIDERS_ALLOWING_CUSTOM_BASE_URL.has(provider)) {
|
|
44
44
|
log.warn(
|
|
45
45
|
{ provider, baseUrl: safeBaseUrl },
|
|
46
|
-
`Stripping baseUrl for provider "${provider}"
|
|
46
|
+
`Stripping baseUrl for provider "${provider}": base_url is only valid for openai-compatible and ollama providers.`,
|
|
47
47
|
);
|
|
48
48
|
safeBaseUrl = null;
|
|
49
49
|
}
|
|
@@ -85,7 +85,7 @@ export async function resolveAuth(
|
|
|
85
85
|
|
|
86
86
|
case "none":
|
|
87
87
|
// Keyless endpoints still need their custom baseUrl threaded through
|
|
88
|
-
// (LM Studio / vLLM openai-compatible servers).
|
|
88
|
+
// (Ollama, LM Studio / vLLM openai-compatible servers).
|
|
89
89
|
return {
|
|
90
90
|
ok: true,
|
|
91
91
|
resolved: {
|
|
@@ -12,13 +12,21 @@
|
|
|
12
12
|
* can keep reading `block.source.data` exactly as before.
|
|
13
13
|
*
|
|
14
14
|
* Blocks that already carry base64 (a live, in-flight turn) pass through
|
|
15
|
-
* untouched
|
|
15
|
+
* untouched (no disk read) so only reloaded history pays the resolution cost,
|
|
16
16
|
* and only on its first send after reload. The walk is pure: it returns fresh
|
|
17
17
|
* block/message objects and never mutates the caller's in-memory history.
|
|
18
|
+
*
|
|
19
|
+
* The walk is also the last place an image block can be checked before a
|
|
20
|
+
* provider serializes it, so it is where an image the provider cannot decode
|
|
21
|
+
* is replaced by a text note: see {@link UNSENDABLE_IMAGE_FORMAT_NOTE}.
|
|
18
22
|
*/
|
|
19
23
|
|
|
20
24
|
import { optimizeImageForTransport } from "../agent/image-optimize.js";
|
|
21
25
|
import { getAttachmentContent } from "../persistence/attachments-store.js";
|
|
26
|
+
import {
|
|
27
|
+
heifImageMimeType,
|
|
28
|
+
sniffImageMimeType,
|
|
29
|
+
} from "../util/image-conversion.js";
|
|
22
30
|
import { getLogger } from "../util/logger.js";
|
|
23
31
|
import type {
|
|
24
32
|
Base64MediaSource,
|
|
@@ -43,6 +51,65 @@ export function mediaSourceBytes(source: MediaSource): Buffer | null {
|
|
|
43
51
|
return getAttachmentContent(source.attachmentId);
|
|
44
52
|
}
|
|
45
53
|
|
|
54
|
+
/**
|
|
55
|
+
* Whether an image source's bytes are not one of the formats providers accept,
|
|
56
|
+
* so {@link resolveMediaReferences} replaces the block with
|
|
57
|
+
* {@link UNSENDABLE_IMAGE_FORMAT_NOTE} instead of sending it. Applies the same
|
|
58
|
+
* rule as {@link resolveImageBlock}, including its treatment of a reference
|
|
59
|
+
* whose payload is gone: that is an unavailable attachment, not an unreadable
|
|
60
|
+
* format. Lets a caller holding a persisted block ask whether the model will
|
|
61
|
+
* see the image.
|
|
62
|
+
*
|
|
63
|
+
* Answers for the formats every provider reads, so HEIF counts as unsendable
|
|
64
|
+
* even though a Gemini turn ({@link MediaResolveOptions}) would carry it.
|
|
65
|
+
*/
|
|
66
|
+
export function isUnsendableImageSource(source: MediaSource): boolean {
|
|
67
|
+
const bytes =
|
|
68
|
+
source.type === "base64"
|
|
69
|
+
? imageHeadBytes(source.data)
|
|
70
|
+
: getAttachmentContent(source.attachmentId);
|
|
71
|
+
return bytes != null && sendableImageMimeType(bytes, {}) === null;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Media types a provider accepts for an image block beyond the four every
|
|
76
|
+
* provider reads (PNG, JPEG, GIF, WebP).
|
|
77
|
+
*
|
|
78
|
+
* HEIF is the one format where providers genuinely disagree: Gemini decodes
|
|
79
|
+
* `image/heic` and `image/heif` directly
|
|
80
|
+
* (https://ai.google.dev/gemini-api/docs/image-understanding), while
|
|
81
|
+
* OpenAI-compatible and Anthropic endpoints answer HTTP 400 for the whole
|
|
82
|
+
* request. HEIF bytes reach a provider whenever transcoding was unavailable
|
|
83
|
+
* (`convertImageToJpeg` is backed by macOS `sips`), so a Linux-hosted assistant
|
|
84
|
+
* on Gemini depends on them passing through.
|
|
85
|
+
*/
|
|
86
|
+
export interface MediaResolveOptions {
|
|
87
|
+
acceptsHeif?: boolean;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// 16 base64 chars decode to the 12 bytes the longest signature needs.
|
|
91
|
+
function imageHeadBytes(dataBase64: string): Buffer {
|
|
92
|
+
return Buffer.from(dataBase64.slice(0, 16), "base64");
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Media type the provider should be told an image is, or null when it cannot
|
|
97
|
+
* read the bytes at all.
|
|
98
|
+
*/
|
|
99
|
+
function sendableImageMimeType(
|
|
100
|
+
bytes: Uint8Array,
|
|
101
|
+
options: MediaResolveOptions,
|
|
102
|
+
): string | null {
|
|
103
|
+
const sniffed = sniffImageMimeType(bytes);
|
|
104
|
+
if (sniffed) {
|
|
105
|
+
return sniffed;
|
|
106
|
+
}
|
|
107
|
+
if (options.acceptsHeif) {
|
|
108
|
+
return heifImageMimeType(bytes);
|
|
109
|
+
}
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
|
|
46
113
|
/**
|
|
47
114
|
* Narrow a media source to its base64 arm. After {@link resolveMediaReferences}
|
|
48
115
|
* runs, provider transforms only ever see base64 sources; this asserts that
|
|
@@ -96,9 +163,57 @@ export function resolveMediaSourceData(
|
|
|
96
163
|
return { data: bytes.toString("base64"), media_type: source.media_type };
|
|
97
164
|
}
|
|
98
165
|
|
|
99
|
-
|
|
166
|
+
/**
|
|
167
|
+
* Note that replaces an image block whose bytes are not one of the formats
|
|
168
|
+
* providers accept (PNG, JPEG, GIF, WebP). Sending the block instead costs the
|
|
169
|
+
* user the whole turn: an OpenAI-compatible endpoint answers HTTP 400 ("The
|
|
170
|
+
* image data you provided does not represent a valid image") for the entire
|
|
171
|
+
* request, so one unreadable image in a batch of eight blocks the reply to all
|
|
172
|
+
* of them. The note keeps the turn alive and tells the model what it is
|
|
173
|
+
* missing, and the daemon posts a system card naming the file so the user
|
|
174
|
+
* knows too.
|
|
175
|
+
*/
|
|
176
|
+
export const UNSENDABLE_IMAGE_FORMAT_NOTE =
|
|
177
|
+
"[Image omitted: its format is not one the model can read (PNG, JPEG, GIF, and WebP are)]";
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Resolve an image block into one the provider can accept, keyed on the bytes
|
|
181
|
+
* rather than on the declared media type.
|
|
182
|
+
*
|
|
183
|
+
* The declaration is only a claim: web clients derive it from the filename
|
|
184
|
+
* extension, so a HEIC photo saved as `.png` arrives declared `image/png`. One
|
|
185
|
+
* sniff covers both failure modes that claim produces. Bytes that sniff as
|
|
186
|
+
* nothing the target provider reads are unsendable (a corrupt head, or a format
|
|
187
|
+
* outside both the universal four and the provider's own extras: AVIF, BMP,
|
|
188
|
+
* TIFF, SVG) and become {@link UNSENDABLE_IMAGE_FORMAT_NOTE}. Bytes that sniff
|
|
189
|
+
* as something other than the declaration are relabeled, since a mismatch is
|
|
190
|
+
* fatal on its own (Anthropic answers "image does not match the provided media
|
|
191
|
+
* type").
|
|
192
|
+
*
|
|
193
|
+
* The sniff runs ahead of the transport optimization so sendability cannot
|
|
194
|
+
* depend on an unrelated size threshold: optimization rewrites only the images
|
|
195
|
+
* it decides to rescale.
|
|
196
|
+
*/
|
|
197
|
+
async function resolveImageBlock(
|
|
198
|
+
block: ImageContent,
|
|
199
|
+
options: MediaResolveOptions,
|
|
200
|
+
): Promise<ContentBlock> {
|
|
100
201
|
if (block.source.type === "base64") {
|
|
101
|
-
|
|
202
|
+
const sniffed = sendableImageMimeType(
|
|
203
|
+
imageHeadBytes(block.source.data),
|
|
204
|
+
options,
|
|
205
|
+
);
|
|
206
|
+
if (!sniffed) {
|
|
207
|
+
log.warn(
|
|
208
|
+
{ declaredMediaType: block.source.media_type },
|
|
209
|
+
"Inline image is not in a provider-readable format; substituting a text note",
|
|
210
|
+
);
|
|
211
|
+
return { type: "text", text: UNSENDABLE_IMAGE_FORMAT_NOTE };
|
|
212
|
+
}
|
|
213
|
+
return {
|
|
214
|
+
type: "image",
|
|
215
|
+
source: { type: "base64", media_type: sniffed, data: block.source.data },
|
|
216
|
+
};
|
|
102
217
|
}
|
|
103
218
|
const bytes = getAttachmentContent(block.source.attachmentId);
|
|
104
219
|
if (!bytes) {
|
|
@@ -111,11 +226,22 @@ async function resolveImageBlock(block: ImageContent): Promise<ContentBlock> {
|
|
|
111
226
|
text: "[Attachment unavailable: image could not be loaded]",
|
|
112
227
|
};
|
|
113
228
|
}
|
|
229
|
+
const sniffed = sendableImageMimeType(bytes, options);
|
|
230
|
+
if (!sniffed) {
|
|
231
|
+
log.warn(
|
|
232
|
+
{
|
|
233
|
+
attachmentId: block.source.attachmentId,
|
|
234
|
+
declaredMediaType: block.source.media_type,
|
|
235
|
+
},
|
|
236
|
+
"Referenced image is not in a provider-readable format; substituting a text note",
|
|
237
|
+
);
|
|
238
|
+
return { type: "text", text: UNSENDABLE_IMAGE_FORMAT_NOTE };
|
|
239
|
+
}
|
|
114
240
|
// Apply the same transport optimization the inline-base64 path used, so a
|
|
115
241
|
// reloaded (reference) turn sends the model the same bytes a live turn would.
|
|
116
242
|
const { data, mediaType } = await optimizeImageForTransport(
|
|
117
243
|
bytes.toString("base64"),
|
|
118
|
-
|
|
244
|
+
sniffed,
|
|
119
245
|
);
|
|
120
246
|
return {
|
|
121
247
|
type: "image",
|
|
@@ -161,10 +287,13 @@ function resolveFileBlock(block: FileContent): ContentBlock {
|
|
|
161
287
|
};
|
|
162
288
|
}
|
|
163
289
|
|
|
164
|
-
async function resolveBlock(
|
|
290
|
+
async function resolveBlock(
|
|
291
|
+
block: ContentBlock,
|
|
292
|
+
options: MediaResolveOptions,
|
|
293
|
+
): Promise<ContentBlock> {
|
|
165
294
|
switch (block.type) {
|
|
166
295
|
case "image":
|
|
167
|
-
return resolveImageBlock(block);
|
|
296
|
+
return resolveImageBlock(block, options);
|
|
168
297
|
case "file":
|
|
169
298
|
return resolveFileBlock(block);
|
|
170
299
|
case "tool_result": {
|
|
@@ -174,7 +303,9 @@ async function resolveBlock(block: ContentBlock): Promise<ContentBlock> {
|
|
|
174
303
|
}
|
|
175
304
|
return {
|
|
176
305
|
...block,
|
|
177
|
-
contentBlocks: await Promise.all(
|
|
306
|
+
contentBlocks: await Promise.all(
|
|
307
|
+
block.contentBlocks.map((nested) => resolveBlock(nested, options)),
|
|
308
|
+
),
|
|
178
309
|
};
|
|
179
310
|
}
|
|
180
311
|
default:
|
|
@@ -182,13 +313,40 @@ async function resolveBlock(block: ContentBlock): Promise<ContentBlock> {
|
|
|
182
313
|
}
|
|
183
314
|
}
|
|
184
315
|
|
|
185
|
-
|
|
316
|
+
/**
|
|
317
|
+
* Whether an inline image needs rebuilding before it can be sent: its payload
|
|
318
|
+
* is unreadable, or its declared media type is not exactly what the bytes are.
|
|
319
|
+
* The comparison is exact so a non-canonical spelling of a real format is
|
|
320
|
+
* rewritten too (`image/jpg` is not in any provider's accepted set, and clients
|
|
321
|
+
* derive it from a `.jpg` extension). Sniffing here, rather than rebuilding
|
|
322
|
+
* every inline block unconditionally, keeps the clean live turn on the identity
|
|
323
|
+
* fast path below at the cost of decoding a 12-byte head per image.
|
|
324
|
+
*/
|
|
325
|
+
function base64ImageNeedsRewrite(
|
|
326
|
+
source: Base64MediaSource,
|
|
327
|
+
options: MediaResolveOptions,
|
|
328
|
+
): boolean {
|
|
329
|
+
return (
|
|
330
|
+
sendableImageMimeType(imageHeadBytes(source.data), options) !==
|
|
331
|
+
source.media_type
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
function contentNeedsResolution(
|
|
336
|
+
content: ContentBlock[],
|
|
337
|
+
options: MediaResolveOptions,
|
|
338
|
+
): boolean {
|
|
186
339
|
return content.some((block) => {
|
|
187
|
-
if (block.type === "image"
|
|
340
|
+
if (block.type === "image") {
|
|
341
|
+
return block.source.type === "workspace_ref"
|
|
342
|
+
? true
|
|
343
|
+
: base64ImageNeedsRewrite(block.source, options);
|
|
344
|
+
}
|
|
345
|
+
if (block.type === "file") {
|
|
188
346
|
return block.source.type === "workspace_ref";
|
|
189
347
|
}
|
|
190
348
|
if (block.type === "tool_result" && block.contentBlocks?.length) {
|
|
191
|
-
return
|
|
349
|
+
return contentNeedsResolution(block.contentBlocks, options);
|
|
192
350
|
}
|
|
193
351
|
return false;
|
|
194
352
|
});
|
|
@@ -196,21 +354,24 @@ function contentHasReference(content: ContentBlock[]): boolean {
|
|
|
196
354
|
|
|
197
355
|
/**
|
|
198
356
|
* Return a copy of `messages` with every {@link WorkspaceRefMediaSource}
|
|
199
|
-
* resolved to inline base64
|
|
200
|
-
*
|
|
201
|
-
* or disk I/O.
|
|
357
|
+
* resolved to inline base64 and every image block reconciled with its own
|
|
358
|
+
* bytes. Messages that need neither are returned unchanged (same object
|
|
359
|
+
* reference) so the common live turn does no allocation or disk I/O.
|
|
202
360
|
*/
|
|
203
361
|
export function resolveMediaReferences(
|
|
204
362
|
messages: Message[],
|
|
363
|
+
options: MediaResolveOptions = {},
|
|
205
364
|
): Promise<Message[]> {
|
|
206
365
|
return Promise.all(
|
|
207
366
|
messages.map(async (message) => {
|
|
208
|
-
if (!
|
|
367
|
+
if (!contentNeedsResolution(message.content, options)) {
|
|
209
368
|
return message;
|
|
210
369
|
}
|
|
211
370
|
return {
|
|
212
371
|
...message,
|
|
213
|
-
content: await Promise.all(
|
|
372
|
+
content: await Promise.all(
|
|
373
|
+
message.content.map((block) => resolveBlock(block, options)),
|
|
374
|
+
),
|
|
214
375
|
};
|
|
215
376
|
}),
|
|
216
377
|
);
|
|
@@ -2190,6 +2190,16 @@ export function catalogMaxOutputTokens(
|
|
|
2190
2190
|
)?.maxOutputTokens;
|
|
2191
2191
|
}
|
|
2192
2192
|
|
|
2193
|
+
/** The `contextWindowTokens` declared for a (provider, model) catalog entry, if any. */
|
|
2194
|
+
export function catalogContextWindowTokens(
|
|
2195
|
+
provider: string,
|
|
2196
|
+
modelId: string,
|
|
2197
|
+
): number | undefined {
|
|
2198
|
+
return PROVIDER_CATALOG.find((p) => p.id === provider)?.models.find(
|
|
2199
|
+
(m) => m.id === modelId,
|
|
2200
|
+
)?.contextWindowTokens;
|
|
2201
|
+
}
|
|
2202
|
+
|
|
2193
2203
|
/**
|
|
2194
2204
|
* Model IDs (across all catalog providers) flagged
|
|
2195
2205
|
* `supportsPromptCacheBreakpoints`. Consumed by the OpenAI Responses
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
import OpenAI from "openai";
|
|
4
4
|
|
|
5
5
|
import type { NormalizedOpenAIAPIError } from "../api-error-normalization.js";
|
|
6
6
|
import {
|
|
@@ -44,6 +44,22 @@ describe("normalizeOpenAIAPIError", () => {
|
|
|
44
44
|
);
|
|
45
45
|
});
|
|
46
46
|
|
|
47
|
+
test("stamps network_error for SDK connection failures", () => {
|
|
48
|
+
const cause = Object.assign(new Error("connect ECONNREFUSED 127.0.0.1:9"), {
|
|
49
|
+
code: "ECONNREFUSED",
|
|
50
|
+
});
|
|
51
|
+
const n = normalizeOpenAIAPIError(new OpenAI.APIConnectionError({ cause }));
|
|
52
|
+
expect(n.reason).toBe("network_error");
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test("does not stamp network_error for user aborts", () => {
|
|
56
|
+
// APIUserAbortError covers caller cancellation and inner stream
|
|
57
|
+
// deadlines; classifying it as transient would make retry re-run
|
|
58
|
+
// 30-minute deadline failures.
|
|
59
|
+
const n = normalizeOpenAIAPIError(new OpenAI.APIUserAbortError());
|
|
60
|
+
expect(n.reason).not.toBe("network_error");
|
|
61
|
+
});
|
|
62
|
+
|
|
47
63
|
test("extracts OpenAI-shaped error metadata", () => {
|
|
48
64
|
const n = normalizeOpenAIAPIError(
|
|
49
65
|
apiError(401),
|