@vellumai/assistant 0.6.4 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierignore +5 -0
- package/ARCHITECTURE.md +32 -36
- package/Dockerfile +12 -0
- package/README.md +3 -4
- package/bun.lock +8 -3
- package/docs/architecture/integrations.md +1 -20
- package/docs/architecture/security.md +16 -16
- package/docs/error-handling.md +111 -0
- package/docs/skills.md +10 -10
- package/docs/stt-provider-onboarding.md +2 -1
- package/knip.json +9 -2
- package/node_modules/@vellumai/ces-contracts/package.json +2 -1
- package/node_modules/@vellumai/ces-contracts/src/__tests__/trust-rules.test.ts +471 -0
- package/node_modules/@vellumai/ces-contracts/src/trust-rules.ts +398 -4
- package/node_modules/@vellumai/credential-storage/bun.lock +2 -2
- package/node_modules/@vellumai/credential-storage/package.json +2 -2
- package/node_modules/@vellumai/credential-storage/src/oauth-runtime.ts +20 -2
- package/node_modules/@vellumai/egress-proxy/bun.lock +2 -2
- package/node_modules/@vellumai/egress-proxy/package.json +2 -2
- package/openapi.yaml +123 -11
- package/package.json +6 -3
- package/scripts/generate-openapi.ts +50 -11
- package/src/__tests__/agent-loop-callsite-precedence.test.ts +318 -0
- package/src/__tests__/agent-loop-sentry-hygiene.test.ts +137 -0
- package/src/__tests__/agent-loop.test.ts +112 -1
- package/src/__tests__/anthropic-error-formatting.test.ts +98 -0
- package/src/__tests__/anthropic-provider.test.ts +171 -2
- package/src/__tests__/approval-cascade.test.ts +31 -10
- package/src/__tests__/approval-routes-http.test.ts +134 -10
- package/src/__tests__/assistant-attachments.test.ts +44 -0
- package/src/__tests__/assistant-feature-flags-integration.test.ts +29 -0
- package/src/__tests__/browser-fill-credential.test.ts +1 -1
- package/src/__tests__/browser-identifier-parity-guard.test.ts +53 -0
- package/src/__tests__/browser-skill-baseline-tool-payload.test.ts +23 -33
- package/src/__tests__/browser-skill-endstate.test.ts +51 -182
- package/src/__tests__/btw-routes.test.ts +47 -1
- package/src/__tests__/call-controller.test.ts +1 -2
- package/src/__tests__/call-site-routing-provider.test.ts +214 -0
- package/src/__tests__/catalog-cache.test.ts +27 -4
- package/src/__tests__/channel-approval-routes.test.ts +4 -4
- package/src/__tests__/channel-reply-delivery.test.ts +300 -2
- package/src/__tests__/checker.test.ts +428 -501
- package/src/__tests__/cli-command-risk-guard.test.ts +30 -33
- package/src/__tests__/compaction-circuit-breaker.test.ts +336 -0
- package/src/__tests__/compaction.benchmark.test.ts +1 -1
- package/src/__tests__/config-analysis.test.ts +11 -28
- package/src/__tests__/config-loader-backfill.test.ts +174 -0
- package/src/__tests__/config-loader-corrupt.test.ts +183 -0
- package/src/__tests__/config-loader-quarantine-bulletin.test.ts +202 -0
- package/src/__tests__/config-schema-cmd.test.ts +11 -5
- package/src/__tests__/config-schema.test.ts +427 -114
- package/src/__tests__/config-watcher.test.ts +2 -2
- package/src/__tests__/contact-store-user-file.test.ts +72 -73
- package/src/__tests__/contacts-write.test.ts +4 -4
- package/src/__tests__/context-token-estimator.test.ts +191 -1
- package/src/__tests__/context-window-manager.test.ts +530 -2
- package/src/__tests__/conversation-abort-tool-results.test.ts +30 -16
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +61 -17
- package/src/__tests__/conversation-agent-loop.test.ts +412 -82
- package/src/__tests__/conversation-attachments.test.ts +1 -1
- package/src/__tests__/conversation-confirmation-signals.test.ts +30 -9
- package/src/__tests__/conversation-error.test.ts +37 -6
- package/src/__tests__/conversation-history-web-search.test.ts +6 -0
- package/src/__tests__/conversation-init.benchmark.test.ts +36 -0
- package/src/__tests__/conversation-lifecycle.test.ts +336 -0
- package/src/__tests__/conversation-load-history-repair.test.ts +27 -10
- package/src/__tests__/conversation-pre-run-repair.test.ts +30 -16
- package/src/__tests__/conversation-process-callsite.test.ts +306 -0
- package/src/__tests__/conversation-provider-retry-repair.test.ts +30 -16
- package/src/__tests__/conversation-queue.test.ts +41 -26
- package/src/__tests__/conversation-routes-disk-view.test.ts +29 -1
- package/src/__tests__/conversation-routes-slash-commands.test.ts +31 -3
- package/src/__tests__/conversation-runtime-assembly.test.ts +2735 -55
- package/src/__tests__/conversation-runtime-workspace.test.ts +12 -12
- package/src/__tests__/conversation-skill-tools.test.ts +12 -146
- package/src/__tests__/conversation-slash-queue.test.ts +34 -19
- package/src/__tests__/conversation-slash-unknown.test.ts +30 -16
- package/src/__tests__/conversation-speed-override.test.ts +30 -11
- package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +1035 -0
- package/src/__tests__/conversation-surfaces-standalone.test.ts +630 -0
- package/src/__tests__/conversation-title-service.test.ts +2 -2
- package/src/__tests__/conversation-tool-setup-batch-authorized.test.ts +1 -1
- package/src/__tests__/conversation-unread-route.test.ts +2 -2
- package/src/__tests__/conversation-usage.test.ts +3 -1
- package/src/__tests__/conversation-workspace-cache-state.test.ts +31 -10
- package/src/__tests__/conversation-workspace-injection.test.ts +43 -15
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +44 -16
- package/src/__tests__/credential-broker-browser-fill.test.ts +110 -0
- package/src/__tests__/credential-security-invariants.test.ts +3 -0
- package/src/__tests__/credential-storage-oauth-compat.test.ts +18 -0
- package/src/__tests__/credential-storage-static-compat.test.ts +28 -0
- package/src/__tests__/credential-vault-unit.test.ts +135 -19
- package/src/__tests__/credentials-cli.test.ts +1 -9
- package/src/__tests__/cross-provider-web-search.test.ts +84 -0
- package/src/__tests__/daemon-server-persist-and-process-callsite.test.ts +92 -0
- package/src/__tests__/delete-propagation.test.ts +437 -0
- package/src/__tests__/dm-backfill.test.ts +417 -0
- package/src/__tests__/dm-persistence.test.ts +227 -0
- package/src/__tests__/edit-propagation.test.ts +280 -0
- package/src/__tests__/ephemeral-permissions.test.ts +93 -3
- package/src/__tests__/estimator-calibration-integration.test.ts +208 -0
- package/src/__tests__/estimator-calibration.test.ts +213 -0
- package/src/__tests__/extension-id-sync-guard.test.ts +26 -7
- package/src/__tests__/file-write-tool.test.ts +151 -1
- package/src/__tests__/filing-service.test.ts +255 -0
- package/src/__tests__/gemini-provider.test.ts +0 -3
- package/src/__tests__/guardian-grant-minting.test.ts +8 -0
- package/src/__tests__/headless-browser-interactions.test.ts +1 -1
- package/src/__tests__/heartbeat-service.test.ts +96 -15
- package/src/__tests__/host-shell-tool.test.ts +124 -18
- package/src/__tests__/http-user-message-parity.test.ts +29 -1
- package/src/__tests__/inbound-slack-persistence.test.ts +340 -0
- package/src/__tests__/intent-routing.test.ts +1 -40
- package/src/__tests__/llm-catalog-parity.test.ts +174 -0
- package/src/__tests__/llm-context-normalization.test.ts +121 -0
- package/src/__tests__/llm-resolver.test.ts +214 -0
- package/src/__tests__/llm-schema.test.ts +223 -0
- package/src/__tests__/managed-proxy-context.test.ts +6 -2
- package/src/__tests__/messaging-skill-split.test.ts +3 -34
- package/src/__tests__/migration-import-from-url.test.ts +684 -0
- package/src/__tests__/model-intents.test.ts +9 -83
- package/src/__tests__/notification-decision-fallback.test.ts +0 -10
- package/src/__tests__/notification-decision-identity.test.ts +0 -9
- package/src/__tests__/notification-decision-recipient-context.test.ts +0 -9
- package/src/__tests__/oauth-store.test.ts +10 -7
- package/src/__tests__/oauth2-gateway-transport.test.ts +8 -3
- package/src/__tests__/oauth2-refresh-retry.test.ts +279 -0
- package/src/__tests__/openai-provider.test.ts +7 -0
- package/src/__tests__/openai-responses-provider.test.ts +396 -0
- package/src/__tests__/openrouter-provider-only.test.ts +135 -0
- package/src/__tests__/outbound-slack-persistence.test.ts +293 -0
- package/src/__tests__/permission-checker-host-gate.test.ts +1 -1
- package/src/__tests__/permission-mode.test.ts +16 -0
- package/src/__tests__/permission-types.test.ts +0 -1
- package/src/__tests__/persona-resolver.test.ts +13 -13
- package/src/__tests__/pkb-autoinject.test.ts +37 -1
- package/src/__tests__/platform-bash-auto-approve.test.ts +1 -1
- package/src/__tests__/pricing.test.ts +50 -3
- package/src/__tests__/profiler-routes.test.ts +1 -1
- package/src/__tests__/provider-commit-message-generator.test.ts +14 -84
- package/src/__tests__/provider-env-vars-scope.test.ts +52 -0
- package/src/__tests__/provider-error-scenarios.test.ts +135 -6
- package/src/__tests__/provider-managed-proxy-integration.test.ts +42 -11
- package/src/__tests__/provider-registry-ollama.test.ts +1 -2
- package/src/__tests__/proxy-approval-callback.test.ts +0 -1
- package/src/__tests__/reaction-persistence.test.ts +560 -0
- package/src/__tests__/relay-server.test.ts +1 -1
- package/src/__tests__/require-fresh-approval.test.ts +1 -1
- package/src/__tests__/retry-openrouter-only-normalization.test.ts +136 -0
- package/src/__tests__/retry-thinking-tool-choice.test.ts +226 -0
- package/src/__tests__/risk-classifier-parity.test.ts +230 -0
- package/src/__tests__/sanitize-config-for-transfer.test.ts +78 -1
- package/src/__tests__/secret-ingress-http.test.ts +28 -0
- package/src/__tests__/secret-prompter-channel-fallback.test.ts +125 -0
- package/src/__tests__/secret-routes-managed-proxy.test.ts +2 -3
- package/src/__tests__/secret-scanner-executor.test.ts +1 -1
- package/src/__tests__/send-endpoint-busy.test.ts +29 -1
- package/src/__tests__/server-history-render.test.ts +31 -0
- package/src/__tests__/shell-parser-property.test.ts +13 -13
- package/src/__tests__/skill-cache-store.test.ts +182 -0
- package/src/__tests__/skills.test.ts +19 -33
- package/src/__tests__/slack-app-setup-skill-regression.test.ts +3 -1
- package/src/__tests__/slack-skill.test.ts +3 -8
- package/src/__tests__/starter-bundle.test.ts +35 -0
- package/src/__tests__/subagent-call-site-routing.test.ts +280 -0
- package/src/__tests__/suggestion-routes.test.ts +160 -3
- package/src/__tests__/system-prompt.test.ts +22 -35
- package/src/__tests__/task-runner.test.ts +3 -1
- package/src/__tests__/tcc-sandbox-deny.test.ts +198 -0
- package/src/__tests__/terminal-tools.test.ts +8 -0
- package/src/__tests__/test-support/browser-skill-harness.ts +2 -52
- package/src/__tests__/thread-backfill.test.ts +941 -0
- package/src/__tests__/tool-execution-pipeline.benchmark.test.ts +2 -2
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +2 -2
- package/src/__tests__/tool-executor.test.ts +60 -94
- package/src/__tests__/trust-store.test.ts +442 -109
- package/src/__tests__/update-bulletin-job.test.ts +389 -0
- package/src/__tests__/usage-cache-backfill-migration.test.ts +3 -1
- package/src/__tests__/verification-control-plane-policy.test.ts +1 -22
- package/src/__tests__/voice-session-bridge.test.ts +39 -0
- package/src/__tests__/volume-security-guard.test.ts +3 -2
- package/src/__tests__/web-search-history.test.ts +337 -0
- package/src/__tests__/workspace-migration-039-drop-legacy-llm-keys.test.ts +343 -0
- package/src/__tests__/workspace-migration-043-release-notes-latex-rendering.test.ts +202 -0
- package/src/__tests__/workspace-migration-045-release-notes-meet-avatar.test.ts +210 -0
- package/src/__tests__/workspace-migration-drop-user-md.test.ts +11 -11
- package/src/__tests__/workspace-migration-unify-llm-callsite-configs.test.ts +841 -0
- package/src/__tests__/workspace-policy.test.ts +1 -13
- package/src/acp/client-handler.ts +1 -2
- package/src/agent/loop.ts +209 -17
- package/src/avatar/resvg-lazy.test.ts +136 -0
- package/src/avatar/resvg-lazy.ts +82 -9
- package/src/avatar/traits-png-sync.ts +21 -1
- package/src/browser/__tests__/operations.test.ts +163 -0
- package/src/browser/identifiers.ts +51 -0
- package/src/browser/operations.ts +660 -0
- package/src/browser/types.ts +81 -0
- package/src/calls/guardian-question-copy.ts +2 -2
- package/src/calls/telephony-stt-routing.ts +1 -1
- package/src/calls/voice-session-bridge.ts +1 -0
- package/src/cli/AGENTS.md +1 -1
- package/src/cli/commands/__tests__/attachment.test.ts +438 -0
- package/src/cli/commands/__tests__/browser.test.ts +554 -0
- package/src/cli/commands/__tests__/cache.test.ts +623 -0
- package/src/cli/commands/__tests__/email-list.test.ts +6 -0
- package/src/cli/commands/__tests__/email-send.test.ts +93 -1
- package/src/cli/commands/__tests__/image-generation.test.ts +666 -0
- package/src/cli/commands/__tests__/inference-send.test.ts +451 -0
- package/src/cli/commands/__tests__/stt-transcribe.test.ts +454 -0
- package/src/cli/commands/__tests__/task.test.ts +913 -0
- package/src/cli/commands/__tests__/tts-synthesize.test.ts +594 -0
- package/src/cli/commands/__tests__/ui-confirm.test.ts +650 -0
- package/src/cli/commands/__tests__/ui.test.ts +1215 -0
- package/src/cli/commands/__tests__/watchers.test.ts +716 -0
- package/src/cli/commands/attachment.ts +182 -0
- package/src/cli/commands/browser.ts +350 -0
- package/src/cli/commands/cache.ts +341 -0
- package/src/cli/commands/completions.ts +0 -3
- package/src/cli/commands/config.ts +6 -6
- package/src/cli/commands/conversations-import.ts +347 -0
- package/src/cli/commands/conversations.ts +14 -1
- package/src/cli/commands/email.ts +234 -194
- package/src/cli/commands/image-generation.ts +300 -0
- package/src/cli/commands/inference.ts +200 -0
- package/src/cli/commands/memory.ts +127 -17
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +0 -1
- package/src/cli/commands/platform/__tests__/connect.test.ts +0 -1
- package/src/cli/commands/platform/__tests__/disconnect.test.ts +0 -1
- package/src/cli/commands/platform/__tests__/status.test.ts +0 -1
- package/src/cli/commands/stt.ts +339 -0
- package/src/cli/commands/task.ts +795 -0
- package/src/cli/commands/trust.ts +50 -19
- package/src/cli/commands/tts.ts +273 -0
- package/src/cli/commands/ui.ts +670 -0
- package/src/cli/commands/watchers.ts +509 -0
- package/src/cli/lib/daemon-credential-client.ts +0 -19
- package/src/cli/program.ts +23 -4
- package/src/cli.ts +0 -37
- package/src/config/bundled-skills/conversations/tools/rename-conversation.ts +23 -1
- package/src/config/bundled-skills/media-processing/services/reduce.ts +1 -1
- package/src/config/bundled-skills/messaging/SKILL.md +2 -2
- package/src/config/bundled-skills/messaging/TOOLS.json +4 -0
- package/src/config/bundled-skills/messaging/tools/messaging-archive-by-sender.ts +8 -1
- package/src/config/bundled-skills/messaging/tools/messaging-read.ts +15 -1
- package/src/config/bundled-skills/messaging/tools/messaging-search.ts +21 -1
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +11 -12
- package/src/config/bundled-skills/phone-calls/references/CONFIG.md +9 -8
- package/src/config/bundled-skills/settings/TOOLS.json +3 -3
- package/src/config/bundled-tool-registry.ts +0 -175
- package/src/config/env.ts +7 -2
- package/src/config/feature-flag-registry.json +25 -9
- package/src/config/llm-resolver.ts +128 -0
- package/src/config/loader.ts +194 -10
- package/src/config/raw-config-utils.ts +30 -2
- package/src/config/sanitize-for-transfer.ts +35 -0
- package/src/config/schema.ts +30 -41
- package/src/config/schemas/analysis.ts +3 -22
- package/src/config/schemas/calls.ts +0 -4
- package/src/config/schemas/filing.ts +2 -7
- package/src/config/schemas/heartbeat.ts +0 -5
- package/src/config/schemas/inference.ts +3 -23
- package/src/config/schemas/llm.ts +318 -0
- package/src/config/schemas/memory-processing.ts +1 -9
- package/src/config/schemas/notifications.ts +4 -11
- package/src/config/schemas/platform.ts +3 -9
- package/src/config/schemas/security.ts +33 -0
- package/src/config/schemas/services.ts +9 -4
- package/src/config/schemas/stt.ts +1 -0
- package/src/config/schemas/tts.ts +53 -0
- package/src/config/schemas/updates.ts +1 -1
- package/src/config/schemas/workspace-git.ts +3 -40
- package/src/config/skills.ts +2 -2
- package/src/context/__tests__/compact-prompt.test.ts +45 -0
- package/src/context/__tests__/microcompact.test.ts +805 -0
- package/src/context/estimator-calibration.ts +136 -0
- package/src/context/microcompact.ts +443 -0
- package/src/context/prompts/compact.md +12 -0
- package/src/context/token-estimator.ts +61 -3
- package/src/context/window-manager.ts +229 -25
- package/src/credential-execution/approval-bridge.ts +0 -1
- package/src/credential-execution/executable-discovery.ts +19 -8
- package/src/credential-execution/process-manager.test.ts +109 -0
- package/src/credential-execution/process-manager.ts +65 -2
- package/src/daemon/approval-generators.ts +29 -4
- package/src/daemon/assistant-attachments.ts +24 -13
- package/src/daemon/classifier.ts +2 -2
- package/src/daemon/config-watcher.ts +0 -1
- package/src/daemon/context-overflow-reducer.ts +4 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +79 -12
- package/src/daemon/conversation-agent-loop.ts +462 -80
- package/src/daemon/conversation-attachments.ts +2 -6
- package/src/daemon/conversation-error.ts +36 -1
- package/src/daemon/conversation-lifecycle.ts +30 -6
- package/src/daemon/conversation-messaging.ts +73 -4
- package/src/daemon/conversation-process.ts +10 -4
- package/src/daemon/conversation-queue-manager.ts +3 -0
- package/src/daemon/conversation-runtime-assembly.ts +760 -29
- package/src/daemon/conversation-slash.ts +2 -2
- package/src/daemon/conversation-surfaces.ts +389 -1
- package/src/daemon/conversation-tool-setup.ts +10 -5
- package/src/daemon/conversation-usage.ts +1 -1
- package/src/daemon/conversation.ts +118 -30
- package/src/daemon/external-skills-bootstrap.ts +41 -0
- package/src/daemon/guardian-action-generators.ts +34 -14
- package/src/daemon/handlers/config-model.test.ts +86 -0
- package/src/daemon/handlers/config-model.ts +54 -12
- package/src/daemon/handlers/conversations.ts +9 -2
- package/src/daemon/handlers/shared.ts +39 -11
- package/src/daemon/handlers/skills.ts +2 -2
- package/src/daemon/handlers/slack-channel-oauth-install.ts +197 -0
- package/src/daemon/lifecycle.ts +76 -14
- package/src/daemon/message-types/conversations.ts +14 -0
- package/src/daemon/message-types/messages.ts +9 -1
- package/src/daemon/message-types/trust.ts +0 -2
- package/src/daemon/parse-actual-tokens-from-error.test.ts +57 -1
- package/src/daemon/parse-actual-tokens-from-error.ts +66 -0
- package/src/daemon/pkb-context-tracker.test.ts +169 -0
- package/src/daemon/pkb-context-tracker.ts +125 -0
- package/src/daemon/pkb-reminder-builder.test.ts +70 -0
- package/src/daemon/pkb-reminder-builder.ts +31 -0
- package/src/daemon/providers-setup.ts +6 -0
- package/src/daemon/server.ts +117 -9
- package/src/daemon/tool-side-effects.ts +0 -9
- package/src/daemon/watch-handler.ts +4 -4
- package/src/daemon/web-search-history.ts +126 -0
- package/src/events/domain-events.ts +0 -1
- package/src/filing/filing-service.ts +9 -10
- package/src/heartbeat/heartbeat-service.ts +76 -28
- package/src/home/__tests__/feed-scheduler.test.ts +39 -11
- package/src/home/__tests__/rollup-producer.test.ts +44 -0
- package/src/home/assistant-feed-authoring.ts +4 -0
- package/src/home/emit-feed-event.ts +4 -0
- package/src/home/feed-scheduler.ts +20 -4
- package/src/home/feed-types.ts +56 -2
- package/src/home/relationship-state-writer.ts +2 -2
- package/src/home/rollup-producer.ts +34 -5
- package/src/home/suggested-prompts.ts +101 -0
- package/src/ipc/__tests__/attachment-ipc.test.ts +213 -0
- package/src/ipc/__tests__/browser-ipc.test.ts +339 -0
- package/src/ipc/__tests__/cache-ipc.test.ts +266 -0
- package/src/ipc/__tests__/socket-path.test.ts +73 -0
- package/src/ipc/__tests__/task-ipc.test.ts +577 -0
- package/src/ipc/__tests__/ui-request-route.test.ts +495 -0
- package/src/ipc/__tests__/watcher-ipc.test.ts +295 -0
- package/src/ipc/cli-client.ts +2 -1
- package/src/ipc/cli-server.ts +26 -8
- package/src/ipc/gateway-client.ts +4 -4
- package/src/ipc/routes/attachment.ts +114 -0
- package/src/ipc/routes/browser-context.ts +61 -0
- package/src/ipc/routes/browser.ts +96 -0
- package/src/ipc/routes/cache.ts +96 -0
- package/src/ipc/routes/index.ts +17 -1
- package/src/ipc/routes/task-queue.ts +226 -0
- package/src/ipc/routes/task.ts +173 -0
- package/src/ipc/routes/ui-request.ts +50 -0
- package/src/ipc/routes/watcher.ts +203 -0
- package/src/ipc/socket-path.ts +100 -0
- package/src/memory/__tests__/conversation-analyze-job.test.ts +9 -8
- package/src/memory/__tests__/conversation-group-migration.test.ts +99 -0
- package/src/memory/admin.ts +18 -0
- package/src/memory/conversation-analyze-job.ts +14 -13
- package/src/memory/conversation-attention-store.ts +13 -6
- package/src/memory/conversation-crud.ts +103 -3
- package/src/memory/conversation-group-migration.ts +38 -6
- package/src/memory/conversation-title-service.ts +7 -4
- package/src/memory/db-init.ts +2 -0
- package/src/memory/embedding-backend.ts +1 -1
- package/src/memory/graph/compaction.ts +299 -0
- package/src/memory/graph/consolidation.ts +4 -4
- package/src/memory/graph/conversation-graph-memory.ts +89 -29
- package/src/memory/graph/extraction.test.ts +272 -2
- package/src/memory/graph/extraction.ts +173 -51
- package/src/memory/graph/graph-search.test.ts +92 -0
- package/src/memory/graph/graph-search.ts +4 -1
- package/src/memory/graph/narrative.ts +2 -2
- package/src/memory/graph/pattern-scan.ts +2 -2
- package/src/memory/graph/retriever.test.ts +459 -0
- package/src/memory/graph/retriever.ts +230 -48
- package/src/memory/graph/store.ts +41 -0
- package/src/memory/graph/tool-handlers.ts +27 -0
- package/src/memory/graph/tools.ts +6 -1
- package/src/memory/indexer.ts +5 -5
- package/src/memory/job-handlers/conversation-starters.ts +23 -20
- package/src/memory/job-handlers/summarization.ts +2 -2
- package/src/memory/job-utils.ts +7 -1
- package/src/memory/jobs/embed-pkb-file.test.ts +168 -0
- package/src/memory/jobs/embed-pkb-file.ts +54 -0
- package/src/memory/jobs-store.ts +44 -3
- package/src/memory/jobs-worker.ts +4 -0
- package/src/memory/migrations/140-backfill-usage-cache-accounting.ts +1 -1
- package/src/memory/migrations/220-normalize-user-file-by-principal.ts +2 -2
- package/src/memory/migrations/222-strip-placeholder-sentinels-from-messages.ts +82 -0
- package/src/memory/migrations/index.ts +1 -0
- package/src/memory/pkb/pkb-index.test.ts +368 -0
- package/src/memory/pkb/pkb-index.ts +255 -0
- package/src/memory/pkb/pkb-reconcile.test.ts +251 -0
- package/src/memory/pkb/pkb-reconcile.ts +148 -0
- package/src/memory/pkb/pkb-search.test.ts +438 -0
- package/src/memory/pkb/pkb-search.ts +137 -0
- package/src/memory/pkb/types.ts +53 -0
- package/src/memory/qdrant-client.ts +122 -1
- package/src/memory/slack-thread-store.ts +37 -0
- package/src/messaging/providers/gmail/adapter.ts +6 -16
- package/src/messaging/providers/gmail/client.ts +22 -0
- package/src/messaging/providers/gmail/types.ts +7 -0
- package/src/messaging/providers/slack/adapter.ts +14 -2
- package/src/messaging/providers/slack/backfill.test.ts +257 -0
- package/src/messaging/providers/slack/backfill.ts +101 -0
- package/src/messaging/providers/slack/message-metadata.test.ts +316 -0
- package/src/messaging/providers/slack/message-metadata.ts +123 -0
- package/src/messaging/providers/slack/render-transcript.test.ts +1373 -0
- package/src/messaging/providers/slack/render-transcript.ts +443 -0
- package/src/messaging/style-analyzer.ts +5 -2
- package/src/notifications/README.md +9 -5
- package/src/notifications/decision-engine.ts +3 -9
- package/src/notifications/preference-extractor.ts +2 -6
- package/src/oauth/oauth-store.ts +1 -0
- package/src/oauth/platform-connection.test.ts +47 -0
- package/src/oauth/platform-connection.ts +15 -5
- package/src/oauth/seed-providers.ts +4 -2
- package/src/permissions/approval-policy.test.ts +948 -0
- package/src/permissions/approval-policy.ts +257 -0
- package/src/permissions/bash-risk-classifier.test.ts +1208 -0
- package/src/permissions/bash-risk-classifier.ts +707 -0
- package/src/permissions/checker.ts +217 -708
- package/src/permissions/command-registry.test.ts +535 -0
- package/src/permissions/command-registry.ts +825 -0
- package/src/permissions/defaults.ts +26 -78
- package/src/permissions/file-risk-classifier.test.ts +535 -0
- package/src/permissions/file-risk-classifier.ts +274 -0
- package/src/permissions/risk-types.ts +205 -0
- package/src/permissions/secret-prompter.ts +53 -2
- package/src/permissions/skill-risk-classifier.test.ts +311 -0
- package/src/permissions/skill-risk-classifier.ts +214 -0
- package/src/permissions/trust-client.ts +52 -25
- package/src/permissions/trust-store-interface.ts +1 -6
- package/src/permissions/trust-store.ts +161 -62
- package/src/permissions/types.ts +23 -14
- package/src/permissions/web-risk-classifier.test.ts +170 -0
- package/src/permissions/web-risk-classifier.ts +89 -0
- package/src/permissions/workspace-policy.ts +1 -16
- package/src/platform/client.ts +19 -1
- package/src/prompts/persona-resolver.ts +3 -3
- package/src/prompts/system-prompt.ts +19 -20
- package/src/prompts/templates/SOUL.md +2 -2
- package/src/prompts/update-bulletin-job.ts +190 -0
- package/src/providers/__tests__/context-overflow-error.test.ts +328 -0
- package/src/providers/__tests__/provider-env-vars.test.ts +102 -0
- package/src/providers/__tests__/retry-callsite.test.ts +424 -0
- package/src/providers/anthropic/client.ts +183 -14
- package/src/providers/call-site-routing.ts +71 -0
- package/src/providers/gemini/client.ts +65 -2
- package/src/providers/managed-proxy/constants.ts +2 -1
- package/src/providers/model-catalog.ts +501 -33
- package/src/providers/model-intents.ts +4 -4
- package/src/providers/openai/chat-completions-provider.ts +57 -1
- package/src/providers/openai/responses-provider.ts +86 -9
- package/src/providers/openrouter/client.ts +76 -9
- package/src/providers/provider-env-vars.ts +56 -0
- package/src/providers/provider-send-message.ts +22 -5
- package/src/providers/ratelimit.ts +4 -0
- package/src/providers/registry.ts +19 -8
- package/src/providers/retry.ts +174 -39
- package/src/providers/speech-to-text/__tests__/resolve.test.ts +55 -0
- package/src/providers/speech-to-text/google-gemini-live-stream.ts +4 -4
- package/src/providers/speech-to-text/provider-catalog.ts +17 -0
- package/src/providers/speech-to-text/resolve.ts +7 -0
- package/src/providers/speech-to-text/xai-realtime.test.ts +578 -0
- package/src/providers/speech-to-text/xai-realtime.ts +796 -0
- package/src/providers/speech-to-text/xai.test.ts +155 -0
- package/src/providers/speech-to-text/xai.ts +97 -0
- package/src/providers/types.ts +93 -3
- package/src/runtime/AGENTS.md +2 -2
- package/src/runtime/__tests__/agent-wake.test.ts +43 -2
- package/src/runtime/__tests__/interactive-ui.test.ts +673 -0
- package/src/runtime/agent-wake.ts +63 -22
- package/src/runtime/auth/route-policy.ts +4 -0
- package/src/runtime/btw-sidechain.ts +13 -3
- package/src/runtime/channel-reply-delivery.ts +106 -2
- package/src/runtime/decision-token.ts +116 -0
- package/src/runtime/gateway-client.ts +2 -2
- package/src/runtime/http-router.ts +32 -0
- package/src/runtime/http-server.ts +52 -1
- package/src/runtime/http-types.ts +23 -1
- package/src/runtime/interactive-ui.ts +362 -0
- package/src/runtime/invite-instruction-generator.ts +2 -2
- package/src/runtime/migrations/__tests__/gcs-signed-url.test.ts +176 -0
- package/src/runtime/migrations/__tests__/vbundle-metadata-merge-integration.test.ts +390 -0
- package/src/runtime/migrations/__tests__/vbundle-metadata-merge.test.ts +221 -0
- package/src/runtime/migrations/__tests__/vbundle-streaming-importer.test.ts +1540 -0
- package/src/runtime/migrations/__tests__/vbundle-streaming-validator.test.ts +453 -0
- package/src/runtime/migrations/__tests__/vbundle-tar-stream.test.ts +222 -0
- package/src/runtime/migrations/gcs-signed-url.ts +162 -0
- package/src/runtime/migrations/vbundle-importer.ts +154 -9
- package/src/runtime/migrations/vbundle-metadata-merge.ts +124 -0
- package/src/runtime/migrations/vbundle-streaming-importer.ts +2522 -0
- package/src/runtime/migrations/vbundle-streaming-validator.ts +244 -0
- package/src/runtime/migrations/vbundle-tar-stream.ts +217 -0
- package/src/runtime/migrations/vbundle-validator.ts +15 -6
- package/src/runtime/routes/__tests__/home-feed-routes.test.ts +111 -0
- package/src/runtime/routes/__tests__/migration-import-credential-filter.test.ts +114 -75
- package/src/runtime/routes/__tests__/migration-vellum-metadata-reconcile.test.ts +246 -0
- package/src/runtime/routes/approval-prompt-ts-tracker.ts +58 -0
- package/src/runtime/routes/approval-routes.ts +12 -17
- package/src/runtime/routes/approval-strategies/guardian-callback-strategy.ts +9 -0
- package/src/runtime/routes/avatar-routes.ts +20 -4
- package/src/runtime/routes/btw-routes.ts +1 -4
- package/src/runtime/routes/conversation-management-routes.ts +20 -2
- package/src/runtime/routes/conversation-routes.ts +133 -27
- package/src/runtime/routes/debug-routes.ts +1 -1
- package/src/runtime/routes/diagnostics-routes.ts +6 -4
- package/src/runtime/routes/events-routes.ts +16 -0
- package/src/runtime/routes/guardian-approval-interception.ts +33 -3
- package/src/runtime/routes/guardian-approval-prompt.ts +13 -3
- package/src/runtime/routes/home-feed-routes.ts +120 -2
- package/src/runtime/routes/inbound-message-handler.ts +912 -2
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +113 -2
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +61 -3
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +129 -6
- package/src/runtime/routes/integrations/slack/channel.ts +25 -3
- package/src/runtime/routes/llm-context-normalization.ts +23 -1
- package/src/runtime/routes/migration-routes.ts +720 -124
- package/src/runtime/routes/settings-routes.ts +4 -2
- package/src/runtime/routes/trust-rules-routes.ts +30 -14
- package/src/runtime/routes/work-items-routes.test.ts +1 -1
- package/src/runtime/routes/work-items-routes.ts +3 -2
- package/src/runtime/services/__tests__/analyze-conversation.test.ts +25 -43
- package/src/runtime/services/analyze-conversation.ts +12 -16
- package/src/runtime/skill-route-registry.ts +28 -6
- package/src/schedule/scheduler.ts +8 -0
- package/src/security/__tests__/provider-key-env-fallback.test.ts +119 -0
- package/src/security/__tests__/untrusted-content.test.ts +109 -0
- package/src/security/oauth2.ts +98 -35
- package/src/security/secure-keys.ts +7 -8
- package/src/security/token-manager.ts +27 -13
- package/src/security/untrusted-content.ts +102 -0
- package/src/skills/catalog-cache.ts +26 -7
- package/src/skills/catalog-install.ts +31 -3
- package/src/skills/skill-cache-store.ts +97 -0
- package/src/stt/__tests__/daemon-batch-transcriber.test.ts +76 -0
- package/src/stt/daemon-batch-transcriber.ts +33 -0
- package/src/stt/stt-stream-session.ts +8 -1
- package/src/stt/types.ts +5 -1
- package/src/subagent/manager.ts +41 -13
- package/src/tasks/ephemeral-permissions.ts +9 -4
- package/src/telemetry/usage-telemetry-reporter.ts +27 -5
- package/src/tools/browser/__tests__/browser-status.test.ts +45 -2
- package/src/tools/browser/browser-execution.ts +65 -38
- package/src/tools/browser/cdp-client/cdp-inspect/discovery.ts +22 -0
- package/src/tools/credentials/tool-policy.ts +39 -5
- package/src/tools/credentials/vault.ts +9 -4
- package/src/tools/executor.ts +4 -0
- package/src/tools/filesystem/write.ts +52 -0
- package/src/tools/host-terminal/host-shell.ts +45 -5
- package/src/tools/memory/register.test.ts +185 -0
- package/src/tools/memory/register.ts +3 -1
- package/src/tools/network/web-fetch.ts +20 -10
- package/src/tools/network/web-search.ts +19 -4
- package/src/tools/permission-checker.ts +36 -15
- package/src/tools/policy-context.ts +25 -8
- package/src/tools/registry.ts +55 -3
- package/src/tools/side-effects.ts +0 -11
- package/src/tools/skills/execute.ts +2 -2
- package/src/tools/skills/sandbox-runner.ts +5 -2
- package/src/tools/terminal/backends/native.ts +51 -2
- package/src/tools/terminal/safe-env.ts +3 -2
- package/src/tools/terminal/shell.ts +1 -0
- package/src/tools/tool-manifest.ts +6 -21
- package/src/tools/types.ts +12 -3
- package/src/tools/verification-control-plane-policy.ts +1 -1
- package/src/tts/__tests__/provider-adapters.test.ts +240 -13
- package/src/tts/provider-catalog.ts +18 -0
- package/src/tts/providers/index.ts +2 -0
- package/src/tts/providers/xai-provider.ts +224 -0
- package/src/tts/types.ts +46 -0
- package/src/types/tar-stream.d.ts +66 -0
- package/src/util/json.ts +17 -0
- package/src/util/platform.ts +2 -2
- package/src/util/pricing.ts +15 -5
- package/src/watcher/engine.ts +1 -1
- package/src/watcher/providers/google-calendar.ts +134 -8
- package/src/watcher/providers/outlook-calendar.ts +42 -2
- package/src/workspace/git-service.ts +23 -4
- package/src/workspace/migrations/038-unify-llm-callsite-configs.ts +516 -0
- package/src/workspace/migrations/039-drop-legacy-llm-keys.ts +171 -0
- package/src/workspace/migrations/040-seed-latency-callsite-defaults.ts +154 -0
- package/src/workspace/migrations/041-backfill-google-gmail-settings-scope.ts +57 -0
- package/src/workspace/migrations/042-fix-backfill-google-gmail-settings-scope.ts +70 -0
- package/src/workspace/migrations/043-release-notes-latex-rendering.ts +75 -0
- package/src/workspace/migrations/044-bump-stale-provider-stream-timeout.ts +51 -0
- package/src/workspace/migrations/045-release-notes-meet-avatar.ts +130 -0
- package/src/workspace/migrations/AGENTS.md +1 -1
- package/src/workspace/migrations/registry.ts +16 -0
- package/src/workspace/provider-commit-message-generator.ts +19 -38
- package/src/__tests__/gmail-archive-fallback.test.ts +0 -193
- package/src/__tests__/gmail-archive-gate.test.ts +0 -246
- package/src/__tests__/gmail-preferences.test.ts +0 -117
- package/src/__tests__/outlook-attachments.test.ts +0 -301
- package/src/__tests__/outlook-automation-tools.test.ts +0 -425
- package/src/__tests__/outlook-categories.test.ts +0 -212
- package/src/__tests__/outlook-compose-tools.test.ts +0 -325
- package/src/__tests__/outlook-declutter-tools.test.ts +0 -585
- package/src/__tests__/outlook-follow-up.test.ts +0 -196
- package/src/__tests__/outlook-trash.test.ts +0 -77
- package/src/__tests__/outlook-unsubscribe.test.ts +0 -279
- package/src/__tests__/update-bulletin-format.test.ts +0 -181
- package/src/__tests__/update-bulletin-state.test.ts +0 -135
- package/src/__tests__/update-bulletin.test.ts +0 -478
- package/src/__tests__/update-template-contract.test.ts +0 -29
- package/src/cli/commands/doctor.ts +0 -341
- package/src/config/bundled-skills/browser/SKILL.md +0 -88
- package/src/config/bundled-skills/browser/TOOLS.json +0 -516
- package/src/config/bundled-skills/browser/tools/browser-attach.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-click.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-close.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-detach.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-extract.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-fill-credential.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-hover.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-navigate.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-press-key.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-screenshot.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-scroll.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-select-option.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-snapshot.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-status.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-type.ts +0 -12
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +0 -49
- package/src/config/bundled-skills/browser/tools/browser-wait-for.ts +0 -12
- package/src/config/bundled-skills/chatgpt-import/SKILL.md +0 -27
- package/src/config/bundled-skills/chatgpt-import/TOOLS.json +0 -27
- package/src/config/bundled-skills/chatgpt-import/tools/chatgpt-import.ts +0 -378
- package/src/config/bundled-skills/gmail/SKILL.md +0 -221
- package/src/config/bundled-skills/gmail/TOOLS.json +0 -588
- package/src/config/bundled-skills/gmail/tools/gmail-archive.ts +0 -256
- package/src/config/bundled-skills/gmail/tools/gmail-attachments.ts +0 -112
- package/src/config/bundled-skills/gmail/tools/gmail-draft.ts +0 -44
- package/src/config/bundled-skills/gmail/tools/gmail-filters.ts +0 -81
- package/src/config/bundled-skills/gmail/tools/gmail-follow-up.ts +0 -108
- package/src/config/bundled-skills/gmail/tools/gmail-forward.ts +0 -146
- package/src/config/bundled-skills/gmail/tools/gmail-label.ts +0 -53
- package/src/config/bundled-skills/gmail/tools/gmail-outreach-scan.ts +0 -347
- package/src/config/bundled-skills/gmail/tools/gmail-preferences-tool.ts +0 -59
- package/src/config/bundled-skills/gmail/tools/gmail-preferences.ts +0 -82
- package/src/config/bundled-skills/gmail/tools/gmail-send-draft.ts +0 -26
- package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +0 -347
- package/src/config/bundled-skills/gmail/tools/gmail-trash.ts +0 -29
- package/src/config/bundled-skills/gmail/tools/gmail-unsubscribe.ts +0 -122
- package/src/config/bundled-skills/gmail/tools/gmail-vacation.ts +0 -67
- package/src/config/bundled-skills/gmail/tools/scan-result-store.ts +0 -100
- package/src/config/bundled-skills/gmail/tools/shared.ts +0 -47
- package/src/config/bundled-skills/google-calendar/SKILL.md +0 -51
- package/src/config/bundled-skills/google-calendar/TOOLS.json +0 -226
- package/src/config/bundled-skills/google-calendar/calendar-client.ts +0 -223
- package/src/config/bundled-skills/google-calendar/tools/calendar-check-availability.ts +0 -27
- package/src/config/bundled-skills/google-calendar/tools/calendar-create-event.ts +0 -48
- package/src/config/bundled-skills/google-calendar/tools/calendar-get-event.ts +0 -19
- package/src/config/bundled-skills/google-calendar/tools/calendar-list-events.ts +0 -36
- package/src/config/bundled-skills/google-calendar/tools/calendar-rsvp.ts +0 -58
- package/src/config/bundled-skills/google-calendar/tools/shared.ts +0 -17
- package/src/config/bundled-skills/google-calendar/types.ts +0 -97
- package/src/config/bundled-skills/outlook/SKILL.md +0 -196
- package/src/config/bundled-skills/outlook/TOOLS.json +0 -530
- package/src/config/bundled-skills/outlook/tools/outlook-attachments.ts +0 -85
- package/src/config/bundled-skills/outlook/tools/outlook-categories.ts +0 -77
- package/src/config/bundled-skills/outlook/tools/outlook-draft.ts +0 -84
- package/src/config/bundled-skills/outlook/tools/outlook-follow-up.ts +0 -94
- package/src/config/bundled-skills/outlook/tools/outlook-forward.ts +0 -49
- package/src/config/bundled-skills/outlook/tools/outlook-outreach-scan.ts +0 -237
- package/src/config/bundled-skills/outlook/tools/outlook-rules.ts +0 -161
- package/src/config/bundled-skills/outlook/tools/outlook-send-draft.ts +0 -32
- package/src/config/bundled-skills/outlook/tools/outlook-sender-digest.ts +0 -272
- package/src/config/bundled-skills/outlook/tools/outlook-trash.ts +0 -29
- package/src/config/bundled-skills/outlook/tools/outlook-unsubscribe.ts +0 -129
- package/src/config/bundled-skills/outlook/tools/outlook-vacation.ts +0 -87
- package/src/config/bundled-skills/outlook/tools/shared.ts +0 -20
- package/src/config/bundled-skills/outlook-calendar/SKILL.md +0 -51
- package/src/config/bundled-skills/outlook-calendar/TOOLS.json +0 -221
- package/src/config/bundled-skills/outlook-calendar/calendar-client.ts +0 -252
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-check-availability.ts +0 -53
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-create-event.ts +0 -74
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-get-event.ts +0 -18
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-list-events.ts +0 -46
- package/src/config/bundled-skills/outlook-calendar/tools/outlook-calendar-rsvp.ts +0 -36
- package/src/config/bundled-skills/outlook-calendar/tools/shared.ts +0 -17
- package/src/config/bundled-skills/outlook-calendar/types.ts +0 -120
- package/src/config/bundled-skills/slack/SKILL.md +0 -108
- package/src/config/bundled-skills/tasks/SKILL.md +0 -37
- package/src/config/bundled-skills/tasks/TOOLS.json +0 -353
- package/src/config/bundled-skills/tasks/icon.svg +0 -34
- package/src/config/bundled-skills/tasks/tools/task-delete.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-add.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-remove.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-show.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list-update.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-list.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-queue-run.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-run.ts +0 -12
- package/src/config/bundled-skills/tasks/tools/task-save.ts +0 -12
- package/src/config/bundled-skills/watcher/SKILL.md +0 -31
- package/src/config/bundled-skills/watcher/TOOLS.json +0 -167
- package/src/config/bundled-skills/watcher/tools/watcher-create.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-delete.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-digest.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-list.ts +0 -12
- package/src/config/bundled-skills/watcher/tools/watcher-update.ts +0 -12
- package/src/prompts/templates/UPDATES.md +0 -50
- package/src/prompts/update-bulletin-format.ts +0 -85
- package/src/prompts/update-bulletin-state.ts +0 -58
- package/src/prompts/update-bulletin-template-path.ts +0 -13
- package/src/prompts/update-bulletin.ts +0 -139
- package/src/shared/provider-env-vars.ts +0 -19
- package/src/tools/watcher/create.ts +0 -86
- package/src/tools/watcher/delete.ts +0 -36
- package/src/tools/watcher/digest.ts +0 -54
- package/src/tools/watcher/list.ts +0 -83
- package/src/tools/watcher/update.ts +0 -71
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* - OpenAI Whisper (`openai-whisper`)
|
|
11
11
|
* - Deepgram (`deepgram`)
|
|
12
12
|
* - Google Gemini (`google-gemini`)
|
|
13
|
+
* - xAI (`xai`)
|
|
13
14
|
*/
|
|
14
15
|
|
|
15
16
|
import type {
|
|
@@ -120,6 +121,36 @@ class GoogleGeminiBatchTranscriber implements BatchTranscriber {
|
|
|
120
121
|
}
|
|
121
122
|
}
|
|
122
123
|
|
|
124
|
+
// ---------------------------------------------------------------------------
|
|
125
|
+
// xAI adapter — implements BatchTranscriber on top of the xAI audio
|
|
126
|
+
// transcription provider.
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Wraps `XAIProvider` behind the `BatchTranscriber` contract.
|
|
131
|
+
*
|
|
132
|
+
* Same error-propagation semantics as WhisperBatchTranscriber: raw provider
|
|
133
|
+
* errors pass through unchanged.
|
|
134
|
+
*/
|
|
135
|
+
class XAIBatchTranscriber implements BatchTranscriber {
|
|
136
|
+
readonly providerId = "xai" as const;
|
|
137
|
+
readonly boundaryId = "daemon-batch" as const;
|
|
138
|
+
|
|
139
|
+
private readonly apiKey: string;
|
|
140
|
+
|
|
141
|
+
constructor(apiKey: string) {
|
|
142
|
+
this.apiKey = apiKey;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async transcribe(
|
|
146
|
+
request: SttTranscribeRequest,
|
|
147
|
+
): Promise<SttTranscribeResult> {
|
|
148
|
+
const { XAIProvider } = await import("../providers/speech-to-text/xai.js");
|
|
149
|
+
const provider = new XAIProvider(this.apiKey);
|
|
150
|
+
return provider.transcribe(request.audio, request.mimeType, request.signal);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
123
154
|
// ---------------------------------------------------------------------------
|
|
124
155
|
// Error normalization
|
|
125
156
|
// ---------------------------------------------------------------------------
|
|
@@ -185,6 +216,8 @@ export function createDaemonBatchTranscriber(
|
|
|
185
216
|
return new DeepgramBatchTranscriber(apiKey);
|
|
186
217
|
case "google-gemini":
|
|
187
218
|
return new GoogleGeminiBatchTranscriber(apiKey);
|
|
219
|
+
case "xai":
|
|
220
|
+
return new XAIBatchTranscriber(apiKey);
|
|
188
221
|
default: {
|
|
189
222
|
// Exhaustive check — compile error if a new SttProviderId is added
|
|
190
223
|
// without a corresponding case here.
|
|
@@ -30,6 +30,10 @@
|
|
|
30
30
|
* configurable window.
|
|
31
31
|
*/
|
|
32
32
|
|
|
33
|
+
import {
|
|
34
|
+
listProviderIds,
|
|
35
|
+
supportsBoundary,
|
|
36
|
+
} from "../providers/speech-to-text/provider-catalog.js";
|
|
33
37
|
import { getLogger } from "../util/logger.js";
|
|
34
38
|
import type { StreamingTranscriber, SttStreamServerEvent } from "./types.js";
|
|
35
39
|
|
|
@@ -172,10 +176,13 @@ export class SttStreamSession {
|
|
|
172
176
|
{ provider: this.provider },
|
|
173
177
|
"Streaming transcriber unavailable for provider",
|
|
174
178
|
);
|
|
179
|
+
const streamingProviders = listProviderIds()
|
|
180
|
+
.filter((id) => supportsBoundary(id, "daemon-streaming"))
|
|
181
|
+
.join(", ");
|
|
175
182
|
this.sendEvent({
|
|
176
183
|
type: "error",
|
|
177
184
|
category: "provider-error",
|
|
178
|
-
message: `Streaming transcription is not supported for provider "${this.provider}". Supported providers:
|
|
185
|
+
message: `Streaming transcription is not supported for provider "${this.provider}". Supported providers: ${streamingProviders}.`,
|
|
179
186
|
});
|
|
180
187
|
this.sendEvent({ type: "closed" });
|
|
181
188
|
this.state = "closed";
|
package/src/stt/types.ts
CHANGED
|
@@ -20,7 +20,11 @@
|
|
|
20
20
|
* Canonical provider identifiers for daemon-hosted STT backends.
|
|
21
21
|
* Extend this union as new providers are integrated.
|
|
22
22
|
*/
|
|
23
|
-
export type SttProviderId =
|
|
23
|
+
export type SttProviderId =
|
|
24
|
+
| "openai-whisper"
|
|
25
|
+
| "deepgram"
|
|
26
|
+
| "google-gemini"
|
|
27
|
+
| "xai";
|
|
24
28
|
|
|
25
29
|
/**
|
|
26
30
|
* Telephony-specific STT capability class.
|
package/src/subagent/manager.ts
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
} from "../daemon/conversation.js";
|
|
18
18
|
import type { ServerMessage } from "../daemon/message-protocol.js";
|
|
19
19
|
import { bootstrapConversation } from "../memory/conversation-bootstrap.js";
|
|
20
|
+
import { CallSiteRoutingProvider } from "../providers/call-site-routing.js";
|
|
20
21
|
import { RateLimitProvider } from "../providers/ratelimit.js";
|
|
21
22
|
import { getProvider } from "../providers/registry.js";
|
|
22
23
|
import { createAbortReason } from "../util/abort-reasons.js";
|
|
@@ -155,7 +156,9 @@ export class SubagentManager {
|
|
|
155
156
|
* `onSubagentFinished`. Used by fork spawn to resolve the parent's
|
|
156
157
|
* system prompt when `config.parentSystemPrompt` is not provided.
|
|
157
158
|
*/
|
|
158
|
-
resolveParentConversation?: (
|
|
159
|
+
resolveParentConversation?: (
|
|
160
|
+
conversationId: string,
|
|
161
|
+
) => Conversation | undefined;
|
|
159
162
|
|
|
160
163
|
// ── Spawn ───────────────────────────────────────────────────────────
|
|
161
164
|
|
|
@@ -211,7 +214,19 @@ export class SubagentManager {
|
|
|
211
214
|
|
|
212
215
|
// ── Build conversation dependencies ─────────────────────────────
|
|
213
216
|
const appConfig = getConfig();
|
|
214
|
-
let provider = getProvider(appConfig.
|
|
217
|
+
let provider = getProvider(appConfig.llm.default.provider);
|
|
218
|
+
// Per-call `options.config.callSite` (e.g. `subagentSpawn`) can resolve
|
|
219
|
+
// to a provider name that differs from `llm.default.provider`. Wrap the
|
|
220
|
+
// default provider so the actual transport routes correctly per call,
|
|
221
|
+
// rather than only forwarding metadata to the default's HTTP client.
|
|
222
|
+
// See `providers/call-site-routing.ts`.
|
|
223
|
+
provider = new CallSiteRoutingProvider(provider, (name) => {
|
|
224
|
+
try {
|
|
225
|
+
return getProvider(name);
|
|
226
|
+
} catch {
|
|
227
|
+
return undefined;
|
|
228
|
+
}
|
|
229
|
+
});
|
|
215
230
|
const { rateLimit } = appConfig;
|
|
216
231
|
if (rateLimit.maxRequestsPerMinute > 0) {
|
|
217
232
|
provider = new RateLimitProvider(
|
|
@@ -227,19 +242,21 @@ export class SubagentManager {
|
|
|
227
242
|
if (config.parentSystemPrompt) {
|
|
228
243
|
systemPrompt = config.parentSystemPrompt;
|
|
229
244
|
} else if (this.resolveParentConversation) {
|
|
230
|
-
const parentConv = this.resolveParentConversation(
|
|
245
|
+
const parentConv = this.resolveParentConversation(
|
|
246
|
+
config.parentConversationId,
|
|
247
|
+
);
|
|
231
248
|
const resolved = parentConv?.getCurrentSystemPrompt();
|
|
232
249
|
if (!resolved) {
|
|
233
250
|
throw new Error(
|
|
234
251
|
"Fork spawn requires a parent system prompt but neither config.parentSystemPrompt " +
|
|
235
|
-
|
|
252
|
+
"nor resolveParentConversation yielded one.",
|
|
236
253
|
);
|
|
237
254
|
}
|
|
238
255
|
systemPrompt = resolved;
|
|
239
256
|
} else {
|
|
240
257
|
throw new Error(
|
|
241
258
|
"Fork spawn requires a parent system prompt but neither config.parentSystemPrompt " +
|
|
242
|
-
|
|
259
|
+
"is set nor resolveParentConversation callback is wired.",
|
|
243
260
|
);
|
|
244
261
|
}
|
|
245
262
|
} else {
|
|
@@ -247,7 +264,7 @@ export class SubagentManager {
|
|
|
247
264
|
config.systemPromptOverride ??
|
|
248
265
|
buildSubagentSystemPrompt({ ...config, id: subagentId }, role);
|
|
249
266
|
}
|
|
250
|
-
const maxTokens = appConfig.maxTokens;
|
|
267
|
+
const maxTokens = appConfig.llm.default.maxTokens;
|
|
251
268
|
const workingDir = getSandboxWorkingDir();
|
|
252
269
|
|
|
253
270
|
const memoryPolicy: ConversationMemoryPolicy = isFork
|
|
@@ -266,10 +283,16 @@ export class SubagentManager {
|
|
|
266
283
|
const now = Date.now();
|
|
267
284
|
// For forks, default sendResultToUser to false (silent) unless explicitly true.
|
|
268
285
|
const resolvedSendResultToUser = isFork
|
|
269
|
-
?
|
|
286
|
+
? config.sendResultToUser === true
|
|
287
|
+
? true
|
|
288
|
+
: false
|
|
270
289
|
: config.sendResultToUser;
|
|
271
290
|
const state: SubagentState = {
|
|
272
|
-
config: {
|
|
291
|
+
config: {
|
|
292
|
+
...config,
|
|
293
|
+
id: subagentId,
|
|
294
|
+
sendResultToUser: resolvedSendResultToUser,
|
|
295
|
+
},
|
|
273
296
|
status: "pending",
|
|
274
297
|
conversationId: conversationRecord.id,
|
|
275
298
|
isFork,
|
|
@@ -413,7 +436,9 @@ export class SubagentManager {
|
|
|
413
436
|
// This prepends the inherited context so the fork has full conversational
|
|
414
437
|
// awareness while the objective becomes the latest user turn.
|
|
415
438
|
if (managed.state.isFork && managed.state.config.parentMessages) {
|
|
416
|
-
conversation.injectInheritedContext(
|
|
439
|
+
conversation.injectInheritedContext(
|
|
440
|
+
managed.state.config.parentMessages,
|
|
441
|
+
);
|
|
417
442
|
// Release the parent message arrays now that they've been injected — holding
|
|
418
443
|
// them in SubagentState.config would retain significant memory until the TTL
|
|
419
444
|
// sweep disposes this entry (up to 30 minutes for terminal subagents).
|
|
@@ -439,7 +464,9 @@ export class SubagentManager {
|
|
|
439
464
|
].join("\n")
|
|
440
465
|
: objective;
|
|
441
466
|
const messageId = await conversation.persistUserMessage(message, []);
|
|
442
|
-
await conversation.runAgentLoop(message, messageId, onEvent
|
|
467
|
+
await conversation.runAgentLoop(message, messageId, onEvent, {
|
|
468
|
+
callSite: "subagentSpawn",
|
|
469
|
+
});
|
|
443
470
|
|
|
444
471
|
// Agent loop completed successfully.
|
|
445
472
|
// Copy usage stats from the conversation before sending status (which includes usage).
|
|
@@ -634,7 +661,9 @@ export class SubagentManager {
|
|
|
634
661
|
const conversation = managed.conversation;
|
|
635
662
|
const messageId = await conversation.persistUserMessage(trimmed, []);
|
|
636
663
|
conversation
|
|
637
|
-
.runAgentLoop(trimmed, messageId, onEvent
|
|
664
|
+
.runAgentLoop(trimmed, messageId, onEvent, {
|
|
665
|
+
callSite: "subagentSpawn",
|
|
666
|
+
})
|
|
638
667
|
.catch((err) => {
|
|
639
668
|
log.error({ subagentId, err }, "Subagent message processing failed");
|
|
640
669
|
});
|
|
@@ -900,8 +929,7 @@ export class SubagentManager {
|
|
|
900
929
|
const prefix = managed.state.isFork ? "Fork" : "Subagent";
|
|
901
930
|
let notificationString = `[${prefix} "${info.label}" — ${urgency}] ${message}`;
|
|
902
931
|
if (urgency === "blocked") {
|
|
903
|
-
notificationString +=
|
|
904
|
-
`\nUse subagent_message to send guidance to this ${prefix.toLowerCase()}.`;
|
|
932
|
+
notificationString += `\nUse subagent_message to send guidance to this ${prefix.toLowerCase()}.`;
|
|
905
933
|
}
|
|
906
934
|
|
|
907
935
|
try {
|
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
import { SCOPED_TOOLS } from "@vellumai/ces-contracts";
|
|
2
|
+
|
|
1
3
|
import type { TrustRule } from "../permissions/types.js";
|
|
2
4
|
|
|
5
|
+
/** O(1) lookup set for scoped tool names. */
|
|
6
|
+
const SCOPED_TOOLS_SET: ReadonlySet<string> = new Set(SCOPED_TOOLS);
|
|
7
|
+
|
|
3
8
|
// In-memory map: task_run_id -> ephemeral rules
|
|
4
9
|
const activeTaskRules = new Map<string, TrustRule[]>();
|
|
5
10
|
|
|
@@ -29,9 +34,8 @@ export function clearTaskRunRules(taskRunId: string): void {
|
|
|
29
34
|
* default rules (50) so pre-approved tools aren't shadowed by default
|
|
30
35
|
* `ask` rules (which would trigger prompting and auto-deny in
|
|
31
36
|
* non-interactive task runs), but below user rules (100) so user deny
|
|
32
|
-
* rules still take precedence.
|
|
33
|
-
*
|
|
34
|
-
* path rather than being blanket-approved.
|
|
37
|
+
* rules still take precedence. High-risk tool invocations still flow through
|
|
38
|
+
* the normal risk-classification path rather than being blanket-approved.
|
|
35
39
|
*/
|
|
36
40
|
export function buildTaskRules(
|
|
37
41
|
taskRunId: string,
|
|
@@ -42,7 +46,8 @@ export function buildTaskRules(
|
|
|
42
46
|
id: `ephemeral:${taskRunId}:${tool}`,
|
|
43
47
|
tool,
|
|
44
48
|
pattern: "**",
|
|
45
|
-
scope
|
|
49
|
+
// Only include scope for scoped tools — non-scoped tools don't carry scope.
|
|
50
|
+
...(SCOPED_TOOLS_SET.has(tool) ? { scope: "everywhere" } : {}),
|
|
46
51
|
decision: "allow" as const,
|
|
47
52
|
priority: 75,
|
|
48
53
|
createdAt: Date.now(),
|
|
@@ -45,6 +45,7 @@ const CHECKPOINT_KEY_LIFECYCLE_WATERMARK =
|
|
|
45
45
|
const CHECKPOINT_KEY_LIFECYCLE_WATERMARK_ID =
|
|
46
46
|
"telemetry:lifecycle:last_reported_id";
|
|
47
47
|
const REPORT_INTERVAL_MS = 5 * 60 * 1000;
|
|
48
|
+
const INITIAL_FLUSH_DELAY_MS = 30_000; // Delay first flush to let CES handshake complete
|
|
48
49
|
const BATCH_SIZE = 500;
|
|
49
50
|
const MAX_CONSECUTIVE_BATCHES = 10;
|
|
50
51
|
const TELEMETRY_PATH = "/v1/telemetry/ingest/";
|
|
@@ -54,13 +55,21 @@ const TELEMETRY_PATH = "/v1/telemetry/ingest/";
|
|
|
54
55
|
// ---------------------------------------------------------------------------
|
|
55
56
|
|
|
56
57
|
export class UsageTelemetryReporter {
|
|
58
|
+
private initialFlushTimer: ReturnType<typeof setTimeout> | null = null;
|
|
57
59
|
private timer: ReturnType<typeof setInterval> | null = null;
|
|
58
60
|
private activeFlush: Promise<void> | null = null;
|
|
59
61
|
|
|
60
62
|
start(): void {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
63
|
+
// Delay the first flush to allow the credential infrastructure (CES
|
|
64
|
+
// handshake) to complete. Without this delay, VellumPlatformClient.create()
|
|
65
|
+
// returns null because the credential backend hasn't resolved yet, causing
|
|
66
|
+
// telemetry to fall back to anonymous mode permanently.
|
|
67
|
+
this.initialFlushTimer = setTimeout(() => {
|
|
68
|
+
this.initialFlushTimer = null;
|
|
69
|
+
this.flush().catch((err) => {
|
|
70
|
+
log.warn({ err }, "Initial usage telemetry flush failed");
|
|
71
|
+
});
|
|
72
|
+
}, INITIAL_FLUSH_DELAY_MS);
|
|
64
73
|
this.timer = setInterval(() => {
|
|
65
74
|
this.flush().catch((err) => {
|
|
66
75
|
log.warn({ err }, "Scheduled usage telemetry flush failed");
|
|
@@ -69,6 +78,10 @@ export class UsageTelemetryReporter {
|
|
|
69
78
|
}
|
|
70
79
|
|
|
71
80
|
async stop(): Promise<void> {
|
|
81
|
+
if (this.initialFlushTimer) {
|
|
82
|
+
clearTimeout(this.initialFlushTimer);
|
|
83
|
+
this.initialFlushTimer = null;
|
|
84
|
+
}
|
|
72
85
|
if (this.timer) {
|
|
73
86
|
clearInterval(this.timer);
|
|
74
87
|
this.timer = null;
|
|
@@ -157,6 +170,15 @@ export class UsageTelemetryReporter {
|
|
|
157
170
|
// Resolve auth context — authenticated path uses client, anonymous path
|
|
158
171
|
// sends unauthenticated (telemetry endpoints are public).
|
|
159
172
|
const client = await VellumPlatformClient.create();
|
|
173
|
+
log.debug(
|
|
174
|
+
{
|
|
175
|
+
authenticated: !!client,
|
|
176
|
+
usageCount: events.length,
|
|
177
|
+
turnCount: turnEvents.length,
|
|
178
|
+
lifecycleCount: lifecycleEvents.length,
|
|
179
|
+
},
|
|
180
|
+
"Telemetry flush: resolved auth context",
|
|
181
|
+
);
|
|
160
182
|
|
|
161
183
|
// Build payload
|
|
162
184
|
const typedEvents: TelemetryEvent[] = [
|
|
@@ -222,9 +244,9 @@ export class UsageTelemetryReporter {
|
|
|
222
244
|
}
|
|
223
245
|
|
|
224
246
|
if (!resp.ok) {
|
|
225
|
-
await resp.text();
|
|
247
|
+
const body = await resp.text();
|
|
226
248
|
log.warn(
|
|
227
|
-
{ status: resp.status },
|
|
249
|
+
{ status: resp.status, authenticated: !!client, body },
|
|
228
250
|
"Usage telemetry POST failed — will retry next cycle",
|
|
229
251
|
);
|
|
230
252
|
return;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe, expect, mock, test } from "bun:test";
|
|
1
|
+
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
|
2
2
|
|
|
3
3
|
import type { ToolContext } from "../../types.js";
|
|
4
4
|
import {
|
|
@@ -14,6 +14,11 @@ const probeOutcomes: Record<string, ProbeOutcome> = {
|
|
|
14
14
|
[BROWSER_STATUS_MODE.CDP_INSPECT]: "ok",
|
|
15
15
|
[BROWSER_STATUS_MODE.LOCAL]: "ok",
|
|
16
16
|
};
|
|
17
|
+
const probeErrors: Record<string, CdpError | null> = {
|
|
18
|
+
[BROWSER_STATUS_MODE.EXTENSION]: null,
|
|
19
|
+
[BROWSER_STATUS_MODE.CDP_INSPECT]: null,
|
|
20
|
+
[BROWSER_STATUS_MODE.LOCAL]: null,
|
|
21
|
+
};
|
|
17
22
|
|
|
18
23
|
const buildCandidateListMock = mock((_context: ToolContext) => [
|
|
19
24
|
{ kind: BROWSER_STATUS_MODE.EXTENSION, reason: "mock" },
|
|
@@ -30,7 +35,10 @@ const getCdpClientMock = mock(
|
|
|
30
35
|
conversationId: "test-conversation",
|
|
31
36
|
send: mock(async () => {
|
|
32
37
|
if (outcome === "fail") {
|
|
33
|
-
throw
|
|
38
|
+
throw (
|
|
39
|
+
probeErrors[mode] ??
|
|
40
|
+
new CdpError("transport_error", `${mode} probe failed`)
|
|
41
|
+
);
|
|
34
42
|
}
|
|
35
43
|
return { result: { value: "complete" } };
|
|
36
44
|
}),
|
|
@@ -86,6 +94,15 @@ function makeContext(overrides: Partial<ToolContext> = {}): ToolContext {
|
|
|
86
94
|
}
|
|
87
95
|
|
|
88
96
|
describe("executeBrowserStatus", () => {
|
|
97
|
+
beforeEach(() => {
|
|
98
|
+
probeOutcomes[BROWSER_STATUS_MODE.EXTENSION] = "ok";
|
|
99
|
+
probeOutcomes[BROWSER_STATUS_MODE.CDP_INSPECT] = "ok";
|
|
100
|
+
probeOutcomes[BROWSER_STATUS_MODE.LOCAL] = "ok";
|
|
101
|
+
probeErrors[BROWSER_STATUS_MODE.EXTENSION] = null;
|
|
102
|
+
probeErrors[BROWSER_STATUS_MODE.CDP_INSPECT] = null;
|
|
103
|
+
probeErrors[BROWSER_STATUS_MODE.LOCAL] = null;
|
|
104
|
+
});
|
|
105
|
+
|
|
89
106
|
test("reports extension preflight-unavailable when no host browser proxy is bound", async () => {
|
|
90
107
|
const result = await executeBrowserStatus({}, makeContext());
|
|
91
108
|
expect(result.isError).toBe(false);
|
|
@@ -120,4 +137,30 @@ describe("executeBrowserStatus", () => {
|
|
|
120
137
|
`${BROWSER_STATUS_INPUT_FIELD.CHECK_LOCAL_LAUNCH} must be a boolean`,
|
|
121
138
|
);
|
|
122
139
|
});
|
|
140
|
+
|
|
141
|
+
test("reports extension as connected when probe fails on restricted chrome:// page", async () => {
|
|
142
|
+
probeOutcomes[BROWSER_STATUS_MODE.EXTENSION] = "fail";
|
|
143
|
+
probeErrors[BROWSER_STATUS_MODE.EXTENSION] = new CdpError(
|
|
144
|
+
"cdp_error",
|
|
145
|
+
"Cannot access a chrome:// URL",
|
|
146
|
+
);
|
|
147
|
+
|
|
148
|
+
const result = await executeBrowserStatus(
|
|
149
|
+
{},
|
|
150
|
+
makeContext({
|
|
151
|
+
hostBrowserProxy: {
|
|
152
|
+
isAvailable: () => true,
|
|
153
|
+
} as ToolContext["hostBrowserProxy"],
|
|
154
|
+
}),
|
|
155
|
+
);
|
|
156
|
+
expect(result.isError).toBe(false);
|
|
157
|
+
const payload = JSON.parse(result.content);
|
|
158
|
+
const extension = payload.modes.find(
|
|
159
|
+
(m: { mode: string }) => m.mode === BROWSER_STATUS_MODE.EXTENSION,
|
|
160
|
+
);
|
|
161
|
+
expect(extension).toBeDefined();
|
|
162
|
+
expect(extension.available).toBe(true);
|
|
163
|
+
expect(extension.verified).toBe("active_probe");
|
|
164
|
+
expect(extension.details.restrictedActiveTab).toBe(true);
|
|
165
|
+
});
|
|
123
166
|
});
|
|
@@ -4,6 +4,7 @@ import { getLogger } from "../../util/logger.js";
|
|
|
4
4
|
import { truncate } from "../../util/truncate.js";
|
|
5
5
|
import { safeStringSlice } from "../../util/unicode.js";
|
|
6
6
|
import { credentialBroker } from "../credentials/broker.js";
|
|
7
|
+
import { BROWSER_FILL_CAPABILITY } from "../credentials/tool-policy.js";
|
|
7
8
|
import {
|
|
8
9
|
isPrivateOrLocalHost,
|
|
9
10
|
parseUrl,
|
|
@@ -86,19 +87,22 @@ type StatusCheckMode = BrowserStatusMode;
|
|
|
86
87
|
|
|
87
88
|
const MODE_TRADEOFFS: Record<StatusCheckMode, string[]> = {
|
|
88
89
|
[BROWSER_STATUS_MODE.EXTENSION]: [
|
|
89
|
-
"
|
|
90
|
+
"This is the preferred approach for all things browser-use.",
|
|
91
|
+
"It requires a one-time install of the Vellum Assistant Chrome Extension.",
|
|
92
|
+
"More secure than relying on Chrome's native remote debugging functionality.",
|
|
90
93
|
"Requires the Vellum extension to be paired and actively connected.",
|
|
91
|
-
"Best when the user wants the assistant to operate in their real browser session.",
|
|
92
94
|
],
|
|
93
95
|
[BROWSER_STATUS_MODE.CDP_INSPECT]: [
|
|
94
|
-
"
|
|
95
|
-
"
|
|
96
|
-
"
|
|
96
|
+
"This is the second-best approach for all things browser-use, after the native Vellum Assistant Chrome Extension.",
|
|
97
|
+
"It requires Chrome version 146 or greater",
|
|
98
|
+
"It requires toggling on remote debugging in Chrome Settings",
|
|
99
|
+
"It's prone to phishing attacks from other local processes that may try to do their own remote debugging.",
|
|
97
100
|
],
|
|
98
101
|
[BROWSER_STATUS_MODE.LOCAL]: [
|
|
99
|
-
"
|
|
100
|
-
"
|
|
102
|
+
"The least-preferred approach for all things browser-use.",
|
|
103
|
+
"Considered a last-resort fallback if the user has not installed the Chrome Extension or enabled remote debugging in Chrome, and has indicated that they do not want to.",
|
|
101
104
|
"Does not use the user's existing browser profile, so sessions/cookies may differ.",
|
|
105
|
+
"Requires that Playwright and Chromium are installed on the host machine,",
|
|
102
106
|
],
|
|
103
107
|
};
|
|
104
108
|
|
|
@@ -192,9 +196,9 @@ const REMEDIATION_HINTS: Record<string, string[]> = {
|
|
|
192
196
|
],
|
|
193
197
|
// cdp-inspect backend — discovery-level failures
|
|
194
198
|
"cdp-inspect:unreachable": [
|
|
195
|
-
"Ensure Chrome
|
|
199
|
+
"Ensure that Chrome is on version 146 or higher by going to chrome://settings/help.",
|
|
200
|
+
'Ensure that you have toggled on "Allow remote debugging for this browser instance" by going to chrome://inspect/#remote-debugging',
|
|
196
201
|
"Verify no firewall or antivirus is blocking localhost:9222.",
|
|
197
|
-
"Try: /Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome --remote-debugging-port=9222",
|
|
198
202
|
],
|
|
199
203
|
"cdp-inspect:non_chrome": [
|
|
200
204
|
"The process listening on the configured port is not Chrome/Chromium.",
|
|
@@ -322,6 +326,14 @@ function collectRemediationHints(
|
|
|
322
326
|
return hints;
|
|
323
327
|
}
|
|
324
328
|
|
|
329
|
+
/**
|
|
330
|
+
* Detect the common extension CDP failure where the active tab is a
|
|
331
|
+
* restricted Chrome internal page (e.g. `chrome://newtab`).
|
|
332
|
+
*/
|
|
333
|
+
function isRestrictedChromePageProbeError(error: CdpError): boolean {
|
|
334
|
+
return error.message.toLowerCase().includes("chrome://");
|
|
335
|
+
}
|
|
336
|
+
|
|
325
337
|
/**
|
|
326
338
|
* Parse browser_mode from input and acquire a CdpClient. Returns
|
|
327
339
|
* either a `{ cdp, browserMode }` pair on success or a pre-formatted
|
|
@@ -530,7 +542,7 @@ export function resolveElement(
|
|
|
530
542
|
}
|
|
531
543
|
return {
|
|
532
544
|
resolved: null,
|
|
533
|
-
error: `Error: element_id "${elementId}" not found. Run
|
|
545
|
+
error: `Error: element_id "${elementId}" not found. Run a snapshot first to get current element IDs.`,
|
|
534
546
|
};
|
|
535
547
|
}
|
|
536
548
|
|
|
@@ -740,8 +752,8 @@ export async function executeBrowserNavigate(
|
|
|
740
752
|
).blockedAddress)
|
|
741
753
|
) {
|
|
742
754
|
// Navigate the page away from the private target to prevent
|
|
743
|
-
// follow-up
|
|
744
|
-
//
|
|
755
|
+
// follow-up operations (e.g. snapshot) from reading the
|
|
756
|
+
// already-loaded private content.
|
|
745
757
|
try {
|
|
746
758
|
await navigateAndWait(
|
|
747
759
|
cdp,
|
|
@@ -847,7 +859,7 @@ export async function executeBrowserNavigate(
|
|
|
847
859
|
|
|
848
860
|
if (navigationTimedOut) {
|
|
849
861
|
lines.push(
|
|
850
|
-
`Note: Page is still loading (document.readyState timed out). The page should still be interactive -
|
|
862
|
+
`Note: Page is still loading (document.readyState timed out). The page should still be interactive - take a snapshot to check.`,
|
|
851
863
|
);
|
|
852
864
|
}
|
|
853
865
|
|
|
@@ -917,14 +929,12 @@ export async function executeBrowserNavigate(
|
|
|
917
929
|
lines.push("");
|
|
918
930
|
lines.push(formatAuthChallenge(postCaptchaAuth));
|
|
919
931
|
lines.push("");
|
|
932
|
+
lines.push("Handle this by interacting with the login form:");
|
|
920
933
|
lines.push(
|
|
921
|
-
"
|
|
922
|
-
);
|
|
923
|
-
lines.push(
|
|
924
|
-
"1. Use browser_snapshot to find the sign-in form elements",
|
|
934
|
+
"1. Take a snapshot to find the sign-in form elements",
|
|
925
935
|
);
|
|
926
936
|
lines.push(
|
|
927
|
-
"2. Use
|
|
937
|
+
"2. Use credential fill to enter email/password from credential_store",
|
|
928
938
|
);
|
|
929
939
|
lines.push(
|
|
930
940
|
"3. For email verification codes, use ui_show with a form to ask the user for the code mid-turn",
|
|
@@ -944,18 +954,14 @@ export async function executeBrowserNavigate(
|
|
|
944
954
|
}
|
|
945
955
|
} else {
|
|
946
956
|
// Login / 2FA / OAuth - the agent should handle these itself
|
|
947
|
-
// using browser
|
|
957
|
+
// using browser operations + credential_store. Don't hand off.
|
|
948
958
|
lines.push("");
|
|
949
959
|
lines.push(formatAuthChallenge(challenge));
|
|
950
960
|
lines.push("");
|
|
961
|
+
lines.push("Handle this by interacting with the login form:");
|
|
962
|
+
lines.push("1. Take a snapshot to find the sign-in form elements");
|
|
951
963
|
lines.push(
|
|
952
|
-
"
|
|
953
|
-
);
|
|
954
|
-
lines.push(
|
|
955
|
-
"1. Use browser_snapshot to find the sign-in form elements",
|
|
956
|
-
);
|
|
957
|
-
lines.push(
|
|
958
|
-
"2. Use browser_fill_credential to fill email/password from credential_store",
|
|
964
|
+
"2. Use credential fill to enter email/password from credential_store",
|
|
959
965
|
);
|
|
960
966
|
lines.push(
|
|
961
967
|
"3. For email verification codes, use ui_show with a form to ask the user for the code mid-turn",
|
|
@@ -1006,7 +1012,7 @@ export async function executeBrowserNavigate(
|
|
|
1006
1012
|
}
|
|
1007
1013
|
}
|
|
1008
1014
|
|
|
1009
|
-
// ──
|
|
1015
|
+
// ── snapshot ─────────────────────────────────────────────────────────
|
|
1010
1016
|
|
|
1011
1017
|
export async function executeBrowserSnapshot(
|
|
1012
1018
|
_input: Record<string, unknown>,
|
|
@@ -1938,7 +1944,7 @@ export async function executeBrowserExtract(
|
|
|
1938
1944
|
}
|
|
1939
1945
|
}
|
|
1940
1946
|
|
|
1941
|
-
// ──
|
|
1947
|
+
// ── browser credential fill ──────────────────────────────────────────
|
|
1942
1948
|
|
|
1943
1949
|
export async function executeBrowserFillCredential(
|
|
1944
1950
|
input: Record<string, unknown>,
|
|
@@ -1996,7 +2002,7 @@ export async function executeBrowserFillCredential(
|
|
|
1996
2002
|
const result = await credentialBroker.browserFill({
|
|
1997
2003
|
service,
|
|
1998
2004
|
field,
|
|
1999
|
-
toolName:
|
|
2005
|
+
toolName: BROWSER_FILL_CAPABILITY,
|
|
2000
2006
|
domain: pageDomain,
|
|
2001
2007
|
fill: async (value) => {
|
|
2002
2008
|
// Clear-then-focus-then-insert via the shared helper. We
|
|
@@ -2087,24 +2093,22 @@ function modeTradeoffs(mode: StatusCheckMode): string[] {
|
|
|
2087
2093
|
|
|
2088
2094
|
function extensionSetupActions(): string[] {
|
|
2089
2095
|
return [
|
|
2090
|
-
"Install
|
|
2091
|
-
"Open the extension
|
|
2092
|
-
"Keep the extension connected to the assistant relay.",
|
|
2096
|
+
"Install the Vellum Assistant Chrome extension from the Chrome Web Store: https://chromewebstore.google.com/detail/vellum-assistant-browser/hphbdmpffeigpcdjkckleobjmhhokpne",
|
|
2097
|
+
"Open the extension and pair with your assistant.",
|
|
2093
2098
|
];
|
|
2094
2099
|
}
|
|
2095
2100
|
|
|
2096
2101
|
function cdpInspectSetupActions(): string[] {
|
|
2097
2102
|
return [
|
|
2098
|
-
"
|
|
2099
|
-
"
|
|
2100
|
-
|
|
2103
|
+
"Update Chrome to the latest version by going to chrome://settings/help",
|
|
2104
|
+
"Navigate directly to chrome://inspect/#remote-debugging",
|
|
2105
|
+
'Check the box next to "Allow remote debugging for this browser instance"',
|
|
2101
2106
|
];
|
|
2102
2107
|
}
|
|
2103
2108
|
|
|
2104
2109
|
function localSetupActions(): string[] {
|
|
2105
2110
|
return [
|
|
2106
|
-
"
|
|
2107
|
-
"Install Chromium for Playwright: bunx playwright install chromium.",
|
|
2111
|
+
"Ask your assistant to install playwright and chromium on your host machine.",
|
|
2108
2112
|
];
|
|
2109
2113
|
}
|
|
2110
2114
|
|
|
@@ -2299,6 +2303,29 @@ async function checkExtensionModeStatus(
|
|
|
2299
2303
|
};
|
|
2300
2304
|
}
|
|
2301
2305
|
|
|
2306
|
+
if (isRestrictedChromePageProbeError(probe.error)) {
|
|
2307
|
+
return {
|
|
2308
|
+
mode: BROWSER_STATUS_MODE.EXTENSION,
|
|
2309
|
+
available: true,
|
|
2310
|
+
verified: "active_probe",
|
|
2311
|
+
autoCandidate,
|
|
2312
|
+
summary:
|
|
2313
|
+
"Extension mode transport is connected, but the active Chrome tab is a restricted chrome:// page. Switch to a regular website tab if browser actions fail.",
|
|
2314
|
+
userActions: [
|
|
2315
|
+
"Switch Chrome to a regular http(s) tab (not chrome://...) and retry.",
|
|
2316
|
+
],
|
|
2317
|
+
tradeoffs: modeTradeoffs(BROWSER_STATUS_MODE.EXTENSION),
|
|
2318
|
+
details: {
|
|
2319
|
+
proxyBound,
|
|
2320
|
+
proxyConnected,
|
|
2321
|
+
restrictedActiveTab: true,
|
|
2322
|
+
errorCode: probe.error.code,
|
|
2323
|
+
diagnostic: probe.diagnostic,
|
|
2324
|
+
attemptDiagnostics: probe.error.attemptDiagnostics ?? [],
|
|
2325
|
+
},
|
|
2326
|
+
};
|
|
2327
|
+
}
|
|
2328
|
+
|
|
2302
2329
|
return {
|
|
2303
2330
|
mode: BROWSER_STATUS_MODE.EXTENSION,
|
|
2304
2331
|
available: false,
|
|
@@ -2547,7 +2574,7 @@ export async function executeBrowserStatus(
|
|
|
2547
2574
|
} catch (err) {
|
|
2548
2575
|
const msg = err instanceof Error ? err.message : String(err);
|
|
2549
2576
|
return {
|
|
2550
|
-
content: `Error:
|
|
2577
|
+
content: `Error: browser status check failed: ${msg}`,
|
|
2551
2578
|
isError: true,
|
|
2552
2579
|
};
|
|
2553
2580
|
}
|