@vellumai/assistant 0.7.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +32 -49
- package/Dockerfile +1 -0
- package/README.md +1 -2
- package/__tests__/permissions/gateway-threshold-reader.test.ts +9 -3
- package/bun.lock +26 -26
- package/docs/architecture/security.md +20 -0
- package/docs/plugins.md +7 -9
- package/knip.json +1 -0
- package/node_modules/@vellumai/gateway-client/src/index.ts +1 -0
- package/node_modules/@vellumai/gateway-client/src/ipc-client.ts +39 -1
- package/node_modules/@vellumai/gateway-client/src/types.ts +11 -0
- package/node_modules/@vellumai/service-contracts/package.json +2 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/contracts.test.ts +4 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +107 -0
- package/node_modules/@vellumai/service-contracts/src/index.ts +5 -1
- package/node_modules/@vellumai/service-contracts/src/ingress.ts +24 -0
- package/node_modules/@vellumai/service-contracts/src/twilio-ingress.ts +84 -0
- package/node_modules/@vellumai/skill-host-contracts/src/assistant-event.ts +9 -0
- package/node_modules/@vellumai/twilio-client/bun.lock +24 -0
- package/node_modules/@vellumai/twilio-client/package.json +18 -0
- package/node_modules/@vellumai/twilio-client/src/__tests__/twilio-client.test.ts +128 -0
- package/node_modules/@vellumai/twilio-client/src/index.ts +179 -0
- package/node_modules/@vellumai/twilio-client/tsconfig.json +20 -0
- package/openapi.yaml +565 -12
- package/package.json +6 -3
- package/src/__tests__/app-builder-tool-scripts.test.ts +3 -3
- package/src/__tests__/app-bundler.test.ts +170 -1
- package/src/__tests__/app-control-flow.test.ts +374 -0
- package/src/__tests__/app-control-no-global-cgevent.test.ts +98 -0
- package/src/__tests__/app-control-tool-schemas.test.ts +621 -0
- package/src/__tests__/app-executors.test.ts +30 -43
- package/src/__tests__/approval-routes-http.test.ts +23 -6
- package/src/__tests__/assistant-event-hub-machine-name.test.ts +146 -0
- package/src/__tests__/assistant-event-hub-targeted.test.ts +257 -0
- package/src/__tests__/assistant-event-hub.test.ts +109 -2
- package/src/__tests__/assistant-event.test.ts +10 -0
- package/src/__tests__/assistant-events-sse-hardening.test.ts +7 -2
- package/src/__tests__/assistant-feature-flags-integration.test.ts +11 -7
- package/src/__tests__/background-shell-host-bash.test.ts +14 -15
- package/src/__tests__/bootstrap-turn-cleanup.test.ts +44 -0
- package/src/__tests__/btw-routes.test.ts +13 -4
- package/src/__tests__/call-controller.test.ts +49 -1
- package/src/__tests__/call-domain.test.ts +0 -2
- package/src/__tests__/call-routes-http.test.ts +0 -2
- package/src/__tests__/channel-readiness-service.test.ts +59 -1
- package/src/__tests__/checker.test.ts +3 -4
- package/src/__tests__/config-loader-backfill.test.ts +90 -155
- package/src/__tests__/config-loader-platform-defaults.test.ts +196 -0
- package/src/__tests__/config-schema-cmd.test.ts +0 -1
- package/src/__tests__/config-set-platform-guard.test.ts +48 -4
- package/src/__tests__/config-watcher-cleanup-throttle.test.ts +2 -2
- package/src/__tests__/config-watcher.test.ts +2 -2
- package/src/__tests__/conversation-app-control-instantiation.test.ts +392 -0
- package/src/__tests__/conversation-app-control-lifecycle.test.ts +237 -0
- package/src/__tests__/conversation-init.benchmark.test.ts +0 -2
- package/src/__tests__/conversation-lifecycle.test.ts +36 -0
- package/src/__tests__/conversation-process-app-control-preactivation.test.ts +283 -0
- package/src/__tests__/conversation-routes-disk-view.test.ts +6 -0
- package/src/__tests__/conversation-routes-guardian-reply.test.ts +120 -72
- package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -0
- package/src/__tests__/conversation-slash-commands.test.ts +0 -4
- package/src/__tests__/conversation-surfaces-action-delivery.test.ts +202 -0
- package/src/__tests__/conversation-surfaces-app-control.test.ts +317 -0
- package/src/__tests__/credential-execution-feature-gates.test.ts +5 -12
- package/src/__tests__/credential-execution-managed-contract.test.ts +3 -131
- package/src/__tests__/credentials-cli.test.ts +5 -12
- package/src/__tests__/cu-unified-flow.test.ts +185 -23
- package/src/__tests__/daemon-credential-client.test.ts +101 -19
- package/src/__tests__/db-schedule-syntax-migration.test.ts +2 -0
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +0 -1
- package/src/__tests__/gateway-only-enforcement.test.ts +0 -1
- package/src/__tests__/guardian-verification-voice-binding.test.ts +0 -2
- package/src/__tests__/handlers-skills-memory-v2-reseed.test.ts +0 -2
- package/src/__tests__/handlers-user-message-approval-consumption.test.ts +0 -1
- package/src/__tests__/heartbeat-service.test.ts +718 -1
- package/src/__tests__/helpers/call-route-handler.ts +7 -1
- package/src/__tests__/host-app-control-proxy.test.ts +602 -0
- package/src/__tests__/host-app-control-routes.test.ts +263 -0
- package/src/__tests__/host-bash-proxy.test.ts +246 -47
- package/src/__tests__/host-bash-routes.test.ts +294 -0
- package/src/__tests__/host-browser-proxy.test.ts +24 -22
- package/src/__tests__/host-browser-routes.test.ts +39 -13
- package/src/__tests__/host-cu-proxy.test.ts +41 -52
- package/src/__tests__/host-cu-routes-targeted.test.ts +300 -0
- package/src/__tests__/host-file-edit-tool.test.ts +47 -1
- package/src/__tests__/host-file-proxy-targeted.test.ts +339 -0
- package/src/__tests__/host-file-proxy.test.ts +37 -43
- package/src/__tests__/host-file-read-tool.test.ts +17 -0
- package/src/__tests__/host-file-routes-targeted.test.ts +262 -0
- package/src/__tests__/host-file-write-tool.test.ts +42 -1
- package/src/__tests__/host-proxy-base.test.ts +312 -0
- package/src/__tests__/host-shell-tool.test.ts +22 -4
- package/src/__tests__/host-transfer-proxy-targeted.test.ts +583 -0
- package/src/__tests__/host-transfer-proxy.test.ts +121 -22
- package/src/__tests__/host-transfer-routes-targeted.test.ts +447 -0
- package/src/__tests__/http-user-message-parity.test.ts +1 -0
- package/src/__tests__/identity-intro-cache.test.ts +29 -0
- package/src/__tests__/identity-routes.test.ts +103 -1
- package/src/__tests__/init-feature-flag-overrides.test.ts +26 -3
- package/src/__tests__/inline-command-runner.test.ts +0 -1
- package/src/__tests__/inline-skill-load-permissions.test.ts +5 -11
- package/src/__tests__/integration-status.test.ts +85 -5
- package/src/__tests__/intent-routing.test.ts +0 -1
- package/src/__tests__/jobs-store-qdrant-breaker.test.ts +95 -5
- package/src/__tests__/lifecycle-memory-v2-seed.test.ts +17 -0
- package/src/__tests__/managed-skill-lifecycle.test.ts +0 -1
- package/src/__tests__/mcp-auth-routes.test.ts +197 -0
- package/src/__tests__/mcp-cli.test.ts +338 -2
- package/src/__tests__/memory-jobs-worker-lanes.test.ts +188 -0
- package/src/__tests__/migration-import-commit-http.test.ts +108 -2
- package/src/__tests__/mock-gateway-ipc.ts +1 -0
- package/src/__tests__/oauth-cli.test.ts +0 -2
- package/src/__tests__/oauth2-gateway-transport.test.ts +0 -1
- package/src/__tests__/persistence-secret-redaction.test.ts +299 -0
- package/src/__tests__/platform-bash-auto-approve.test.ts +5 -9
- package/src/__tests__/prechat-onboarding-contract.test.ts +3 -1
- package/src/__tests__/process-message-background-slack.test.ts +2 -0
- package/src/__tests__/provider-commit-message-generator.test.ts +0 -1
- package/src/__tests__/public-ingress-urls.test.ts +97 -0
- package/src/__tests__/require-fresh-approval.test.ts +0 -1
- package/src/__tests__/retry-backoff.test.ts +87 -0
- package/src/__tests__/runtime-events-sse.test.ts +10 -6
- package/src/__tests__/sanitize-config-for-transfer.test.ts +24 -2
- package/src/__tests__/schedule-retry.test.ts +715 -0
- package/src/__tests__/script-proxy-mitm-handler.test.ts +1 -1
- package/src/__tests__/secret-ingress-http.test.ts +1 -0
- package/src/__tests__/send-endpoint-busy.test.ts +3 -0
- package/src/__tests__/shell-tool-proxy-mode.test.ts +0 -1
- package/src/__tests__/skill-feature-flags.test.ts +43 -41
- package/src/__tests__/skill-load-feature-flag.test.ts +13 -14
- package/src/__tests__/skill-load-inline-command.test.ts +0 -51
- package/src/__tests__/skill-load-inline-includes.test.ts +0 -43
- package/src/__tests__/skill-projection.benchmark.test.ts +0 -1
- package/src/__tests__/skill-script-runner-sandbox.test.ts +0 -1
- package/src/__tests__/slack-channel-config.test.ts +9 -14
- package/src/__tests__/system-prompt-ask-mode.test.ts +0 -1
- package/src/__tests__/system-prompt.test.ts +0 -1
- package/src/__tests__/telegram-config.test.ts +0 -1
- package/src/__tests__/test-preload.ts +8 -0
- package/src/__tests__/tool-approval-handler.test.ts +3 -4
- package/src/__tests__/tool-audit-listener.test.ts +48 -0
- package/src/__tests__/tool-execute-pipeline.test.ts +0 -1
- package/src/__tests__/tool-execution-abort-cleanup.test.ts +0 -1
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +0 -1
- package/src/__tests__/tool-executor.test.ts +0 -1
- package/src/__tests__/twilio-config.test.ts +3 -16
- package/src/__tests__/twilio-routes.test.ts +3 -5
- package/src/__tests__/twilio-validation.test.ts +93 -0
- package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +1 -4
- package/src/__tests__/verification-control-plane-policy.test.ts +2 -4
- package/src/__tests__/voice-ingress-preflight.test.ts +19 -0
- package/src/__tests__/workspace-migration-006-services-config.test.ts +3 -2
- package/src/__tests__/workspace-migration-backfill-installation-id.test.ts +1 -5
- package/src/__tests__/workspace-migration-down-functions.test.ts +8 -8
- package/src/__tests__/workspace-migration-unify-llm-callsite-configs.test.ts +10 -6
- package/src/backup/__tests__/paths.test.ts +0 -22
- package/src/backup/__tests__/restore.test.ts +51 -151
- package/src/backup/paths.ts +2 -18
- package/src/backup/restore.ts +107 -231
- package/src/bundler/app-bundler.ts +51 -3
- package/src/calls/relay-server.ts +4 -44
- package/src/calls/twilio-config.ts +2 -17
- package/src/calls/twilio-rest.ts +33 -105
- package/src/calls/twilio-routes.ts +11 -12
- package/src/channels/types.ts +8 -7
- package/src/cli/commands/__tests__/backup.test.ts +6 -277
- package/src/cli/commands/__tests__/gateway.test.ts +288 -0
- package/src/cli/commands/__tests__/memory-v2.test.ts +4 -0
- package/src/cli/commands/__tests__/webhooks.test.ts +0 -1
- package/src/cli/commands/backup.ts +6 -331
- package/src/cli/commands/clients.ts +36 -37
- package/src/cli/commands/contacts.ts +73 -0
- package/src/cli/commands/conversations.ts +2 -5
- package/src/cli/commands/credentials.ts +15 -7
- package/src/cli/commands/domain.ts +66 -15
- package/src/cli/commands/gateway.ts +183 -0
- package/src/cli/commands/keys.ts +9 -6
- package/src/cli/commands/mcp.ts +116 -156
- package/src/cli/commands/memory-v2.ts +296 -1
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +0 -1
- package/src/cli/commands/platform/__tests__/connect.test.ts +0 -2
- package/src/cli/commands/platform/__tests__/disconnect.test.ts +0 -2
- package/src/cli/commands/platform/__tests__/status.test.ts +13 -15
- package/src/cli/commands/platform/disconnect.ts +5 -4
- package/src/cli/commands/platform/index.ts +0 -18
- package/src/cli/lib/daemon-credential-client.ts +110 -28
- package/src/cli/program.ts +2 -0
- package/src/config/assistant-feature-flags.ts +67 -10
- package/src/config/bundled-skills/acp/SKILL.md +6 -0
- package/src/config/bundled-skills/acp/TOOLS.json +1 -22
- package/src/config/bundled-skills/app-builder/SKILL.md +14 -109
- package/src/config/bundled-skills/app-builder/TOOLS.json +1 -28
- package/src/config/bundled-skills/app-builder/tools/app-create.ts +1 -10
- package/src/config/bundled-skills/app-control/SKILL.md +75 -0
- package/src/config/bundled-skills/app-control/TOOLS.json +299 -0
- package/src/config/bundled-skills/app-control/tools/app-control-click.ts +12 -0
- package/src/config/bundled-skills/app-control/tools/app-control-combo.ts +12 -0
- package/src/config/bundled-skills/app-control/tools/app-control-drag.ts +12 -0
- package/src/config/bundled-skills/app-control/tools/app-control-observe.ts +12 -0
- package/src/config/bundled-skills/app-control/tools/app-control-press.ts +12 -0
- package/src/config/bundled-skills/app-control/tools/app-control-sequence.ts +12 -0
- package/src/config/bundled-skills/app-control/tools/app-control-start.ts +12 -0
- package/src/config/bundled-skills/app-control/tools/app-control-stop.ts +12 -0
- package/src/config/bundled-skills/app-control/tools/app-control-type.ts +12 -0
- package/src/config/bundled-skills/computer-use/SKILL.md +6 -0
- package/src/config/bundled-skills/computer-use/TOOLS.json +67 -43
- package/src/config/bundled-skills/contacts/TOOLS.json +0 -16
- package/src/config/bundled-skills/document/TOOLS.json +0 -8
- package/src/config/bundled-skills/followups/TOOLS.json +0 -12
- package/src/config/bundled-skills/image-studio/SKILL.md +4 -0
- package/src/config/bundled-skills/image-studio/TOOLS.json +0 -4
- package/src/config/bundled-skills/media-processing/TOOLS.json +0 -24
- package/src/config/bundled-skills/messaging/TOOLS.json +0 -40
- package/src/config/bundled-skills/phone-calls/TOOLS.json +0 -12
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +19 -4
- package/src/config/bundled-skills/playbooks/TOOLS.json +0 -16
- package/src/config/bundled-skills/schedule/TOOLS.json +14 -14
- package/src/config/bundled-skills/sequences/TOOLS.json +0 -36
- package/src/config/bundled-skills/settings/SKILL.md +4 -0
- package/src/config/bundled-skills/settings/TOOLS.json +0 -12
- package/src/config/bundled-skills/skill-management/SKILL.md +6 -0
- package/src/config/bundled-skills/skill-management/TOOLS.json +0 -8
- package/src/config/bundled-skills/subagent/SKILL.md +6 -2
- package/src/config/bundled-skills/subagent/TOOLS.json +0 -20
- package/src/config/bundled-skills/transcribe/SKILL.md +4 -0
- package/src/config/bundled-skills/transcribe/TOOLS.json +0 -4
- package/src/config/bundled-tool-registry.ts +21 -0
- package/src/config/env-registry.ts +0 -2
- package/src/config/env.ts +19 -12
- package/src/config/feature-flag-registry.json +21 -133
- package/src/config/loader.ts +73 -99
- package/src/config/sanitize-for-transfer.ts +2 -0
- package/src/config/schemas/__tests__/memory-lifecycle.test.ts +80 -0
- package/src/config/schemas/__tests__/memory-v2.test.ts +7 -4
- package/src/config/schemas/calls.ts +0 -9
- package/src/config/schemas/heartbeat.ts +63 -0
- package/src/config/schemas/ingress.ts +10 -6
- package/src/config/schemas/llm.ts +5 -10
- package/src/config/schemas/memory-lifecycle.ts +77 -24
- package/src/config/schemas/memory-v2.ts +48 -4
- package/src/config/schemas/platform.ts +6 -0
- package/src/config/schemas/services.ts +1 -15
- package/src/config/schemas/skills.ts +0 -6
- package/src/config/seed-inference-profiles.ts +1 -1
- package/src/contacts/contact-store.ts +0 -30
- package/src/contacts/contacts-write.ts +0 -27
- package/src/context/window-manager.ts +1 -2
- package/src/credential-execution/feature-gates.ts +10 -10
- package/src/credential-execution/process-manager.ts +12 -41
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +126 -5
- package/src/daemon/bootstrap-turn-cleanup.ts +45 -0
- package/src/daemon/config-watcher.ts +4 -3
- package/src/daemon/conversation-agent-loop-handlers.ts +21 -3
- package/src/daemon/conversation-agent-loop.ts +32 -28
- package/src/daemon/conversation-lifecycle.ts +8 -1
- package/src/daemon/conversation-process.ts +16 -11
- package/src/daemon/conversation-runtime-assembly.ts +2 -2
- package/src/daemon/conversation-surfaces.ts +125 -4
- package/src/daemon/conversation-tool-setup.ts +16 -55
- package/src/daemon/conversation.ts +21 -2
- package/src/daemon/doordash-steps.ts +1 -1
- package/src/daemon/handlers/shared.ts +4 -1
- package/src/daemon/host-app-control-proxy.ts +293 -0
- package/src/daemon/host-bash-proxy.ts +84 -74
- package/src/daemon/host-browser-proxy.ts +67 -82
- package/src/daemon/host-cu-proxy.ts +81 -86
- package/src/daemon/host-file-proxy.ts +93 -69
- package/src/daemon/host-proxy-base.ts +294 -0
- package/src/daemon/host-proxy-preactivation.ts +82 -0
- package/src/daemon/host-transfer-proxy.ts +247 -129
- package/src/daemon/lifecycle.ts +115 -117
- package/src/daemon/message-protocol.ts +3 -8
- package/src/daemon/message-types/contacts.ts +23 -1
- package/src/daemon/message-types/conversations.ts +11 -8
- package/src/daemon/message-types/host-app-control.ts +150 -0
- package/src/daemon/message-types/host-bash.ts +4 -0
- package/src/daemon/message-types/host-cu.ts +2 -0
- package/src/daemon/message-types/host-file.ts +4 -0
- package/src/daemon/message-types/host-transfer.ts +3 -0
- package/src/daemon/message-types/schedules.ts +8 -3
- package/src/daemon/message-types/skills.ts +2 -2
- package/src/daemon/process-message.ts +18 -1
- package/src/daemon/shutdown-handlers.ts +0 -3
- package/src/daemon/tool-setup-types.ts +51 -0
- package/src/daemon/tool-side-effects.ts +1 -1
- package/src/events/tool-audit-listener.ts +2 -1
- package/src/heartbeat/__tests__/heartbeat-feed-event.test.ts +15 -7
- package/src/heartbeat/__tests__/heartbeat-run-store.test.ts +216 -0
- package/src/heartbeat/heartbeat-run-store.ts +236 -0
- package/src/heartbeat/heartbeat-service.ts +280 -49
- package/src/home/__tests__/post-connect-feed.test.ts +99 -0
- package/src/home/__tests__/relationship-state-writer.test.ts +11 -9
- package/src/home/__tests__/suggested-prompts.test.ts +89 -0
- package/src/home/post-connect-feed.ts +68 -0
- package/src/home/relationship-state-writer.ts +17 -92
- package/src/home/suggested-prompts.ts +46 -10
- package/src/inbound/public-ingress-urls.ts +32 -34
- package/src/ipc/__tests__/route-error-envelope.test.ts +80 -0
- package/src/ipc/assistant-server.ts +14 -1
- package/src/ipc/cli-client.ts +32 -1
- package/src/live-voice/live-voice-metrics.ts +10 -10
- package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +304 -0
- package/src/mcp/mcp-auth-orchestrator.ts +213 -0
- package/src/mcp/mcp-auth-state.ts +133 -0
- package/src/mcp/mcp-oauth-provider.ts +19 -0
- package/src/memory/__tests__/jobs-store-job-classes.test.ts +24 -0
- package/src/memory/__tests__/qdrant-client-sentinel.test.ts +49 -0
- package/src/memory/__tests__/sparse-tokenize.test.ts +66 -0
- package/src/memory/anisotropy.test.ts +247 -0
- package/src/memory/anisotropy.ts +443 -0
- package/src/memory/auto-analysis-constants.ts +17 -0
- package/src/memory/auto-analysis-guard.ts +5 -15
- package/src/memory/canonical-guardian-store.ts +7 -7
- package/src/memory/context-search/__tests__/agent-runner-redaction.test.ts +122 -0
- package/src/memory/context-search/agent-protocol.ts +6 -6
- package/src/memory/context-search/agent-runner.ts +32 -7
- package/src/memory/context-search/sources/memory-v2.ts +17 -5
- package/src/memory/conversation-crud.ts +1 -1
- package/src/memory/conversation-key-store.ts +2 -15
- package/src/memory/db-init.ts +4 -0
- package/src/memory/embedding-backend.ts +9 -21
- package/src/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +49 -4
- package/src/memory/graph/conversation-graph-memory.ts +1 -24
- package/src/memory/graph/graph-search.ts +8 -0
- package/src/memory/graph/retriever.ts +28 -0
- package/src/memory/graph/tools.ts +1 -1
- package/src/memory/jobs/__tests__/embed-concept-page.test.ts +8 -2
- package/src/memory/jobs/embed-concept-page.ts +28 -2
- package/src/memory/jobs/embed-pkb-file.test.ts +2 -2
- package/src/memory/jobs-store.ts +66 -22
- package/src/memory/jobs-worker.ts +112 -63
- package/src/memory/memory-v2-activation-log-store.ts +1 -1
- package/src/memory/migrations/237-heartbeat-runs.ts +45 -0
- package/src/memory/migrations/238-schedule-retry-policy.ts +20 -0
- package/src/memory/migrations/index.ts +5 -0
- package/src/memory/migrations/registry.ts +8 -0
- package/src/memory/pkb/pkb-search.ts +7 -0
- package/src/memory/qdrant-client.ts +50 -20
- package/src/memory/schema/infrastructure.ts +15 -0
- package/src/memory/search/semantic.ts +7 -0
- package/src/memory/sparse-tokenize.ts +49 -0
- package/src/memory/v2/__tests__/activation.test.ts +77 -95
- package/src/memory/v2/__tests__/injection.test.ts +43 -21
- package/src/memory/v2/__tests__/sim.test.ts +166 -6
- package/src/memory/v2/__tests__/sparse-bm25.test.ts +292 -0
- package/src/memory/v2/__tests__/static-context.test.ts +0 -1
- package/src/memory/v2/activation.ts +69 -88
- package/src/memory/v2/consolidation-job.ts +3 -5
- package/src/memory/v2/constants.ts +7 -0
- package/src/memory/v2/injection.ts +86 -53
- package/src/memory/v2/prompts/consolidation.ts +312 -91
- package/src/memory/v2/qdrant.ts +99 -1
- package/src/memory/v2/sim.ts +126 -16
- package/src/memory/v2/skill-qdrant.ts +12 -3
- package/src/memory/v2/skill-store.ts +16 -1
- package/src/memory/v2/sparse-bm25.ts +245 -0
- package/src/memory/v2/static-context.ts +6 -5
- package/src/messaging/providers/gmail/types.ts +0 -49
- package/src/messaging/providers/slack/adapter.ts +1 -31
- package/src/messaging/providers/slack/types.ts +0 -32
- package/src/notifications/README.md +10 -10
- package/src/notifications/broadcaster.ts +1 -1
- package/src/notifications/guardian-question-mode.ts +5 -5
- package/src/oauth/connect-orchestrator.ts +4 -0
- package/src/oauth/credential-token-resolver.ts +1 -3
- package/src/oauth/manual-token-connection.ts +0 -4
- package/src/outbound-proxy/index.ts +1 -37
- package/src/outbound-proxy/logging.ts +1 -1
- package/src/outbound-proxy/policy.ts +6 -5
- package/src/outbound-proxy/router.ts +2 -1
- package/src/permissions/approval-policy.test.ts +6 -275
- package/src/permissions/approval-policy.ts +0 -51
- package/src/permissions/checker.test.ts +0 -1
- package/src/permissions/checker.ts +3 -17
- package/src/permissions/gateway-threshold-reader.ts +2 -0
- package/src/permissions/prompter.ts +34 -1
- package/src/permissions/secret-prompter.ts +6 -2
- package/src/prompts/bootstrap-cleanup.ts +27 -0
- package/src/prompts/system-prompt.ts +3 -18
- package/src/prompts/templates/SOUL.md +13 -1
- package/src/providers/speech-to-text/provider-catalog.ts +7 -8
- package/src/runtime/assistant-event-hub.ts +118 -96
- package/src/runtime/assistant-event.ts +1 -0
- package/src/runtime/auth/__tests__/middleware.test.ts +11 -56
- package/src/runtime/auth/middleware.ts +0 -96
- package/src/runtime/auth/route-policy.ts +19 -0
- package/src/runtime/btw-sidechain.ts +2 -3
- package/src/runtime/channel-invite-transport.ts +2 -48
- package/src/runtime/channel-invite-transports/email.ts +1 -1
- package/src/runtime/channel-invite-transports/slack.ts +1 -1
- package/src/runtime/channel-invite-transports/telegram.ts +1 -1
- package/src/runtime/channel-invite-transports/voice.ts +1 -1
- package/src/runtime/channel-invite-transports/whatsapp.ts +1 -1
- package/src/runtime/channel-invite-types.ts +54 -0
- package/src/runtime/channel-readiness-service.ts +32 -13
- package/src/runtime/http-server.ts +3 -329
- package/src/runtime/http-types.ts +0 -5
- package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +413 -0
- package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +260 -0
- package/src/runtime/migrations/__tests__/vbundle-import-version-compat.test.ts +189 -0
- package/src/runtime/migrations/__tests__/vbundle-streaming-importer.test.ts +153 -1
- package/src/runtime/migrations/__tests__/vbundle-symlink-importer.test.ts +451 -0
- package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -0
- package/src/runtime/migrations/__tests__/vbundle-symlink-streaming.test.ts +515 -0
- package/src/runtime/migrations/__tests__/vbundle-symlink-tar.test.ts +437 -0
- package/src/runtime/migrations/__tests__/vbundle-symlink-walker.test.ts +319 -0
- package/src/runtime/migrations/__tests__/vbundle-validator-v1-schema.test.ts +51 -1
- package/src/runtime/migrations/migration-transport.ts +7 -7
- package/src/runtime/migrations/vbundle-builder.ts +327 -60
- package/src/runtime/migrations/vbundle-import-analyzer.ts +4 -4
- package/src/runtime/migrations/vbundle-import-policy.ts +172 -0
- package/src/runtime/migrations/vbundle-importer.ts +245 -68
- package/src/runtime/migrations/vbundle-streaming-importer.ts +326 -35
- package/src/runtime/migrations/vbundle-streaming-validator.ts +157 -4
- package/src/runtime/migrations/vbundle-tar-stream.ts +15 -6
- package/src/runtime/migrations/vbundle-validator.ts +114 -0
- package/src/runtime/pending-interactions.ts +35 -9
- package/src/runtime/routes/__tests__/backup-routes.test.ts +22 -150
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +98 -0
- package/src/runtime/routes/__tests__/gateway-log-routes.test.ts +242 -0
- package/src/runtime/routes/__tests__/heartbeat-routes.test.ts +112 -0
- package/src/runtime/routes/approval-interception-types.ts +13 -0
- package/src/runtime/routes/approval-strategies/guardian-text-engine-strategy.ts +1 -1
- package/src/runtime/routes/backup-routes.ts +15 -38
- package/src/runtime/routes/btw-routes.ts +14 -37
- package/src/runtime/routes/client-routes.ts +1 -0
- package/src/runtime/routes/contact-prompt-routes.ts +183 -0
- package/src/runtime/routes/conversation-query-routes.ts +36 -1
- package/src/runtime/routes/conversation-routes.ts +30 -13
- package/src/runtime/routes/document-pdf-renderer.ts +165 -0
- package/src/runtime/routes/documents-routes.ts +30 -0
- package/src/runtime/routes/errors.ts +19 -4
- package/src/runtime/routes/events-routes.ts +12 -6
- package/src/runtime/routes/gateway-log-routes.ts +79 -0
- package/src/runtime/routes/guardian-approval-interception.ts +2 -8
- package/src/runtime/routes/heartbeat-routes.ts +103 -38
- package/src/runtime/routes/host-app-control-routes.ts +134 -0
- package/src/runtime/routes/host-bash-routes.ts +36 -6
- package/src/runtime/routes/host-browser-routes.ts +108 -13
- package/src/runtime/routes/host-cu-routes.ts +44 -14
- package/src/runtime/routes/host-file-routes.ts +33 -10
- package/src/runtime/routes/host-transfer-routes.ts +64 -24
- package/src/runtime/routes/http-adapter.ts +1 -0
- package/src/runtime/routes/identity-intro-cache.ts +30 -0
- package/src/runtime/routes/identity-routes.ts +15 -43
- package/src/runtime/routes/inbound-message-handler.ts +1 -9
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +0 -7
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +0 -8
- package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +0 -20
- package/src/runtime/routes/inbound-stages/transcribe-audio.ts +5 -13
- package/src/runtime/routes/index.ts +8 -0
- package/src/runtime/routes/mcp-auth-routes.ts +132 -0
- package/src/runtime/routes/memory-item-routes.ts +10 -12
- package/src/runtime/routes/memory-v2-routes.ts +441 -1
- package/src/runtime/routes/migration-routes.ts +96 -0
- package/src/runtime/routes/schedule-routes.ts +7 -0
- package/src/runtime/verification-templates.ts +4 -7
- package/src/schedule/integration-status.ts +66 -2
- package/src/schedule/recurrence-engine.ts +4 -1
- package/src/schedule/retry-backoff.ts +18 -0
- package/src/schedule/retry-policy.ts +82 -0
- package/src/schedule/schedule-recovery.ts +64 -0
- package/src/schedule/schedule-store.ts +106 -2
- package/src/schedule/scheduler-types.ts +25 -0
- package/src/schedule/scheduler.ts +63 -38
- package/src/security/oauth-callback-registry.ts +8 -0
- package/src/sequence/analytics.ts +5 -5
- package/src/sequence/engine.ts +1 -1
- package/src/skills/catalog-files.ts +2 -8
- package/src/skills/include-graph.ts +5 -5
- package/src/skills/remote-skill-policy.ts +5 -5
- package/src/skills/skill-file-provider.ts +1 -1
- package/src/skills/skill-file-types.ts +13 -0
- package/src/skills/skillssh-audit-types.ts +28 -0
- package/src/skills/skillssh-registry.ts +8 -21
- package/src/telemetry/types.ts +2 -0
- package/src/telemetry/usage-telemetry-reporter.test.ts +21 -0
- package/src/telemetry/usage-telemetry-reporter.ts +1 -0
- package/src/tools/app-control/skill-proxy-bridge.ts +28 -0
- package/src/tools/apps/executors.ts +56 -69
- package/src/tools/browser/__tests__/browser-status.test.ts +21 -18
- package/src/tools/browser/browser-execution.ts +2 -2
- package/src/tools/browser/cdp-client/__tests__/factory.test.ts +55 -4
- package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +12 -6
- package/src/tools/browser/cdp-client/factory.ts +23 -24
- package/src/tools/browser/cdp-client/index.ts +1 -14
- package/src/tools/computer-use/definitions.ts +42 -20
- package/src/tools/executor.ts +2 -0
- package/src/tools/host-filesystem/edit.ts +26 -0
- package/src/tools/host-filesystem/read.ts +26 -0
- package/src/tools/host-filesystem/transfer.ts +31 -1
- package/src/tools/host-filesystem/write.ts +26 -0
- package/src/tools/host-terminal/host-shell.ts +58 -0
- package/src/tools/schedule/create.ts +6 -0
- package/src/tools/schedule/list.ts +2 -0
- package/src/tools/schedule/update.ts +10 -0
- package/src/tools/shared/filesystem/file-ops-service.ts +2 -0
- package/src/tools/shared/filesystem/path-policy.ts +25 -1
- package/src/tools/skills/load.ts +0 -32
- package/src/tools/tool-approval-handler.ts +1 -5
- package/src/tools/types.ts +4 -0
- package/src/usage/pricing.ts +1 -1
- package/src/workspace/hatched-date.ts +86 -0
- package/src/workspace/migrations/003-seed-device-id.ts +1 -1
- package/src/workspace/migrations/006-services-config.ts +8 -5
- package/src/workspace/migrations/016-extract-feature-flags-to-protected.ts +3 -9
- package/src/workspace/migrations/021-move-signals-to-workspace.ts +4 -10
- package/src/workspace/migrations/022-move-hooks-to-workspace.ts +4 -10
- package/src/workspace/migrations/023-move-config-files-to-workspace.ts +4 -11
- package/src/workspace/migrations/024-move-runtime-files-to-workspace.ts +3 -10
- package/src/workspace/migrations/040-seed-latency-callsite-defaults.ts +3 -2
- package/src/workspace/migrations/050-seed-main-agent-opus-callsite.ts +2 -1
- package/src/workspace/migrations/059-move-pid-to-workspace.ts +3 -8
- package/src/workspace/migrations/061-move-backup-key-to-workspace.ts +3 -8
- package/src/workspace/migrations/AGENTS.md +1 -1
- package/src/workspace/migrations/migrate-to-workspace-volume.ts +4 -10
- package/src/workspace/migrations/utils.ts +21 -0
- package/src/__tests__/host-browser-e2e-cloud.test.ts +0 -443
- package/src/__tests__/host-browser-e2e-self-hosted-capability.test.ts +0 -226
- package/src/__tests__/host-browser-ws-events-e2e.test.ts +0 -427
- package/src/__tests__/twilio-rest.test.ts +0 -34
- package/src/backup/__tests__/backup-key.test.ts +0 -152
- package/src/backup/__tests__/backup-worker.test.ts +0 -782
- package/src/backup/__tests__/offsite-writer.test.ts +0 -641
- package/src/backup/__tests__/stream-crypt.test.ts +0 -228
- package/src/backup/backup-key.ts +0 -137
- package/src/backup/backup-worker.ts +0 -472
- package/src/backup/offsite-writer.ts +0 -222
- package/src/backup/stream-crypt.ts +0 -263
- package/src/daemon/message-types/pairing.ts +0 -58
- package/src/outbound-proxy/config.ts +0 -20
- package/src/outbound-proxy/health.ts +0 -18
- package/src/outbound-proxy/types.ts +0 -150
- package/src/runtime/capability-tokens.ts +0 -190
- package/src/signals/mcp-reload.ts +0 -18
|
@@ -4,13 +4,13 @@ import type { RecallEvidence, RecallSource } from "./types.js";
|
|
|
4
4
|
|
|
5
5
|
export type RecallAgentConfidence = "high" | "medium" | "low";
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
interface RecallAgentToolDefinition {
|
|
8
8
|
name: string;
|
|
9
9
|
description: string;
|
|
10
10
|
input_schema: Record<string, unknown>;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
interface RecallAgentPromptOptions {
|
|
14
14
|
query: string;
|
|
15
15
|
availableSources?: readonly RecallSource[];
|
|
16
16
|
evidence: readonly RecallEvidence[];
|
|
@@ -18,7 +18,7 @@ export interface RecallAgentPromptOptions {
|
|
|
18
18
|
maxSearchCalls?: number;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
|
|
21
|
+
interface RecallAgentPromptBundle {
|
|
22
22
|
prompt: string;
|
|
23
23
|
evidence: RecallEvidence[];
|
|
24
24
|
}
|
|
@@ -30,14 +30,14 @@ export interface RecallAgentFinish {
|
|
|
30
30
|
unresolved?: string[];
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
-
|
|
33
|
+
type RecallFinishFallbackReason =
|
|
34
34
|
| "malformed_finish_payload"
|
|
35
35
|
| "invalid_confidence"
|
|
36
36
|
| "invalid_citation_ids"
|
|
37
37
|
| "unknown_citation_ids"
|
|
38
38
|
| "empty_answer";
|
|
39
39
|
|
|
40
|
-
|
|
40
|
+
type RecallFinishValidationResult =
|
|
41
41
|
| { ok: true; finish: RecallAgentFinish }
|
|
42
42
|
| {
|
|
43
43
|
ok: false;
|
|
@@ -46,7 +46,7 @@ export type RecallFinishValidationResult =
|
|
|
46
46
|
missingCitationIds?: string[];
|
|
47
47
|
};
|
|
48
48
|
|
|
49
|
-
|
|
49
|
+
interface RecallCitationValidationResult {
|
|
50
50
|
ok: boolean;
|
|
51
51
|
validCitationIds: string[];
|
|
52
52
|
missingCitationIds: string[];
|
|
@@ -5,6 +5,7 @@ import type {
|
|
|
5
5
|
ProviderResponse,
|
|
6
6
|
ToolUseContent,
|
|
7
7
|
} from "../../providers/types.js";
|
|
8
|
+
import { redactSecrets } from "../../security/secret-scanner.js";
|
|
8
9
|
import {
|
|
9
10
|
buildRecallAgentPromptBundle,
|
|
10
11
|
FINISH_RECALL_TOOL_DEFINITION,
|
|
@@ -42,7 +43,7 @@ import type {
|
|
|
42
43
|
RecallSource,
|
|
43
44
|
} from "./types.js";
|
|
44
45
|
|
|
45
|
-
|
|
46
|
+
type AgenticRecallFallbackReason =
|
|
46
47
|
| "no_provider"
|
|
47
48
|
| "provider_error"
|
|
48
49
|
| "timeout"
|
|
@@ -51,7 +52,7 @@ export type AgenticRecallFallbackReason =
|
|
|
51
52
|
| "citation_validation_failed"
|
|
52
53
|
| "finish_answer_validation_failed";
|
|
53
54
|
|
|
54
|
-
|
|
55
|
+
interface AgenticRecallSearchDebug {
|
|
55
56
|
round: number;
|
|
56
57
|
query: string;
|
|
57
58
|
sources: RecallSource[];
|
|
@@ -61,7 +62,7 @@ export interface AgenticRecallSearchDebug {
|
|
|
61
62
|
error?: string;
|
|
62
63
|
}
|
|
63
64
|
|
|
64
|
-
|
|
65
|
+
interface AgenticRecallInspectDebug {
|
|
65
66
|
round: number;
|
|
66
67
|
paths: string[];
|
|
67
68
|
reason: string;
|
|
@@ -69,7 +70,7 @@ export interface AgenticRecallInspectDebug {
|
|
|
69
70
|
errors?: Array<{ path: string; reason: string }>;
|
|
70
71
|
}
|
|
71
72
|
|
|
72
|
-
|
|
73
|
+
interface AgenticRecallDebug {
|
|
73
74
|
mode: "agentic" | "deterministic_fallback";
|
|
74
75
|
normalizedInput: NormalizedRecallInput;
|
|
75
76
|
roundLimit: number;
|
|
@@ -86,12 +87,12 @@ export interface AgenticRecallDebug {
|
|
|
86
87
|
fallbackDetail?: string;
|
|
87
88
|
}
|
|
88
89
|
|
|
89
|
-
|
|
90
|
+
interface AgenticRecallAnswer extends RecallAnswer {
|
|
90
91
|
content: string;
|
|
91
92
|
debug: AgenticRecallDebug;
|
|
92
93
|
}
|
|
93
94
|
|
|
94
|
-
|
|
95
|
+
interface RunAgenticRecallOptions {
|
|
95
96
|
searchOptions?: DeterministicRecallSearchOptions;
|
|
96
97
|
}
|
|
97
98
|
|
|
@@ -414,6 +415,30 @@ export async function runAgenticRecall(
|
|
|
414
415
|
);
|
|
415
416
|
}
|
|
416
417
|
|
|
418
|
+
/**
|
|
419
|
+
* Redact secrets from workspace-sourced evidence excerpts before they are
|
|
420
|
+
* serialised into a prompt that will be sent to an external LLM provider.
|
|
421
|
+
*
|
|
422
|
+
* Memory, PKB, and conversation evidence is already controlled content —
|
|
423
|
+
* only workspace files can contain arbitrary secrets (API keys, tokens, etc.)
|
|
424
|
+
* written by the user or by tools. This runs the same pattern-based scanner
|
|
425
|
+
* used for shell command summaries and approval prompts, replacing any
|
|
426
|
+
* detected secrets with `<redacted type="…" />` markers.
|
|
427
|
+
*
|
|
428
|
+
* The original evidence array is not mutated; citations and local fallback
|
|
429
|
+
* paths continue to reference unredacted values.
|
|
430
|
+
*/
|
|
431
|
+
export function redactWorkspaceEvidence(
|
|
432
|
+
evidence: readonly RecallEvidence[],
|
|
433
|
+
): readonly RecallEvidence[] {
|
|
434
|
+
return evidence.map((item) => {
|
|
435
|
+
if (item.source !== "workspace") return item;
|
|
436
|
+
const redacted = redactSecrets(item.excerpt);
|
|
437
|
+
if (redacted === item.excerpt) return item;
|
|
438
|
+
return { ...item, excerpt: redacted };
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
|
|
417
442
|
function buildPromptBundle(
|
|
418
443
|
input: NormalizedRecallInput,
|
|
419
444
|
evidence: readonly RecallEvidence[],
|
|
@@ -422,7 +447,7 @@ function buildPromptBundle(
|
|
|
422
447
|
return buildRecallAgentPromptBundle({
|
|
423
448
|
query: input.query,
|
|
424
449
|
availableSources: input.sources,
|
|
425
|
-
evidence,
|
|
450
|
+
evidence: redactWorkspaceEvidence(evidence),
|
|
426
451
|
maxSearchCalls: roundLimit,
|
|
427
452
|
});
|
|
428
453
|
}
|
|
@@ -62,11 +62,20 @@ export function isMemoryV2ReadActive(config: AssistantConfig): boolean {
|
|
|
62
62
|
const log = getLogger("context-search-memory-v2-source");
|
|
63
63
|
|
|
64
64
|
/**
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
65
|
+
* Sentinel passed to Qdrant when `config.memory.v2.ann_candidate_limit` is
|
|
66
|
+
* `null` (unlimited). Qdrant's query API requires an explicit numeric
|
|
67
|
+
* `limit`, so unlimited is represented as a number large enough that any
|
|
68
|
+
* realistic concept-page collection is returned in full.
|
|
69
|
+
*
|
|
70
|
+
* Why not `Number.MAX_SAFE_INTEGER`: Qdrant's sparse-vector `SearchContext`
|
|
71
|
+
* pre-allocates `limit * 16` bytes per query, so passing `MAX_SAFE_INTEGER`
|
|
72
|
+
* triggers a ~144 PB allocation and SIGABRTs the Qdrant process. 1_000_000
|
|
73
|
+
* is ~16 MB of pre-allocation in Qdrant — generous headroom over realistic
|
|
74
|
+
* concept-page counts (low thousands today) while staying well clear of
|
|
75
|
+
* the OOM cliff. Bump explicitly via `ann_candidate_limit` if you ever
|
|
76
|
+
* outgrow it.
|
|
68
77
|
*/
|
|
69
|
-
const
|
|
78
|
+
const UNLIMITED_ANN_CANDIDATE_LIMIT = 1_000_000;
|
|
70
79
|
|
|
71
80
|
/** Cap individual concept-page files we are willing to read for lexical scan. */
|
|
72
81
|
const MEMORY_V2_LEXICAL_MAX_FILE_SIZE_BYTES = 256 * 1024;
|
|
@@ -181,10 +190,13 @@ async function activationEvidence(
|
|
|
181
190
|
if (!denseVector || denseVector.length === 0) return [];
|
|
182
191
|
const sparseVector = generateSparseEmbedding(trimmedQuery);
|
|
183
192
|
|
|
193
|
+
const annLimit =
|
|
194
|
+
context.config.memory.v2.ann_candidate_limit ??
|
|
195
|
+
UNLIMITED_ANN_CANDIDATE_LIMIT;
|
|
184
196
|
const hits = await hybridQueryConceptPages(
|
|
185
197
|
denseVector,
|
|
186
198
|
sparseVector,
|
|
187
|
-
|
|
199
|
+
annLimit,
|
|
188
200
|
);
|
|
189
201
|
if (hits.length === 0) return [];
|
|
190
202
|
|
|
@@ -32,7 +32,7 @@ import {
|
|
|
32
32
|
deleteOrphanAttachments,
|
|
33
33
|
linkAttachmentToMessage,
|
|
34
34
|
} from "./attachments-store.js";
|
|
35
|
-
import { AUTO_ANALYSIS_SOURCE } from "./auto-analysis-
|
|
35
|
+
import { AUTO_ANALYSIS_SOURCE } from "./auto-analysis-constants.js";
|
|
36
36
|
import {
|
|
37
37
|
projectAssistantMessage,
|
|
38
38
|
seedForkedConversationAttention,
|
|
@@ -7,20 +7,15 @@
|
|
|
7
7
|
* first contact.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
import { existsSync, unlinkSync } from "node:fs";
|
|
11
|
-
|
|
12
10
|
import { eq } from "drizzle-orm";
|
|
13
11
|
import { v4 as uuid } from "uuid";
|
|
14
12
|
|
|
15
|
-
import {
|
|
16
|
-
import { getWorkspacePromptPath } from "../util/platform.js";
|
|
13
|
+
import { cleanupBootstrapFiles } from "../prompts/bootstrap-cleanup.js";
|
|
17
14
|
import { initConversationDir } from "./conversation-disk-view.js";
|
|
18
15
|
import { GENERATING_TITLE } from "./conversation-title-service.js";
|
|
19
16
|
import { getDb } from "./db-connection.js";
|
|
20
17
|
import { conversationKeys, conversations } from "./schema.js";
|
|
21
18
|
|
|
22
|
-
const log = getLogger("conversation-key-store");
|
|
23
|
-
|
|
24
19
|
/** Set after the first conversation is created so BOOTSTRAP.md is deleted on the second. */
|
|
25
20
|
let firstConversationSeen = false;
|
|
26
21
|
|
|
@@ -204,15 +199,7 @@ export function getOrCreateConversation(
|
|
|
204
199
|
// for its entire duration. Any subsequent conversation means
|
|
205
200
|
// onboarding is over.
|
|
206
201
|
if (firstConversationSeen) {
|
|
207
|
-
|
|
208
|
-
if (existsSync(bp)) {
|
|
209
|
-
try {
|
|
210
|
-
unlinkSync(bp);
|
|
211
|
-
log.info("Deleted BOOTSTRAP.md — onboarding conversation ended");
|
|
212
|
-
} catch {
|
|
213
|
-
// Best-effort
|
|
214
|
-
}
|
|
215
|
-
}
|
|
202
|
+
cleanupBootstrapFiles("new conversation created after onboarding");
|
|
216
203
|
}
|
|
217
204
|
firstConversationSeen = true;
|
|
218
205
|
|
package/src/memory/db-init.ts
CHANGED
|
@@ -102,6 +102,7 @@ import {
|
|
|
102
102
|
migrateGuardianTimestampsEpochMs,
|
|
103
103
|
migrateGuardianVerificationPurpose,
|
|
104
104
|
migrateGuardianVerificationSessions,
|
|
105
|
+
migrateHeartbeatRuns,
|
|
105
106
|
migrateInviteCodeHashColumn,
|
|
106
107
|
migrateInviteContactId,
|
|
107
108
|
migrateLlmRequestLogMessageId,
|
|
@@ -153,6 +154,7 @@ import {
|
|
|
153
154
|
migrateRenameVoiceToPhone,
|
|
154
155
|
migrateScheduleOneShotRouting,
|
|
155
156
|
migrateScheduleQuietFlag,
|
|
157
|
+
migrateScheduleRetryPolicy,
|
|
156
158
|
migrateScheduleReuseConversation,
|
|
157
159
|
migrateScheduleScriptColumn,
|
|
158
160
|
migrateScheduleWakeConversationId,
|
|
@@ -401,9 +403,11 @@ export function initializeDb(): void {
|
|
|
401
403
|
migrateLlmUsageAttribution,
|
|
402
404
|
migrateSlackCompactionWatermark,
|
|
403
405
|
migrateToolInvocationsMatchedRuleId,
|
|
406
|
+
migrateHeartbeatRuns,
|
|
404
407
|
function migrateBackfillAppConversationIds() {
|
|
405
408
|
backfillAppConversationIds();
|
|
406
409
|
},
|
|
410
|
+
migrateScheduleRetryPolicy,
|
|
407
411
|
];
|
|
408
412
|
|
|
409
413
|
// Run each migration step, catching and logging individual failures so one
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
type SparseEmbedding,
|
|
22
22
|
type TextEmbeddingInput,
|
|
23
23
|
} from "./embedding-types.js";
|
|
24
|
+
import { SPARSE_VOCAB_SIZE, tokenHash, tokenize } from "./sparse-tokenize.js";
|
|
24
25
|
|
|
25
26
|
export type { EmbeddingInput, MultimodalEmbeddingInput, TextEmbeddingInput };
|
|
26
27
|
|
|
@@ -787,31 +788,18 @@ async function isOllamaConfigured(config: AssistantConfig): Promise<boolean> {
|
|
|
787
788
|
// Simple tokenizer + TF-IDF sparse encoder. Produces a SparseEmbedding
|
|
788
789
|
// with term indices (hashed to a fixed vocabulary) and TF-IDF weights.
|
|
789
790
|
// Can be upgraded to a learned sparse encoder (e.g. SPLADE) later.
|
|
790
|
-
|
|
791
|
-
|
|
791
|
+
// Tokenization primitives (`tokenize`, `tokenHash`, `SPARSE_VOCAB_SIZE`) live
|
|
792
|
+
// in `./sparse-tokenize.ts` so the BM25 encoder can share them without
|
|
793
|
+
// transitively depending on this module.
|
|
792
794
|
|
|
793
795
|
/**
|
|
794
796
|
* Bump this version whenever the sparse embedding algorithm changes
|
|
795
|
-
* (e.g. hash function fix, tokenizer change)
|
|
796
|
-
*
|
|
797
|
+
* (e.g. hash function fix, tokenizer change). Now inert metadata — the v1
|
|
798
|
+
* Qdrant sentinel was decoupled from this constant, so a bump no longer
|
|
799
|
+
* forces an automatic rebuild. Operators must explicitly run
|
|
800
|
+
* `assistant memory v2 reembed` to rematerialize the v2 sparse index.
|
|
797
801
|
*/
|
|
798
|
-
export const SPARSE_EMBEDDING_VERSION =
|
|
799
|
-
|
|
800
|
-
/** Tokenize text into lowercase alphanumeric tokens (Unicode-aware). */
|
|
801
|
-
function tokenize(text: string): string[] {
|
|
802
|
-
return text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
803
|
-
}
|
|
804
|
-
|
|
805
|
-
/** Hash a token to a stable index in [0, vocabSize). */
|
|
806
|
-
function tokenHash(token: string, vocabSize: number): number {
|
|
807
|
-
// FNV-1a 32-bit hash for speed
|
|
808
|
-
let hash = 0x811c9dc5;
|
|
809
|
-
for (let i = 0; i < token.length; i++) {
|
|
810
|
-
hash ^= token.charCodeAt(i);
|
|
811
|
-
hash = Math.imul(hash, 0x01000193) >>> 0;
|
|
812
|
-
}
|
|
813
|
-
return hash % vocabSize;
|
|
814
|
-
}
|
|
802
|
+
export const SPARSE_EMBEDDING_VERSION = 4;
|
|
815
803
|
|
|
816
804
|
/**
|
|
817
805
|
* Generate a TF-IDF-based sparse embedding for the given text.
|
|
@@ -333,16 +333,54 @@ describe("ConversationGraphMemory.prepareMemory — v2 routing (per-turn path)",
|
|
|
333
333
|
|
|
334
334
|
expect(result.mode).toBe("per-turn");
|
|
335
335
|
expect(result.injectedBlockText).not.toBeNull();
|
|
336
|
-
expect(result.injectedBlockText).toContain("<memory>");
|
|
336
|
+
expect(result.injectedBlockText).not.toContain("<memory>");
|
|
337
337
|
expect(result.injectedBlockText).toContain("### alice-vscode");
|
|
338
338
|
|
|
339
|
-
// The leading content block on the user message is the v2 block
|
|
339
|
+
// The leading content block on the user message is the v2 block,
|
|
340
|
+
// wrapped exactly once.
|
|
340
341
|
const lastMsg = result.runMessages[result.runMessages.length - 1];
|
|
341
342
|
expect(lastMsg?.role).toBe("user");
|
|
342
343
|
const firstBlock = lastMsg?.content[0];
|
|
343
344
|
expect(firstBlock?.type).toBe("text");
|
|
344
345
|
if (firstBlock?.type !== "text") throw new Error("unexpected block type");
|
|
345
|
-
expect(firstBlock.text
|
|
346
|
+
expect(firstBlock.text.startsWith("<memory>\n")).toBe(true);
|
|
347
|
+
expect(firstBlock.text.endsWith("\n</memory>")).toBe(true);
|
|
348
|
+
// No nested wrapper.
|
|
349
|
+
expect(firstBlock.text.match(/<memory>/g)?.length).toBe(1);
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
test("reinjectCachedMemory after v2 injection wraps exactly once (no double-wrap)", async () => {
|
|
353
|
+
// Regression for the double-wrap bug: v2 cached `lastInjectedBlock`
|
|
354
|
+
// already wrapped, then `reinjectCachedMemory` re-wrapped via
|
|
355
|
+
// `injectTextBlock`, producing `<memory>\n<memory>\n...\n</memory>\n</memory>`.
|
|
356
|
+
_setOverridesForTesting({ "memory-v2-enabled": true });
|
|
357
|
+
stageTurn([{ slug: "alice-vscode", denseScore: 0.9 }]);
|
|
358
|
+
|
|
359
|
+
const memory = makeMemory();
|
|
360
|
+
const config = makeConfig(true);
|
|
361
|
+
const messages = makeMessages("Tell me about Alice's editor preferences");
|
|
362
|
+
|
|
363
|
+
const initial = await memory.prepareMemory(
|
|
364
|
+
messages,
|
|
365
|
+
config,
|
|
366
|
+
new AbortController().signal,
|
|
367
|
+
noopEvent,
|
|
368
|
+
);
|
|
369
|
+
expect(initial.injectedBlockText).not.toBeNull();
|
|
370
|
+
|
|
371
|
+
// Simulate post-compaction: caller re-runs `applyRuntimeInjections`
|
|
372
|
+
// (which strips memory injections) and then asks for the cached
|
|
373
|
+
// memory to be re-prepended.
|
|
374
|
+
const reinjected = memory.reinjectCachedMemory(messages);
|
|
375
|
+
const lastMsg = reinjected.runMessages[reinjected.runMessages.length - 1];
|
|
376
|
+
const firstBlock = lastMsg?.content[0];
|
|
377
|
+
expect(firstBlock?.type).toBe("text");
|
|
378
|
+
if (firstBlock?.type !== "text") throw new Error("unexpected block type");
|
|
379
|
+
expect(firstBlock.text.startsWith("<memory>\n")).toBe(true);
|
|
380
|
+
expect(firstBlock.text.endsWith("\n</memory>")).toBe(true);
|
|
381
|
+
expect(firstBlock.text.match(/<memory>/g)?.length).toBe(1);
|
|
382
|
+
expect(firstBlock.text.match(/<\/memory>/g)?.length).toBe(1);
|
|
383
|
+
expect(firstBlock.text).toContain("### alice-vscode");
|
|
346
384
|
});
|
|
347
385
|
|
|
348
386
|
test("flag on + config on with empty Qdrant hits → no v2 block, v1 fallback skipped", async () => {
|
|
@@ -384,7 +422,14 @@ describe("ConversationGraphMemory.prepareMemory — v2 routing (context-load pat
|
|
|
384
422
|
|
|
385
423
|
expect(result.mode).toBe("context-load");
|
|
386
424
|
expect(result.injectedBlockText).not.toBeNull();
|
|
387
|
-
expect(result.injectedBlockText).toContain("
|
|
425
|
+
expect(result.injectedBlockText).toContain("### alice-vscode");
|
|
426
|
+
// injectedBlockText is the unwrapped inner content; the wrapper is
|
|
427
|
+
// applied at injection time on the run message.
|
|
428
|
+
expect(result.injectedBlockText).not.toContain("<memory>");
|
|
429
|
+
const lastMsg = result.runMessages[result.runMessages.length - 1];
|
|
430
|
+
const firstBlock = lastMsg?.content[0];
|
|
431
|
+
if (firstBlock?.type !== "text") throw new Error("unexpected block type");
|
|
432
|
+
expect(firstBlock.text.match(/<memory>/g)?.length).toBe(1);
|
|
388
433
|
});
|
|
389
434
|
|
|
390
435
|
test("flag off → v2 not run on first turn either", async () => {
|
|
@@ -691,7 +691,7 @@ export class ConversationGraphMemory {
|
|
|
691
691
|
|
|
692
692
|
return {
|
|
693
693
|
routed: true,
|
|
694
|
-
runMessages:
|
|
694
|
+
runMessages: injectTextBlock(messages, result.block),
|
|
695
695
|
injectedBlockText: result.block,
|
|
696
696
|
};
|
|
697
697
|
}
|
|
@@ -881,26 +881,3 @@ function readRawUserText(message: Message | undefined): string | null {
|
|
|
881
881
|
if (texts.length === 0) return null;
|
|
882
882
|
return texts.join(" ");
|
|
883
883
|
}
|
|
884
|
-
|
|
885
|
-
/**
|
|
886
|
-
* Prepend a pre-rendered `<memory>` block (produced by
|
|
887
|
-
* `injectMemoryV2Block`) to the last user message. Unlike v1's
|
|
888
|
-
* {@link injectMemoryBlock}, the input here is already wrapped — we
|
|
889
|
-
* just need to attach it as a leading text block. We still strip any
|
|
890
|
-
* pre-existing memory injections first so the layer is idempotent
|
|
891
|
-
* across compaction-driven re-injection.
|
|
892
|
-
*/
|
|
893
|
-
function prependMemoryV2Block(messages: Message[], block: string): Message[] {
|
|
894
|
-
if (block.trim().length === 0) return messages;
|
|
895
|
-
if (messages.length === 0) return messages;
|
|
896
|
-
const cleaned = stripExistingMemoryInjections(messages);
|
|
897
|
-
const userTail = cleaned[cleaned.length - 1];
|
|
898
|
-
if (!userTail || userTail.role !== "user") return messages;
|
|
899
|
-
return [
|
|
900
|
-
...cleaned.slice(0, -1),
|
|
901
|
-
{
|
|
902
|
-
...userTail,
|
|
903
|
-
content: [{ type: "text" as const, text: block }, ...userTail.content],
|
|
904
|
-
},
|
|
905
|
-
];
|
|
906
|
-
}
|
|
@@ -2,8 +2,10 @@
|
|
|
2
2
|
// Memory Graph — Qdrant vector search for graph nodes
|
|
3
3
|
// ---------------------------------------------------------------------------
|
|
4
4
|
|
|
5
|
+
import { getConfig } from "../../config/loader.js";
|
|
5
6
|
import type { AssistantConfig } from "../../config/types.js";
|
|
6
7
|
import { getLogger } from "../../util/logger.js";
|
|
8
|
+
import { isMemoryV2ReadActive } from "../context-search/sources/memory-v2.js";
|
|
7
9
|
import { selectedBackendSupportsMultimodal } from "../embedding-backend.js";
|
|
8
10
|
import type { EmbeddingInput } from "../embedding-types.js";
|
|
9
11
|
import { embedAndUpsert } from "../job-utils.js";
|
|
@@ -48,6 +50,12 @@ export async function searchGraphNodes(
|
|
|
48
50
|
dateRange?: { afterMs?: number; beforeMs?: number },
|
|
49
51
|
excludeScopeIds?: string[],
|
|
50
52
|
): Promise<GraphSearchResult[]> {
|
|
53
|
+
// v2 owns the read path when both gates are on. The v1 `memory` collection
|
|
54
|
+
// is in active retirement and a corrupted sparse segment can OOM-crash the
|
|
55
|
+
// shared Qdrant process — short-circuiting here keeps v1 background work
|
|
56
|
+
// and stale callers from taking v2 down with them.
|
|
57
|
+
if (isMemoryV2ReadActive(getConfig())) return [];
|
|
58
|
+
|
|
51
59
|
if (isQdrantBreakerOpen()) {
|
|
52
60
|
log.warn("Qdrant circuit breaker open, skipping graph search");
|
|
53
61
|
return [];
|
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
} from "../../providers/provider-send-message.js";
|
|
15
15
|
import type { ContentBlock, ImageContent } from "../../providers/types.js";
|
|
16
16
|
import { getLogger } from "../../util/logger.js";
|
|
17
|
+
import { isMemoryV2ReadActive } from "../context-search/sources/memory-v2.js";
|
|
17
18
|
import { embedWithRetry } from "../embed.js";
|
|
18
19
|
import {
|
|
19
20
|
generateSparseEmbedding,
|
|
@@ -425,6 +426,33 @@ export interface ContextLoadResult {
|
|
|
425
426
|
export async function loadContextMemory(
|
|
426
427
|
opts: ContextLoadOpts,
|
|
427
428
|
): Promise<ContextLoadResult> {
|
|
429
|
+
// v2 owns the read path when both gates are on. The v1 collection is in
|
|
430
|
+
// active retirement and querying it can OOM-crash Qdrant via a corrupted
|
|
431
|
+
// sparse segment, so we skip the embedding work and downstream searches
|
|
432
|
+
// entirely. Caller (`runContextLoad`) sees zero nodes and routes to the
|
|
433
|
+
// v2 activation pipeline.
|
|
434
|
+
if (isMemoryV2ReadActive(opts.config)) {
|
|
435
|
+
return {
|
|
436
|
+
nodes: [],
|
|
437
|
+
serendipityNodes: [],
|
|
438
|
+
triggeredNodes: [],
|
|
439
|
+
latencyMs: 0,
|
|
440
|
+
metrics: {
|
|
441
|
+
semanticHits: 0,
|
|
442
|
+
mergedCount: 0,
|
|
443
|
+
selectedCount: 0,
|
|
444
|
+
tier1Count: 0,
|
|
445
|
+
tier2Count: 0,
|
|
446
|
+
hybridSearchLatencyMs: 0,
|
|
447
|
+
sparseVectorUsed: false,
|
|
448
|
+
embeddingProvider: null,
|
|
449
|
+
embeddingModel: null,
|
|
450
|
+
queryContext: null,
|
|
451
|
+
topCandidates: [],
|
|
452
|
+
},
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
|
|
428
456
|
const start = Date.now();
|
|
429
457
|
const ctxLoadCfg = opts.config.memory.retrieval.injection.contextLoad;
|
|
430
458
|
const maxNodes = opts.maxNodes ?? ctxLoadCfg.maxNodes;
|
|
@@ -76,7 +76,7 @@ export const graphRememberDefinition: ToolDefinition = {
|
|
|
76
76
|
finish_turn: {
|
|
77
77
|
type: "boolean",
|
|
78
78
|
description:
|
|
79
|
-
"
|
|
79
|
+
"When you have nothing else to say and want to hand control back to the user you MUST set this to true. When true, your turn ends after this tool call. It's critical that you do this in order to avoid unnecessary LLM calls.",
|
|
80
80
|
},
|
|
81
81
|
},
|
|
82
82
|
required: ["content"],
|
|
@@ -103,6 +103,12 @@ mock.module("../../v2/qdrant.js", () => ({
|
|
|
103
103
|
deleteConceptPageEmbedding: async (slug: string) => {
|
|
104
104
|
deleteCalls.push(slug);
|
|
105
105
|
},
|
|
106
|
+
// Other exports from the real module — stubbed so transitive imports
|
|
107
|
+
// don't crash on missing names when the mock replaces the module wholesale.
|
|
108
|
+
hybridQueryConceptPages: async () => [],
|
|
109
|
+
_resetMemoryV2QdrantForTests: () => {},
|
|
110
|
+
ensureConceptPageCollection: async () => {},
|
|
111
|
+
MEMORY_V2_COLLECTION: "memory_v2_concept_pages",
|
|
106
112
|
}));
|
|
107
113
|
|
|
108
114
|
// ── Workspace setup ────────────────────────────────────────────────
|
|
@@ -321,7 +327,7 @@ describe("enqueueEmbedConceptPageJob", () => {
|
|
|
321
327
|
const id = enqueueEmbedConceptPageJob({ slug: "alice-prefers-vs-code" });
|
|
322
328
|
expect(id).toBeTruthy();
|
|
323
329
|
|
|
324
|
-
const claimed = claimMemoryJobs(10);
|
|
330
|
+
const claimed = claimMemoryJobs({ slowLlm: 10, fast: 10, embed: 10 });
|
|
325
331
|
expect(claimed).toHaveLength(1);
|
|
326
332
|
const [job] = claimed;
|
|
327
333
|
expect(job.type).toBe("embed_concept_page");
|
|
@@ -337,7 +343,7 @@ describe("enqueueEmbedConceptPageJob", () => {
|
|
|
337
343
|
|
|
338
344
|
enqueueEmbedConceptPageJob({ slug: "round-trip-slug" });
|
|
339
345
|
|
|
340
|
-
const claimed = claimMemoryJobs(10);
|
|
346
|
+
const claimed = claimMemoryJobs({ slowLlm: 10, fast: 10, embed: 10 });
|
|
341
347
|
expect(claimed).toHaveLength(1);
|
|
342
348
|
const [job] = claimed;
|
|
343
349
|
expect(job.type).toBe("embed_concept_page");
|
|
@@ -24,6 +24,7 @@ import type { AssistantConfig } from "../../config/types.js";
|
|
|
24
24
|
import { BackendUnavailableError } from "../../util/errors.js";
|
|
25
25
|
import { getLogger } from "../../util/logger.js";
|
|
26
26
|
import { getWorkspaceDir } from "../../util/platform.js";
|
|
27
|
+
import { applyCorrectionIfCalibrated } from "../anisotropy.js";
|
|
27
28
|
import { getDb } from "../db-connection.js";
|
|
28
29
|
import {
|
|
29
30
|
embedWithBackend,
|
|
@@ -39,6 +40,10 @@ import {
|
|
|
39
40
|
deleteConceptPageEmbedding,
|
|
40
41
|
upsertConceptPageEmbedding,
|
|
41
42
|
} from "../v2/qdrant.js";
|
|
43
|
+
import {
|
|
44
|
+
generateBm25DocEmbedding,
|
|
45
|
+
getConceptPageCorpusStats,
|
|
46
|
+
} from "../v2/sparse-bm25.js";
|
|
42
47
|
|
|
43
48
|
const log = getLogger("memory-v2-embed-concept-page");
|
|
44
49
|
|
|
@@ -141,7 +146,17 @@ export async function embedConceptPageJob(
|
|
|
141
146
|
|
|
142
147
|
// Sparse is cheap (in-process tokenization) and changes any time the body
|
|
143
148
|
// changes, so we always recompute it rather than caching alongside dense.
|
|
144
|
-
|
|
149
|
+
// BM25 weights live on the doc side; queries embed binary occurrence in
|
|
150
|
+
// sim.ts. When corpus stats aren't built yet (cold daemon, walking the
|
|
151
|
+
// corpus for the first time), fall back to the legacy TF-only encoding —
|
|
152
|
+
// the next reembed pass overwrites the page once stats are available.
|
|
153
|
+
const corpusStats = getConceptPageCorpusStats();
|
|
154
|
+
const sparse = corpusStats
|
|
155
|
+
? generateBm25DocEmbedding(text, corpusStats, {
|
|
156
|
+
k1: config.memory.v2.bm25_k1,
|
|
157
|
+
b: config.memory.v2.bm25_b,
|
|
158
|
+
})
|
|
159
|
+
: generateSparseEmbedding(text);
|
|
145
160
|
|
|
146
161
|
const now = Date.now();
|
|
147
162
|
// Persist freshly embedded vectors for cross-restart reuse. On cache hit
|
|
@@ -189,9 +204,20 @@ export async function embedConceptPageJob(
|
|
|
189
204
|
}
|
|
190
205
|
}
|
|
191
206
|
|
|
207
|
+
// Apply anisotropy correction at the boundary between the (raw) cached
|
|
208
|
+
// dense vector and the Qdrant collection. Storing raw in SQLite and
|
|
209
|
+
// corrected in Qdrant means a recalibration just needs a reembed pass —
|
|
210
|
+
// the cache survives and the (cheap) correction math reruns over each
|
|
211
|
+
// cached vector. Pass-through when no calibration is fit yet.
|
|
212
|
+
const correctedDense = await applyCorrectionIfCalibrated(
|
|
213
|
+
dense,
|
|
214
|
+
provider,
|
|
215
|
+
model,
|
|
216
|
+
);
|
|
217
|
+
|
|
192
218
|
await upsertConceptPageEmbedding({
|
|
193
219
|
slug,
|
|
194
|
-
dense,
|
|
220
|
+
dense: correctedDense,
|
|
195
221
|
sparse,
|
|
196
222
|
updatedAt: now,
|
|
197
223
|
});
|
|
@@ -134,7 +134,7 @@ describe("enqueuePkbIndexJob", () => {
|
|
|
134
134
|
});
|
|
135
135
|
expect(id).toBeTruthy();
|
|
136
136
|
|
|
137
|
-
const claimed = claimMemoryJobs(10);
|
|
137
|
+
const claimed = claimMemoryJobs({ slowLlm: 10, fast: 10, embed: 10 });
|
|
138
138
|
expect(claimed).toHaveLength(1);
|
|
139
139
|
const [job] = claimed;
|
|
140
140
|
const expectedType: MemoryJobType = "embed_pkb_file";
|
|
@@ -153,7 +153,7 @@ describe("enqueuePkbIndexJob", () => {
|
|
|
153
153
|
memoryScopeId: "scope-rt",
|
|
154
154
|
});
|
|
155
155
|
|
|
156
|
-
const claimed = claimMemoryJobs(10);
|
|
156
|
+
const claimed = claimMemoryJobs({ slowLlm: 10, fast: 10, embed: 10 });
|
|
157
157
|
expect(claimed).toHaveLength(1);
|
|
158
158
|
const [job] = claimed;
|
|
159
159
|
expect(job.type).toBe("embed_pkb_file");
|