@vellumai/assistant 0.5.6 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +16 -2
- package/ARCHITECTURE.md +6 -75
- package/Dockerfile +3 -2
- package/README.md +0 -2
- package/bun.lock +0 -414
- package/docker-entrypoint.sh +9 -0
- package/docs/architecture/keychain-broker.md +45 -240
- package/docs/architecture/memory.md +13 -11
- package/docs/architecture/security.md +0 -17
- package/docs/credential-execution-service.md +2 -2
- package/node_modules/@vellumai/ces-contracts/package.json +1 -0
- package/node_modules/@vellumai/ces-contracts/src/error.ts +1 -1
- package/node_modules/@vellumai/ces-contracts/src/grants.ts +1 -1
- package/node_modules/@vellumai/ces-contracts/src/handles.ts +1 -1
- package/node_modules/@vellumai/ces-contracts/src/index.ts +1 -1
- package/node_modules/@vellumai/ces-contracts/src/rpc.ts +120 -1
- package/node_modules/@vellumai/credential-storage/package.json +1 -0
- package/node_modules/@vellumai/egress-proxy/package.json +1 -0
- package/package.json +2 -3
- package/src/__tests__/actor-token-service.test.ts +0 -114
- package/src/__tests__/approval-cascade.test.ts +0 -1
- package/src/__tests__/assistant-feature-flags-integration.test.ts +30 -29
- package/src/__tests__/browser-fill-credential.test.ts +1 -1
- package/src/__tests__/browser-skill-endstate.test.ts +6 -5
- package/src/__tests__/btw-routes.test.ts +0 -39
- package/src/__tests__/call-controller.test.ts +0 -1
- package/src/__tests__/call-domain.test.ts +0 -128
- package/src/__tests__/ces-rpc-credential-backend.test.ts +199 -0
- package/src/__tests__/ces-startup-timeout.test.ts +40 -0
- package/src/__tests__/channel-approval-routes.test.ts +0 -5
- package/src/__tests__/channel-readiness-service.test.ts +1 -60
- package/src/__tests__/checker.test.ts +4 -2
- package/src/__tests__/cli-command-risk-guard.test.ts +112 -0
- package/src/__tests__/config-schema-cmd.test.ts +0 -2
- package/src/__tests__/config-schema.test.ts +3 -1
- package/src/__tests__/conversation-abort-tool-results.test.ts +0 -1
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +0 -2
- package/src/__tests__/conversation-agent-loop.test.ts +2 -4
- package/src/__tests__/conversation-attention-telegram.test.ts +0 -5
- package/src/__tests__/conversation-confirmation-signals.test.ts +0 -1
- package/src/__tests__/conversation-error.test.ts +15 -1
- package/src/__tests__/conversation-init.benchmark.test.ts +0 -2
- package/src/__tests__/conversation-messaging-secret-redirect.test.ts +1 -1
- package/src/__tests__/conversation-pre-run-repair.test.ts +0 -1
- package/src/__tests__/conversation-provider-retry-repair.test.ts +0 -1
- package/src/__tests__/conversation-queue.test.ts +0 -1
- package/src/__tests__/conversation-skill-tools.test.ts +0 -54
- package/src/__tests__/conversation-slash-queue.test.ts +0 -1
- package/src/__tests__/conversation-slash-unknown.test.ts +0 -1
- package/src/__tests__/conversation-title-service.test.ts +87 -0
- package/src/__tests__/conversation-workspace-injection.test.ts +0 -1
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +0 -1
- package/src/__tests__/credential-execution-client.test.ts +5 -2
- package/src/__tests__/credential-execution-feature-gates.test.ts +59 -30
- package/src/__tests__/credential-execution-managed-contract.test.ts +35 -20
- package/src/__tests__/credential-security-e2e.test.ts +1 -67
- package/src/__tests__/credential-security-invariants.test.ts +6 -50
- package/src/__tests__/credentials-cli.test.ts +82 -3
- package/src/__tests__/daemon-credential-client.test.ts +123 -0
- package/src/__tests__/db-migration-rollback.test.ts +2015 -1
- package/src/__tests__/deterministic-verification-control-plane.test.ts +1 -0
- package/src/__tests__/docker-signing-key-bootstrap.test.ts +34 -143
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +6 -4
- package/src/__tests__/gateway-client-managed-outbound.test.ts +79 -1
- package/src/__tests__/guardian-routing-state.test.ts +0 -5
- package/src/__tests__/host-shell-tool.test.ts +6 -7
- package/src/__tests__/http-user-message-parity.test.ts +3 -103
- package/src/__tests__/inbound-invite-redemption.test.ts +0 -4
- package/src/__tests__/inline-skill-load-permissions.test.ts +6 -8
- package/src/__tests__/intent-routing.test.ts +0 -13
- package/src/__tests__/jobs-store-qdrant-breaker.test.ts +178 -0
- package/src/__tests__/journal-context.test.ts +335 -0
- package/src/__tests__/keychain-broker-client.test.ts +161 -22
- package/src/__tests__/memory-context-benchmark.benchmark.test.ts +0 -3
- package/src/__tests__/memory-jobs-worker-backoff.test.ts +150 -0
- package/src/__tests__/memory-lifecycle-e2e.test.ts +70 -25
- package/src/__tests__/memory-recall-quality.test.ts +48 -17
- package/src/__tests__/memory-regressions.test.ts +408 -363
- package/src/__tests__/memory-retrieval.benchmark.test.ts +0 -3
- package/src/__tests__/migration-export-http.test.ts +2 -2
- package/src/__tests__/migration-import-commit-http.test.ts +2 -2
- package/src/__tests__/migration-import-preflight-http.test.ts +2 -2
- package/src/__tests__/migration-validate-http.test.ts +2 -2
- package/src/__tests__/non-member-access-request.test.ts +2 -7
- package/src/__tests__/notification-decision-fallback.test.ts +4 -0
- package/src/__tests__/notification-decision-identity.test.ts +4 -0
- package/src/__tests__/notification-decision-strategy.test.ts +71 -0
- package/src/__tests__/oauth-cli.test.ts +5 -1
- package/src/__tests__/permission-types.test.ts +1 -0
- package/src/__tests__/provider-commit-message-generator.test.ts +0 -37
- package/src/__tests__/provider-error-scenarios.test.ts +0 -267
- package/src/__tests__/provider-managed-proxy-integration.test.ts +5 -6
- package/src/__tests__/provider-streaming.benchmark.test.ts +2 -81
- package/src/__tests__/qdrant-manager.test.ts +28 -2
- package/src/__tests__/registry.test.ts +0 -6
- package/src/__tests__/relay-server.test.ts +1 -2
- package/src/__tests__/runtime-attachment-metadata.test.ts +0 -4
- package/src/__tests__/script-proxy-injection-runtime.test.ts +1 -1
- package/src/__tests__/secret-onetime-send.test.ts +1 -1
- package/src/__tests__/secret-routes-managed-proxy.test.ts +0 -4
- package/src/__tests__/secure-keys.test.ts +95 -272
- package/src/__tests__/shell-identity.test.ts +96 -6
- package/src/__tests__/skill-feature-flags-integration.test.ts +22 -14
- package/src/__tests__/skill-feature-flags.test.ts +46 -45
- package/src/__tests__/skill-load-feature-flag.test.ts +7 -10
- package/src/__tests__/skill-load-inline-command.test.ts +8 -12
- package/src/__tests__/skill-load-inline-includes.test.ts +6 -10
- package/src/__tests__/skill-load-tool.test.ts +0 -2
- package/src/__tests__/skill-memory.test.ts +17 -3
- package/src/__tests__/skill-projection-feature-flag.test.ts +33 -29
- package/src/__tests__/skills.test.ts +0 -2
- package/src/__tests__/slack-inbound-verification.test.ts +0 -4
- package/src/__tests__/stale-approval-dedup.test.ts +171 -0
- package/src/__tests__/stt-hints.test.ts +437 -0
- package/src/__tests__/suggestion-routes.test.ts +1 -32
- package/src/__tests__/system-prompt.test.ts +0 -1
- package/src/__tests__/task-memory-cleanup.test.ts +14 -0
- package/src/__tests__/tool-executor-shell-integration.test.ts +5 -3
- package/src/__tests__/trusted-contact-lifecycle-notifications.test.ts +0 -5
- package/src/__tests__/trusted-contact-multichannel.test.ts +0 -4
- package/src/__tests__/twilio-routes-twiml.test.ts +139 -1
- package/src/__tests__/update-bulletin.test.ts +0 -2
- package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +6 -9
- package/src/__tests__/voice-quality.test.ts +58 -0
- package/src/__tests__/voice-scoped-grant-consumer.test.ts +0 -7
- package/src/__tests__/workspace-migration-015-migrate-credentials-to-keychain.test.ts +252 -0
- package/src/__tests__/workspace-migration-016-migrate-credentials-from-keychain.test.ts +220 -0
- package/src/__tests__/workspace-migration-down-functions.test.ts +1009 -0
- package/src/__tests__/workspace-migrations-runner.test.ts +114 -0
- package/src/acp/agent-process.ts +9 -1
- package/src/agent/loop.ts +1 -1
- package/src/approvals/guardian-request-resolvers.ts +164 -38
- package/src/calls/__tests__/tts-text-sanitizer.test.ts +254 -0
- package/src/calls/audio-store.test.ts +97 -0
- package/src/calls/audio-store.ts +205 -0
- package/src/calls/call-controller.ts +90 -8
- package/src/calls/call-domain.ts +3 -0
- package/src/calls/call-store.ts +10 -3
- package/src/calls/fish-audio-client.ts +129 -0
- package/src/calls/relay-server.ts +27 -0
- package/src/calls/stt-hints.ts +189 -0
- package/src/calls/tts-text-sanitizer.ts +61 -0
- package/src/calls/twilio-routes.ts +34 -5
- package/src/calls/types.ts +1 -0
- package/src/calls/voice-ingress-preflight.ts +0 -42
- package/src/calls/voice-quality.ts +38 -5
- package/src/calls/voice-session-bridge.ts +7 -12
- package/src/cli/commands/avatar.ts +2 -2
- package/src/cli/commands/config.ts +1 -4
- package/src/cli/commands/credentials.ts +128 -82
- package/src/cli/commands/doctor.ts +2 -2
- package/src/cli/commands/keys.ts +7 -7
- package/src/cli/commands/memory.ts +1 -1
- package/src/cli/commands/oauth/connections.ts +11 -29
- package/src/cli/commands/oauth/index.ts +7 -0
- package/src/cli/commands/oauth/platform.ts +525 -0
- package/src/cli/commands/platform.ts +3 -3
- package/src/cli/lib/daemon-credential-client.ts +284 -0
- package/src/cli.ts +1 -1
- package/src/config/assistant-feature-flags.ts +186 -5
- package/src/config/bundled-skills/AGENTS.md +34 -0
- package/src/config/bundled-skills/acp/SKILL.md +10 -0
- package/src/config/bundled-skills/app-builder/SKILL.md +0 -4
- package/src/config/bundled-skills/messaging/SKILL.md +5 -5
- package/src/config/bundled-skills/messaging/tools/messaging-analyze-style.ts +2 -2
- package/src/config/bundled-skills/phone-calls/TOOLS.json +4 -0
- package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +1 -0
- package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +1 -0
- package/src/config/bundled-skills/settings/SKILL.md +15 -2
- package/src/config/bundled-skills/settings/TOOLS.json +47 -2
- package/src/config/bundled-skills/settings/tools/avatar-remove.ts +59 -0
- package/src/config/bundled-skills/settings/tools/avatar-update.ts +80 -0
- package/src/config/bundled-skills/settings/tools/voice-config-update.ts +42 -0
- package/src/config/bundled-skills/slack/SKILL.md +1 -1
- package/src/config/bundled-tool-registry.ts +5 -11
- package/src/config/defaults.ts +0 -2
- package/src/config/env-registry.ts +5 -5
- package/src/config/env.ts +21 -14
- package/src/config/feature-flag-registry.json +49 -9
- package/src/config/loader.ts +106 -42
- package/src/config/schema.ts +9 -29
- package/src/config/schemas/calls.ts +30 -0
- package/src/config/schemas/fish-audio.ts +39 -0
- package/src/config/schemas/inference.ts +2 -2
- package/src/config/schemas/journal.ts +16 -0
- package/src/config/schemas/memory-processing.ts +2 -2
- package/src/config/schemas/security.ts +0 -4
- package/src/config/types.ts +1 -1
- package/src/contacts/contact-store.ts +39 -0
- package/src/contacts/types.ts +2 -0
- package/src/credential-execution/approval-bridge.ts +1 -0
- package/src/credential-execution/executable-discovery.ts +28 -4
- package/src/credential-execution/feature-gates.ts +16 -0
- package/src/credential-execution/process-manager.ts +38 -0
- package/src/credential-execution/startup-timeout.ts +36 -0
- package/src/daemon/approval-generators.ts +3 -9
- package/src/daemon/assistant-attachments.ts +9 -0
- package/src/daemon/config-watcher.ts +5 -0
- package/src/daemon/conversation-error.ts +13 -1
- package/src/daemon/conversation-memory.ts +1 -2
- package/src/daemon/conversation-process.ts +18 -1
- package/src/daemon/conversation-surfaces.ts +30 -1
- package/src/daemon/conversation-tool-setup.ts +0 -105
- package/src/daemon/conversation.ts +21 -1
- package/src/daemon/guardian-action-generators.ts +3 -9
- package/src/daemon/handlers/config-vercel.ts +92 -0
- package/src/daemon/handlers/skills.ts +2 -15
- package/src/daemon/install-symlink.ts +195 -0
- package/src/daemon/lifecycle.ts +234 -51
- package/src/daemon/message-types/conversations.ts +4 -4
- package/src/daemon/message-types/diagnostics.ts +3 -22
- package/src/daemon/message-types/messages.ts +0 -2
- package/src/daemon/message-types/upgrades.ts +8 -0
- package/src/daemon/server.ts +32 -95
- package/src/events/domain-events.ts +2 -1
- package/src/inbound/platform-callback-registration.ts +3 -3
- package/src/instrument.ts +8 -5
- package/src/memory/app-store.ts +31 -0
- package/src/memory/conversation-title-service.ts +50 -1
- package/src/memory/db-init.ts +16 -0
- package/src/memory/indexer.ts +19 -10
- package/src/memory/items-extractor.ts +328 -321
- package/src/memory/job-handlers/conversation-starters.ts +4 -1
- package/src/memory/job-handlers/summarization.ts +26 -16
- package/src/memory/jobs-store.ts +63 -6
- package/src/memory/jobs-worker.ts +31 -7
- package/src/memory/journal-memory.ts +214 -0
- package/src/memory/migrations/001-job-deferrals.ts +19 -0
- package/src/memory/migrations/004-entity-relation-dedup.ts +10 -0
- package/src/memory/migrations/005-fingerprint-scope-unique.ts +76 -0
- package/src/memory/migrations/006-scope-salted-fingerprints.ts +50 -0
- package/src/memory/migrations/007-assistant-id-to-self.ts +10 -0
- package/src/memory/migrations/008-remove-assistant-id-columns.ts +34 -0
- package/src/memory/migrations/009-llm-usage-events-drop-assistant-id.ts +26 -0
- package/src/memory/migrations/014-backfill-inbox-thread-state.ts +10 -0
- package/src/memory/migrations/015-drop-active-search-index.ts +17 -0
- package/src/memory/migrations/019-notification-tables-schema-migration.ts +12 -0
- package/src/memory/migrations/020-rename-macos-ios-channel-to-vellum.ts +121 -0
- package/src/memory/migrations/024-embedding-vector-blob.ts +74 -0
- package/src/memory/migrations/026a-embeddings-nullable-vector-json.ts +82 -0
- package/src/memory/migrations/036-normalize-phone-identities.ts +11 -0
- package/src/memory/migrations/116-messages-fts.ts +106 -1
- package/src/memory/migrations/126-backfill-guardian-principal-id.ts +52 -0
- package/src/memory/migrations/127-guardian-principal-id-not-null.ts +77 -0
- package/src/memory/migrations/134-contacts-notes-column.ts +13 -0
- package/src/memory/migrations/135-backfill-contact-interaction-stats.ts +20 -0
- package/src/memory/migrations/136-drop-assistant-id-columns.ts +52 -0
- package/src/memory/migrations/140-backfill-usage-cache-accounting.ts +13 -0
- package/src/memory/migrations/141-rename-verification-table.ts +54 -0
- package/src/memory/migrations/142-rename-verification-session-id-column.ts +25 -0
- package/src/memory/migrations/143-rename-guardian-verification-values.ts +35 -0
- package/src/memory/migrations/144-rename-voice-to-phone.ts +136 -0
- package/src/memory/migrations/145-drop-accounts-table.ts +32 -0
- package/src/memory/migrations/147-migrate-reminders-to-schedules.ts +14 -1
- package/src/memory/migrations/148-drop-reminders-table.ts +35 -1
- package/src/memory/migrations/150-oauth-apps-client-secret-path.ts +69 -1
- package/src/memory/migrations/162-guardian-timestamps-epoch-ms.ts +290 -0
- package/src/memory/migrations/169-rename-gmail-provider-key-to-google.ts +51 -1
- package/src/memory/migrations/174-rename-thread-starters-table.ts +47 -1
- package/src/memory/migrations/176-drop-capability-card-state.ts +13 -0
- package/src/memory/migrations/180-backfill-inline-attachments-to-disk.ts +16 -0
- package/src/memory/migrations/181-rename-thread-starters-checkpoints.ts +28 -1
- package/src/memory/migrations/190-call-session-skip-disclosure.ts +15 -0
- package/src/memory/migrations/191-backfill-audio-attachment-mime-types.ts +64 -0
- package/src/memory/migrations/192-contacts-user-file-column.ts +15 -0
- package/src/memory/migrations/193-add-source-type-columns.ts +81 -0
- package/src/memory/migrations/index.ts +5 -0
- package/src/memory/migrations/registry.ts +98 -0
- package/src/memory/migrations/validate-migration-state.ts +137 -11
- package/src/memory/qdrant-circuit-breaker.ts +9 -0
- package/src/memory/qdrant-manager.ts +64 -7
- package/src/memory/retriever.test.ts +37 -25
- package/src/memory/retriever.ts +24 -49
- package/src/memory/schema/calls.ts +1 -0
- package/src/memory/schema/contacts.ts +1 -0
- package/src/memory/schema/memory-core.ts +2 -0
- package/src/memory/search/formatting.ts +7 -44
- package/src/memory/search/staleness.ts +4 -0
- package/src/memory/search/tier-classifier.ts +10 -2
- package/src/memory/search/types.ts +2 -5
- package/src/memory/task-memory-cleanup.ts +4 -3
- package/src/notifications/adapters/slack.ts +168 -6
- package/src/notifications/broadcaster.ts +1 -0
- package/src/notifications/copy-composer.ts +59 -2
- package/src/notifications/decision-engine.ts +4 -1
- package/src/notifications/signal.ts +2 -0
- package/src/notifications/types.ts +2 -0
- package/src/oauth/connection-resolver.ts +6 -4
- package/src/permissions/checker.ts +0 -38
- package/src/permissions/shell-identity.ts +76 -22
- package/src/permissions/types.ts +4 -2
- package/src/platform/client.ts +35 -7
- package/src/prompts/journal-context.ts +133 -0
- package/src/prompts/persona-resolver.ts +194 -0
- package/src/prompts/system-prompt.ts +44 -4
- package/src/prompts/templates/SOUL.md +10 -0
- package/src/prompts/templates/users/default.md +1 -0
- package/src/providers/provider-send-message.ts +3 -32
- package/src/providers/registry.ts +29 -179
- package/src/providers/types.ts +1 -1
- package/src/runtime/access-request-helper.ts +4 -0
- package/src/runtime/auth/__tests__/credential-service.test.ts +0 -1
- package/src/runtime/auth/__tests__/external-assistant-id.test.ts +13 -68
- package/src/runtime/auth/__tests__/guard-tests.test.ts +9 -50
- package/src/runtime/auth/external-assistant-id.ts +13 -59
- package/src/runtime/auth/route-policy.ts +17 -1
- package/src/runtime/auth/token-service.ts +43 -138
- package/src/runtime/channel-readiness-service.ts +1 -16
- package/src/runtime/gateway-client.ts +47 -4
- package/src/runtime/guardian-decision-types.ts +45 -4
- package/src/runtime/http-server.ts +31 -3
- package/src/runtime/middleware/error-handler.ts +1 -9
- package/src/runtime/routes/access-request-decision.ts +2 -2
- package/src/runtime/routes/app-management-routes.ts +2 -1
- package/src/runtime/routes/approval-strategies/guardian-callback-strategy.ts +219 -30
- package/src/runtime/routes/approval-strategies/guardian-text-engine-strategy.ts +37 -14
- package/src/runtime/routes/audio-routes.ts +40 -0
- package/src/runtime/routes/btw-routes.ts +0 -17
- package/src/runtime/routes/channel-readiness-routes.ts +9 -4
- package/src/runtime/routes/conversation-query-routes.ts +63 -1
- package/src/runtime/routes/conversation-routes.ts +4 -44
- package/src/runtime/routes/debug-routes.ts +12 -9
- package/src/runtime/routes/diagnostics-routes.ts +1 -477
- package/src/runtime/routes/guardian-approval-interception.ts +168 -11
- package/src/runtime/routes/guardian-approval-prompt.ts +6 -1
- package/src/runtime/routes/guardian-approval-reply-helpers.ts +103 -21
- package/src/runtime/routes/identity-routes.ts +19 -30
- package/src/runtime/routes/inbound-message-handler.ts +31 -1
- package/src/runtime/routes/inbound-stages/acl-enforcement.ts +64 -5
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +52 -40
- package/src/runtime/routes/inbound-stages/secret-ingress-check.ts +4 -33
- package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +1 -1
- package/src/runtime/routes/integrations/twilio.ts +52 -10
- package/src/runtime/routes/integrations/vercel.ts +89 -0
- package/src/runtime/routes/log-export-routes.ts +5 -0
- package/src/runtime/routes/memory-item-routes.test.ts +3 -3
- package/src/runtime/routes/memory-item-routes.ts +46 -14
- package/src/runtime/routes/migration-rollback-routes.ts +209 -0
- package/src/runtime/routes/migration-routes.ts +17 -1
- package/src/runtime/routes/notification-routes.ts +58 -0
- package/src/runtime/routes/schedule-routes.ts +65 -0
- package/src/runtime/routes/secret-routes.ts +141 -10
- package/src/runtime/routes/settings-routes.ts +41 -1
- package/src/runtime/routes/tts-routes.ts +96 -0
- package/src/runtime/routes/upgrade-broadcast-routes.ts +26 -2
- package/src/runtime/routes/workspace-commit-routes.ts +62 -0
- package/src/runtime/routes/workspace-routes.test.ts +22 -1
- package/src/runtime/routes/workspace-routes.ts +1 -1
- package/src/runtime/routes/workspace-utils.ts +86 -2
- package/src/security/ces-credential-client.ts +75 -29
- package/src/security/ces-rpc-credential-backend.ts +86 -0
- package/src/security/credential-backend.ts +22 -92
- package/src/security/keychain-broker-client.ts +10 -2
- package/src/security/secure-keys.ts +113 -115
- package/src/skills/catalog-install.ts +6 -32
- package/src/skills/skill-memory.ts +1 -0
- package/src/subagent/manager.ts +2 -5
- package/src/telemetry/usage-telemetry-reporter.ts +4 -2
- package/src/tools/acp/spawn.ts +78 -1
- package/src/tools/calls/call-start.ts +1 -0
- package/src/tools/credentials/vault.ts +5 -3
- package/src/tools/executor.ts +0 -4
- package/src/tools/memory/definitions.ts +3 -2
- package/src/tools/memory/handlers.ts +10 -7
- package/src/tools/network/script-proxy/session-manager.ts +19 -4
- package/src/tools/network/web-fetch.ts +3 -1
- package/src/tools/skills/execute.ts +1 -1
- package/src/tools/terminal/safe-env.ts +1 -0
- package/src/tools/types.ts +0 -8
- package/src/util/browser.ts +15 -0
- package/src/util/errors.ts +0 -12
- package/src/util/platform.ts +4 -51
- package/src/workspace/git-service.ts +5 -2
- package/src/workspace/migrations/001-avatar-rename.ts +15 -0
- package/src/workspace/migrations/003-seed-device-id.ts +17 -1
- package/src/workspace/migrations/004-extract-collect-usage-data.ts +33 -0
- package/src/workspace/migrations/005-add-send-diagnostics.ts +3 -0
- package/src/workspace/migrations/006-services-config.ts +49 -0
- package/src/workspace/migrations/007-web-search-provider-rename.ts +27 -0
- package/src/workspace/migrations/008-voice-timeout-and-max-steps.ts +3 -0
- package/src/workspace/migrations/009-backfill-conversation-disk-view.ts +4 -0
- package/src/workspace/migrations/010-app-dir-rename.ts +78 -0
- package/src/workspace/migrations/011-backfill-installation-id.ts +11 -0
- package/src/workspace/migrations/012-rename-conversation-disk-view-dirs.ts +44 -0
- package/src/workspace/migrations/013-repair-conversation-disk-view.ts +5 -0
- package/src/workspace/migrations/015-migrate-credentials-to-keychain.ts +153 -0
- package/src/workspace/migrations/016-extract-feature-flags-to-protected.ts +156 -0
- package/src/workspace/migrations/016-migrate-credentials-from-keychain.ts +150 -0
- package/src/workspace/migrations/017-seed-persona-dirs.ts +96 -0
- package/src/workspace/migrations/018-rekey-compound-credential-keys.ts +184 -0
- package/src/workspace/migrations/019-scope-journal-to-guardian.ts +103 -0
- package/src/workspace/migrations/migrate-to-workspace-volume.ts +27 -5
- package/src/workspace/migrations/registry.ts +12 -0
- package/src/workspace/migrations/runner.ts +106 -2
- package/src/workspace/migrations/types.ts +4 -0
- package/src/workspace/provider-commit-message-generator.ts +12 -21
- package/src/__tests__/claude-code-skill-regression.test.ts +0 -206
- package/src/__tests__/claude-code-tool-profiles.test.ts +0 -99
- package/src/__tests__/diagnostics-export.test.ts +0 -288
- package/src/__tests__/local-gateway-health.test.ts +0 -209
- package/src/__tests__/provider-fail-open-selection.test.ts +0 -271
- package/src/__tests__/provider-failover-actual-provider.test.ts +0 -66
- package/src/__tests__/secret-ingress-handler.test.ts +0 -120
- package/src/__tests__/swarm-conversation-integration.test.ts +0 -358
- package/src/__tests__/swarm-dag-pathological.test.ts +0 -547
- package/src/__tests__/swarm-orchestrator.test.ts +0 -463
- package/src/__tests__/swarm-plan-validator.test.ts +0 -384
- package/src/__tests__/swarm-recursion.test.ts +0 -197
- package/src/__tests__/swarm-router-planner.test.ts +0 -234
- package/src/__tests__/swarm-tool.test.ts +0 -185
- package/src/__tests__/swarm-worker-backend.test.ts +0 -144
- package/src/__tests__/swarm-worker-runner.test.ts +0 -288
- package/src/commands/__tests__/cc-command-registry.test.ts +0 -396
- package/src/commands/cc-command-registry.ts +0 -248
- package/src/config/bundled-skills/claude-code/SKILL.md +0 -53
- package/src/config/bundled-skills/claude-code/TOOLS.json +0 -47
- package/src/config/bundled-skills/claude-code/tools/claude-code.ts +0 -12
- package/src/config/bundled-skills/orchestration/SKILL.md +0 -33
- package/src/config/bundled-skills/orchestration/TOOLS.json +0 -35
- package/src/config/bundled-skills/orchestration/tools/swarm-delegate.ts +0 -12
- package/src/config/schemas/swarm.ts +0 -82
- package/src/logfire.ts +0 -135
- package/src/memory/search/lexical.ts +0 -48
- package/src/providers/failover.ts +0 -186
- package/src/runtime/local-gateway-health.ts +0 -275
- package/src/security/secret-ingress.ts +0 -68
- package/src/swarm/backend-claude-code.ts +0 -225
- package/src/swarm/checkpoint.ts +0 -137
- package/src/swarm/graph-utils.ts +0 -53
- package/src/swarm/index.ts +0 -55
- package/src/swarm/limits.ts +0 -66
- package/src/swarm/orchestrator.ts +0 -424
- package/src/swarm/plan-validator.ts +0 -117
- package/src/swarm/router-planner.ts +0 -162
- package/src/swarm/router-prompts.ts +0 -39
- package/src/swarm/synthesizer.ts +0 -81
- package/src/swarm/types.ts +0 -72
- package/src/swarm/worker-backend.ts +0 -131
- package/src/swarm/worker-prompts.ts +0 -80
- package/src/swarm/worker-runner.ts +0 -170
- package/src/tools/claude-code/claude-code.ts +0 -610
- package/src/tools/swarm/delegate.ts +0 -205
|
@@ -1,3 +1,44 @@
|
|
|
1
|
+
import type { DrizzleDb } from "../db-connection.js";
|
|
2
|
+
import { downJobDeferrals } from "./001-job-deferrals.js";
|
|
3
|
+
import { downMemoryEntityRelationDedup } from "./004-entity-relation-dedup.js";
|
|
4
|
+
import { downMemoryItemsFingerprintScopeUnique } from "./005-fingerprint-scope-unique.js";
|
|
5
|
+
import { downMemoryItemsScopeSaltedFingerprints } from "./006-scope-salted-fingerprints.js";
|
|
6
|
+
import { downAssistantIdToSelf } from "./007-assistant-id-to-self.js";
|
|
7
|
+
import { downRemoveAssistantIdColumns } from "./008-remove-assistant-id-columns.js";
|
|
8
|
+
import { downLlmUsageEventsDropAssistantId } from "./009-llm-usage-events-drop-assistant-id.js";
|
|
9
|
+
import { downBackfillInboxThreadState } from "./014-backfill-inbox-thread-state.js";
|
|
10
|
+
import { downDropActiveSearchIndex } from "./015-drop-active-search-index.js";
|
|
11
|
+
import { downNotificationTablesSchema } from "./019-notification-tables-schema-migration.js";
|
|
12
|
+
import { downRenameChannelToVellum } from "./020-rename-macos-ios-channel-to-vellum.js";
|
|
13
|
+
import { downEmbeddingVectorBlob } from "./024-embedding-vector-blob.js";
|
|
14
|
+
import { downEmbeddingsNullableVectorJson } from "./026a-embeddings-nullable-vector-json.js";
|
|
15
|
+
import { downNormalizePhoneIdentities } from "./036-normalize-phone-identities.js";
|
|
16
|
+
import { downBackfillGuardianPrincipalId } from "./126-backfill-guardian-principal-id.js";
|
|
17
|
+
import { downGuardianPrincipalIdNotNull } from "./127-guardian-principal-id-not-null.js";
|
|
18
|
+
import { downContactsNotesColumn } from "./134-contacts-notes-column.js";
|
|
19
|
+
import { downBackfillContactInteractionStats } from "./135-backfill-contact-interaction-stats.js";
|
|
20
|
+
import { downDropAssistantIdColumns } from "./136-drop-assistant-id-columns.js";
|
|
21
|
+
import { downBackfillUsageCacheAccounting } from "./140-backfill-usage-cache-accounting.js";
|
|
22
|
+
import { downRenameVerificationTable } from "./141-rename-verification-table.js";
|
|
23
|
+
import { downRenameVerificationSessionIdColumn } from "./142-rename-verification-session-id-column.js";
|
|
24
|
+
import { downRenameGuardianVerificationValues } from "./143-rename-guardian-verification-values.js";
|
|
25
|
+
import { downRenameVoiceToPhone } from "./144-rename-voice-to-phone.js";
|
|
26
|
+
import { migrateDropAccountsTableDown } from "./145-drop-accounts-table.js";
|
|
27
|
+
import { migrateRemindersToSchedulesDown } from "./147-migrate-reminders-to-schedules.js";
|
|
28
|
+
import { migrateDropRemindersTableDown } from "./148-drop-reminders-table.js";
|
|
29
|
+
import { migrateOAuthAppsClientSecretPathDown } from "./150-oauth-apps-client-secret-path.js";
|
|
30
|
+
import {
|
|
31
|
+
migrateGuardianTimestampsEpochMsDown,
|
|
32
|
+
migrateGuardianTimestampsRebuildDown,
|
|
33
|
+
} from "./162-guardian-timestamps-epoch-ms.js";
|
|
34
|
+
import { migrateRenameGmailProviderKeyToGoogleDown } from "./169-rename-gmail-provider-key-to-google.js";
|
|
35
|
+
import { migrateRenameThreadStartersTableDown } from "./174-rename-thread-starters-table.js";
|
|
36
|
+
import { migrateDropCapabilityCardStateDown } from "./176-drop-capability-card-state.js";
|
|
37
|
+
import { migrateBackfillInlineAttachmentsToDiskDown } from "./180-backfill-inline-attachments-to-disk.js";
|
|
38
|
+
import { migrateRenameThreadStartersCheckpointsDown } from "./181-rename-thread-starters-checkpoints.js";
|
|
39
|
+
import { migrateBackfillAudioAttachmentMimeTypesDown } from "./191-backfill-audio-attachment-mime-types.js";
|
|
40
|
+
import { migrateAddSourceTypeColumnsDown } from "./193-add-source-type-columns.js";
|
|
41
|
+
|
|
1
42
|
export interface MigrationRegistryEntry {
|
|
2
43
|
/** The checkpoint key written to memory_checkpoints on completion. */
|
|
3
44
|
key: string;
|
|
@@ -7,6 +48,8 @@ export interface MigrationRegistryEntry {
|
|
|
7
48
|
dependsOn?: string[];
|
|
8
49
|
/** Human-readable description for diagnostics and future authorship guidance. */
|
|
9
50
|
description: string;
|
|
51
|
+
/** Reverse the migration. Must be idempotent — safe to re-run. */
|
|
52
|
+
down: (database: DrizzleDb) => void;
|
|
10
53
|
}
|
|
11
54
|
|
|
12
55
|
// ---------------------------------------------------------------------------
|
|
@@ -26,18 +69,21 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
|
|
|
26
69
|
version: 1,
|
|
27
70
|
description:
|
|
28
71
|
"Reconcile legacy deferral history from attempts column into deferrals column",
|
|
72
|
+
down: downJobDeferrals,
|
|
29
73
|
},
|
|
30
74
|
{
|
|
31
75
|
key: "migration_memory_entity_relations_dedup_v1",
|
|
32
76
|
version: 2,
|
|
33
77
|
description:
|
|
34
78
|
"Deduplicate entity relation edges before enforcing the (source, target, relation) unique index",
|
|
79
|
+
down: downMemoryEntityRelationDedup,
|
|
35
80
|
},
|
|
36
81
|
{
|
|
37
82
|
key: "migration_memory_items_fingerprint_scope_unique_v1",
|
|
38
83
|
version: 3,
|
|
39
84
|
description:
|
|
40
85
|
"Replace column-level UNIQUE on fingerprint with compound (fingerprint, scope_id) unique index",
|
|
86
|
+
down: downMemoryItemsFingerprintScopeUnique,
|
|
41
87
|
},
|
|
42
88
|
{
|
|
43
89
|
key: "migration_memory_items_scope_salted_fingerprints_v1",
|
|
@@ -45,12 +91,14 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
|
|
|
45
91
|
dependsOn: ["migration_memory_items_fingerprint_scope_unique_v1"],
|
|
46
92
|
description:
|
|
47
93
|
"Recompute memory item fingerprints to include scope_id prefix after schema change",
|
|
94
|
+
down: downMemoryItemsScopeSaltedFingerprints,
|
|
48
95
|
},
|
|
49
96
|
{
|
|
50
97
|
key: "migration_normalize_assistant_id_to_self_v1",
|
|
51
98
|
version: 5,
|
|
52
99
|
description:
|
|
53
100
|
'Normalize all assistant_id values in scoped tables to the implicit "self" single-tenant identity',
|
|
101
|
+
down: downAssistantIdToSelf,
|
|
54
102
|
},
|
|
55
103
|
{
|
|
56
104
|
key: "migration_remove_assistant_id_columns_v1",
|
|
@@ -58,6 +106,7 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
|
|
|
58
106
|
dependsOn: ["migration_normalize_assistant_id_to_self_v1"],
|
|
59
107
|
description:
|
|
60
108
|
"Rebuild four tables to drop the assistant_id column after normalization",
|
|
109
|
+
down: downRemoveAssistantIdColumns,
|
|
61
110
|
},
|
|
62
111
|
{
|
|
63
112
|
key: "migration_remove_assistant_id_lue_v1",
|
|
@@ -65,36 +114,42 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
|
|
|
65
114
|
dependsOn: ["migration_normalize_assistant_id_to_self_v1"],
|
|
66
115
|
description:
|
|
67
116
|
"Remove assistant_id column from llm_usage_events (separate checkpoint from the four-table migration)",
|
|
117
|
+
down: downLlmUsageEventsDropAssistantId,
|
|
68
118
|
},
|
|
69
119
|
{
|
|
70
120
|
key: "backfill_inbox_thread_state_from_bindings",
|
|
71
121
|
version: 8,
|
|
72
122
|
description:
|
|
73
123
|
"Seed assistant_inbox_thread_state from external_conversation_bindings",
|
|
124
|
+
down: downBackfillInboxThreadState,
|
|
74
125
|
},
|
|
75
126
|
{
|
|
76
127
|
key: "drop_active_search_index_v1",
|
|
77
128
|
version: 9,
|
|
78
129
|
description:
|
|
79
130
|
"Drop old idx_memory_items_active_search so it can be recreated with updated covering columns",
|
|
131
|
+
down: downDropActiveSearchIndex,
|
|
80
132
|
},
|
|
81
133
|
{
|
|
82
134
|
key: "migration_notification_tables_schema_v1",
|
|
83
135
|
version: 10,
|
|
84
136
|
description:
|
|
85
137
|
"Drop legacy enum-based notification tables so they can be recreated with the new signal-contract schema",
|
|
138
|
+
down: downNotificationTablesSchema,
|
|
86
139
|
},
|
|
87
140
|
{
|
|
88
141
|
key: "migration_rename_macos_ios_channel_to_vellum_v1",
|
|
89
142
|
version: 11,
|
|
90
143
|
description:
|
|
91
144
|
"Rename macos and ios channel identifiers to vellum across all tables",
|
|
145
|
+
down: downRenameChannelToVellum,
|
|
92
146
|
},
|
|
93
147
|
{
|
|
94
148
|
key: "migration_embedding_vector_blob_v1",
|
|
95
149
|
version: 12,
|
|
96
150
|
description:
|
|
97
151
|
"Add vector_blob BLOB column to memory_embeddings and backfill from vector_json for compact binary storage",
|
|
152
|
+
down: downEmbeddingVectorBlob,
|
|
98
153
|
},
|
|
99
154
|
{
|
|
100
155
|
key: "migration_embeddings_nullable_vector_json_v1",
|
|
@@ -102,18 +157,21 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
|
|
|
102
157
|
dependsOn: ["migration_embedding_vector_blob_v1"],
|
|
103
158
|
description:
|
|
104
159
|
"Rebuild memory_embeddings to make vector_json nullable (pre-100 DBs had NOT NULL)",
|
|
160
|
+
down: downEmbeddingsNullableVectorJson,
|
|
105
161
|
},
|
|
106
162
|
{
|
|
107
163
|
key: "migration_normalize_phone_identities_v1",
|
|
108
164
|
version: 14,
|
|
109
165
|
description:
|
|
110
166
|
"Normalize phone-like identity fields to E.164 format across guardian bindings, verification challenges, canonical requests, ingress members, and rate limits",
|
|
167
|
+
down: downNormalizePhoneIdentities,
|
|
111
168
|
},
|
|
112
169
|
{
|
|
113
170
|
key: "migration_backfill_guardian_principal_id_v3",
|
|
114
171
|
version: 15,
|
|
115
172
|
description:
|
|
116
173
|
"Backfill guardianPrincipalId for existing channel_guardian_bindings and canonical_guardian_requests rows, expire unresolvable pending requests",
|
|
174
|
+
down: downBackfillGuardianPrincipalId,
|
|
117
175
|
},
|
|
118
176
|
{
|
|
119
177
|
key: "migration_guardian_principal_id_not_null_v1",
|
|
@@ -121,18 +179,21 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
|
|
|
121
179
|
dependsOn: ["migration_backfill_guardian_principal_id_v3"],
|
|
122
180
|
description:
|
|
123
181
|
"Enforce NOT NULL on channel_guardian_bindings.guardian_principal_id after backfill",
|
|
182
|
+
down: downGuardianPrincipalIdNotNull,
|
|
124
183
|
},
|
|
125
184
|
{
|
|
126
185
|
key: "migration_contacts_notes_column_v1",
|
|
127
186
|
version: 17,
|
|
128
187
|
description:
|
|
129
188
|
"Consolidate relationship/importance/response_expectation/preferred_tone into a single notes TEXT column, then drop the legacy columns",
|
|
189
|
+
down: downContactsNotesColumn,
|
|
130
190
|
},
|
|
131
191
|
{
|
|
132
192
|
key: "backfill_contact_interaction_stats",
|
|
133
193
|
version: 18,
|
|
134
194
|
description:
|
|
135
195
|
"Backfill contacts.last_interaction from the max lastSeenAt across each contact's channels",
|
|
196
|
+
down: downBackfillContactInteractionStats,
|
|
136
197
|
},
|
|
137
198
|
{
|
|
138
199
|
key: "migration_drop_assistant_id_columns_v1",
|
|
@@ -140,48 +201,56 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
|
|
|
140
201
|
dependsOn: ["migration_normalize_assistant_id_to_self_v1"],
|
|
141
202
|
description:
|
|
142
203
|
"Drop assistant_id columns from all 16 daemon tables after normalization to single-tenant identity",
|
|
204
|
+
down: downDropAssistantIdColumns,
|
|
143
205
|
},
|
|
144
206
|
{
|
|
145
207
|
key: "migration_backfill_usage_cache_accounting_v1",
|
|
146
208
|
version: 20,
|
|
147
209
|
description:
|
|
148
210
|
"Backfill historical Anthropic llm_usage_events rows from llm_request_logs with cache-aware pricing",
|
|
211
|
+
down: downBackfillUsageCacheAccounting,
|
|
149
212
|
},
|
|
150
213
|
{
|
|
151
214
|
key: "migration_rename_verification_table_v1",
|
|
152
215
|
version: 21,
|
|
153
216
|
description:
|
|
154
217
|
"Rename channel_guardian_verification_challenges table to channel_verification_sessions and update indexes",
|
|
218
|
+
down: downRenameVerificationTable,
|
|
155
219
|
},
|
|
156
220
|
{
|
|
157
221
|
key: "migration_rename_verification_session_id_column_v1",
|
|
158
222
|
version: 22,
|
|
159
223
|
description:
|
|
160
224
|
"Rename guardian_verification_session_id column in call_sessions to verification_session_id",
|
|
225
|
+
down: downRenameVerificationSessionIdColumn,
|
|
161
226
|
},
|
|
162
227
|
{
|
|
163
228
|
key: "migration_rename_guardian_verification_values_v1",
|
|
164
229
|
version: 23,
|
|
165
230
|
description:
|
|
166
231
|
"Rename persisted guardian_verification call_mode and guardian_voice_verification_* event_type values to drop the guardian_ prefix",
|
|
232
|
+
down: downRenameGuardianVerificationValues,
|
|
167
233
|
},
|
|
168
234
|
{
|
|
169
235
|
key: "migration_rename_voice_to_phone_v1",
|
|
170
236
|
version: 24,
|
|
171
237
|
description:
|
|
172
238
|
'Rename stored "voice" channel values to "phone" across all tables with channel text columns',
|
|
239
|
+
down: downRenameVoiceToPhone,
|
|
173
240
|
},
|
|
174
241
|
{
|
|
175
242
|
key: "migration_drop_accounts_table_v1",
|
|
176
243
|
version: 25,
|
|
177
244
|
description:
|
|
178
245
|
"Drop the unused legacy accounts table and its leftover indexes after account_manage removal",
|
|
246
|
+
down: migrateDropAccountsTableDown,
|
|
179
247
|
},
|
|
180
248
|
{
|
|
181
249
|
key: "migration_reminders_to_schedules_v1",
|
|
182
250
|
version: 26,
|
|
183
251
|
description:
|
|
184
252
|
"Copy all existing reminders into cron_jobs as one-shot schedules with correct status and field mapping",
|
|
253
|
+
down: migrateRemindersToSchedulesDown,
|
|
185
254
|
},
|
|
186
255
|
{
|
|
187
256
|
key: "migration_drop_reminders_table_v1",
|
|
@@ -189,18 +258,21 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
|
|
|
189
258
|
dependsOn: ["migration_reminders_to_schedules_v1"],
|
|
190
259
|
description:
|
|
191
260
|
"Drop the legacy reminders table and its index after data migration to cron_jobs",
|
|
261
|
+
down: migrateDropRemindersTableDown,
|
|
192
262
|
},
|
|
193
263
|
{
|
|
194
264
|
key: "migration_oauth_apps_client_secret_path_v1",
|
|
195
265
|
version: 28,
|
|
196
266
|
description:
|
|
197
267
|
"Add client_secret_credential_path column to oauth_apps and backfill existing rows with convention-based paths",
|
|
268
|
+
down: migrateOAuthAppsClientSecretPathDown,
|
|
198
269
|
},
|
|
199
270
|
{
|
|
200
271
|
key: "migration_guardian_timestamps_epoch_ms_v1",
|
|
201
272
|
version: 29,
|
|
202
273
|
description:
|
|
203
274
|
"Convert guardian table timestamps from ISO 8601 text to epoch ms integers for consistency with all other tables",
|
|
275
|
+
down: migrateGuardianTimestampsEpochMsDown,
|
|
204
276
|
},
|
|
205
277
|
{
|
|
206
278
|
key: "migration_guardian_timestamps_rebuild_v1",
|
|
@@ -208,18 +280,21 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
|
|
|
208
280
|
dependsOn: ["migration_guardian_timestamps_epoch_ms_v1"],
|
|
209
281
|
description:
|
|
210
282
|
"Rebuild guardian tables so timestamp columns have INTEGER affinity instead of TEXT",
|
|
283
|
+
down: migrateGuardianTimestampsRebuildDown,
|
|
211
284
|
},
|
|
212
285
|
{
|
|
213
286
|
key: "migration_rename_gmail_provider_key_to_google_v1",
|
|
214
287
|
version: 31,
|
|
215
288
|
description:
|
|
216
289
|
"Rename integration:gmail provider key to integration:google across oauth_providers, oauth_apps, and oauth_connections",
|
|
290
|
+
down: migrateRenameGmailProviderKeyToGoogleDown,
|
|
217
291
|
},
|
|
218
292
|
{
|
|
219
293
|
key: "migration_rename_thread_starters_table_v1",
|
|
220
294
|
version: 32,
|
|
221
295
|
description:
|
|
222
296
|
"Rename thread_starters table to conversation_starters and recreate indexes with new names",
|
|
297
|
+
down: migrateRenameThreadStartersTableDown,
|
|
223
298
|
},
|
|
224
299
|
{
|
|
225
300
|
key: "migration_drop_capability_card_state_v1",
|
|
@@ -227,12 +302,14 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
|
|
|
227
302
|
dependsOn: ["migration_rename_thread_starters_table_v1"],
|
|
228
303
|
description:
|
|
229
304
|
"Remove deleted capability-card rows, jobs, checkpoints, and category state",
|
|
305
|
+
down: migrateDropCapabilityCardStateDown,
|
|
230
306
|
},
|
|
231
307
|
{
|
|
232
308
|
key: "migration_backfill_inline_attachments_v1",
|
|
233
309
|
version: 34,
|
|
234
310
|
description:
|
|
235
311
|
"Backfill existing inline base64 attachments to on-disk storage and clear dataBase64",
|
|
312
|
+
down: migrateBackfillInlineAttachmentsToDiskDown,
|
|
236
313
|
},
|
|
237
314
|
{
|
|
238
315
|
key: "migration_rename_thread_starters_checkpoints_v1",
|
|
@@ -240,12 +317,33 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
|
|
|
240
317
|
dependsOn: ["migration_rename_thread_starters_table_v1"],
|
|
241
318
|
description:
|
|
242
319
|
"Rename checkpoint keys from thread_starters: to conversation_starters: prefix so renamed code paths find existing generation state",
|
|
320
|
+
down: migrateRenameThreadStartersCheckpointsDown,
|
|
321
|
+
},
|
|
322
|
+
{
|
|
323
|
+
key: "migration_backfill_audio_attachment_mime_types_v1",
|
|
324
|
+
version: 36,
|
|
325
|
+
description:
|
|
326
|
+
"Backfill correct MIME types for audio attachments stored as application/octet-stream due to missing extension map entries",
|
|
327
|
+
down: migrateBackfillAudioAttachmentMimeTypesDown,
|
|
328
|
+
},
|
|
329
|
+
{
|
|
330
|
+
key: "migration_add_source_type_columns_v1",
|
|
331
|
+
version: 37,
|
|
332
|
+
description:
|
|
333
|
+
"Add source_type and source_message_role columns to memory_items with backfill from verification_state and source messages",
|
|
334
|
+
down: migrateAddSourceTypeColumnsDown,
|
|
243
335
|
},
|
|
244
336
|
];
|
|
245
337
|
|
|
338
|
+
export function getMaxMigrationVersion(): number {
|
|
339
|
+
return Math.max(...MIGRATION_REGISTRY.map((e) => e.version));
|
|
340
|
+
}
|
|
341
|
+
|
|
246
342
|
export interface MigrationValidationResult {
|
|
247
343
|
/** Keys of migrations whose checkpoint has value 'started' — started but never completed. */
|
|
248
344
|
crashed: string[];
|
|
249
345
|
/** Pairs where a completed migration's declared prerequisite is missing from checkpoints. */
|
|
250
346
|
dependencyViolations: Array<{ migration: string; missingDependency: string }>;
|
|
347
|
+
/** Checkpoint keys present in the database but absent from the migration registry — likely from a newer version. */
|
|
348
|
+
unknownCheckpoints: string[];
|
|
251
349
|
}
|
|
@@ -33,7 +33,9 @@ export function recoverCrashedMigrations(database: DrizzleDb): string[] {
|
|
|
33
33
|
return [];
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
const crashed = rows
|
|
36
|
+
const crashed = rows
|
|
37
|
+
.filter((r) => r.value === "started" || r.value === "rolling_back")
|
|
38
|
+
.map((r) => r.key);
|
|
37
39
|
if (crashed.length === 0) return [];
|
|
38
40
|
|
|
39
41
|
log.error(
|
|
@@ -83,7 +85,12 @@ export function withCrashRecovery(
|
|
|
83
85
|
const existing = raw
|
|
84
86
|
.query(`SELECT value FROM memory_checkpoints WHERE key = ?`)
|
|
85
87
|
.get(checkpointKey) as { value: string } | null;
|
|
86
|
-
if (
|
|
88
|
+
if (
|
|
89
|
+
existing &&
|
|
90
|
+
existing.value !== "started" &&
|
|
91
|
+
existing.value !== "rolling_back"
|
|
92
|
+
)
|
|
93
|
+
return;
|
|
87
94
|
|
|
88
95
|
raw
|
|
89
96
|
.query(
|
|
@@ -126,12 +133,14 @@ export function validateMigrationState(
|
|
|
126
133
|
.all() as Array<{ key: string; value: string }>;
|
|
127
134
|
} catch {
|
|
128
135
|
// memory_checkpoints may not exist on a very old database; skip.
|
|
129
|
-
return { crashed: [], dependencyViolations: [] };
|
|
136
|
+
return { crashed: [], dependencyViolations: [], unknownCheckpoints: [] };
|
|
130
137
|
}
|
|
131
138
|
|
|
132
|
-
// Any remaining 'started' checkpoints after recovery +
|
|
133
|
-
// indicate a migration that was retried but failed again.
|
|
134
|
-
const crashed = rows
|
|
139
|
+
// Any remaining 'started' or 'rolling_back' checkpoints after recovery +
|
|
140
|
+
// migration execution indicate a migration that was retried but failed again.
|
|
141
|
+
const crashed = rows
|
|
142
|
+
.filter((r) => r.value === "started" || r.value === "rolling_back")
|
|
143
|
+
.map((r) => r.key);
|
|
135
144
|
if (crashed.length > 0) {
|
|
136
145
|
log.error(
|
|
137
146
|
{ crashed },
|
|
@@ -149,11 +158,14 @@ export function validateMigrationState(
|
|
|
149
158
|
);
|
|
150
159
|
}
|
|
151
160
|
|
|
152
|
-
// Only rows whose value is NOT 'started' represent truly
|
|
153
|
-
// In-progress/crashed checkpoints
|
|
154
|
-
// dependencies — the migration never finished, so its postconditions
|
|
161
|
+
// Only rows whose value is NOT 'started' or 'rolling_back' represent truly
|
|
162
|
+
// completed migrations. In-progress/crashed checkpoints must not count as
|
|
163
|
+
// applied dependencies — the migration never finished, so its postconditions
|
|
164
|
+
// are unmet.
|
|
155
165
|
const completed = new Set(
|
|
156
|
-
rows
|
|
166
|
+
rows
|
|
167
|
+
.filter((r) => r.value !== "started" && r.value !== "rolling_back")
|
|
168
|
+
.map((r) => r.key),
|
|
157
169
|
);
|
|
158
170
|
|
|
159
171
|
const dependencyViolations: Array<{
|
|
@@ -191,5 +203,119 @@ export function validateMigrationState(
|
|
|
191
203
|
);
|
|
192
204
|
}
|
|
193
205
|
|
|
194
|
-
|
|
206
|
+
// Detect checkpoints that exist in the database but have no corresponding
|
|
207
|
+
// registry entry — these are from a newer version of the daemon.
|
|
208
|
+
//
|
|
209
|
+
// The memory_checkpoints table is a general-purpose key-value store also
|
|
210
|
+
// used by non-migration subsystems (e.g., "identity:intro:text",
|
|
211
|
+
// "conversation_starters:item_count_at_last_gen"). Filter to only keys
|
|
212
|
+
// that follow migration naming conventions before comparing against the
|
|
213
|
+
// registry to avoid false-positive warnings.
|
|
214
|
+
const registryKeys = new Set(MIGRATION_REGISTRY.map((e) => e.key));
|
|
215
|
+
const isMigrationKey = (k: string): boolean =>
|
|
216
|
+
k.startsWith("migration_") ||
|
|
217
|
+
k.startsWith("backfill_") ||
|
|
218
|
+
k.startsWith("drop_");
|
|
219
|
+
const unknownCheckpoints = [...completed].filter(
|
|
220
|
+
(k) => isMigrationKey(k) && !registryKeys.has(k),
|
|
221
|
+
);
|
|
222
|
+
|
|
223
|
+
if (unknownCheckpoints.length > 0) {
|
|
224
|
+
log.warn(
|
|
225
|
+
{ unknownCheckpoints },
|
|
226
|
+
`Database contains ${unknownCheckpoints.length} migration checkpoint(s) from a newer version. Data may be incompatible.`,
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
return { crashed, dependencyViolations, unknownCheckpoints };
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Roll back all completed memory (database) migrations with version > targetVersion.
|
|
235
|
+
*
|
|
236
|
+
* Iterates eligible migrations in reverse version order. For each:
|
|
237
|
+
* 1. Marks the checkpoint as `"rolling_back"` for crash recovery.
|
|
238
|
+
* 2. Calls `entry.down(database)` — each down() manages its own transactions.
|
|
239
|
+
* (`down` is required on `MigrationRegistryEntry` at the type level.)
|
|
240
|
+
* 3. Deletes the checkpoint from `memory_checkpoints`.
|
|
241
|
+
*
|
|
242
|
+
* **Usage**: Pass the target version number you want to roll back *to*. All
|
|
243
|
+
* migrations with a higher version number that have been applied will be
|
|
244
|
+
* reversed. For example, `rollbackMemoryMigration(db, 5)` rolls back all
|
|
245
|
+
* applied migrations with version > 5.
|
|
246
|
+
*
|
|
247
|
+
* **Checkpoint state**: Each rolled-back migration's checkpoint is deleted
|
|
248
|
+
* from `memory_checkpoints`. If the process crashes mid-rollback, the
|
|
249
|
+
* `"rolling_back"` marker is detected and cleared by
|
|
250
|
+
* `recoverCrashedMigrations` on the next startup.
|
|
251
|
+
*
|
|
252
|
+
* **Warning — data loss**: Some down() migrations may not fully restore the
|
|
253
|
+
* original state (e.g., DROP TABLE migrations recreate the table but cannot
|
|
254
|
+
* recover the original data). Review each migration's down() implementation
|
|
255
|
+
* before calling this function programmatically.
|
|
256
|
+
*
|
|
257
|
+
* **Important**: Stop the assistant before running rollbacks. Rolling back
|
|
258
|
+
* migrations while the assistant is running may cause schema mismatches,
|
|
259
|
+
* query failures, or data corruption.
|
|
260
|
+
*
|
|
261
|
+
* @param database The Drizzle database instance.
|
|
262
|
+
* @param targetVersion Roll back to this version (exclusive — all migrations
|
|
263
|
+
* with version > targetVersion are reversed).
|
|
264
|
+
* @returns The list of rolled-back migration keys.
|
|
265
|
+
*/
|
|
266
|
+
export function rollbackMemoryMigration(
|
|
267
|
+
database: DrizzleDb,
|
|
268
|
+
targetVersion: number,
|
|
269
|
+
): string[] {
|
|
270
|
+
const raw = getSqliteFrom(database);
|
|
271
|
+
|
|
272
|
+
// Read completed checkpoints to determine which migrations have been applied.
|
|
273
|
+
let rows: Array<{ key: string; value: string }>;
|
|
274
|
+
try {
|
|
275
|
+
rows = raw
|
|
276
|
+
.query(`SELECT key, value FROM memory_checkpoints`)
|
|
277
|
+
.all() as Array<{ key: string; value: string }>;
|
|
278
|
+
} catch {
|
|
279
|
+
return [];
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
const completedKeys = new Set(
|
|
283
|
+
rows
|
|
284
|
+
.filter((r) => r.value !== "started" && r.value !== "rolling_back")
|
|
285
|
+
.map((r) => r.key),
|
|
286
|
+
);
|
|
287
|
+
|
|
288
|
+
// Find registry entries with version > targetVersion that have completed checkpoints.
|
|
289
|
+
const toRollback = MIGRATION_REGISTRY.filter(
|
|
290
|
+
(entry) => entry.version > targetVersion && completedKeys.has(entry.key),
|
|
291
|
+
).sort((a, b) => b.version - a.version); // reverse version order
|
|
292
|
+
|
|
293
|
+
const rolledBack: string[] = [];
|
|
294
|
+
|
|
295
|
+
for (const entry of toRollback) {
|
|
296
|
+
// Mark as rolling_back for crash recovery — if the process crashes here,
|
|
297
|
+
// recoverCrashedMigrations will clear this checkpoint on next startup.
|
|
298
|
+
raw
|
|
299
|
+
.query(
|
|
300
|
+
`UPDATE memory_checkpoints SET value = 'rolling_back', updated_at = ? WHERE key = ?`,
|
|
301
|
+
)
|
|
302
|
+
.run(Date.now(), entry.key);
|
|
303
|
+
|
|
304
|
+
// Execute the down migration — let it manage its own transaction lifecycle.
|
|
305
|
+
// Many down() functions call BEGIN/COMMIT internally or use PRAGMA statements
|
|
306
|
+
// that are no-ops inside a transaction.
|
|
307
|
+
entry.down(database);
|
|
308
|
+
|
|
309
|
+
// Delete the checkpoint after down() succeeds — outside any transaction
|
|
310
|
+
// so it's not affected by down()'s internal transaction management.
|
|
311
|
+
raw.query(`DELETE FROM memory_checkpoints WHERE key = ?`).run(entry.key);
|
|
312
|
+
|
|
313
|
+
log.info(
|
|
314
|
+
{ key: entry.key, version: entry.version },
|
|
315
|
+
`Rolled back migration "${entry.key}" (version ${entry.version})`,
|
|
316
|
+
);
|
|
317
|
+
rolledBack.push(entry.key);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
return rolledBack;
|
|
195
321
|
}
|
|
@@ -101,6 +101,15 @@ export function isQdrantBreakerOpen(): boolean {
|
|
|
101
101
|
return breakerState !== "closed";
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
+
/**
|
|
105
|
+
* Returns true when the breaker is open and the cooldown has elapsed,
|
|
106
|
+
* meaning the next call to `withQdrantBreaker` will transition to half-open.
|
|
107
|
+
* Use this to allow a single probe job through when embed jobs are otherwise skipped.
|
|
108
|
+
*/
|
|
109
|
+
export function shouldAllowQdrantProbe(): boolean {
|
|
110
|
+
return breakerState === "open" && Date.now() - openedAt >= COOLDOWN_MS;
|
|
111
|
+
}
|
|
112
|
+
|
|
104
113
|
/** @internal Test-only: reset circuit breaker state */
|
|
105
114
|
export function _resetQdrantBreaker(): void {
|
|
106
115
|
breakerState = "closed";
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
existsSync,
|
|
5
5
|
mkdirSync,
|
|
6
6
|
readFileSync,
|
|
7
|
+
symlinkSync,
|
|
7
8
|
unlinkSync,
|
|
8
9
|
writeFileSync,
|
|
9
10
|
} from "node:fs";
|
|
@@ -12,7 +13,7 @@ import { dirname, join } from "node:path";
|
|
|
12
13
|
|
|
13
14
|
import type { Subprocess } from "bun";
|
|
14
15
|
|
|
15
|
-
import { getQdrantUrlEnv } from "../config/env.js";
|
|
16
|
+
import { getQdrantReadyzTimeoutMs, getQdrantUrlEnv } from "../config/env.js";
|
|
16
17
|
import { getLogger } from "../util/logger.js";
|
|
17
18
|
import { getDataDir } from "../util/platform.js";
|
|
18
19
|
|
|
@@ -47,6 +48,8 @@ export interface QdrantManagerConfig {
|
|
|
47
48
|
*/
|
|
48
49
|
export class QdrantManager {
|
|
49
50
|
private process: Subprocess | null = null;
|
|
51
|
+
private stderrBuffer = "";
|
|
52
|
+
private stderrDrained: Promise<void> = Promise.resolve();
|
|
50
53
|
private readonly url: string;
|
|
51
54
|
private readonly host: string;
|
|
52
55
|
private readonly port: number;
|
|
@@ -67,7 +70,8 @@ export class QdrantManager {
|
|
|
67
70
|
|
|
68
71
|
this.readyzPollIntervalMs =
|
|
69
72
|
config.readyzPollIntervalMs ?? READYZ_POLL_INTERVAL_MS;
|
|
70
|
-
this.readyzTimeoutMs =
|
|
73
|
+
this.readyzTimeoutMs =
|
|
74
|
+
config.readyzTimeoutMs ?? getQdrantReadyzTimeoutMs() ?? READYZ_TIMEOUT_MS;
|
|
71
75
|
this.shutdownGraceMs = config.shutdownGraceMs ?? SHUTDOWN_GRACE_MS;
|
|
72
76
|
|
|
73
77
|
// External mode only if QDRANT_URL is explicitly set
|
|
@@ -92,13 +96,15 @@ export class QdrantManager {
|
|
|
92
96
|
await this.installBinary(binaryPath);
|
|
93
97
|
}
|
|
94
98
|
|
|
99
|
+
const spawnPath = this.ensureVellumSymlink(binaryPath);
|
|
100
|
+
|
|
95
101
|
log.info(
|
|
96
|
-
{ binaryPath, storagePath: this.storagePath, port: this.port },
|
|
102
|
+
{ binaryPath: spawnPath, storagePath: this.storagePath, port: this.port },
|
|
97
103
|
"Starting Qdrant",
|
|
98
104
|
);
|
|
99
105
|
|
|
100
|
-
|
|
101
|
-
cmd: [
|
|
106
|
+
const proc = Bun.spawn({
|
|
107
|
+
cmd: [spawnPath],
|
|
102
108
|
env: {
|
|
103
109
|
...process.env,
|
|
104
110
|
QDRANT__SERVICE__HOST: this.host,
|
|
@@ -109,8 +115,10 @@ export class QdrantManager {
|
|
|
109
115
|
QDRANT__LOG_LEVEL: "WARN",
|
|
110
116
|
},
|
|
111
117
|
stdout: "ignore",
|
|
112
|
-
stderr: "
|
|
118
|
+
stderr: "pipe",
|
|
113
119
|
});
|
|
120
|
+
this.process = proc;
|
|
121
|
+
this.drainStderrFrom(proc.stderr);
|
|
114
122
|
|
|
115
123
|
if (this.process.pid) {
|
|
116
124
|
this.writePid(this.process.pid);
|
|
@@ -150,6 +158,7 @@ export class QdrantManager {
|
|
|
150
158
|
}
|
|
151
159
|
|
|
152
160
|
this.process = null;
|
|
161
|
+
this.stderrBuffer = "";
|
|
153
162
|
this.cleanupPid();
|
|
154
163
|
log.info("Qdrant stopped");
|
|
155
164
|
}
|
|
@@ -259,6 +268,15 @@ export class QdrantManager {
|
|
|
259
268
|
private async waitForReady(): Promise<void> {
|
|
260
269
|
const start = Date.now();
|
|
261
270
|
while (Date.now() - start < this.readyzTimeoutMs) {
|
|
271
|
+
// Fail fast if the managed process exited before becoming ready
|
|
272
|
+
if (this.process != null && this.process.exitCode != null) {
|
|
273
|
+
await this.stderrDrained;
|
|
274
|
+
const stderr = this.stderrBuffer.trim();
|
|
275
|
+
throw new Error(
|
|
276
|
+
`Qdrant process exited with code ${this.process.exitCode} before becoming ready` +
|
|
277
|
+
(stderr ? `\nstderr:\n${stderr}` : ""),
|
|
278
|
+
);
|
|
279
|
+
}
|
|
262
280
|
try {
|
|
263
281
|
const res = await fetch(`${this.url}/readyz`);
|
|
264
282
|
if (res.ok) return;
|
|
@@ -267,11 +285,32 @@ export class QdrantManager {
|
|
|
267
285
|
}
|
|
268
286
|
await Bun.sleep(this.readyzPollIntervalMs);
|
|
269
287
|
}
|
|
288
|
+
const stderr = this.stderrBuffer.trim();
|
|
270
289
|
throw new Error(
|
|
271
|
-
`Qdrant did not become ready within ${this.readyzTimeoutMs}ms at ${this.url}
|
|
290
|
+
`Qdrant did not become ready within ${this.readyzTimeoutMs}ms at ${this.url}` +
|
|
291
|
+
(stderr ? `\nstderr:\n${stderr}` : ""),
|
|
272
292
|
);
|
|
273
293
|
}
|
|
274
294
|
|
|
295
|
+
private drainStderrFrom(stream: ReadableStream<Uint8Array>): void {
|
|
296
|
+
const reader = stream.getReader();
|
|
297
|
+
const decoder = new TextDecoder();
|
|
298
|
+
this.stderrDrained = (async () => {
|
|
299
|
+
try {
|
|
300
|
+
for (;;) {
|
|
301
|
+
const { done, value } = await reader.read();
|
|
302
|
+
if (done) break;
|
|
303
|
+
this.stderrBuffer += decoder.decode(value, { stream: true });
|
|
304
|
+
if (this.stderrBuffer.length > 4096) {
|
|
305
|
+
this.stderrBuffer = this.stderrBuffer.slice(-4096);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
} catch {
|
|
309
|
+
// Stream closed or error — expected during shutdown
|
|
310
|
+
}
|
|
311
|
+
})();
|
|
312
|
+
}
|
|
313
|
+
|
|
275
314
|
private getBinaryPath(): string {
|
|
276
315
|
return join(getDataDir(), "qdrant", "bin", "qdrant");
|
|
277
316
|
}
|
|
@@ -314,4 +353,22 @@ export class QdrantManager {
|
|
|
314
353
|
}
|
|
315
354
|
}
|
|
316
355
|
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Ensures a `vellum-qdrant` symlink exists next to the real binary so that
|
|
359
|
+
* `lsof` reports "vellum-qdrant" in the COMMAND column, making the process
|
|
360
|
+
* discoverable by tools that scan for "vellum" in process names.
|
|
361
|
+
*/
|
|
362
|
+
private ensureVellumSymlink(binaryPath: string): string {
|
|
363
|
+
const symlinkPath = join(dirname(binaryPath), "vellum-qdrant");
|
|
364
|
+
if (!existsSync(symlinkPath)) {
|
|
365
|
+
try {
|
|
366
|
+
symlinkSync(binaryPath, symlinkPath);
|
|
367
|
+
} catch {
|
|
368
|
+
// Fall back to the real binary if symlink creation fails
|
|
369
|
+
return binaryPath;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
return symlinkPath;
|
|
373
|
+
}
|
|
317
374
|
}
|