@vellumai/assistant 0.11.1 → 0.11.2-staging.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +1 -3
- package/README.md +1 -1
- package/eslint.config.mjs +28 -8
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/stripe-currency.test.ts +37 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/stripe-currency.ts +55 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/stripe-currency.test.ts +37 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/stripe-currency.ts +55 -0
- package/node_modules/@vellumai/service-contracts/package.json +1 -0
- package/node_modules/@vellumai/service-contracts/src/__tests__/stripe-currency.test.ts +37 -0
- package/node_modules/@vellumai/service-contracts/src/stripe-currency.ts +55 -0
- package/node_modules/@vellumai/slack-text/src/index.test.ts +48 -0
- package/node_modules/@vellumai/slack-text/src/index.ts +25 -6
- package/openapi.yaml +292 -5
- package/package.json +1 -1
- package/src/__tests__/agent-loop-resume-interrupted.test.ts +223 -0
- package/src/__tests__/byok-default-profile-ensure.test.ts +43 -0
- package/src/__tests__/cli-logger-boundary-guard.test.ts +87 -0
- package/src/__tests__/compaction-events.test.ts +139 -2
- package/src/__tests__/config-loader-backfill.test.ts +35 -3
- package/src/__tests__/config-schema.test.ts +1 -0
- package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +30 -0
- package/src/__tests__/conversation-agent-loop.test.ts +180 -7
- package/src/__tests__/conversation-error.test.ts +211 -4
- package/src/__tests__/conversation-options-turn-scoped-transport.test.ts +64 -0
- package/src/__tests__/conversation-queue.test.ts +126 -0
- package/src/__tests__/conversation-retry-route.test.ts +46 -7
- package/src/__tests__/conversation-slash.test.ts +4 -2
- package/src/__tests__/conversation-summarize-route.test.ts +7 -1
- package/src/__tests__/conversation-summarize-up-to.test.ts +71 -5
- package/src/__tests__/document-create-dedupe.test.ts +2 -1
- package/src/__tests__/document-find-replace.test.ts +2 -1
- package/src/__tests__/document-tool-security.test.ts +2 -1
- package/src/__tests__/document-update-default-surface.test.ts +2 -1
- package/src/__tests__/document-workspace-file.test.ts +467 -0
- package/src/__tests__/emit-signal-routing-intent.test.ts +290 -49
- package/src/__tests__/guardian-card-withdrawal.test.ts +91 -3
- package/src/__tests__/http-user-message-parity.test.ts +66 -0
- package/src/__tests__/list-messages-attachments.test.ts +87 -0
- package/src/__tests__/list-messages-provider-error.test.ts +143 -0
- package/src/__tests__/list-messages-system-card.test.ts +100 -0
- package/src/__tests__/llm-resolver.test.ts +19 -9
- package/src/__tests__/managed-profile-guard.test.ts +23 -0
- package/src/__tests__/notification-platform-adapter.test.ts +130 -2
- package/src/__tests__/notification-telegram-adapter.test.ts +6 -0
- package/src/__tests__/notification-vellum-adapter.test.ts +45 -0
- package/src/__tests__/plugin-api-model-profiles.test.ts +10 -1
- package/src/__tests__/plugin-import-boundary-guard.test.ts +3 -0
- package/src/__tests__/provider-error-scenarios.test.ts +140 -0
- package/src/__tests__/provider-send-message-override-profile.test.ts +95 -0
- package/src/__tests__/run-conversation-turn-persistence.test.ts +66 -3
- package/src/__tests__/scripted-turn-metadata-persistence.test.ts +209 -0
- package/src/__tests__/skills.test.ts +27 -0
- package/src/__tests__/slack-channels-routes.test.ts +0 -2
- package/src/__tests__/slack-share-routes.test.ts +0 -3
- package/src/__tests__/slack-users-routes.test.ts +0 -2
- package/src/__tests__/subagent-tools.test.ts +11 -0
- package/src/__tests__/tool-preview-lifecycle.test.ts +58 -0
- package/src/__tests__/tool-result-spool.test.ts +5 -4
- package/src/__tests__/turn-boundary-resolution.test.ts +53 -0
- package/src/__tests__/turn-events-store.test.ts +26 -0
- package/src/__tests__/ui-visual-surface.test.ts +695 -0
- package/src/__tests__/unified-turn-context-visible-app.test.ts +99 -0
- package/src/__tests__/visible-app-context.test.ts +189 -0
- package/src/__tests__/workspace-git-service.test.ts +173 -33
- package/src/__tests__/workspace-migration-137-repair-retired-fireworks-minimax-model-id.test.ts +157 -0
- package/src/__tests__/workspace-migration-138-backfill-home-feed-titles.test.ts +373 -0
- package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +137 -0
- package/src/agent/loop.ts +36 -7
- package/src/api/events/context-window-usage.ts +31 -0
- package/src/api/events/notification-intent.ts +8 -0
- package/src/api/events/ui-surface-pending.ts +35 -0
- package/src/api/index.ts +14 -0
- package/src/api/responses/conversation-message.ts +25 -4
- package/src/api/surfaces.ts +90 -1
- package/src/approvals/guardian-card-withdrawal.ts +66 -31
- package/src/approvals/guardian-decision-primitive.ts +4 -0
- package/src/calls/__tests__/call-setup-router.test.ts +156 -26
- package/src/calls/__tests__/voice-session-bridge.test.ts +201 -12
- package/src/calls/call-setup-router.ts +80 -36
- package/src/calls/voice-session-bridge.ts +173 -26
- package/src/cli/commands/inference.help.ts +3 -3
- package/src/cli/commands/notifications.help.ts +3 -2
- package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +42 -128
- package/src/cli/commands/platform/__tests__/credits.test.ts +9 -78
- package/src/cli/commands/platform/__tests__/helpers.ts +90 -0
- package/src/cli/commands/platform/__tests__/invoices.test.ts +238 -0
- package/src/cli/commands/platform/__tests__/plans.test.ts +9 -88
- package/src/cli/commands/platform/__tests__/status.test.ts +12 -87
- package/src/cli/commands/platform/__tests__/subscription.test.ts +9 -86
- package/src/cli/commands/platform/index.help.ts +92 -0
- package/src/cli/commands/platform/index.ts +7 -0
- package/src/cli/commands/platform/invoices.ts +132 -0
- package/src/cli/commands/usage.help.ts +1 -1
- package/src/cli/lib/list-installed-plugins.ts +2 -1
- package/src/config/__tests__/default-profile-catalog.test.ts +7 -10
- package/src/config/__tests__/deployment-context-defaults.test.ts +27 -5
- package/src/config/assistant-feature-flags.ts +7 -2
- package/src/config/bundled-skills/app-builder/SKILL.md +3 -2
- package/src/config/bundled-skills/subagent/SKILL.md +3 -1
- package/src/config/bundled-skills/subagent/TOOLS.json +3 -3
- package/src/config/bundled-skills/visualize/SKILL.md +163 -0
- package/src/config/call-site-defaults.ts +3 -2
- package/src/config/default-profile-catalog.ts +75 -70
- package/src/config/default-profile-names.ts +13 -28
- package/src/config/env-registry.ts +1 -0
- package/src/config/feature-flag-registry.json +16 -0
- package/src/config/llm-resolver.ts +21 -5
- package/src/config/loader.ts +18 -11
- package/src/config/schemas/llm.ts +1 -9
- package/src/config/schemas/memory-retrospective.ts +9 -0
- package/src/config/schemas/monitoring.ts +28 -2
- package/src/config/schemas/workspace-git.ts +15 -0
- package/src/config/seed-inference-profiles.ts +19 -5
- package/src/context/post-turn-tool-result-truncation.ts +2 -2
- package/src/conversations/__tests__/message-consolidation.test.ts +54 -0
- package/src/conversations/message-consolidation.ts +17 -15
- package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +182 -0
- package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +181 -0
- package/src/daemon/conversation-agent-loop-handlers.ts +90 -7
- package/src/daemon/conversation-agent-loop.ts +107 -22
- package/src/daemon/conversation-error.ts +169 -48
- package/src/daemon/conversation-messaging.ts +87 -4
- package/src/daemon/conversation-process.ts +57 -45
- package/src/daemon/conversation-runtime-assembly.ts +57 -0
- package/src/daemon/conversation-store.ts +36 -1
- package/src/daemon/conversation-surfaces.ts +68 -6
- package/src/daemon/conversation-turn-finalize.ts +76 -18
- package/src/daemon/conversation.ts +122 -27
- package/src/daemon/lifecycle.ts +9 -0
- package/src/daemon/message-types/conversations.ts +8 -0
- package/src/daemon/message-types/surfaces.ts +3 -0
- package/src/documents/document-store.ts +247 -10
- package/src/home/__tests__/feed-types.test.ts +43 -8
- package/src/home/__tests__/feed-writer.test.ts +59 -0
- package/src/home/feed-types.ts +34 -7
- package/src/home/feed-writer.ts +15 -6
- package/src/live-voice/__tests__/activity-label.test.ts +95 -0
- package/src/live-voice/__tests__/live-activity-reporter.test.ts +86 -5
- package/src/live-voice/__tests__/live-voice-agent-turn.test.ts +310 -25
- package/src/live-voice/__tests__/live-voice-events.test.ts +4 -4
- package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +8 -4
- package/src/live-voice/__tests__/live-voice-vad.test.ts +9 -1
- package/src/live-voice/activity-label.ts +169 -0
- package/src/live-voice/live-activity-reporter.ts +35 -6
- package/src/live-voice/live-voice-session.ts +282 -41
- package/src/live-voice/protocol.ts +35 -0
- package/src/messaging/providers/slack/__tests__/adapter-mention-rendering.test.ts +84 -2
- package/src/messaging/providers/slack/adapter.ts +86 -19
- package/src/messaging/providers/slack/api.test.ts +85 -1
- package/src/messaging/providers/slack/api.ts +121 -288
- package/src/messaging/providers/slack/client.ts +20 -251
- package/src/messaging/providers/slack/send.test.ts +4 -9
- package/src/messaging/providers/slack/send.ts +1 -1
- package/src/messaging/providers/slack/types.ts +5 -0
- package/src/messaging/providers/slack/web-api-transport.test.ts +181 -0
- package/src/messaging/providers/slack/web-api-transport.ts +367 -0
- package/src/messaging/providers/slack/withdraw.ts +6 -14
- package/src/messaging/providers/telegram-bot/send.test.ts +31 -1
- package/src/messaging/providers/telegram-bot/send.ts +23 -2
- package/src/messaging/providers/telegram-bot/withdraw.test.ts +152 -0
- package/src/messaging/providers/telegram-bot/withdraw.ts +166 -0
- package/src/monitoring/__tests__/db-integrity-sample.test.ts +18 -4
- package/src/monitoring/__tests__/file-descriptors.test.ts +144 -0
- package/src/monitoring/file-descriptors.ts +262 -0
- package/src/monitoring/process-memory.ts +4 -11
- package/src/monitoring/resource-sampler.ts +5 -0
- package/src/monitoring/worker.ts +12 -0
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +740 -0
- package/src/notifications/__tests__/broadcaster.test.ts +347 -7
- package/src/notifications/__tests__/copy-composer.test.ts +53 -3
- package/src/notifications/__tests__/decision-engine.test.ts +349 -29
- package/src/notifications/__tests__/deterministic-checks.test.ts +21 -0
- package/src/notifications/__tests__/edit-notification.test.ts +330 -0
- package/src/notifications/__tests__/guardian-delivery-recorder.test.ts +71 -0
- package/src/notifications/__tests__/home-feed-side-effect.test.ts +170 -12
- package/src/notifications/adapters/macos.ts +3 -0
- package/src/notifications/adapters/platform.ts +90 -14
- package/src/notifications/adapters/telegram.ts +6 -4
- package/src/notifications/assistant-reply-producer.ts +219 -0
- package/src/notifications/broadcaster.ts +523 -317
- package/src/notifications/copy-composer.ts +27 -6
- package/src/notifications/decision-engine.ts +154 -77
- package/src/notifications/deterministic-checks.ts +6 -7
- package/src/notifications/edit-notification.ts +8 -4
- package/src/notifications/emit-signal.ts +49 -16
- package/src/notifications/guardian-delivery-recorder.ts +8 -2
- package/src/notifications/home-feed-side-effect.ts +43 -6
- package/src/notifications/notification-utils.ts +39 -4
- package/src/notifications/signal.ts +10 -0
- package/src/notifications/types.ts +17 -0
- package/src/persistence/bookmark-crud.ts +18 -9
- package/src/persistence/conversation-attention-store.ts +26 -0
- package/src/persistence/conversation-crud.ts +79 -32
- package/src/persistence/conversation-queries.ts +30 -15
- package/src/persistence/conversation-title-service.ts +5 -170
- package/src/persistence/conversation-types.ts +192 -1
- package/src/persistence/db-init.ts +14 -1
- package/src/persistence/db-maintenance.ts +5 -4
- package/src/persistence/embeddings/__tests__/plugin-index-qdrant-init.test.ts +219 -0
- package/src/persistence/embeddings/__tests__/plugin-index.test.ts +22 -0
- package/src/persistence/embeddings/__tests__/worker-script-version.test.ts +76 -0
- package/src/persistence/embeddings/embedding-local.ts +2 -1
- package/src/persistence/embeddings/embedding-runtime-manager.ts +29 -5
- package/src/persistence/embeddings/plugin-index.ts +84 -12
- package/src/persistence/migrations/360-add-document-workspace-path.test.ts +110 -0
- package/src/persistence/migrations/360-add-document-workspace-path.ts +39 -0
- package/src/persistence/planner-statistics.ts +14 -0
- package/src/persistence/schema/documents.ts +26 -11
- package/src/persistence/steps.ts +2 -0
- package/src/platform/client.test.ts +172 -2
- package/src/platform/client.ts +178 -55
- package/src/plugin-api/conversation-turn.ts +13 -0
- package/src/plugin-api/model-profiles.test.ts +6 -2
- package/src/plugins/defaults/memory/__tests__/bookmark-crud.test.ts +34 -0
- package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +22 -0
- package/src/plugins/defaults/memory/__tests__/jobs-store-enqueue-gate.test.ts +11 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-accounting.test.ts +199 -0
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +98 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +105 -1
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-sweep.test.ts +24 -2
- package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +524 -0
- package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +1 -1
- package/src/plugins/defaults/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +9 -4
- package/src/plugins/defaults/memory/host-utils.ts +5 -0
- package/src/plugins/defaults/memory/memory-retrospective-accounting.ts +105 -1
- package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +64 -5
- package/src/plugins/defaults/memory/memory-retrospective-job.ts +36 -1
- package/src/plugins/defaults/memory/memory-retrospective-sweep.ts +12 -3
- package/src/plugins/defaults/memory/src/__tests__/memory-v2-simulate-route.test.ts +1 -0
- package/src/plugins/defaults/memory/substrate/__tests__/skill-store.test.ts +195 -1
- package/src/plugins/defaults/memory/substrate/consolidation-job.ts +1 -1
- package/src/plugins/defaults/memory/substrate/skill-store.ts +167 -29
- package/src/plugins/defaults/memory/v2/__tests__/injection.test.ts +131 -0
- package/src/plugins/defaults/memory/v2/__tests__/router.test.ts +1 -0
- package/src/plugins/defaults/memory/v2/activation-log-store.ts +4 -0
- package/src/plugins/defaults/memory/v2/injection.ts +48 -3
- package/src/plugins/defaults/memory/v2/rerank-local.ts +6 -2
- package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +1 -1
- package/src/plugins/defaults/platform-hosted/routes/reengage.ts +1 -1
- package/src/plugins/defaults/turn-context/injectors.ts +1 -0
- package/src/plugins/defaults/turn-context/unified-turn-context.ts +26 -0
- package/src/plugins/types.ts +17 -0
- package/src/prompts/templates/system-sections.ts +2 -2
- package/src/providers/__tests__/dispatch-connection-routing.test.ts +43 -2
- package/src/providers/__tests__/vellum-mismatch-routing.test.ts +8 -0
- package/src/providers/call-site-routing.ts +77 -14
- package/src/providers/connection-resolution.ts +42 -2
- package/src/providers/inference/__tests__/adapter-factory-openai-compatible.test.ts +127 -1
- package/src/providers/inference/adapter-factory.ts +70 -12
- package/src/providers/model-catalog.ts +0 -11
- package/src/providers/model-intents.ts +11 -0
- package/src/providers/registry.ts +8 -0
- package/src/providers/retry.ts +85 -17
- package/src/providers/types.ts +8 -1
- package/src/runtime/__tests__/agent-wake.test.ts +181 -0
- package/src/runtime/agent-wake.ts +132 -9
- package/src/runtime/channel-approval-types.ts +25 -0
- package/src/runtime/routes/__tests__/connection-routes-vs-cli-parity.test.ts +2 -2
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +11 -0
- package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +121 -9
- package/src/runtime/routes/__tests__/platform-invoice-routes.test.ts +392 -0
- package/src/runtime/routes/__tests__/slack-channel-routes.test.ts +7 -0
- package/src/runtime/routes/__tests__/workspace-commit-routes.test.ts +63 -0
- package/src/runtime/routes/consolidation-routes.ts +1 -1
- package/src/runtime/routes/conversation-management-routes.ts +18 -6
- package/src/runtime/routes/conversation-query-routes.ts +14 -23
- package/src/runtime/routes/conversation-routes.ts +113 -8
- package/src/runtime/routes/credential-routes.ts +1 -5
- package/src/runtime/routes/documents-routes.ts +207 -12
- package/src/runtime/routes/identity-routes.ts +3 -93
- package/src/runtime/routes/inbound-stages/admission-policy.test.ts +31 -1
- package/src/runtime/routes/inbound-stages/admission-policy.ts +18 -0
- package/src/runtime/routes/inference-provider-connection-routes.ts +52 -3
- package/src/runtime/routes/notification-routes.ts +4 -1
- package/src/runtime/routes/platform-routes.ts +238 -3
- package/src/runtime/routes/playground/guard.ts +1 -2
- package/src/runtime/routes/slack-channel-routes.ts +2 -4
- package/src/runtime/routes/workspace-commit-routes.ts +49 -2
- package/src/runtime/routes/workspace-routes.ts +11 -12
- package/src/runtime/routes/workspace-utils.ts +49 -2
- package/src/runtime/services/conversation-serializer.ts +2 -4
- package/src/subagent/__tests__/consult-context-gating.test.ts +98 -0
- package/src/subagent/__tests__/consult-context-skills.test.ts +82 -0
- package/src/subagent/__tests__/consult-context.test.ts +84 -0
- package/src/subagent/__tests__/consult-prompt.test.ts +36 -0
- package/src/subagent/consult-context.ts +410 -0
- package/src/subagent/consult-prompt.ts +38 -5
- package/src/subagent/manager.ts +9 -4
- package/src/subagent/types.ts +8 -0
- package/src/telemetry/telemetry-event-sources.ts +7 -0
- package/src/telemetry/telemetry-wire-source.json +1 -1
- package/src/telemetry/telemetry-wire.generated.ts +1 -0
- package/src/telemetry/turn-events-store.ts +30 -1
- package/src/telemetry/types.ts +24 -0
- package/src/telemetry/usage-telemetry-reporter.test.ts +48 -0
- package/src/tools/browser/pinned-tabs.ts +3 -1
- package/src/tools/subagent/spawn.ts +23 -1
- package/src/tools/terminal/safe-env.ts +1 -0
- package/src/tools/ui-surface/definitions.ts +39 -1
- package/src/tools/ui-surface/surface-shape-docs.ts +38 -4
- package/src/tools/ui-surface/visual-validation.ts +787 -0
- package/src/tools/workflows/run-workflow.test.ts +1 -0
- package/src/util/__tests__/short-title.test.ts +229 -0
- package/src/util/__tests__/worker-compute.test.ts +65 -0
- package/src/util/cgroup-cpu.ts +93 -0
- package/src/util/errors.ts +27 -0
- package/src/util/process-tree.ts +19 -0
- package/src/util/short-title.ts +189 -0
- package/src/util/worker-compute.ts +83 -0
- package/src/workspace/byok-default-profile-ensure.ts +25 -15
- package/src/workspace/git-service.ts +79 -30
- package/src/workspace/migrations/137-repair-retired-fireworks-minimax-model-id.ts +131 -0
- package/src/workspace/migrations/138-backfill-home-feed-titles.ts +179 -0
- package/src/workspace/migrations/139-clear-renamed-cost-profile-label.ts +92 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/plugins/defaults/memory/substrate/constants.ts +0 -8
|
@@ -123,8 +123,26 @@ const fakeQdrant = {
|
|
|
123
123
|
},
|
|
124
124
|
};
|
|
125
125
|
|
|
126
|
+
// The client is already initialized in this process, so the ops resolve it
|
|
127
|
+
// straight from the singleton. `plugin-index-qdrant-init.test.ts` covers the
|
|
128
|
+
// other side: a process where it is not.
|
|
126
129
|
mock.module("../qdrant-client.js", () => ({
|
|
127
130
|
getQdrantClient: () => fakeQdrant,
|
|
131
|
+
initQdrantClient: () => fakeQdrant,
|
|
132
|
+
resolveQdrantUrl: () => "http://127.0.0.1:6333",
|
|
133
|
+
}));
|
|
134
|
+
mock.module("../../../config/loader.js", () => ({
|
|
135
|
+
getConfig: () => ({
|
|
136
|
+
memory: {
|
|
137
|
+
qdrant: {
|
|
138
|
+
collection: "vellum_memory",
|
|
139
|
+
vectorSize: 3,
|
|
140
|
+
onDisk: true,
|
|
141
|
+
quantization: "none",
|
|
142
|
+
},
|
|
143
|
+
embeddings: { provider: "gemini" },
|
|
144
|
+
},
|
|
145
|
+
}),
|
|
128
146
|
}));
|
|
129
147
|
mock.module("../qdrant-circuit-breaker.js", () => ({
|
|
130
148
|
withQdrantBreaker: <T>(fn: () => Promise<T>) => fn(),
|
|
@@ -137,6 +155,10 @@ mock.module("../embed.js", () => ({
|
|
|
137
155
|
}),
|
|
138
156
|
}));
|
|
139
157
|
mock.module("../embedding-backend.js", () => ({
|
|
158
|
+
selectEmbeddingBackend: async () => ({
|
|
159
|
+
backend: { provider: "gemini", model: "test-embed-model" },
|
|
160
|
+
reason: null,
|
|
161
|
+
}),
|
|
140
162
|
getMemoryBackendStatus: async () => ({
|
|
141
163
|
enabled: true,
|
|
142
164
|
degraded: false,
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tripwire coupling the generated worker scripts to the cache-invalidation
|
|
3
|
+
* version that ships them.
|
|
4
|
+
*
|
|
5
|
+
* `isReady()` short-circuits `ensureInstalled()` whenever the on-disk manifest
|
|
6
|
+
* matches RUNTIME_VERSION, and the worker scripts are only written from
|
|
7
|
+
* `install()`. So editing a generator without bumping the trailing
|
|
8
|
+
* `_workers-vN` suffix leaves every existing install running the old scripts,
|
|
9
|
+
* and the change silently does nothing for anyone who already downloaded the
|
|
10
|
+
* runtime. That is a shipped no-op with green CI, which is why it needs a
|
|
11
|
+
* mechanical check rather than reviewer memory.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { describe, expect, test } from "bun:test";
|
|
15
|
+
|
|
16
|
+
import {
|
|
17
|
+
generateRerankWorkerScript,
|
|
18
|
+
generateWorkerScript,
|
|
19
|
+
RUNTIME_VERSION,
|
|
20
|
+
} from "../embedding-runtime-manager.js";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Fingerprint of the two generated scripts. Update this together with the
|
|
24
|
+
* `_workers-vN` bump below, never on its own.
|
|
25
|
+
*/
|
|
26
|
+
const EXPECTED_WORKER_SCRIPTS_HASH = "4f85253eaebd315";
|
|
27
|
+
|
|
28
|
+
/** Suffix the fingerprint above was taken at. */
|
|
29
|
+
const EXPECTED_WORKERS_VERSION = "workers-v4";
|
|
30
|
+
|
|
31
|
+
function workerScriptsHash(): string {
|
|
32
|
+
// The two scripts are joined by a NUL, written as an escape so this file
|
|
33
|
+
// stays ASCII text rather than tripping git's binary heuristic. NUL cannot
|
|
34
|
+
// occur inside either generated script, so text moving from one worker to
|
|
35
|
+
// the other can never hash the same as the unchanged pair.
|
|
36
|
+
return Bun.hash(
|
|
37
|
+
generateWorkerScript() + "\u0000" + generateRerankWorkerScript(),
|
|
38
|
+
).toString(16);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
describe("worker script cache invalidation", () => {
|
|
42
|
+
test("a change to either generator comes with a _workers-vN bump", () => {
|
|
43
|
+
const actual = workerScriptsHash();
|
|
44
|
+
expect(
|
|
45
|
+
actual,
|
|
46
|
+
[
|
|
47
|
+
"The generated worker scripts changed.",
|
|
48
|
+
"",
|
|
49
|
+
"Existing installs only regenerate these scripts when RUNTIME_VERSION",
|
|
50
|
+
"changes, so bump the trailing `_workers-vN` suffix in",
|
|
51
|
+
"embedding-runtime-manager.ts, then update both constants in this test:",
|
|
52
|
+
"",
|
|
53
|
+
` EXPECTED_WORKER_SCRIPTS_HASH = "${actual}"`,
|
|
54
|
+
` EXPECTED_WORKERS_VERSION = "workers-v<N+1>"`,
|
|
55
|
+
"",
|
|
56
|
+
"Skipping the bump ships a silent no-op for every existing install.",
|
|
57
|
+
].join("\n"),
|
|
58
|
+
).toBe(EXPECTED_WORKER_SCRIPTS_HASH);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test("RUNTIME_VERSION carries the workers suffix the hash was taken at", () => {
|
|
62
|
+
expect(RUNTIME_VERSION.endsWith(`_${EXPECTED_WORKERS_VERSION}`)).toBe(true);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("both workers read the intra-op thread cap the host sets", () => {
|
|
66
|
+
// Guards the JARVIS-1398 wiring specifically: the host computing a cap is
|
|
67
|
+
// useless if the scripts on disk never consult it.
|
|
68
|
+
for (const script of [
|
|
69
|
+
generateWorkerScript(),
|
|
70
|
+
generateRerankWorkerScript(),
|
|
71
|
+
]) {
|
|
72
|
+
expect(script).toContain("VELLUM_ONNX_INTRA_OP_THREADS");
|
|
73
|
+
expect(script).toContain("session_options: sessionOptions");
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
});
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
getEmbedWorkerPidPath,
|
|
16
16
|
} from "../../util/platform.js";
|
|
17
17
|
import { PromiseGuard } from "../../util/promise-guard.js";
|
|
18
|
+
import { workerComputeEnv } from "../../util/worker-compute.js";
|
|
18
19
|
import { workerMemoryEnv } from "../../util/worker-memory.js";
|
|
19
20
|
import { EmbeddingRuntimeManager } from "./embedding-runtime-manager.js";
|
|
20
21
|
import {
|
|
@@ -509,7 +510,7 @@ export class LocalEmbeddingBackend implements EmbeddingBackend {
|
|
|
509
510
|
|
|
510
511
|
const proc = Bun.spawn({
|
|
511
512
|
cmd: [bunPath, "--smol", workerPath, this.model, modelCacheDir],
|
|
512
|
-
env: workerMemoryEnv(),
|
|
513
|
+
env: { ...workerMemoryEnv(), ...workerComputeEnv() },
|
|
513
514
|
stdin: "pipe",
|
|
514
515
|
stdout: "pipe",
|
|
515
516
|
stderr: "pipe",
|
|
@@ -43,8 +43,13 @@ const JINJA_VERSION = "0.5.5";
|
|
|
43
43
|
* `_workers-vN` suffix forces existing installs to regenerate the worker
|
|
44
44
|
* scripts when the worker IPC contract or spawn-args list changes (without
|
|
45
45
|
* requiring an `@huggingface/transformers` version bump).
|
|
46
|
+
*
|
|
47
|
+
* v4: workers read `VELLUM_ONNX_INTRA_OP_THREADS` and pass
|
|
48
|
+
* `session_options.intraOpNumThreads` (JARVIS-1398). The environment the host
|
|
49
|
+
* spawns them with counts as part of the contract, so an install that keeps
|
|
50
|
+
* the v3 scripts would silently ignore the cap.
|
|
46
51
|
*/
|
|
47
|
-
const RUNTIME_VERSION = `ort-${ONNXRUNTIME_NODE_VERSION}_hf-${TRANSFORMERS_VERSION}_jinja-${JINJA_VERSION}_workers-
|
|
52
|
+
export const RUNTIME_VERSION = `ort-${ONNXRUNTIME_NODE_VERSION}_hf-${TRANSFORMERS_VERSION}_jinja-${JINJA_VERSION}_workers-v4`;
|
|
48
53
|
|
|
49
54
|
const WORKER_FILENAME = "embed-worker.mjs";
|
|
50
55
|
const RERANK_WORKER_FILENAME = "rerank-worker.mjs";
|
|
@@ -116,7 +121,7 @@ async function downloadAndExtract(
|
|
|
116
121
|
|
|
117
122
|
// ── Worker script content ───────────────────────────────────────────
|
|
118
123
|
|
|
119
|
-
function generateWorkerScript(): string {
|
|
124
|
+
export function generateWorkerScript(): string {
|
|
120
125
|
// This script is run by a standalone bun process (not the compiled daemon).
|
|
121
126
|
// Because it runs in a real bun runtime, bare specifier resolution works
|
|
122
127
|
// normally — node_modules/ in the same directory is found automatically.
|
|
@@ -130,9 +135,19 @@ const model = process.argv[2];
|
|
|
130
135
|
const cacheDir = process.argv[3];
|
|
131
136
|
if (cacheDir && env) env.cacheDir = cacheDir;
|
|
132
137
|
|
|
138
|
+
// Cap the ONNX intra-op thread pool. Left unset, ONNX takes one thread per
|
|
139
|
+
// physical core for the whole batch and starves foreground work; the host
|
|
140
|
+
// computes the cap in util/worker-compute.ts and passes it in the environment.
|
|
141
|
+
// A missing or unparseable value means no cap, i.e. the ONNX default.
|
|
142
|
+
const intraOpNumThreads = Number(process.env.VELLUM_ONNX_INTRA_OP_THREADS);
|
|
143
|
+
const sessionOptions =
|
|
144
|
+
Number.isInteger(intraOpNumThreads) && intraOpNumThreads > 0
|
|
145
|
+
? { intraOpNumThreads }
|
|
146
|
+
: undefined;
|
|
147
|
+
|
|
133
148
|
let extractor;
|
|
134
149
|
try {
|
|
135
|
-
extractor = await pipeline('feature-extraction', model, { dtype: 'fp32' });
|
|
150
|
+
extractor = await pipeline('feature-extraction', model, { dtype: 'fp32', session_options: sessionOptions });
|
|
136
151
|
process.stdout.write(JSON.stringify({ type: 'ready' }) + '\\n');
|
|
137
152
|
} catch (err) {
|
|
138
153
|
process.stdout.write(JSON.stringify({ type: 'error', error: err.message || String(err) }) + '\\n');
|
|
@@ -177,7 +192,7 @@ process.stdin.on('end', () => process.exit(0));
|
|
|
177
192
|
`;
|
|
178
193
|
}
|
|
179
194
|
|
|
180
|
-
function generateRerankWorkerScript(): string {
|
|
195
|
+
export function generateRerankWorkerScript(): string {
|
|
181
196
|
// Cross-encoder rerank worker. Loads a sequence-classification model and
|
|
182
197
|
// scores paired (queries[i], passages[i]) tuples in one batched ONNX
|
|
183
198
|
// inference call. Mirrors the embed worker's lifecycle (ready signal,
|
|
@@ -204,11 +219,20 @@ const cacheDir = process.argv[3];
|
|
|
204
219
|
const dtype = process.argv[4] || 'q8';
|
|
205
220
|
if (cacheDir && env) env.cacheDir = cacheDir;
|
|
206
221
|
|
|
222
|
+
// Cap the ONNX intra-op thread pool. See the embed worker for the rationale;
|
|
223
|
+
// the host computes the cap in util/worker-compute.ts and passes it in the
|
|
224
|
+
// environment. A missing or unparseable value means no cap, i.e. the default.
|
|
225
|
+
const intraOpNumThreads = Number(process.env.VELLUM_ONNX_INTRA_OP_THREADS);
|
|
226
|
+
const sessionOptions =
|
|
227
|
+
Number.isInteger(intraOpNumThreads) && intraOpNumThreads > 0
|
|
228
|
+
? { intraOpNumThreads }
|
|
229
|
+
: undefined;
|
|
230
|
+
|
|
207
231
|
let tokenizer;
|
|
208
232
|
let session;
|
|
209
233
|
try {
|
|
210
234
|
tokenizer = await AutoTokenizer.from_pretrained(model);
|
|
211
|
-
session = await AutoModelForSequenceClassification.from_pretrained(model, { dtype });
|
|
235
|
+
session = await AutoModelForSequenceClassification.from_pretrained(model, { dtype, session_options: sessionOptions });
|
|
212
236
|
process.stdout.write(JSON.stringify({ type: 'ready' }) + '\\n');
|
|
213
237
|
} catch (err) {
|
|
214
238
|
process.stdout.write(JSON.stringify({ type: 'error', error: err.message || String(err) }) + '\\n');
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
|
|
30
30
|
import { randomUUID } from "node:crypto";
|
|
31
31
|
|
|
32
|
+
import { getConfig } from "../../config/loader.js";
|
|
32
33
|
import type { AssistantConfig } from "../../config/types.js";
|
|
33
34
|
import { BackendUnavailableError } from "../../util/errors.js";
|
|
34
35
|
import { getLogger } from "../../util/logger.js";
|
|
@@ -36,6 +37,7 @@ import { embedWithRetry } from "./embed.js";
|
|
|
36
37
|
import {
|
|
37
38
|
generateSparseEmbedding,
|
|
38
39
|
getMemoryBackendStatus,
|
|
40
|
+
selectEmbeddingBackend,
|
|
39
41
|
} from "./embedding-backend.js";
|
|
40
42
|
import {
|
|
41
43
|
type EmbeddingInput,
|
|
@@ -43,7 +45,12 @@ import {
|
|
|
43
45
|
type SparseEmbedding,
|
|
44
46
|
} from "./embedding-types.js";
|
|
45
47
|
import { withQdrantBreaker } from "./qdrant-circuit-breaker.js";
|
|
46
|
-
import {
|
|
48
|
+
import {
|
|
49
|
+
getQdrantClient,
|
|
50
|
+
initQdrantClient,
|
|
51
|
+
resolveQdrantUrl,
|
|
52
|
+
type VellumQdrantClient,
|
|
53
|
+
} from "./qdrant-client.js";
|
|
47
54
|
|
|
48
55
|
const log = getLogger("plugin-index");
|
|
49
56
|
|
|
@@ -124,13 +131,78 @@ export interface IndexedDocument {
|
|
|
124
131
|
|
|
125
132
|
// ── Internal helpers ───────────────────────────────────────────────────────
|
|
126
133
|
|
|
127
|
-
/**
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
134
|
+
/**
|
|
135
|
+
* Config fingerprint of the client {@link resolveQdrant} last built, so a live
|
|
136
|
+
* config change is not served by a stale client. `VellumQdrantClient` captures
|
|
137
|
+
* its collection, dimension, and model identity at construction, so a client
|
|
138
|
+
* built before the change would keep writing at the old dimension (rejected by
|
|
139
|
+
* a resized collection) or quietly mix vectors from two models into one
|
|
140
|
+
* collection without triggering the sentinel migration.
|
|
141
|
+
*/
|
|
142
|
+
let clientConfigIdentity: string | null = null;
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* The Qdrant client for the shared collection, initializing it from the live
|
|
146
|
+
* workspace config when this process has not.
|
|
147
|
+
*
|
|
148
|
+
* The eager `initQdrantClient` in `runMemoryStartup`
|
|
149
|
+
* (`plugins/defaults/memory/startup.ts`) cannot be relied on here: it runs in
|
|
150
|
+
* the daemon process only, and only while memory v1 is the live tier — on a
|
|
151
|
+
* default workspace (`memory.v2.enabled`, or `memory.v3.live`) it is skipped
|
|
152
|
+
* entirely, because no v1 lane reads or writes the collection in that state.
|
|
153
|
+
* The plugin index is not a memory tier: it is plugin-owned search that must
|
|
154
|
+
* work on every tier and in any process that runs plugin code, so it resolves
|
|
155
|
+
* the client itself rather than depending on a memory-tier decision. Mirrors
|
|
156
|
+
* `resolveLexicalIndex` in `persistence/job-handlers/message-lexical.ts`.
|
|
157
|
+
*
|
|
158
|
+
* Cheap: `initQdrantClient` only constructs a client — the collection is
|
|
159
|
+
* created lazily inside each operation — and the steady-state path is a
|
|
160
|
+
* fingerprint comparison, with the embedding-backend lookup reached only when
|
|
161
|
+
* the config it derives from has actually changed.
|
|
162
|
+
*
|
|
163
|
+
* The dense embedding identity is passed through so a collection created or
|
|
164
|
+
* reused from here keeps the same model-sentinel semantics as the v1 path: a
|
|
165
|
+
* model or dimension change recreates the collection, which is the durability
|
|
166
|
+
* contract documented at the top of this file.
|
|
167
|
+
*/
|
|
168
|
+
async function resolveQdrant(): Promise<VellumQdrantClient> {
|
|
169
|
+
const config = getConfig();
|
|
170
|
+
const url = resolveQdrantUrl(config);
|
|
171
|
+
const { collection, vectorSize, onDisk, quantization } = config.memory.qdrant;
|
|
172
|
+
// `memory.embeddings` rather than the resolved backend: it is what the
|
|
173
|
+
// resolution below reads, and comparing it keeps that lookup off the hot
|
|
174
|
+
// path. Over-sensitive by a field or two (a `required` flip re-inits), which
|
|
175
|
+
// costs one redundant construction and never a wrong client.
|
|
176
|
+
const identity = JSON.stringify([
|
|
177
|
+
url,
|
|
178
|
+
collection,
|
|
179
|
+
vectorSize,
|
|
180
|
+
onDisk,
|
|
181
|
+
quantization,
|
|
182
|
+
config.memory.embeddings,
|
|
183
|
+
]);
|
|
184
|
+
|
|
185
|
+
if (identity === clientConfigIdentity) {
|
|
186
|
+
try {
|
|
187
|
+
return getQdrantClient();
|
|
188
|
+
} catch {
|
|
189
|
+
// Fingerprint outlived the singleton (another module reset it) — rebuild.
|
|
190
|
+
}
|
|
133
191
|
}
|
|
192
|
+
|
|
193
|
+
const selection = await selectEmbeddingBackend(config);
|
|
194
|
+
const client = initQdrantClient({
|
|
195
|
+
url,
|
|
196
|
+
collection,
|
|
197
|
+
vectorSize,
|
|
198
|
+
onDisk,
|
|
199
|
+
quantization,
|
|
200
|
+
embeddingModel: selection.backend
|
|
201
|
+
? `${selection.backend.provider}:${selection.backend.model}`
|
|
202
|
+
: undefined,
|
|
203
|
+
});
|
|
204
|
+
clientConfigIdentity = identity;
|
|
205
|
+
return client;
|
|
134
206
|
}
|
|
135
207
|
|
|
136
208
|
/** Embed a single input to a dense vector, failing loudly if no backend is up. */
|
|
@@ -208,7 +280,7 @@ export async function indexDocument(
|
|
|
208
280
|
const normalized = normalizeEmbeddingInput(input);
|
|
209
281
|
const documentId = opts?.documentId ?? randomUUID();
|
|
210
282
|
const now = opts?.createdAt ?? Date.now();
|
|
211
|
-
const qdrant =
|
|
283
|
+
const qdrant = await resolveQdrant();
|
|
212
284
|
|
|
213
285
|
await withQdrantBreaker(() =>
|
|
214
286
|
qdrant.upsert(
|
|
@@ -249,7 +321,7 @@ export async function queryIndex(
|
|
|
249
321
|
const embedding = await embedDense(config, query);
|
|
250
322
|
const sparseVector = sparseFor(query);
|
|
251
323
|
const limit = opts?.limit ?? 10;
|
|
252
|
-
const qdrant =
|
|
324
|
+
const qdrant = await resolveQdrant();
|
|
253
325
|
|
|
254
326
|
const filter = {
|
|
255
327
|
must: [
|
|
@@ -290,7 +362,7 @@ export async function getDocument(
|
|
|
290
362
|
plugin: string,
|
|
291
363
|
documentId: string,
|
|
292
364
|
): Promise<IndexedDocument | null> {
|
|
293
|
-
const qdrant =
|
|
365
|
+
const qdrant = await resolveQdrant();
|
|
294
366
|
const found = await withQdrantBreaker(() =>
|
|
295
367
|
qdrant.getByTarget(
|
|
296
368
|
PLUGIN_INDEX_TARGET_TYPE,
|
|
@@ -315,7 +387,7 @@ export async function removeDocument(
|
|
|
315
387
|
plugin: string,
|
|
316
388
|
documentId: string,
|
|
317
389
|
): Promise<void> {
|
|
318
|
-
const qdrant =
|
|
390
|
+
const qdrant = await resolveQdrant();
|
|
319
391
|
await withQdrantBreaker(() =>
|
|
320
392
|
qdrant.deleteByTargetAndPlugin(
|
|
321
393
|
PLUGIN_INDEX_TARGET_TYPE,
|
|
@@ -333,7 +405,7 @@ export async function removeDocument(
|
|
|
333
405
|
*/
|
|
334
406
|
export async function purgeEmbeddingsForPlugin(plugin: string): Promise<void> {
|
|
335
407
|
try {
|
|
336
|
-
const qdrant =
|
|
408
|
+
const qdrant = await resolveQdrant();
|
|
337
409
|
await withQdrantBreaker(() => qdrant.deleteByPlugin(plugin));
|
|
338
410
|
} catch (err) {
|
|
339
411
|
log.warn({ err, plugin }, "Failed to purge plugin embeddings");
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { Database } from "bun:sqlite";
|
|
2
|
+
import { describe, expect, test } from "bun:test";
|
|
3
|
+
|
|
4
|
+
import { drizzle } from "drizzle-orm/bun-sqlite";
|
|
5
|
+
|
|
6
|
+
import * as schema from "../schema.js";
|
|
7
|
+
import { migrateAddDocumentWorkspacePath } from "./360-add-document-workspace-path.js";
|
|
8
|
+
|
|
9
|
+
/** Pre-360 shape: the documents table exactly as core tables created it. */
|
|
10
|
+
function createTestDb() {
|
|
11
|
+
const sqlite = new Database(":memory:");
|
|
12
|
+
sqlite.exec(/*sql*/ `
|
|
13
|
+
CREATE TABLE conversations (
|
|
14
|
+
id TEXT PRIMARY KEY,
|
|
15
|
+
created_at INTEGER NOT NULL
|
|
16
|
+
);
|
|
17
|
+
|
|
18
|
+
CREATE TABLE documents (
|
|
19
|
+
surface_id TEXT PRIMARY KEY,
|
|
20
|
+
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
|
|
21
|
+
title TEXT NOT NULL,
|
|
22
|
+
content TEXT NOT NULL,
|
|
23
|
+
word_count INTEGER NOT NULL DEFAULT 0,
|
|
24
|
+
created_at INTEGER NOT NULL,
|
|
25
|
+
updated_at INTEGER NOT NULL
|
|
26
|
+
);
|
|
27
|
+
|
|
28
|
+
INSERT INTO conversations (id, created_at) VALUES ('conv-1', 1000);
|
|
29
|
+
`);
|
|
30
|
+
return { sqlite, db: drizzle(sqlite, { schema }) };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function columnInfo(sqlite: Database) {
|
|
34
|
+
return sqlite.query("PRAGMA table_info(documents)").all() as Array<{
|
|
35
|
+
name: string;
|
|
36
|
+
notnull: number;
|
|
37
|
+
}>;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function insertDocument(
|
|
41
|
+
sqlite: Database,
|
|
42
|
+
surfaceId: string,
|
|
43
|
+
workspacePath: string | null,
|
|
44
|
+
): void {
|
|
45
|
+
sqlite
|
|
46
|
+
.query(
|
|
47
|
+
/*sql*/ `INSERT INTO documents (surface_id, conversation_id, title, content, word_count, created_at, updated_at, workspace_path)
|
|
48
|
+
VALUES (?, 'conv-1', 'Doc', '', 0, 1000, 1000, ?)`,
|
|
49
|
+
)
|
|
50
|
+
.run(surfaceId, workspacePath);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
describe("migration 360: documents.workspace_path", () => {
|
|
54
|
+
test("adds the nullable column to an existing database", () => {
|
|
55
|
+
const { sqlite, db } = createTestDb();
|
|
56
|
+
expect(columnInfo(sqlite).map((c) => c.name)).not.toContain(
|
|
57
|
+
"workspace_path",
|
|
58
|
+
);
|
|
59
|
+
|
|
60
|
+
migrateAddDocumentWorkspacePath(db);
|
|
61
|
+
|
|
62
|
+
const column = columnInfo(sqlite).find((c) => c.name === "workspace_path");
|
|
63
|
+
expect(column).toBeDefined();
|
|
64
|
+
expect(column?.notnull).toBe(0);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test("rows written before the column existed read back null", () => {
|
|
68
|
+
const { sqlite, db } = createTestDb();
|
|
69
|
+
sqlite.exec(/*sql*/ `
|
|
70
|
+
INSERT INTO documents (surface_id, conversation_id, title, content, word_count, created_at, updated_at)
|
|
71
|
+
VALUES ('doc-old', 'conv-1', 'Old', 'body', 1, 1000, 1000)
|
|
72
|
+
`);
|
|
73
|
+
|
|
74
|
+
migrateAddDocumentWorkspacePath(db);
|
|
75
|
+
|
|
76
|
+
const row = sqlite
|
|
77
|
+
.query(
|
|
78
|
+
"SELECT workspace_path FROM documents WHERE surface_id = 'doc-old'",
|
|
79
|
+
)
|
|
80
|
+
.get() as { workspace_path: unknown };
|
|
81
|
+
expect(row.workspace_path).toBeNull();
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test("rejects a second document bound to the same file", () => {
|
|
85
|
+
const { sqlite, db } = createTestDb();
|
|
86
|
+
migrateAddDocumentWorkspacePath(db);
|
|
87
|
+
|
|
88
|
+
insertDocument(sqlite, "doc-a", "notes/plan.md");
|
|
89
|
+
|
|
90
|
+
expect(() => insertDocument(sqlite, "doc-b", "notes/plan.md")).toThrow();
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("leaves file-less documents unconstrained", () => {
|
|
94
|
+
const { sqlite, db } = createTestDb();
|
|
95
|
+
migrateAddDocumentWorkspacePath(db);
|
|
96
|
+
|
|
97
|
+
insertDocument(sqlite, "doc-a", null);
|
|
98
|
+
expect(() => insertDocument(sqlite, "doc-b", null)).not.toThrow();
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("is idempotent: re-run is a no-op", () => {
|
|
102
|
+
const { sqlite, db } = createTestDb();
|
|
103
|
+
|
|
104
|
+
migrateAddDocumentWorkspacePath(db);
|
|
105
|
+
expect(() => migrateAddDocumentWorkspacePath(db)).not.toThrow();
|
|
106
|
+
|
|
107
|
+
const names = columnInfo(sqlite).map((c) => c.name);
|
|
108
|
+
expect(names.filter((n) => n === "workspace_path")).toHaveLength(1);
|
|
109
|
+
});
|
|
110
|
+
});
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { DrizzleDb } from "../db-connection.js";
|
|
2
|
+
import { tableHasColumn } from "./schema-introspection.js";
|
|
3
|
+
|
|
4
|
+
const COLUMN_NAME = "workspace_path";
|
|
5
|
+
const COLUMN_DEFINITION = "workspace_path TEXT";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Add `workspace_path` to `documents` and enforce one document per file.
|
|
9
|
+
*
|
|
10
|
+
* The column holds the workspace-relative path of the markdown file a document
|
|
11
|
+
* is bound to. It is set when a client opens a workspace `.md` file through
|
|
12
|
+
* `POST /v1/documents/for-workspace-file`, which gives the file a document
|
|
13
|
+
* identity so comments, assistant iteration, and PDF export — all keyed off a
|
|
14
|
+
* document id — work on it. Documents that have no file behind them (assistant
|
|
15
|
+
* drafts, client-created surfaces) stay NULL.
|
|
16
|
+
*
|
|
17
|
+
* The unique index is partial (`WHERE workspace_path IS NOT NULL`) so the many
|
|
18
|
+
* NULL rows stay unconstrained while a given file resolves to exactly one
|
|
19
|
+
* document. The find-or-create route depends on that uniqueness for its
|
|
20
|
+
* idempotency.
|
|
21
|
+
*
|
|
22
|
+
* Nullable with no backfill: nothing recorded a file binding before this
|
|
23
|
+
* column existed, so pre-existing rows correctly stay NULL.
|
|
24
|
+
*
|
|
25
|
+
* Idempotent: the column add is guarded with `tableHasColumn` and the index
|
|
26
|
+
* with `IF NOT EXISTS`, so a crash between the two does not break the next
|
|
27
|
+
* boot.
|
|
28
|
+
*/
|
|
29
|
+
export function migrateAddDocumentWorkspacePath(database: DrizzleDb): void {
|
|
30
|
+
if (!tableHasColumn(database, "documents", COLUMN_NAME)) {
|
|
31
|
+
database.run(`ALTER TABLE documents ADD COLUMN ${COLUMN_DEFINITION}`);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
database.run(/*sql*/ `
|
|
35
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_documents_workspace_path
|
|
36
|
+
ON documents(workspace_path)
|
|
37
|
+
WHERE workspace_path IS NOT NULL
|
|
38
|
+
`);
|
|
39
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `PRAGMA optimize` mask run by the daily maintenance sweep and by
|
|
3
|
+
* `db-init` after a boot that applied migrations.
|
|
4
|
+
*
|
|
5
|
+
* Both run on connections with no useful query history, so `0x10000` is what
|
|
6
|
+
* lets optimize look past tables missing from `sqlite_stat1` and reconsider
|
|
7
|
+
* ones whose entries have gone stale. `0x2` keeps the default "analyze where
|
|
8
|
+
* it helps" behavior and `0x10` bounds each ANALYZE with a temporary
|
|
9
|
+
* `analysis_limit`. Releases before 3.46 ignore bits they do not recognize
|
|
10
|
+
* rather than failing, so the mask is safe to send to any version.
|
|
11
|
+
*
|
|
12
|
+
* https://sqlite.org/lang_analyze.html#recommended_usage_patterns
|
|
13
|
+
*/
|
|
14
|
+
export const PLANNER_OPTIMIZE_PRAGMA = "PRAGMA optimize=0x10012";
|
|
@@ -1,24 +1,39 @@
|
|
|
1
|
+
import { sql } from "drizzle-orm";
|
|
1
2
|
import {
|
|
2
3
|
index,
|
|
3
4
|
integer,
|
|
4
5
|
primaryKey,
|
|
5
6
|
sqliteTable,
|
|
6
7
|
text,
|
|
8
|
+
uniqueIndex,
|
|
7
9
|
} from "drizzle-orm/sqlite-core";
|
|
8
10
|
|
|
9
11
|
import { conversations } from "./conversations.js";
|
|
10
12
|
|
|
11
|
-
export const documents = sqliteTable(
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
.
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
13
|
+
export const documents = sqliteTable(
|
|
14
|
+
"documents",
|
|
15
|
+
{
|
|
16
|
+
surfaceId: text("surface_id").primaryKey(),
|
|
17
|
+
conversationId: text("conversation_id")
|
|
18
|
+
.notNull()
|
|
19
|
+
.references(() => conversations.id, { onDelete: "cascade" }),
|
|
20
|
+
title: text("title").notNull(),
|
|
21
|
+
content: text("content").notNull(),
|
|
22
|
+
wordCount: integer("word_count").notNull().default(0),
|
|
23
|
+
createdAt: integer("created_at").notNull(),
|
|
24
|
+
updatedAt: integer("updated_at").notNull(),
|
|
25
|
+
// Workspace-relative path of the markdown file this document is bound to.
|
|
26
|
+
// NULL for documents with no file behind them.
|
|
27
|
+
workspacePath: text("workspace_path"),
|
|
28
|
+
},
|
|
29
|
+
(table) => [
|
|
30
|
+
// Partial so the many file-less documents stay unconstrained while a given
|
|
31
|
+
// file resolves to exactly one document.
|
|
32
|
+
uniqueIndex("idx_documents_workspace_path")
|
|
33
|
+
.on(table.workspacePath)
|
|
34
|
+
.where(sql`workspace_path IS NOT NULL`),
|
|
35
|
+
],
|
|
36
|
+
);
|
|
22
37
|
|
|
23
38
|
// Junction table mapping a document surface to every conversation it appears in.
|
|
24
39
|
export const documentConversations = sqliteTable(
|
package/src/persistence/steps.ts
CHANGED
|
@@ -467,6 +467,7 @@ import { migrateAddSubagentParentToolUseId } from "./migrations/356-add-subagent
|
|
|
467
467
|
import { migrateMoveMemorySegmentsToMemoryDb } from "./migrations/357-move-memory-segments-to-memory-db.js";
|
|
468
468
|
import { migrateMoveMemoryEmbeddingsToMemoryDb } from "./migrations/358-move-memory-embeddings-to-memory-db.js";
|
|
469
469
|
import { migrateMoveMemorySummariesToMemoryDb } from "./migrations/359-move-memory-summaries-to-memory-db.js";
|
|
470
|
+
import { migrateAddDocumentWorkspacePath } from "./migrations/360-add-document-workspace-path.js";
|
|
470
471
|
import type { MigrationStep } from "./migrations/run-migrations.js";
|
|
471
472
|
|
|
472
473
|
export const migrationSteps: MigrationStep[] = [
|
|
@@ -1541,4 +1542,5 @@ export const migrationSteps: MigrationStep[] = [
|
|
|
1541
1542
|
"migrateMemorySummariesScopeUpdatedIndex",
|
|
1542
1543
|
],
|
|
1543
1544
|
},
|
|
1545
|
+
migrateAddDocumentWorkspacePath,
|
|
1544
1546
|
];
|