@vellumai/assistant 0.8.1 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +2 -7
- package/Dockerfile +75 -1
- package/bun.lock +11 -1
- package/docker-entrypoint.sh +5 -0
- package/docker-init-apt-root.sh +94 -0
- package/docker-kata-apt-env.sh +39 -0
- package/docs/plugins.md +88 -47
- package/docs/skills.md +9 -7
- package/examples/plugins/echo/README.md +27 -27
- package/examples/plugins/echo/package.json +3 -0
- package/examples/plugins/echo/register.ts +31 -31
- package/node_modules/@vellumai/slack-text/src/index.test.ts +114 -14
- package/node_modules/@vellumai/slack-text/src/index.ts +82 -18
- package/openapi.yaml +325 -3
- package/package.json +3 -1
- package/scripts/generate-openapi.ts +83 -10
- package/scripts/sync-llm-catalog.ts +2 -2
- package/scripts/sync-web-search-catalog.ts +47 -25
- package/src/__tests__/agent-image-optimize.test.ts +11 -3
- package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +131 -0
- package/src/__tests__/anthropic-provider.test.ts +45 -0
- package/src/__tests__/app-builder-tool-scripts.test.ts +9 -3
- package/src/__tests__/app-executors.test.ts +220 -4
- package/src/__tests__/auto-analysis-end-to-end.test.ts +35 -0
- package/src/__tests__/bundled-asset.test.ts +6 -6
- package/src/__tests__/channel-availability-routes.test.ts +206 -0
- package/src/__tests__/channel-delivery-store.test.ts +289 -1
- package/src/__tests__/circuit-breaker-pipeline.test.ts +0 -1
- package/src/__tests__/clawhub.test.ts +75 -16
- package/src/__tests__/compactor-tail-resolution.test.ts +41 -0
- package/src/__tests__/config-schema.test.ts +21 -0
- package/src/__tests__/config-set-route.test.ts +80 -0
- package/src/__tests__/config-sounds-sync.test.ts +97 -0
- package/src/__tests__/config-watcher-skill-reseed.test.ts +453 -0
- package/src/__tests__/context-search-conversations-source.test.ts +117 -2
- package/src/__tests__/context-search-memory-v2-source.test.ts +0 -1
- package/src/__tests__/context-search-workspace-source.test.ts +7 -0
- package/src/__tests__/context-token-estimator.test.ts +1 -0
- package/src/__tests__/conversation-abort-tool-results.test.ts +4 -1
- package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +1 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +92 -92
- package/src/__tests__/conversation-agent-loop.test.ts +2 -0
- package/src/__tests__/conversation-error.test.ts +42 -3
- package/src/__tests__/conversation-fork-crud.test.ts +82 -0
- package/src/__tests__/conversation-inference-profile-route.test.ts +40 -4
- package/src/__tests__/conversation-lifecycle.test.ts +173 -0
- package/src/__tests__/conversation-message-sync-tags.test.ts +97 -0
- package/src/__tests__/conversation-pairing.test.ts +54 -0
- package/src/__tests__/conversation-process-callsite.test.ts +4 -1
- package/src/__tests__/conversation-provider-retry-repair.test.ts +5 -1
- package/src/__tests__/conversation-queue.test.ts +4 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +76 -9
- package/src/__tests__/conversation-slash-queue.test.ts +59 -1
- package/src/__tests__/conversation-slash-unknown.test.ts +4 -1
- package/src/__tests__/conversation-surfaces-table-action.test.ts +360 -0
- package/src/__tests__/conversation-sync-tags.test.ts +235 -0
- package/src/__tests__/conversation-workspace-injection.test.ts +5 -1
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +5 -1
- package/src/__tests__/credential-security-invariants.test.ts +3 -2
- package/src/__tests__/db-slack-external-content-normalization.test.ts +301 -0
- package/src/__tests__/delete-managed-skill-tool.test.ts +55 -13
- package/src/__tests__/disk-pressure-tools.test.ts +1 -0
- package/src/__tests__/dm-backfill.test.ts +121 -10
- package/src/__tests__/document-tool-security.test.ts +258 -0
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +0 -1
- package/src/__tests__/edit-propagation.test.ts +33 -0
- package/src/__tests__/empty-response-pipeline.test.ts +0 -4
- package/src/__tests__/external-plugin-loader.test.ts +60 -36
- package/src/__tests__/filing-service.test.ts +140 -0
- package/src/__tests__/get-skill-detail-audit.test.ts +0 -4
- package/src/__tests__/handlers-skills-memory-v2-reseed.test.ts +43 -62
- package/src/__tests__/helpers/tar-fixtures.ts +39 -0
- package/src/__tests__/helpers/wait-for.ts +21 -0
- package/src/__tests__/history-repair-pipeline.test.ts +0 -3
- package/src/__tests__/history-repair.test.ts +73 -0
- package/src/__tests__/host-app-control-proxy.test.ts +266 -10
- package/src/__tests__/image-credentials.test.ts +1 -1
- package/src/__tests__/inbound-slack-persistence.test.ts +2 -0
- package/src/__tests__/inference-no-mode-boot-e2e.test.ts +1 -1
- package/src/__tests__/inference-profile-reaper.test.ts +4 -2
- package/src/__tests__/inference-profile-session-handler.test.ts +18 -6
- package/src/__tests__/inference-profile-session-ipc.test.ts +17 -5
- package/src/__tests__/injector-chain.test.ts +10 -8
- package/src/__tests__/install-skill-routing.test.ts +155 -37
- package/src/__tests__/lifecycle-memory-v2-seed.test.ts +92 -3
- package/src/__tests__/list-messages-page-latest.test.ts +55 -0
- package/src/__tests__/llm-call-pipeline.test.ts +0 -3
- package/src/__tests__/llm-catalog-parity.test.ts +55 -13
- package/src/__tests__/llm-request-log-source-clickhouse.test.ts +34 -0
- package/src/__tests__/llm-request-log-source-factory.test.ts +29 -53
- package/src/__tests__/llm-usage-store.test.ts +114 -0
- package/src/__tests__/managed-profile-guard.test.ts +31 -29
- package/src/__tests__/managed-skill-lifecycle.test.ts +109 -18
- package/src/__tests__/managed-store.test.ts +84 -192
- package/src/__tests__/media-generate-image.test.ts +1 -1
- package/src/__tests__/memory-retrieval-pipeline.test.ts +0 -2
- package/src/__tests__/messages-after-tiebreaker.test.ts +122 -0
- package/src/__tests__/oauth-commands-routes.test.ts +168 -16
- package/src/__tests__/oauth-provider-profiles.test.ts +9 -0
- package/src/__tests__/openai-provider.test.ts +24 -0
- package/src/__tests__/openai-responses-cutover-guard.test.ts +17 -9
- package/src/__tests__/overflow-reduce-pipeline.test.ts +0 -2
- package/src/__tests__/persistence-pipeline.test.ts +0 -2
- package/src/__tests__/{managed-proxy-context.test.ts → platform-proxy-context.test.ts} +1 -1
- package/src/__tests__/platform.test.ts +2 -0
- package/src/__tests__/plugin-api-shim.test.ts +125 -0
- package/src/__tests__/plugin-bootstrap.test.ts +10 -36
- package/src/__tests__/plugin-external-api.test.ts +68 -0
- package/src/__tests__/plugin-registry.test.ts +0 -77
- package/src/__tests__/plugin-route-contribution.test.ts +0 -1
- package/src/__tests__/plugin-skill-contribution.test.ts +0 -2
- package/src/__tests__/plugin-tool-contribution.test.ts +16 -15
- package/src/__tests__/plugin-types.test.ts +3 -13
- package/src/__tests__/process-message-background-slack.test.ts +8 -1
- package/src/__tests__/process-message-display-content.test.ts +421 -0
- package/src/__tests__/provider-catalog-visibility.test.ts +142 -0
- package/src/__tests__/provider-error-scenarios.test.ts +111 -0
- package/src/__tests__/{provider-managed-proxy-integration.test.ts → provider-platform-proxy-integration.test.ts} +8 -8
- package/src/__tests__/scaffold-managed-skill-tool.test.ts +65 -13
- package/src/__tests__/schedule-routes.test.ts +50 -3
- package/src/__tests__/schedule-store.test.ts +94 -0
- package/src/__tests__/scheduler-reuse-conversation.test.ts +54 -7
- package/src/__tests__/schema-transforms.test.ts +20 -0
- package/src/__tests__/search-skills-unified.test.ts +0 -5
- package/src/__tests__/server-history-render.test.ts +43 -0
- package/src/__tests__/skill-load-feature-flag.test.ts +0 -12
- package/src/__tests__/skill-load-tool.test.ts +27 -89
- package/src/__tests__/skill-memory.test.ts +23 -3
- package/src/__tests__/skills-file-content-endpoint.test.ts +9 -38
- package/src/__tests__/skills-files-catalog-fallback.test.ts +0 -3
- package/src/__tests__/skills-install-extract.test.ts +49 -38
- package/src/__tests__/skills-install-staging.test.ts +159 -0
- package/src/__tests__/skills-uninstall.test.ts +9 -41
- package/src/__tests__/skills.test.ts +51 -58
- package/src/__tests__/slack-channel-config.test.ts +9 -0
- package/src/__tests__/subagent-tool-filtering.test.ts +50 -0
- package/src/__tests__/system-prompt.test.ts +737 -63
- package/src/__tests__/terminal-tools.test.ts +28 -1
- package/src/__tests__/thread-backfill.test.ts +557 -27
- package/src/__tests__/title-generate-pipeline.test.ts +0 -13
- package/src/__tests__/token-estimate-pipeline.test.ts +0 -3
- package/src/__tests__/tool-error-pipeline.test.ts +0 -3
- package/src/__tests__/tool-execute-pipeline.test.ts +0 -5
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +1 -1
- package/src/__tests__/tool-executor.test.ts +16 -4
- package/src/__tests__/tool-result-truncate-pipeline.test.ts +0 -12
- package/src/__tests__/turn-events-store.test.ts +256 -0
- package/src/__tests__/twilio-routes.test.ts +4 -0
- package/src/__tests__/user-plugin-loader.test.ts +0 -7
- package/src/__tests__/voice-session-bridge.test.ts +198 -0
- package/src/__tests__/web-search-catalog-parity.test.ts +32 -10
- package/src/__tests__/workspace-migration-057-repair-stale-gemini-model-ids.test.ts +115 -3
- package/src/__tests__/workspace-migration-072-seed-reply-suggestion-callsite.test.ts +50 -0
- package/src/__tests__/workspace-migration-073-repair-recall-callsite-empty-profile.test.ts +153 -0
- package/src/__tests__/workspace-migration-085-memory-v2-bm25-b-reembed-disabled-v2-pages.test.ts +220 -0
- package/src/__tests__/workspace-migration-086-revert-stale-gemini-mis-rewrites.test.ts +269 -0
- package/src/__tests__/workspace-migration-remove-legacy-skills-index.test.ts +309 -0
- package/src/__tests__/workspace-migrations-runner.test.ts +111 -3
- package/src/acp/resolve-agent.ts +1 -1
- package/src/agent/image-optimize.ts +13 -5
- package/src/calls/voice-session-bridge.ts +61 -42
- package/src/channels/types.ts +108 -0
- package/src/cli/__tests__/unknown-command.test.ts +24 -0
- package/src/cli/commands/__tests__/changelog.test.ts +304 -319
- package/src/cli/commands/__tests__/schedules.test.ts +491 -0
- package/src/cli/commands/changelog.ts +106 -42
- package/src/cli/commands/conversations.ts +102 -17
- package/src/cli/commands/default-action.ts +10 -53
- package/src/cli/commands/notifications.ts +329 -317
- package/src/cli/commands/plugins.ts +185 -0
- package/src/cli/commands/schedules.ts +391 -0
- package/src/cli/commands/telemetry.ts +40 -0
- package/src/cli/lib/__tests__/cli-colors.test.ts +48 -0
- package/src/cli/lib/__tests__/confirm-prompt.test.ts +159 -0
- package/src/cli/lib/__tests__/install-from-github.test.ts +355 -0
- package/src/cli/lib/__tests__/list-installed-plugins.test.ts +154 -0
- package/src/cli/lib/__tests__/uninstall-plugin.test.ts +124 -0
- package/src/cli/lib/__tests__/unknown-command.test.ts +106 -0
- package/src/cli/lib/cli-colors.ts +12 -0
- package/src/cli/lib/confirm-prompt.ts +79 -0
- package/src/cli/lib/install-from-github.ts +304 -0
- package/src/cli/lib/list-installed-plugins.ts +137 -0
- package/src/cli/lib/uninstall-plugin.ts +82 -0
- package/src/cli/lib/unknown-command.ts +111 -0
- package/src/cli/program.ts +38 -2
- package/src/config/bundled-skills/app-builder/SKILL.md +23 -21
- package/src/config/bundled-skills/app-builder/TOOLS.json +7 -0
- package/src/config/bundled-skills/computer-use/TOOLS.json +15 -52
- package/src/config/bundled-skills/document/SKILL.md +23 -3
- package/src/config/bundled-skills/document/TOOLS.json +53 -0
- package/src/config/bundled-skills/document/tools/document-delete.ts +12 -0
- package/src/config/bundled-skills/document/tools/document-list.ts +12 -0
- package/src/config/bundled-skills/document/tools/document-read.ts +12 -0
- package/src/config/bundled-skills/skill-management/SKILL.md +2 -2
- package/src/config/bundled-skills/skill-management/TOOLS.json +7 -7
- package/src/config/bundled-tool-registry.ts +6 -0
- package/src/config/feature-flag-registry.json +41 -1
- package/src/config/loader.ts +64 -38
- package/src/config/schema.ts +7 -10
- package/src/config/schemas/__tests__/llm-request-logs.test.ts +36 -0
- package/src/config/schemas/channels.ts +8 -0
- package/src/config/schemas/compaction.ts +28 -0
- package/src/config/schemas/heartbeat.ts +9 -0
- package/src/config/schemas/llm-request-logs.ts +31 -7
- package/src/config/schemas/llm.ts +3 -0
- package/src/config/schemas/memory-retrieval.ts +18 -0
- package/src/config/schemas/tools.ts +14 -0
- package/src/config/skills.ts +3 -96
- package/src/context/compactor.ts +1047 -0
- package/src/context/token-estimator.ts +2 -2
- package/src/context/window-manager.ts +197 -1520
- package/src/credential-execution/managed-catalog.ts +37 -0
- package/src/credential-health/credential-health-service.ts +280 -19
- package/src/daemon/__tests__/conversation-lifecycle-auto-analyze.test.ts +34 -0
- package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +138 -0
- package/src/daemon/__tests__/conversation-tool-setup.test.ts +74 -0
- package/src/daemon/approval-generators.ts +8 -6
- package/src/daemon/config-watcher.ts +94 -31
- package/src/daemon/conversation-agent-loop.ts +169 -9
- package/src/daemon/conversation-error.ts +171 -37
- package/src/daemon/conversation-lifecycle.ts +53 -40
- package/src/daemon/conversation-messaging.ts +25 -6
- package/src/daemon/conversation-process.ts +49 -12
- package/src/daemon/conversation-runtime-assembly.ts +16 -1
- package/src/daemon/conversation-slash.ts +12 -5
- package/src/daemon/conversation-store.ts +11 -4
- package/src/daemon/conversation-tool-setup.ts +39 -7
- package/src/daemon/conversation.ts +33 -1
- package/src/daemon/external-plugins-bootstrap.ts +217 -181
- package/src/daemon/first-greeting.ts +22 -2
- package/src/daemon/handlers/config-model.ts +6 -5
- package/src/daemon/handlers/config-slack-channel.ts +15 -3
- package/src/daemon/handlers/shared.ts +14 -5
- package/src/daemon/handlers/skills.ts +111 -108
- package/src/daemon/history-repair.ts +28 -1
- package/src/daemon/host-app-control-proxy.ts +98 -23
- package/src/daemon/lifecycle.ts +45 -35
- package/src/daemon/meet-host-supervisor.ts +5 -4
- package/src/daemon/memory-v2-startup.ts +49 -0
- package/src/daemon/message-protocol.ts +1 -0
- package/src/daemon/message-types/conversations.ts +25 -0
- package/src/daemon/message-types/messages.ts +61 -0
- package/src/daemon/message-types/subagents.ts +1 -0
- package/src/daemon/message-types/sync.ts +1 -0
- package/src/daemon/pkb-reminder-builder.test.ts +1 -1
- package/src/daemon/pkb-reminder-builder.ts +1 -1
- package/src/daemon/plugin-source-watcher.ts +146 -0
- package/src/daemon/process-message.ts +21 -3
- package/src/daemon/server.ts +11 -2
- package/src/daemon/skill-memory-refresh.ts +29 -0
- package/src/documents/document-store.ts +221 -3
- package/src/embedded/plugin-api.ts +40 -0
- package/src/filing/filing-service.ts +39 -0
- package/src/heartbeat/__tests__/heartbeat-service.test.ts +91 -6
- package/src/heartbeat/heartbeat-run-store.ts +2 -1
- package/src/heartbeat/heartbeat-service.ts +41 -0
- package/src/home/__tests__/feed-types.test.ts +40 -0
- package/src/home/feed-types.ts +22 -0
- package/src/home/post-connect-feed.ts +1 -0
- package/src/index.ts +18 -1
- package/src/live-voice/__tests__/live-voice-stt.test.ts +57 -0
- package/src/mcp/client.ts +20 -4
- package/src/media/image-credentials.ts +3 -3
- package/src/memory/__tests__/bookmark-crud.test.ts +33 -27
- package/src/memory/__tests__/conversation-queries.test.ts +263 -0
- package/src/memory/__tests__/jobs-worker-v2-graph-trigger-embed.test.ts +113 -0
- package/src/memory/__tests__/memory-retrospective-startup-cleanup.test.ts +119 -14
- package/src/memory/__tests__/message-content.test.ts +35 -0
- package/src/memory/bookmark-crud.ts +42 -10
- package/src/memory/context-search/sources/conversations.ts +62 -2
- package/src/memory/context-search/sources/workspace.ts +4 -0
- package/src/memory/conversation-crud.ts +63 -19
- package/src/memory/conversation-queries.ts +110 -10
- package/src/memory/db-init.ts +6 -0
- package/src/memory/delivery-crud.ts +152 -5
- package/src/memory/embedding-backend.ts +4 -4
- package/src/memory/external-conversation-store.ts +66 -5
- package/src/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +66 -9
- package/src/memory/graph/conversation-graph-memory.ts +31 -15
- package/src/memory/graph/tools.ts +3 -3
- package/src/memory/indexer.ts +34 -29
- package/src/memory/jobs/__tests__/embed-concept-page.test.ts +73 -0
- package/src/memory/jobs/embed-concept-page.ts +20 -11
- package/src/memory/jobs-worker.ts +6 -1
- package/src/memory/llm-request-log-source-clickhouse.ts +17 -10
- package/src/memory/llm-request-log-source.ts +19 -52
- package/src/memory/llm-usage-store.ts +125 -5
- package/src/memory/memory-retrospective-startup-cleanup.ts +72 -5
- package/src/memory/message-content.ts +1 -1
- package/src/memory/migrations/109-external-conversation-bindings.ts +15 -4
- package/src/memory/migrations/229-delete-private-conversations.test.ts +38 -1
- package/src/memory/migrations/229-delete-private-conversations.ts +7 -0
- package/src/memory/migrations/247-external-conversation-binding-thread-id.ts +78 -0
- package/src/memory/migrations/248-create-onboarding-events.ts +21 -0
- package/src/memory/migrations/249-normalize-slack-external-content.ts +240 -0
- package/src/memory/migrations/index.ts +6 -0
- package/src/memory/migrations/registry.ts +8 -0
- package/src/memory/onboarding-events-store.ts +106 -0
- package/src/memory/schema/bookmarks.ts +0 -2
- package/src/memory/schema/calls.ts +1 -0
- package/src/memory/schema/inference.ts +1 -3
- package/src/memory/schema/infrastructure.ts +12 -0
- package/src/memory/turn-events-store.ts +127 -2
- package/src/memory/v2/__tests__/activation.test.ts +0 -8
- package/src/memory/v2/__tests__/injection.test.ts +98 -8
- package/src/memory/v2/__tests__/migration.test.ts +87 -0
- package/src/memory/v2/__tests__/page-index.test.ts +83 -0
- package/src/memory/v2/__tests__/prompts-router.test.ts +58 -6
- package/src/memory/v2/__tests__/qdrant.test.ts +66 -3
- package/src/memory/v2/__tests__/router.test.ts +15 -0
- package/src/memory/v2/__tests__/skill-store.test.ts +387 -8
- package/src/memory/v2/injection.ts +32 -6
- package/src/memory/v2/migration.ts +49 -19
- package/src/memory/v2/page-index.ts +35 -5
- package/src/memory/v2/prompts/router.ts +11 -8
- package/src/memory/v2/prompts/sweep.ts +2 -2
- package/src/memory/v2/qdrant.ts +135 -7
- package/src/memory/v2/router.ts +9 -8
- package/src/memory/v2/skill-store.ts +120 -35
- package/src/messaging/providers/slack/__tests__/adapter-token-routing.test.ts +45 -5
- package/src/messaging/providers/slack/__tests__/download.test.ts +231 -0
- package/src/messaging/providers/slack/adapter.ts +43 -5
- package/src/messaging/providers/slack/client.ts +27 -0
- package/src/messaging/providers/slack/deep-link.ts +65 -0
- package/src/messaging/providers/slack/download.ts +104 -0
- package/src/messaging/providers/slack/message-metadata.test.ts +32 -0
- package/src/messaging/providers/slack/message-metadata.ts +27 -0
- package/src/messaging/providers/slack/render-transcript.test.ts +134 -0
- package/src/messaging/providers/slack/render-transcript.ts +69 -5
- package/src/messaging/providers/slack/types.ts +20 -1
- package/src/notifications/conversation-pairing.ts +2 -1
- package/src/notifications/decision-engine.ts +2 -1
- package/src/notifications/emit-signal.ts +20 -1
- package/src/notifications/home-feed-side-effect.ts +54 -0
- package/src/notifications/signal.ts +3 -1
- package/src/oauth/connection-resolver.ts +8 -4
- package/src/oauth/platform-connection.ts +6 -2
- package/src/oauth/seed-providers.ts +10 -1
- package/src/permissions/checker.ts +2 -0
- package/src/permissions/ipc-risk-types.ts +1 -0
- package/src/permissions/question-prompter.test.ts +416 -0
- package/src/permissions/question-prompter.ts +294 -0
- package/src/platform/client.test.ts +1 -1
- package/src/platform/client.ts +1 -1
- package/src/plugin-api/constants.ts +26 -0
- package/src/plugin-api/index.ts +34 -1
- package/src/plugin-api/types.ts +104 -22
- package/src/plugins/defaults/circuit-breaker.ts +0 -5
- package/src/plugins/defaults/compaction.ts +0 -4
- package/src/plugins/defaults/empty-response.ts +0 -2
- package/src/plugins/defaults/history-repair.ts +0 -2
- package/src/plugins/defaults/injectors.ts +36 -3
- package/src/plugins/defaults/llm-call.ts +0 -2
- package/src/plugins/defaults/memory-retrieval.ts +0 -1
- package/src/plugins/defaults/overflow-reduce.ts +0 -1
- package/src/plugins/defaults/persistence.ts +0 -2
- package/src/plugins/defaults/title-generate.ts +0 -5
- package/src/plugins/defaults/token-estimate.ts +0 -2
- package/src/plugins/defaults/tool-error.ts +0 -7
- package/src/plugins/defaults/tool-execute.ts +0 -2
- package/src/plugins/defaults/tool-result-truncate.ts +0 -4
- package/src/plugins/ensure-plugin-api-shim.ts +96 -0
- package/src/plugins/external-api.ts +104 -0
- package/src/plugins/external-plugin-loader.ts +105 -32
- package/src/plugins/feature-gate.ts +22 -0
- package/src/plugins/pipeline.ts +37 -0
- package/src/plugins/registry.ts +48 -80
- package/src/plugins/types.ts +31 -26
- package/src/plugins/user-loader.ts +21 -2
- package/src/proactive-artifact/aux-message-injector.ts +11 -0
- package/src/proactive-artifact/job.test.ts +37 -5
- package/src/prompts/__tests__/system-prompt.test.ts +12 -0
- package/src/prompts/__tests__/task-progress-hint-section.test.ts +99 -0
- package/src/prompts/normalize-onboarding.ts +27 -0
- package/src/prompts/sections.ts +302 -0
- package/src/prompts/system-prompt.ts +63 -166
- package/src/prompts/templates/BOOTSTRAP.md +17 -1
- package/src/prompts/templates/system-sections.ts +173 -0
- package/src/providers/__tests__/inference.test.ts +22 -7
- package/src/providers/anthropic/client.ts +28 -28
- package/src/providers/connection-resolution.ts +7 -0
- package/src/providers/inference/adapter-factory.ts +41 -4
- package/src/providers/inference/connections.ts +74 -29
- package/src/providers/inference/resolve-auth.ts +12 -4
- package/src/providers/model-catalog.ts +294 -12
- package/src/providers/openai/chat-completions-provider.ts +10 -2
- package/src/providers/openrouter/client.ts +7 -0
- package/src/providers/{managed-proxy → platform-proxy}/constants.ts +4 -1
- package/src/providers/{managed-proxy → platform-proxy}/context.ts +3 -3
- package/src/providers/provider-availability.ts +17 -2
- package/src/providers/provider-catalog-visibility.ts +36 -0
- package/src/providers/registry.ts +22 -14
- package/src/providers/retry.ts +47 -1
- package/src/runtime/__tests__/agent-wake.test.ts +152 -0
- package/src/runtime/agent-wake.ts +42 -14
- package/src/runtime/auth/route-policy.ts +8 -1
- package/src/runtime/btw-sidechain.ts +2 -0
- package/src/runtime/http-types.ts +19 -0
- package/src/runtime/migrations/origin-mode.ts +1 -1
- package/src/runtime/pending-interactions.ts +1 -0
- package/src/runtime/routes/__tests__/bookmark-routes.test.ts +17 -0
- package/src/runtime/routes/__tests__/conversation-management-routes.test.ts +5 -1
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +107 -20
- package/src/runtime/routes/__tests__/question-routes.test.ts +395 -0
- package/src/runtime/routes/__tests__/tts-routes.test.ts +64 -1
- package/src/runtime/routes/acp-routes-list.test.ts +143 -0
- package/src/runtime/routes/acp-routes.ts +5 -3
- package/src/runtime/routes/auth-routes.ts +1 -1
- package/src/runtime/routes/bookmark-routes.ts +5 -3
- package/src/runtime/routes/btw-routes.ts +5 -1
- package/src/runtime/routes/channel-availability-routes.ts +121 -0
- package/src/runtime/routes/conversation-cli-routes.ts +44 -3
- package/src/runtime/routes/conversation-list-routes.ts +3 -20
- package/src/runtime/routes/conversation-management-routes.ts +17 -42
- package/src/runtime/routes/conversation-query-routes.ts +40 -35
- package/src/runtime/routes/conversation-routes.ts +90 -11
- package/src/runtime/routes/documents-routes.ts +25 -86
- package/src/runtime/routes/group-routes.ts +5 -0
- package/src/runtime/routes/inbound-conversation.ts +28 -8
- package/src/runtime/routes/inbound-message-handler.ts +236 -41
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +111 -0
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +32 -1
- package/src/runtime/routes/inbound-stages/edit-intercept.ts +17 -4
- package/src/runtime/routes/index.ts +6 -0
- package/src/runtime/routes/inference-profile-session-handler.ts +17 -44
- package/src/runtime/routes/inference-profile-session-reaper.ts +7 -21
- package/src/runtime/routes/inference-provider-connection-routes.ts +65 -21
- package/src/runtime/routes/integrations/slack/share.ts +4 -52
- package/src/runtime/routes/integrations/slack/token.ts +43 -0
- package/src/runtime/routes/integrations/twilio.ts +6 -13
- package/src/runtime/routes/notification-routes.ts +1 -1
- package/src/runtime/routes/oauth-commands-routes.ts +105 -15
- package/src/runtime/routes/oauth-lifecycle-routes.ts +43 -0
- package/src/runtime/routes/question-routes.ts +259 -0
- package/src/runtime/routes/rename-conversation-routes.ts +2 -33
- package/src/runtime/routes/schedule-routes.ts +4 -7
- package/src/runtime/routes/subagents-routes.ts +57 -18
- package/src/runtime/routes/telemetry-routes.ts +27 -0
- package/src/runtime/routes/tts-routes.ts +27 -2
- package/src/runtime/routes/workspace-routes.test.ts +43 -0
- package/src/runtime/routes/workspace-routes.ts +28 -0
- package/src/runtime/services/conversation-serializer.ts +39 -7
- package/src/runtime/sync/resource-sync-events.ts +93 -1
- package/src/schedule/schedule-store.ts +27 -2
- package/src/schedule/scheduler.ts +9 -1
- package/src/security/__tests__/untrusted-content.test.ts +86 -0
- package/src/security/untrusted-content.ts +93 -8
- package/src/skills/catalog-files.ts +1 -1
- package/src/skills/catalog-install.ts +233 -116
- package/src/skills/clawhub.ts +70 -13
- package/src/skills/managed-store.ts +4 -119
- package/src/skills/skillssh-registry.ts +27 -48
- package/src/subagent/manager.ts +15 -7
- package/src/telemetry/types.ts +113 -1
- package/src/telemetry/usage-telemetry-reporter.test.ts +312 -5
- package/src/telemetry/usage-telemetry-reporter.ts +113 -7
- package/src/tools/apps/executors.ts +58 -7
- package/src/tools/ask-question/ask-question-tool.test.ts +509 -0
- package/src/tools/ask-question/ask-question-tool.ts +304 -0
- package/src/tools/browser/browser-execution.ts +15 -11
- package/src/tools/computer-use/definitions.ts +3 -3
- package/src/tools/credentials/vault.ts +1 -1
- package/src/tools/document/document-tool.ts +124 -1
- package/src/tools/filesystem/edit.ts +1 -1
- package/src/tools/filesystem/list.ts +1 -1
- package/src/tools/filesystem/read.ts +1 -1
- package/src/tools/filesystem/write.ts +5 -2
- package/src/tools/host-filesystem/transfer.ts +1 -1
- package/src/tools/host-terminal/host-shell.ts +1 -1
- package/src/tools/permission-checker.ts +1 -1
- package/src/tools/registry.ts +17 -7
- package/src/tools/schedule/create.ts +2 -2
- package/src/tools/schema-transforms.ts +7 -2
- package/src/tools/side-effects.ts +1 -0
- package/src/tools/skills/delete-managed.ts +4 -4
- package/src/tools/skills/execute.ts +1 -1
- package/src/tools/skills/scaffold-managed.ts +3 -2
- package/src/tools/subagent/notify-parent.ts +1 -1
- package/src/tools/system/request-permission.ts +2 -2
- package/src/tools/terminal/safe-env.ts +60 -1
- package/src/tools/tool-manifest.ts +2 -0
- package/src/tools/types.ts +72 -21
- package/src/tools/ui-surface/definitions.ts +6 -5
- package/src/tts/__tests__/provider-adapters.test.ts +76 -2
- package/src/tts/providers/elevenlabs-provider.ts +75 -1
- package/src/types/onboarding-context.ts +2 -0
- package/src/util/errors.ts +17 -0
- package/src/util/platform.ts +10 -0
- package/src/watcher/__tests__/engine.test.ts +22 -0
- package/src/watcher/engine.ts +6 -2
- package/src/workspace/migrations/057-repair-stale-gemini-model-ids.ts +80 -15
- package/src/workspace/migrations/072-seed-reply-suggestion-callsite.ts +35 -22
- package/src/workspace/migrations/073-repair-recall-callsite-empty-profile.ts +3 -1
- package/src/workspace/migrations/083-system-prompt-prefix-to-file.ts +191 -0
- package/src/workspace/migrations/084-remove-legacy-skills-index.ts +276 -0
- package/src/workspace/migrations/085-memory-v2-bm25-b-reembed-disabled-v2-pages.ts +137 -0
- package/src/workspace/migrations/086-revert-stale-gemini-mis-rewrites.ts +198 -0
- package/src/workspace/migrations/registry.ts +8 -0
- package/src/workspace/migrations/runner.ts +39 -9
- package/src/workspace/migrations/types.ts +4 -0
- package/examples/plugins/echo/bun.lock +0 -25
- package/src/__tests__/context-window-manager.test.ts +0 -2481
- package/src/context/__tests__/compact-prompt.test.ts +0 -63
- package/src/context/prompts/compact.md +0 -26
- package/src/prompts/__tests__/build-cli-reference-section.test.ts +0 -37
- /package/src/__tests__/{secret-routes-managed-proxy.test.ts → secret-routes-platform-proxy.test.ts} +0 -0
|
@@ -30,6 +30,16 @@ const log = getLogger("memory-v2-page-index");
|
|
|
30
30
|
|
|
31
31
|
const SUMMARY_MAX_LENGTH = 200;
|
|
32
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Collapse every run of whitespace (including embedded newlines and tabs) to a
|
|
35
|
+
* single space and trim. The router prompt renders one entry per line, so an
|
|
36
|
+
* embedded newline anywhere in `summary` would split that entry across lines
|
|
37
|
+
* and corrupt the format the router parses.
|
|
38
|
+
*/
|
|
39
|
+
function normalizeSummary(raw: string): string {
|
|
40
|
+
return raw.replace(/\s+/g, " ").trim().slice(0, SUMMARY_MAX_LENGTH);
|
|
41
|
+
}
|
|
42
|
+
|
|
33
43
|
/**
|
|
34
44
|
* One row in the rendered page index. `id` is a 1-based dense integer that is
|
|
35
45
|
* stable within a single index version (i.e. a single build). It changes when
|
|
@@ -97,6 +107,21 @@ export async function getPageIndex(workspaceDir: string): Promise<PageIndex> {
|
|
|
97
107
|
outgoingSlugs: string[];
|
|
98
108
|
}
|
|
99
109
|
|
|
110
|
+
const { listSkillEntries, SKILL_SLUG_PREFIX } =
|
|
111
|
+
await import("./skill-store.js");
|
|
112
|
+
|
|
113
|
+
// Build the skill-slug set first so we can drop colliding concept pages.
|
|
114
|
+
// Collision policy: **skill entries win**. Skill rows are seeded from the
|
|
115
|
+
// curated catalog and the router needs them to be reachable under their
|
|
116
|
+
// canonical slugs; a hand-authored page sitting under `skills/<id>` is
|
|
117
|
+
// either a stale leftover from a prior write or a user mistake. `bySlug`
|
|
118
|
+
// is last-writer-wins, so without explicit dedupe one side would silently
|
|
119
|
+
// shadow the other depending on iteration order.
|
|
120
|
+
const skillEntries = listSkillEntries();
|
|
121
|
+
const skillSlugs = new Set(
|
|
122
|
+
skillEntries.map((entry) => `${SKILL_SLUG_PREFIX}${entry.id}`),
|
|
123
|
+
);
|
|
124
|
+
|
|
100
125
|
const drafts: DraftEntry[] = [];
|
|
101
126
|
for (let i = 0; i < settled.length; i++) {
|
|
102
127
|
const result = settled[i];
|
|
@@ -110,20 +135,25 @@ export async function getPageIndex(workspaceDir: string): Promise<PageIndex> {
|
|
|
110
135
|
}
|
|
111
136
|
const page = result.value;
|
|
112
137
|
if (!page) continue;
|
|
138
|
+
if (skillSlugs.has(slug)) {
|
|
139
|
+
log.warn(
|
|
140
|
+
{ slug },
|
|
141
|
+
"Dropping concept page from index — slug collides with a seeded skill entry; skill wins",
|
|
142
|
+
);
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
113
145
|
const summarySource = page.frontmatter.summary?.trim() || page.body.trim();
|
|
114
146
|
drafts.push({
|
|
115
147
|
slug,
|
|
116
|
-
summary: summarySource
|
|
148
|
+
summary: normalizeSummary(summarySource),
|
|
117
149
|
outgoingSlugs: page.frontmatter.edges,
|
|
118
150
|
});
|
|
119
151
|
}
|
|
120
152
|
|
|
121
|
-
const
|
|
122
|
-
await import("./skill-store.js");
|
|
123
|
-
for (const entry of listSkillEntries()) {
|
|
153
|
+
for (const entry of skillEntries) {
|
|
124
154
|
drafts.push({
|
|
125
155
|
slug: `${SKILL_SLUG_PREFIX}${entry.id}`,
|
|
126
|
-
summary: entry.content
|
|
156
|
+
summary: normalizeSummary(entry.content),
|
|
127
157
|
outgoingSlugs: [],
|
|
128
158
|
});
|
|
129
159
|
}
|
|
@@ -27,7 +27,6 @@ import { homedir } from "node:os";
|
|
|
27
27
|
import { isAbsolute, join } from "node:path";
|
|
28
28
|
|
|
29
29
|
import { getLogger } from "../../../util/logger.js";
|
|
30
|
-
import { getWorkspaceDir } from "../../../util/platform.js";
|
|
31
30
|
|
|
32
31
|
const log = getLogger("memory-v2-router-prompt");
|
|
33
32
|
|
|
@@ -102,7 +101,7 @@ export function renderRouterPrompt(opts: RenderRouterPromptOpts): string {
|
|
|
102
101
|
* referenced by `memory.v2.router.router_prompt_path`, then substitute the
|
|
103
102
|
* standard placeholders. Path-resolution rules mirror the consolidation
|
|
104
103
|
* prompt override: absolute paths used as-is, leading `~/` expanded to home,
|
|
105
|
-
* relative paths resolved under
|
|
104
|
+
* relative paths resolved under `workspaceDir`.
|
|
106
105
|
*
|
|
107
106
|
* Failure handling is intentionally permissive — missing file, read error,
|
|
108
107
|
* oversized file, or empty/whitespace-only body all log a warning and fall
|
|
@@ -111,11 +110,12 @@ export function renderRouterPrompt(opts: RenderRouterPromptOpts): string {
|
|
|
111
110
|
*/
|
|
112
111
|
export function resolveRouterPrompt(
|
|
113
112
|
overridePath: string | null,
|
|
113
|
+
workspaceDir: string,
|
|
114
114
|
opts: RenderRouterPromptOpts,
|
|
115
115
|
): string {
|
|
116
116
|
if (overridePath === null) return renderRouterPrompt(opts);
|
|
117
117
|
|
|
118
|
-
const resolvedPath = resolveOverridePath(overridePath);
|
|
118
|
+
const resolvedPath = resolveOverridePath(overridePath, workspaceDir);
|
|
119
119
|
let contents: string;
|
|
120
120
|
try {
|
|
121
121
|
const stat = lstatSync(resolvedPath);
|
|
@@ -178,15 +178,18 @@ function substitutePlaceholders(
|
|
|
178
178
|
const assistant = opts.assistantName?.trim() || "the assistant";
|
|
179
179
|
const user = opts.userName?.trim() || "the user";
|
|
180
180
|
return template
|
|
181
|
-
.replaceAll(ASSISTANT_NAME_PLACEHOLDER, assistant)
|
|
182
|
-
.replaceAll(USER_NAME_PLACEHOLDER, user)
|
|
183
|
-
.replaceAll(PAGE_INDEX_PLACEHOLDER, opts.pageIndexBlock);
|
|
181
|
+
.replaceAll(ASSISTANT_NAME_PLACEHOLDER, () => assistant)
|
|
182
|
+
.replaceAll(USER_NAME_PLACEHOLDER, () => user)
|
|
183
|
+
.replaceAll(PAGE_INDEX_PLACEHOLDER, () => opts.pageIndexBlock);
|
|
184
184
|
}
|
|
185
185
|
|
|
186
|
-
function resolveOverridePath(
|
|
186
|
+
function resolveOverridePath(
|
|
187
|
+
overridePath: string,
|
|
188
|
+
workspaceDir: string,
|
|
189
|
+
): string {
|
|
187
190
|
if (overridePath.startsWith("~/")) {
|
|
188
191
|
return join(homedir(), overridePath.slice(2));
|
|
189
192
|
}
|
|
190
193
|
if (isAbsolute(overridePath)) return overridePath;
|
|
191
|
-
return join(
|
|
194
|
+
return join(workspaceDir, overridePath);
|
|
192
195
|
}
|
|
@@ -51,6 +51,6 @@ export function renderSweepPrompt(opts: {
|
|
|
51
51
|
const user = opts.userName?.trim() || "the user";
|
|
52
52
|
return SWEEP_PROMPT.replaceAll(
|
|
53
53
|
ASSISTANT_NAME_PLACEHOLDER,
|
|
54
|
-
assistant,
|
|
55
|
-
).replaceAll(USER_NAME_PLACEHOLDER, user);
|
|
54
|
+
() => assistant,
|
|
55
|
+
).replaceAll(USER_NAME_PLACEHOLDER, () => user);
|
|
56
56
|
}
|
package/src/memory/v2/qdrant.ts
CHANGED
|
@@ -200,6 +200,9 @@ async function ensureConceptPageCollectionOnce(): Promise<{
|
|
|
200
200
|
|
|
201
201
|
const missing = missingNamedVectors(info);
|
|
202
202
|
if (missing.length === 0) {
|
|
203
|
+
// Long-lived installs may predate the `kind` payload index; ensure
|
|
204
|
+
// every required index exists before declaring the collection ready.
|
|
205
|
+
await ensurePayloadIndexes();
|
|
203
206
|
_collectionReady = true;
|
|
204
207
|
return { migrated: false };
|
|
205
208
|
}
|
|
@@ -271,17 +274,62 @@ async function ensureConceptPageCollectionOnce(): Promise<{
|
|
|
271
274
|
throw err;
|
|
272
275
|
}
|
|
273
276
|
|
|
274
|
-
|
|
275
|
-
// so upserts and slug-restricted queries don't pay a per-call indexing cost.
|
|
276
|
-
await client.createPayloadIndex(MEMORY_V2_COLLECTION, {
|
|
277
|
-
field_name: "slug",
|
|
278
|
-
field_schema: "keyword",
|
|
279
|
-
});
|
|
277
|
+
await ensurePayloadIndexes();
|
|
280
278
|
|
|
281
279
|
_collectionReady = true;
|
|
282
280
|
return { migrated };
|
|
283
281
|
}
|
|
284
282
|
|
|
283
|
+
/**
|
|
284
|
+
* Idempotently create the payload indexes the collection's query and
|
|
285
|
+
* filter paths rely on:
|
|
286
|
+
*
|
|
287
|
+
* - `slug` (keyword): every slug-restricted query and prefix scan filters on it.
|
|
288
|
+
* - `kind` (keyword): the skill-backfill scroll filters with `is_empty` on
|
|
289
|
+
* `kind`. Strict-mode Qdrant deployments reject filters on unindexed
|
|
290
|
+
* payload fields, so without this the backfill consistently fails and
|
|
291
|
+
* legacy skill points remain untagged.
|
|
292
|
+
*
|
|
293
|
+
* Same-schema `createPayloadIndex` calls are idempotent server-side in
|
|
294
|
+
* Qdrant (200 OK), so the only "already exists" failures we expect are
|
|
295
|
+
* narrow races where a concurrent caller created the same index a moment
|
|
296
|
+
* earlier. Those are benign and swallowed. Every other failure — strict-mode
|
|
297
|
+
* rejection, index-limit, transient network blip — must propagate so the
|
|
298
|
+
* caller does not latch readiness on a collection whose `slug`/`kind`
|
|
299
|
+
* filters will keep rejecting queries until the next daemon restart.
|
|
300
|
+
*/
|
|
301
|
+
async function ensurePayloadIndexes(): Promise<void> {
|
|
302
|
+
const client = getClient();
|
|
303
|
+
const indexes = [
|
|
304
|
+
{ field_name: "slug", field_schema: "keyword" as const },
|
|
305
|
+
{ field_name: "kind", field_schema: "keyword" as const },
|
|
306
|
+
];
|
|
307
|
+
// Parallel so one "already exists" race on a single index doesn't stall
|
|
308
|
+
// the other create round-trip. v1's `qdrant-client.ts` uses the same
|
|
309
|
+
// Promise.all shape.
|
|
310
|
+
await Promise.all(
|
|
311
|
+
indexes.map(async (index) => {
|
|
312
|
+
try {
|
|
313
|
+
await client.createPayloadIndex(MEMORY_V2_COLLECTION, index);
|
|
314
|
+
} catch (err) {
|
|
315
|
+
if (isPayloadIndexAlreadyExists(err)) return;
|
|
316
|
+
throw err;
|
|
317
|
+
}
|
|
318
|
+
}),
|
|
319
|
+
);
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* True when a `createPayloadIndex` error indicates the index already
|
|
324
|
+
* exists with matching parameters — the only failure shape it is safe to
|
|
325
|
+
* swallow. Qdrant returns 4xx with messages like
|
|
326
|
+
* `"Wrong input: Payload field 'kind' already exists ..."`.
|
|
327
|
+
*/
|
|
328
|
+
function isPayloadIndexAlreadyExists(err: unknown): boolean {
|
|
329
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
330
|
+
return /already exists/i.test(msg);
|
|
331
|
+
}
|
|
332
|
+
|
|
285
333
|
/**
|
|
286
334
|
* Return the names of required named vectors absent from the collection's
|
|
287
335
|
* current schema. An empty array means the collection is fully migrated.
|
|
@@ -422,7 +470,10 @@ export async function deleteConceptPageEmbedding(slug: string): Promise<void> {
|
|
|
422
470
|
* `payload.kind` matches are eligible for deletion. This is critical because
|
|
423
471
|
* `validateSlug` permits user-authored concept pages slugged like
|
|
424
472
|
* `skills/foo`; without a kind filter they would collide with the skill
|
|
425
|
-
* namespace and be repeatedly pruned every seed run.
|
|
473
|
+
* namespace and be repeatedly pruned every seed run. The companion
|
|
474
|
+
* {@link backfillKindOnPointsWithPrefix} preserves this invariant for legacy
|
|
475
|
+
* untagged rows by tagging only suffixes the caller knows are skills —
|
|
476
|
+
* user-authored `skills/<slug>` rows stay kindless and outside this prune.
|
|
426
477
|
*
|
|
427
478
|
* Idempotent: when the live `<prefix>*` slugs already match `activeSuffixes`,
|
|
428
479
|
* the function performs a single scroll and no deletes.
|
|
@@ -491,6 +542,83 @@ export async function pruneSlugsWithPrefixExcept(
|
|
|
491
542
|
}
|
|
492
543
|
}
|
|
493
544
|
|
|
545
|
+
/**
|
|
546
|
+
* Set `payload.kind` on every point whose slug starts with `prefix`, whose
|
|
547
|
+
* suffix is in `allowedSuffixes`, and is currently missing the `kind`
|
|
548
|
+
* discriminator. Used to tag legacy rows that predate the kind field so the
|
|
549
|
+
* kind-scoped {@link pruneSlugsWithPrefixExcept} no longer leaves them as
|
|
550
|
+
* orphans.
|
|
551
|
+
*
|
|
552
|
+
* `allowedSuffixes` is required because `validateSlug` permits user-authored
|
|
553
|
+
* concept pages slugged like `skills/my-notes` — those rows also lack `kind`
|
|
554
|
+
* and would otherwise be tagged here and then deleted by the kind-scoped
|
|
555
|
+
* prune. Callers must pass the closed set of legitimate suffixes (e.g. the
|
|
556
|
+
* union of installed + remote-catalog skill IDs) so user pages stay untagged.
|
|
557
|
+
*
|
|
558
|
+
* The "missing kind" predicate is pushed to Qdrant via `is_empty`, so once
|
|
559
|
+
* every legacy row has been tagged the scroll returns the bounded set of
|
|
560
|
+
* other kindless concept pages without ever touching the already-tagged
|
|
561
|
+
* rows. Idempotent across retries: a row tagged by an earlier partial run
|
|
562
|
+
* no longer matches the filter and is silently skipped.
|
|
563
|
+
*/
|
|
564
|
+
export async function backfillKindOnPointsWithPrefix(
|
|
565
|
+
prefix: string,
|
|
566
|
+
kind: string,
|
|
567
|
+
allowedSuffixes: ReadonlySet<string>,
|
|
568
|
+
): Promise<number> {
|
|
569
|
+
if (allowedSuffixes.size === 0) return 0;
|
|
570
|
+
await ensureConceptPageCollection();
|
|
571
|
+
|
|
572
|
+
const client = getClient();
|
|
573
|
+
|
|
574
|
+
const doBackfill = async (): Promise<number> => {
|
|
575
|
+
const pointIds: Array<string | number> = [];
|
|
576
|
+
let offset: string | number | undefined = undefined;
|
|
577
|
+
const maxIterations = 10_000;
|
|
578
|
+
const batchSize = 256;
|
|
579
|
+
for (let i = 0; i < maxIterations; i++) {
|
|
580
|
+
const result = await client.scroll(MEMORY_V2_COLLECTION, {
|
|
581
|
+
limit: batchSize,
|
|
582
|
+
with_payload: true,
|
|
583
|
+
with_vector: false,
|
|
584
|
+
filter: { must: [{ is_empty: { key: "kind" } }] },
|
|
585
|
+
...(offset !== undefined ? { offset } : {}),
|
|
586
|
+
});
|
|
587
|
+
for (const point of result.points) {
|
|
588
|
+
const slug = (point.payload as { slug?: unknown } | null)?.slug;
|
|
589
|
+
if (typeof slug !== "string") continue;
|
|
590
|
+
if (!slug.startsWith(prefix)) continue;
|
|
591
|
+
const suffix = slug.slice(prefix.length);
|
|
592
|
+
if (!allowedSuffixes.has(suffix)) continue;
|
|
593
|
+
pointIds.push(point.id);
|
|
594
|
+
}
|
|
595
|
+
const next = result.next_page_offset;
|
|
596
|
+
if (next == null) break;
|
|
597
|
+
offset = typeof next === "string" ? next : (next as number);
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
if (pointIds.length === 0) return 0;
|
|
601
|
+
|
|
602
|
+
await client.setPayload(MEMORY_V2_COLLECTION, {
|
|
603
|
+
payload: { kind },
|
|
604
|
+
points: pointIds,
|
|
605
|
+
wait: true,
|
|
606
|
+
});
|
|
607
|
+
return pointIds.length;
|
|
608
|
+
};
|
|
609
|
+
|
|
610
|
+
try {
|
|
611
|
+
return await doBackfill();
|
|
612
|
+
} catch (err) {
|
|
613
|
+
if (isCollectionMissing(err)) {
|
|
614
|
+
_collectionReady = false;
|
|
615
|
+
await ensureConceptPageCollection();
|
|
616
|
+
return await doBackfill();
|
|
617
|
+
}
|
|
618
|
+
throw err;
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
|
|
494
622
|
/**
|
|
495
623
|
* Approximate count of points in the v2 concept-page collection. Used by the
|
|
496
624
|
* daemon-startup rebuild hook to detect "collection exists but empty" — the
|
package/src/memory/v2/router.ts
CHANGED
|
@@ -181,6 +181,7 @@ export async function runRouter(
|
|
|
181
181
|
|
|
182
182
|
const systemPrompt = resolveRouterPrompt(
|
|
183
183
|
config.memory?.v2?.router?.router_prompt_path ?? null,
|
|
184
|
+
workspaceDir,
|
|
184
185
|
{
|
|
185
186
|
assistantName: getAssistantName(),
|
|
186
187
|
userName: resolveUserName(workspaceDir),
|
|
@@ -274,22 +275,22 @@ export async function runRouter(
|
|
|
274
275
|
);
|
|
275
276
|
}
|
|
276
277
|
|
|
277
|
-
|
|
278
|
-
|
|
278
|
+
// De-duplicate BEFORE applying the cap — otherwise a duplicate-heavy
|
|
279
|
+
// model output like `[1, 1, 2]` with `max=2` slices to `[1, 1]` and
|
|
280
|
+
// dedupes to `[1]`, under-filling the cap.
|
|
281
|
+
const dedupedIds = Array.from(new Set(inRangeIds));
|
|
282
|
+
|
|
283
|
+
const truncated = dedupedIds.length > maxPageIds;
|
|
284
|
+
const finalIds = truncated ? dedupedIds.slice(0, maxPageIds) : dedupedIds;
|
|
279
285
|
if (truncated) {
|
|
280
286
|
log.warn(
|
|
281
|
-
{ returned:
|
|
287
|
+
{ returned: dedupedIds.length, max: maxPageIds },
|
|
282
288
|
"Router returned more page IDs than max_page_ids; truncating",
|
|
283
289
|
);
|
|
284
290
|
}
|
|
285
291
|
|
|
286
|
-
// De-duplicate while preserving order — the index lookup alone wouldn't
|
|
287
|
-
// catch repeats from the model.
|
|
288
|
-
const seen = new Set<number>();
|
|
289
292
|
const selectedSlugs: string[] = [];
|
|
290
293
|
for (const id of finalIds) {
|
|
291
|
-
if (seen.has(id)) continue;
|
|
292
|
-
seen.add(id);
|
|
293
294
|
const entry = pageIndex.byId.get(id);
|
|
294
295
|
if (!entry) continue;
|
|
295
296
|
selectedSlugs.push(entry.slug);
|
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
} from "../embedding-backend.js";
|
|
37
37
|
import { invalidatePageIndex } from "./page-index.js";
|
|
38
38
|
import {
|
|
39
|
+
backfillKindOnPointsWithPrefix,
|
|
39
40
|
pruneSlugsWithPrefixExcept,
|
|
40
41
|
upsertConceptPageEmbedding,
|
|
41
42
|
} from "./qdrant.js";
|
|
@@ -78,6 +79,11 @@ export function skillSlugFor(id: string): string {
|
|
|
78
79
|
* successful re-seed so callers always see a consistent snapshot.
|
|
79
80
|
*/
|
|
80
81
|
let entries: Map<string, SkillEntry> | null = null;
|
|
82
|
+
let requestedSeedGeneration = 0;
|
|
83
|
+
let processedSeedGeneration = 0;
|
|
84
|
+
let activeSeedDrain: Promise<void> | null = null;
|
|
85
|
+
let lastSeedError: unknown = null;
|
|
86
|
+
const seedWaiters: Array<{ generation: number; resolve: () => void }> = [];
|
|
81
87
|
|
|
82
88
|
/**
|
|
83
89
|
* Seed (or re-seed) skill embeddings into the unified concept-page collection.
|
|
@@ -85,32 +91,19 @@ let entries: Map<string, SkillEntry> | null = null;
|
|
|
85
91
|
* background callers like daemon startup; pass `{ throwOnError: true }` from
|
|
86
92
|
* synchronous CLI-driven paths that need to surface failures to the operator.
|
|
87
93
|
*
|
|
88
|
-
* Single-flight + coalesced: at most one seed runs at a time
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
* the
|
|
94
|
+
* Single-flight + coalesced: at most one seed runs at a time. Requests made
|
|
95
|
+
* while a seed is in flight advance the requested generation; stale in-flight
|
|
96
|
+
* snapshots are skipped before they write embeddings or replace the cache,
|
|
97
|
+
* then the drain loop immediately processes the latest generation. Strict
|
|
98
|
+
* callers observe the awaited generation's latest outcome via `lastSeedError`.
|
|
93
99
|
*/
|
|
94
|
-
let seedTail: Promise<void> = Promise.resolve();
|
|
95
|
-
let seedPending: Promise<void> | null = null;
|
|
96
|
-
let lastSeedError: unknown = null;
|
|
97
100
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
)
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
await runSeedOnce();
|
|
105
|
-
});
|
|
106
|
-
seedTail = next.catch(() => {});
|
|
107
|
-
seedPending = next;
|
|
108
|
-
}
|
|
109
|
-
await seedPending;
|
|
110
|
-
if (opts.throwOnError && lastSeedError) {
|
|
111
|
-
throw lastSeedError;
|
|
112
|
-
}
|
|
113
|
-
}
|
|
101
|
+
/**
|
|
102
|
+
* In-process latch for the legacy `kind` backfill (see
|
|
103
|
+
* {@link backfillKindOnPointsWithPrefix}). New upserts always write `kind`,
|
|
104
|
+
* so once the latch is set there is no follow-up work to do this process.
|
|
105
|
+
*/
|
|
106
|
+
let legacyKindBackfillDone = false;
|
|
114
107
|
|
|
115
108
|
/**
|
|
116
109
|
* Steps (per run):
|
|
@@ -132,7 +125,49 @@ export async function seedV2SkillEntries(
|
|
|
132
125
|
* stale points from prior catalog state (e.g. uninstalled skills).
|
|
133
126
|
* 7. Replace the module-level `entries` cache with the freshly built map.
|
|
134
127
|
*/
|
|
135
|
-
async function
|
|
128
|
+
export async function seedV2SkillEntries(
|
|
129
|
+
opts: { throwOnError?: boolean } = {},
|
|
130
|
+
): Promise<void> {
|
|
131
|
+
const generation = ++requestedSeedGeneration;
|
|
132
|
+
const waiter = new Promise<void>((resolve) => {
|
|
133
|
+
seedWaiters.push({ generation, resolve });
|
|
134
|
+
});
|
|
135
|
+
startSeedDrainIfNeeded();
|
|
136
|
+
await waiter;
|
|
137
|
+
if (opts.throwOnError && lastSeedError) {
|
|
138
|
+
throw lastSeedError;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function startSeedDrainIfNeeded(): void {
|
|
143
|
+
if (activeSeedDrain) return;
|
|
144
|
+
if (processedSeedGeneration >= requestedSeedGeneration) return;
|
|
145
|
+
|
|
146
|
+
activeSeedDrain = drainSeedQueue().finally(() => {
|
|
147
|
+
activeSeedDrain = null;
|
|
148
|
+
startSeedDrainIfNeeded();
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async function drainSeedQueue(): Promise<void> {
|
|
153
|
+
while (processedSeedGeneration < requestedSeedGeneration) {
|
|
154
|
+
const generationToProcess = requestedSeedGeneration;
|
|
155
|
+
await runSeedV2SkillEntries(generationToProcess);
|
|
156
|
+
processedSeedGeneration = generationToProcess;
|
|
157
|
+
resolveSeedWaiters();
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function resolveSeedWaiters(): void {
|
|
162
|
+
for (let i = seedWaiters.length - 1; i >= 0; i -= 1) {
|
|
163
|
+
const waiter = seedWaiters[i]!;
|
|
164
|
+
if (waiter.generation > processedSeedGeneration) continue;
|
|
165
|
+
seedWaiters.splice(i, 1);
|
|
166
|
+
waiter.resolve();
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async function runSeedV2SkillEntries(generation: number): Promise<void> {
|
|
136
171
|
try {
|
|
137
172
|
const config = getConfig();
|
|
138
173
|
const catalog = loadSkillCatalog();
|
|
@@ -160,11 +195,19 @@ async function runSeedOnce(): Promise<void> {
|
|
|
160
195
|
// Seed uninstalled catalog skills so their activation hints are
|
|
161
196
|
// discoverable by intent. Track whether the catalog was available so we
|
|
162
197
|
// can guard pruning below.
|
|
198
|
+
//
|
|
199
|
+
// Build the legacy-backfill allowlist in parallel: every locally
|
|
200
|
+
// installed skill id (regardless of enabled state) plus every remote
|
|
201
|
+
// catalog id. Restricting the backfill to this set keeps user-authored
|
|
202
|
+
// concept pages that happen to live under `skills/<slug>` from being
|
|
203
|
+
// mis-tagged and then pruned. See `backfillKindOnPointsWithPrefix`.
|
|
204
|
+
const knownSkillIds = new Set<string>(installedIds);
|
|
163
205
|
let catalogAvailable = false;
|
|
164
206
|
try {
|
|
165
207
|
const fullCatalog = await getCatalog();
|
|
166
208
|
catalogAvailable = fullCatalog.length > 0;
|
|
167
209
|
for (const entry of fullCatalog) {
|
|
210
|
+
knownSkillIds.add(entry.id);
|
|
168
211
|
if (installedIds.has(entry.id)) continue;
|
|
169
212
|
const flagKey = entry.metadata?.vellum?.["feature-flag"];
|
|
170
213
|
if (flagKey && !isAssistantFeatureFlagEnabled(flagKey, config))
|
|
@@ -184,12 +227,16 @@ async function runSeedOnce(): Promise<void> {
|
|
|
184
227
|
// unavailable embedding backend in the all-disabled case, so pruning and
|
|
185
228
|
// cache replacement still run and clear stale state.
|
|
186
229
|
const nextEntries = new Map<string, SkillEntry>();
|
|
230
|
+
let denseVectors: number[][] = [];
|
|
231
|
+
let encodeSparse: (
|
|
232
|
+
input: string,
|
|
233
|
+
) => ReturnType<typeof generateSparseEmbedding> = generateSparseEmbedding;
|
|
187
234
|
if (seeds.length > 0) {
|
|
188
235
|
const embedded = await embedWithBackend(
|
|
189
236
|
config,
|
|
190
237
|
seeds.map((s) => s.content),
|
|
191
238
|
);
|
|
192
|
-
|
|
239
|
+
denseVectors = await Promise.all(
|
|
193
240
|
embedded.vectors.map((v) =>
|
|
194
241
|
applyCorrectionIfCalibrated(v, embedded.provider, embedded.model),
|
|
195
242
|
),
|
|
@@ -203,14 +250,25 @@ async function runSeedOnce(): Promise<void> {
|
|
|
203
250
|
// entirely. Fall back to the legacy TF encoder only during the cold-start
|
|
204
251
|
// window before corpus stats finish building.
|
|
205
252
|
const corpusStats = getConceptPageCorpusStats();
|
|
206
|
-
|
|
253
|
+
encodeSparse = (input: string) =>
|
|
207
254
|
corpusStats
|
|
208
255
|
? generateBm25DocEmbedding(input, corpusStats, {
|
|
209
256
|
k1: config.memory.v2.bm25_k1,
|
|
210
257
|
b: config.memory.v2.bm25_b,
|
|
211
258
|
})
|
|
212
259
|
: generateSparseEmbedding(input);
|
|
260
|
+
}
|
|
213
261
|
|
|
262
|
+
if (generation !== requestedSeedGeneration) {
|
|
263
|
+
log.info(
|
|
264
|
+
{ generation, latestGeneration: requestedSeedGeneration },
|
|
265
|
+
"Skipping stale v2 skill seed result",
|
|
266
|
+
);
|
|
267
|
+
lastSeedError = null;
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
if (seeds.length > 0) {
|
|
214
272
|
const now = Date.now();
|
|
215
273
|
await Promise.all(
|
|
216
274
|
seeds.map((seed, i) =>
|
|
@@ -233,6 +291,25 @@ async function runSeedOnce(): Promise<void> {
|
|
|
233
291
|
// uninstalled catalog skills should exist, so skip pruning entirely to
|
|
234
292
|
// avoid aggressively removing previously-seeded catalog skill embeddings.
|
|
235
293
|
if (catalogAvailable) {
|
|
294
|
+
// Tag legacy skill points missing `payload.kind` before pruning so the
|
|
295
|
+
// kind-scoped prune can see them. Once-per-process; the backfill is
|
|
296
|
+
// idempotent (server-side `is_empty` filter), so a partial failure
|
|
297
|
+
// converges on retry.
|
|
298
|
+
if (!legacyKindBackfillDone) {
|
|
299
|
+
try {
|
|
300
|
+
await backfillKindOnPointsWithPrefix(
|
|
301
|
+
SKILL_SLUG_PREFIX,
|
|
302
|
+
SKILL_PAYLOAD_KIND,
|
|
303
|
+
knownSkillIds,
|
|
304
|
+
);
|
|
305
|
+
legacyKindBackfillDone = true;
|
|
306
|
+
} catch (err) {
|
|
307
|
+
log.warn(
|
|
308
|
+
{ err },
|
|
309
|
+
"Failed to backfill kind on legacy skill points — pruning may leave orphans this run",
|
|
310
|
+
);
|
|
311
|
+
}
|
|
312
|
+
}
|
|
236
313
|
await pruneSlugsWithPrefixExcept(
|
|
237
314
|
SKILL_SLUG_PREFIX,
|
|
238
315
|
seeds.map((s) => s.id),
|
|
@@ -265,12 +342,16 @@ async function runSeedOnce(): Promise<void> {
|
|
|
265
342
|
* Accepts either a bare skill id (`example-skill`) or its unified-collection
|
|
266
343
|
* slug (`skills/example-skill`) so render-side callers can pass through what
|
|
267
344
|
* they have without a manual prefix strip.
|
|
345
|
+
*
|
|
346
|
+
* Returns a frozen copy so callers cannot mutate the underlying cache entry
|
|
347
|
+
* — matches the defensive-copy contract of `listSkillEntries`.
|
|
268
348
|
*/
|
|
269
349
|
export function getSkillCapability(idOrSlug: string): SkillEntry | null {
|
|
270
350
|
const id = idOrSlug.startsWith(SKILL_SLUG_PREFIX)
|
|
271
351
|
? idOrSlug.slice(SKILL_SLUG_PREFIX.length)
|
|
272
352
|
: idOrSlug;
|
|
273
|
-
|
|
353
|
+
const entry = entries?.get(id);
|
|
354
|
+
return entry ? Object.freeze({ ...entry }) : null;
|
|
274
355
|
}
|
|
275
356
|
|
|
276
357
|
/** True iff the slug refers to a skill entry in the unified collection. */
|
|
@@ -280,8 +361,9 @@ export function isSkillSlug(slug: string): boolean {
|
|
|
280
361
|
|
|
281
362
|
/**
|
|
282
363
|
* Snapshot of the in-process skill cache, sorted by skill id (ASCII order)
|
|
283
|
-
* for determinism. Returns a freshly allocated array
|
|
284
|
-
* cannot mutate the underlying cache
|
|
364
|
+
* for determinism. Returns a freshly allocated array of frozen entry copies
|
|
365
|
+
* on each call, so callers cannot mutate the underlying cache — neither by
|
|
366
|
+
* reassigning the array nor by writing through entry fields.
|
|
285
367
|
*
|
|
286
368
|
* The cache is replaced atomically by `seedV2SkillEntries`, so a snapshot
|
|
287
369
|
* may be stale once a subsequent seed run completes. Callers that need
|
|
@@ -289,15 +371,18 @@ export function isSkillSlug(slug: string): boolean {
|
|
|
289
371
|
*/
|
|
290
372
|
export function listSkillEntries(): SkillEntry[] {
|
|
291
373
|
if (!entries) return [];
|
|
292
|
-
return [...entries.values()]
|
|
293
|
-
a.id < b.id ? -1 : a.id > b.id ? 1 : 0
|
|
294
|
-
|
|
374
|
+
return [...entries.values()]
|
|
375
|
+
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
|
|
376
|
+
.map((entry) => Object.freeze({ ...entry }));
|
|
295
377
|
}
|
|
296
378
|
|
|
297
379
|
/** @internal Test-only: clear the module-level cache. */
|
|
298
380
|
export function _resetSkillStoreForTests(): void {
|
|
299
381
|
entries = null;
|
|
300
|
-
|
|
301
|
-
|
|
382
|
+
requestedSeedGeneration = 0;
|
|
383
|
+
processedSeedGeneration = 0;
|
|
384
|
+
activeSeedDrain = null;
|
|
385
|
+
seedWaiters.splice(0, seedWaiters.length);
|
|
302
386
|
lastSeedError = null;
|
|
387
|
+
legacyKindBackfillDone = false;
|
|
303
388
|
}
|