@vellumai/assistant 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +2 -0
- package/README.md +88 -2
- package/eslint.config.mjs +31 -0
- package/package.json +1 -1
- package/scripts/ipc/check-swift-decoder-drift.ts +4 -1
- package/scripts/ipc/generate-swift.ts +31 -2
- package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +438 -1
- package/src/__tests__/approval-conversation-turn.test.ts +214 -0
- package/src/__tests__/approval-hardcoded-copy-guard.test.ts +41 -0
- package/src/__tests__/approval-message-composer.test.ts +253 -0
- package/src/__tests__/browser-manager.test.ts +1 -0
- package/src/__tests__/call-conversation-messages.test.ts +130 -0
- package/src/__tests__/call-domain.test.ts +12 -2
- package/src/__tests__/call-orchestrator.test.ts +799 -249
- package/src/__tests__/call-pointer-messages.test.ts +148 -0
- package/src/__tests__/call-recovery.test.ts +3 -0
- package/src/__tests__/call-routes-http.test.ts +32 -2
- package/src/__tests__/call-store.test.ts +3 -0
- package/src/__tests__/channel-approval-routes.test.ts +1277 -98
- package/src/__tests__/channel-approval.test.ts +37 -0
- package/src/__tests__/channel-approvals.test.ts +36 -50
- package/src/__tests__/channel-guardian.test.ts +630 -22
- package/src/__tests__/channel-readiness-service.test.ts +324 -0
- package/src/__tests__/checker.test.ts +14 -7
- package/src/__tests__/clarification-resolver.test.ts +44 -24
- package/src/__tests__/commit-message-enrichment-service.test.ts +9 -4
- package/src/__tests__/computer-use-session-working-dir.test.ts +8 -0
- package/src/__tests__/config-schema.test.ts +14 -8
- package/src/__tests__/context-window-manager.test.ts +30 -2
- package/src/__tests__/contradiction-checker.test.ts +20 -5
- package/src/__tests__/credential-security-invariants.test.ts +7 -2
- package/src/__tests__/daemon-lifecycle.test.ts +13 -12
- package/src/__tests__/db-migration-rollback.test.ts +752 -0
- package/src/__tests__/dictation-mode-detection.test.ts +63 -0
- package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +2 -0
- package/src/__tests__/entity-search.test.ts +615 -0
- package/src/__tests__/fuzzy-match-property.test.ts +5 -5
- package/src/__tests__/guardian-action-store.test.ts +123 -0
- package/src/__tests__/guardian-action-sweep.test.ts +277 -0
- package/src/__tests__/guardian-dispatch.test.ts +389 -0
- package/src/__tests__/guardian-question-copy.test.ts +47 -0
- package/src/__tests__/handlers-telegram-config.test.ts +4 -2
- package/src/__tests__/handlers-twilio-config.test.ts +533 -0
- package/src/__tests__/intent-routing.test.ts +2 -0
- package/src/__tests__/ipc-snapshot.test.ts +291 -1
- package/src/__tests__/memory-upsert-concurrency.test.ts +828 -0
- package/src/__tests__/messaging-send-tool.test.ts +65 -0
- package/src/__tests__/model-intents.test.ts +96 -0
- package/src/__tests__/no-direct-anthropic-sdk-imports.test.ts +42 -0
- package/src/__tests__/oauth2-gateway-transport.test.ts +130 -0
- package/src/__tests__/onboarding-starter-tasks.test.ts +2 -0
- package/src/__tests__/provider-commit-message-generator.test.ts +89 -13
- package/src/__tests__/provider-error-scenarios.test.ts +621 -0
- package/src/__tests__/provider-fail-open-selection.test.ts +119 -0
- package/src/__tests__/qdrant-manager.test.ts +27 -20
- package/src/__tests__/relay-server.test.ts +779 -40
- package/src/__tests__/run-orchestrator-assistant-events.test.ts +6 -0
- package/src/__tests__/run-orchestrator.test.ts +42 -4
- package/src/__tests__/runtime-runs-http.test.ts +17 -1
- package/src/__tests__/runtime-runs.test.ts +16 -0
- package/src/__tests__/schedule-store.test.ts +18 -4
- package/src/__tests__/scheduler-recurrence.test.ts +13 -4
- package/src/__tests__/session-abort-tool-results.test.ts +6 -0
- package/src/__tests__/session-agent-loop.test.ts +857 -0
- package/src/__tests__/session-conflict-gate.test.ts +6 -0
- package/src/__tests__/session-pre-run-repair.test.ts +6 -0
- package/src/__tests__/session-profile-injection.test.ts +6 -0
- package/src/__tests__/session-provider-retry-repair.test.ts +6 -0
- package/src/__tests__/session-queue.test.ts +6 -0
- package/src/__tests__/session-runtime-assembly.test.ts +321 -13
- package/src/__tests__/session-slash-known.test.ts +6 -0
- package/src/__tests__/session-slash-queue.test.ts +6 -0
- package/src/__tests__/session-slash-unknown.test.ts +6 -0
- package/src/__tests__/session-surfaces-task-progress.test.ts +2 -0
- package/src/__tests__/session-tool-setup-app-refresh.test.ts +1 -0
- package/src/__tests__/session-tool-setup-memory-scope.test.ts +1 -0
- package/src/__tests__/session-tool-setup-side-effect-flag.test.ts +1 -0
- package/src/__tests__/session-workspace-injection.test.ts +6 -0
- package/src/__tests__/session-workspace-tool-tracking.test.ts +6 -0
- package/src/__tests__/skills.test.ts +2 -0
- package/src/__tests__/sms-messaging-provider.test.ts +126 -0
- package/src/__tests__/starter-task-flow.test.ts +2 -0
- package/src/__tests__/swarm-dag-pathological.test.ts +535 -0
- package/src/__tests__/system-prompt.test.ts +2 -0
- package/src/__tests__/task-management-tools.test.ts +2 -2
- package/src/__tests__/task-runner.test.ts +14 -4
- package/src/__tests__/terminal-tools.test.ts +25 -19
- package/src/__tests__/tool-execution-abort-cleanup.test.ts +545 -0
- package/src/__tests__/tool-executor-shell-integration.test.ts +11 -11
- package/src/__tests__/tool-executor.test.ts +23 -24
- package/src/__tests__/trust-store.test.ts +3 -3
- package/src/__tests__/twilio-rest.test.ts +29 -0
- package/src/__tests__/twilio-routes-elevenlabs.test.ts +3 -0
- package/src/__tests__/twilio-routes-twiml.test.ts +11 -0
- package/src/__tests__/twilio-routes.test.ts +167 -11
- package/src/__tests__/twitter-cli-error-shaping.test.ts +2 -2
- package/src/__tests__/user-reference.test.ts +2 -0
- package/src/__tests__/voice-quality.test.ts +222 -0
- package/src/__tests__/web-search.test.ts +46 -30
- package/src/__tests__/work-item-output.test.ts +110 -0
- package/src/agent/loop.ts +1 -1
- package/src/agent-heartbeat/agent-heartbeat-service.ts +2 -10
- package/src/amazon/client.ts +1418 -0
- package/src/amazon/request-extractor.ts +135 -0
- package/src/amazon/session.ts +109 -0
- package/src/autonomy/autonomy-store.ts +5 -5
- package/src/browser-extension-relay/client.ts +124 -0
- package/src/browser-extension-relay/protocol.ts +63 -0
- package/src/browser-extension-relay/server.ts +177 -0
- package/src/bundler/app-bundler.ts +3 -3
- package/src/bundler/bundle-signer.ts +1 -1
- package/src/bundler/signature-verifier.ts +1 -1
- package/src/calls/call-conversation-messages.ts +33 -0
- package/src/calls/call-domain.ts +114 -10
- package/src/calls/call-orchestrator.ts +268 -59
- package/src/calls/call-pointer-messages.ts +53 -0
- package/src/calls/call-recovery.ts +3 -8
- package/src/calls/call-store.ts +69 -87
- package/src/calls/elevenlabs-config.ts +3 -2
- package/src/calls/guardian-action-sweep.ts +105 -0
- package/src/calls/guardian-dispatch.ts +203 -0
- package/src/calls/guardian-question-copy.ts +133 -0
- package/src/calls/relay-server.ts +466 -8
- package/src/calls/speaker-identification.ts +1 -1
- package/src/calls/twilio-config.ts +22 -14
- package/src/calls/twilio-provider.ts +6 -4
- package/src/calls/twilio-rest.ts +308 -7
- package/src/calls/twilio-routes.ts +65 -12
- package/src/calls/types.ts +3 -1
- package/src/channels/types.ts +25 -0
- package/src/cli/amazon.ts +815 -0
- package/src/cli/config-commands.ts +2 -2
- package/src/cli/core-commands.ts +4 -3
- package/src/cli/influencer.ts +244 -0
- package/src/cli/map.ts +89 -6
- package/src/cli.ts +1 -1
- package/src/config/agent-schema.ts +171 -0
- package/src/config/bundled-skills/amazon/SKILL.md +127 -0
- package/src/config/bundled-skills/amazon/icon.svg +13 -0
- package/src/config/bundled-skills/api-mapping/SKILL.md +78 -0
- package/src/config/bundled-skills/browser/SKILL.md +1 -0
- package/src/config/bundled-skills/browser/TOOLS.json +17 -0
- package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +25 -0
- package/src/config/bundled-skills/doordash/SKILL.md +51 -51
- package/src/config/bundled-skills/email-setup/SKILL.md +14 -5
- package/src/config/bundled-skills/google-oauth-setup/SKILL.md +183 -0
- package/src/config/bundled-skills/influencer/SKILL.md +144 -0
- package/src/config/bundled-skills/knowledge-graph/SKILL.md +15 -0
- package/src/config/bundled-skills/knowledge-graph/TOOLS.json +56 -0
- package/src/config/bundled-skills/knowledge-graph/tools/graph-query.ts +185 -0
- package/src/config/bundled-skills/macos-automation/icon.svg +12 -0
- package/src/config/bundled-skills/media-processing/SKILL.md +176 -0
- package/src/config/bundled-skills/media-processing/TOOLS.json +230 -0
- package/src/config/bundled-skills/media-processing/__tests__/concurrency-pool.test.ts +77 -0
- package/src/config/bundled-skills/media-processing/__tests__/cost-tracker.test.ts +69 -0
- package/src/config/bundled-skills/media-processing/__tests__/preprocess.test.ts +303 -0
- package/src/config/bundled-skills/media-processing/services/concurrency-pool.ts +55 -0
- package/src/config/bundled-skills/media-processing/services/cost-tracker.ts +86 -0
- package/src/config/bundled-skills/media-processing/services/gemini-map.ts +339 -0
- package/src/config/bundled-skills/media-processing/services/preprocess.ts +551 -0
- package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +259 -0
- package/src/config/bundled-skills/media-processing/services/reduce.ts +197 -0
- package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts +136 -0
- package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +59 -0
- package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +195 -0
- package/src/config/bundled-skills/media-processing/tools/ingest-media.ts +197 -0
- package/src/config/bundled-skills/media-processing/tools/media-diagnostics.ts +143 -0
- package/src/config/bundled-skills/media-processing/tools/media-status.ts +75 -0
- package/src/config/bundled-skills/media-processing/tools/query-media-events.ts +65 -0
- package/src/config/bundled-skills/messaging/SKILL.md +33 -8
- package/src/config/bundled-skills/messaging/tools/messaging-analyze-style.ts +4 -7
- package/src/config/bundled-skills/messaging/tools/messaging-reply.ts +2 -1
- package/src/config/bundled-skills/messaging/tools/messaging-send.ts +5 -1
- package/src/config/bundled-skills/phone-calls/SKILL.md +88 -23
- package/src/config/bundled-skills/twitter/SKILL.md +19 -3
- package/src/config/bundled-skills/twitter/icon.svg +14 -0
- package/src/config/bundled-tool-registry.ts +310 -0
- package/src/config/calls-schema.ts +181 -0
- package/src/config/core-schema.ts +309 -0
- package/src/config/defaults.ts +28 -3
- package/src/config/env-registry.ts +162 -0
- package/src/config/env.ts +175 -0
- package/src/config/loader.ts +6 -6
- package/src/config/memory-schema.ts +528 -0
- package/src/config/sandbox-schema.ts +55 -0
- package/src/config/schema.ts +158 -1133
- package/src/config/skill-state.ts +1 -1
- package/src/config/skills-schema.ts +32 -0
- package/src/config/skills.ts +35 -24
- package/src/config/system-prompt.ts +131 -56
- package/src/config/templates/IDENTITY.md +2 -2
- package/src/config/templates/SOUL.md +1 -1
- package/src/config/types.ts +1 -0
- package/src/config/user-reference.ts +4 -9
- package/src/config/vellum-skills/catalog.json +6 -7
- package/src/config/vellum-skills/chatgpt-import/tools/chatgpt-import.ts +5 -1
- package/src/config/vellum-skills/slack-oauth-setup/SKILL.md +4 -3
- package/src/config/vellum-skills/sms-setup/SKILL.md +216 -0
- package/src/config/vellum-skills/twilio-setup/SKILL.md +40 -8
- package/src/context/window-manager.ts +27 -7
- package/src/daemon/approval-generators.ts +186 -0
- package/src/daemon/approved-devices-store.ts +140 -0
- package/src/daemon/assistant-attachments.ts +1 -1
- package/src/daemon/classifier.ts +35 -32
- package/src/daemon/config-watcher.ts +1 -1
- package/src/daemon/daemon-control.ts +217 -0
- package/src/daemon/handlers/apps.ts +2 -3
- package/src/daemon/handlers/config-channels.ts +158 -0
- package/src/daemon/handlers/config-inbox.ts +540 -0
- package/src/daemon/handlers/config-ingress.ts +231 -0
- package/src/daemon/handlers/config-integrations.ts +258 -0
- package/src/daemon/handlers/config-model.ts +143 -0
- package/src/daemon/handlers/config-parental.ts +163 -0
- package/src/daemon/handlers/config-scheduling.ts +172 -0
- package/src/daemon/handlers/config-slack.ts +92 -0
- package/src/daemon/handlers/config-telegram.ts +301 -0
- package/src/daemon/handlers/config-tools.ts +177 -0
- package/src/daemon/handlers/config-trust.ts +104 -0
- package/src/daemon/handlers/config-twilio.ts +1080 -0
- package/src/daemon/handlers/config.ts +53 -1689
- package/src/daemon/handlers/diagnostics.ts +1 -1
- package/src/daemon/handlers/dictation.ts +180 -0
- package/src/daemon/handlers/documents.ts +18 -32
- package/src/daemon/handlers/identity.ts +14 -23
- package/src/daemon/handlers/index.ts +11 -0
- package/src/daemon/handlers/misc.ts +3 -5
- package/src/daemon/handlers/pairing.ts +98 -0
- package/src/daemon/handlers/sessions.ts +56 -5
- package/src/daemon/handlers/shared.ts +6 -1
- package/src/daemon/handlers/skills.ts +1 -1
- package/src/daemon/handlers/twitter-auth.ts +2 -0
- package/src/daemon/handlers/work-items.ts +17 -9
- package/src/daemon/handlers/workspace-files.ts +4 -3
- package/src/daemon/install-cli-launchers.ts +113 -0
- package/src/daemon/ipc-contract/apps.ts +356 -0
- package/src/daemon/ipc-contract/browser.ts +74 -0
- package/src/daemon/ipc-contract/computer-use.ts +151 -0
- package/src/daemon/ipc-contract/diagnostics.ts +56 -0
- package/src/daemon/ipc-contract/documents.ts +74 -0
- package/src/daemon/ipc-contract/inbox.ts +209 -0
- package/src/daemon/ipc-contract/integrations.ts +284 -0
- package/src/daemon/ipc-contract/memory.ts +48 -0
- package/src/daemon/ipc-contract/messages.ts +211 -0
- package/src/daemon/ipc-contract/pairing.ts +45 -0
- package/src/daemon/ipc-contract/parental-control.ts +95 -0
- package/src/daemon/ipc-contract/schedules.ts +97 -0
- package/src/daemon/ipc-contract/sessions.ts +315 -0
- package/src/daemon/ipc-contract/shared.ts +42 -0
- package/src/daemon/ipc-contract/skills.ts +120 -0
- package/src/daemon/ipc-contract/subagents.ts +58 -0
- package/src/daemon/ipc-contract/surfaces.ts +250 -0
- package/src/daemon/ipc-contract/trust.ts +60 -0
- package/src/daemon/ipc-contract/work-items.ts +225 -0
- package/src/daemon/ipc-contract/workspace.ts +113 -0
- package/src/daemon/ipc-contract-inventory.json +70 -0
- package/src/daemon/ipc-contract-inventory.ts +55 -29
- package/src/daemon/ipc-contract.ts +229 -2426
- package/src/daemon/ipc-protocol.ts +1 -1
- package/src/daemon/ipc-validate.ts +7 -0
- package/src/daemon/lifecycle.ts +97 -377
- package/src/daemon/pairing-store.ts +177 -0
- package/src/daemon/providers-setup.ts +43 -0
- package/src/daemon/ride-shotgun-handler.ts +68 -3
- package/src/daemon/server.ts +66 -46
- package/src/daemon/session-agent-loop-handlers.ts +421 -0
- package/src/daemon/session-agent-loop.ts +117 -275
- package/src/daemon/session-dynamic-profile.ts +1 -1
- package/src/daemon/session-history.ts +1 -1
- package/src/daemon/session-media-retry.ts +1 -1
- package/src/daemon/session-messaging.ts +37 -2
- package/src/daemon/session-notifiers.ts +5 -25
- package/src/daemon/session-process.ts +99 -59
- package/src/daemon/session-queue-manager.ts +96 -4
- package/src/daemon/session-runtime-assembly.ts +199 -10
- package/src/daemon/session-surfaces.ts +19 -4
- package/src/daemon/session-tool-setup.ts +30 -30
- package/src/daemon/session-workspace.ts +1 -1
- package/src/daemon/session.ts +35 -2
- package/src/daemon/shutdown-handlers.ts +122 -0
- package/src/daemon/trace-emitter.ts +1 -1
- package/src/daemon/watch-handler.ts +36 -33
- package/src/doordash/cart-queries.ts +787 -0
- package/src/doordash/client.ts +144 -127
- package/src/doordash/order-queries.ts +85 -0
- package/src/doordash/queries.ts +10 -1308
- package/src/doordash/search-queries.ts +203 -0
- package/src/doordash/session.ts +3 -2
- package/src/doordash/store-queries.ts +246 -0
- package/src/doordash/types.ts +367 -0
- package/src/email/providers/agentmail.ts +2 -1
- package/src/email/providers/index.ts +3 -2
- package/src/email/service.ts +3 -2
- package/src/errors.ts +43 -0
- package/src/home-base/prebuilt/seed.ts +1 -1
- package/src/hooks/cli.ts +6 -5
- package/src/hooks/config.ts +6 -8
- package/src/hooks/discovery.ts +6 -5
- package/src/hooks/manager.ts +4 -3
- package/src/hooks/runner.ts +2 -2
- package/src/hooks/templates.ts +5 -5
- package/src/inbound/public-ingress-urls.ts +6 -4
- package/src/index.ts +4 -2
- package/src/influencer/client.ts +1104 -0
- package/src/instrument.ts +4 -3
- package/src/logfire.ts +4 -3
- package/src/memory/admin.ts +25 -35
- package/src/memory/attachments-store.ts +4 -7
- package/src/memory/channel-delivery-store.ts +30 -1
- package/src/memory/channel-guardian-store.ts +202 -2
- package/src/memory/clarification-resolver.ts +37 -33
- package/src/memory/conflict-store.ts +67 -61
- package/src/memory/contradiction-checker.ts +141 -117
- package/src/memory/conversation-store.ts +335 -51
- package/src/memory/db-connection.ts +27 -4
- package/src/memory/db-init.ts +265 -4
- package/src/memory/db.ts +14 -1
- package/src/memory/embedding-backend.ts +27 -5
- package/src/memory/embedding-ollama.ts +2 -1
- package/src/memory/entity-extractor.ts +38 -35
- package/src/memory/guardian-action-store.ts +430 -0
- package/src/memory/inbox-escalation-projection.ts +59 -0
- package/src/memory/inbox-thread-store.ts +218 -0
- package/src/memory/ingress-invite-store.ts +338 -0
- package/src/memory/ingress-member-store.ts +350 -0
- package/src/memory/items-extractor.ts +91 -97
- package/src/memory/job-handlers/index-maintenance.ts +3 -3
- package/src/memory/job-handlers/media-processing.ts +69 -0
- package/src/memory/job-handlers/summarization.ts +32 -26
- package/src/memory/job-utils.ts +3 -10
- package/src/memory/jobs-store.ts +8 -10
- package/src/memory/jobs-worker.ts +55 -36
- package/src/memory/media-store.ts +759 -0
- package/src/memory/migrations/001-job-deferrals.ts +45 -0
- package/src/memory/migrations/002-tool-invocations-fk.ts +43 -0
- package/src/memory/migrations/003-memory-fts-backfill.ts +24 -0
- package/src/memory/migrations/004-entity-relation-dedup.ts +87 -0
- package/src/memory/migrations/005-fingerprint-scope-unique.ts +80 -0
- package/src/memory/migrations/006-scope-salted-fingerprints.ts +62 -0
- package/src/memory/migrations/007-assistant-id-to-self.ts +254 -0
- package/src/memory/migrations/008-remove-assistant-id-columns.ts +208 -0
- package/src/memory/migrations/009-llm-usage-events-drop-assistant-id.ts +83 -0
- package/src/memory/migrations/010-ext-conv-bindings-channel-chat-unique.ts +56 -0
- package/src/memory/migrations/011-call-sessions-provider-sid-dedup.ts +63 -0
- package/src/memory/migrations/012-call-sessions-add-initiated-from.ts +19 -0
- package/src/memory/migrations/013-guardian-action-tables.ts +68 -0
- package/src/memory/migrations/014-backfill-inbox-thread-state.ts +76 -0
- package/src/memory/migrations/015-drop-active-search-index.ts +27 -0
- package/src/memory/migrations/016-memory-segments-indexes.ts +11 -0
- package/src/memory/migrations/017-memory-items-indexes.ts +10 -0
- package/src/memory/migrations/018-remaining-table-indexes.ts +13 -0
- package/src/memory/migrations/index.ts +24 -0
- package/src/memory/migrations/registry.ts +79 -0
- package/src/memory/migrations/validate-migration-state.ts +69 -0
- package/src/memory/qdrant-manager.ts +49 -8
- package/src/memory/query-builder.ts +1 -1
- package/src/memory/raw-query.ts +119 -0
- package/src/memory/recall-cache.ts +4 -1
- package/src/memory/retriever.ts +165 -47
- package/src/memory/schema-migration.ts +25 -984
- package/src/memory/schema.ts +228 -7
- package/src/memory/search/entity.ts +205 -31
- package/src/memory/search/lexical.ts +81 -52
- package/src/memory/search/ranking.ts +27 -23
- package/src/memory/search/semantic.ts +157 -19
- package/src/memory/search/types.ts +24 -0
- package/src/memory/shared-app-links-store.ts +4 -5
- package/src/memory/validation.ts +19 -0
- package/src/messaging/draft-store.ts +5 -6
- package/src/messaging/provider-types.ts +2 -0
- package/src/messaging/providers/sms/adapter.ts +201 -0
- package/src/messaging/providers/sms/client.ts +93 -0
- package/src/messaging/providers/sms/types.ts +7 -0
- package/src/messaging/providers/telegram-bot/adapter.ts +2 -5
- package/src/messaging/providers/whatsapp/adapter.ts +136 -0
- package/src/messaging/providers/whatsapp/client.ts +67 -0
- package/src/messaging/style-analyzer.ts +5 -4
- package/src/messaging/thread-summarizer.ts +61 -69
- package/src/messaging/triage-engine.ts +62 -71
- package/src/migrations/config-merge.ts +53 -0
- package/src/migrations/data-layout.ts +68 -0
- package/src/migrations/data-merge.ts +33 -0
- package/src/migrations/hooks-merge.ts +90 -0
- package/src/migrations/index.ts +6 -0
- package/src/migrations/log.ts +23 -0
- package/src/migrations/skills-merge.ts +33 -0
- package/src/migrations/workspace-layout.ts +79 -0
- package/src/permissions/checker.ts +133 -11
- package/src/permissions/prompter.ts +14 -0
- package/src/permissions/shell-identity.ts +31 -1
- package/src/permissions/trust-store.ts +21 -1
- package/src/providers/anthropic/client.ts +4 -4
- package/src/providers/failover.ts +2 -2
- package/src/providers/model-intents.ts +70 -0
- package/src/providers/ollama/client.ts +2 -1
- package/src/providers/provider-send-message.ts +176 -0
- package/src/providers/registry.ts +71 -30
- package/src/providers/retry.ts +35 -1
- package/src/providers/types.ts +12 -1
- package/src/runtime/approval-conversation-turn.ts +97 -0
- package/src/runtime/approval-message-composer.ts +253 -0
- package/src/runtime/channel-approval-parser.ts +36 -2
- package/src/runtime/channel-approvals.ts +11 -24
- package/src/runtime/channel-guardian-service.ts +88 -21
- package/src/runtime/channel-readiness-service.ts +418 -0
- package/src/runtime/channel-readiness-types.ts +35 -0
- package/src/runtime/channel-retry-sweep.ts +184 -0
- package/src/runtime/guardian-context-resolver.ts +108 -0
- package/src/runtime/http-server.ts +275 -717
- package/src/runtime/http-types.ts +59 -3
- package/src/runtime/middleware/auth.ts +116 -0
- package/src/runtime/middleware/error-handler.ts +33 -0
- package/src/runtime/middleware/twilio-validation.ts +127 -0
- package/src/runtime/routes/app-routes.ts +1 -1
- package/src/runtime/routes/call-routes.ts +51 -7
- package/src/runtime/routes/channel-delivery-routes.ts +170 -0
- package/src/runtime/routes/channel-guardian-routes.ts +1191 -0
- package/src/runtime/routes/channel-inbound-routes.ts +1152 -0
- package/src/runtime/routes/channel-route-shared.ts +144 -0
- package/src/runtime/routes/channel-routes.ts +32 -1588
- package/src/runtime/routes/conversation-routes.ts +50 -7
- package/src/runtime/routes/events-routes.ts +2 -2
- package/src/runtime/routes/identity-routes.ts +126 -0
- package/src/runtime/routes/pairing-routes.ts +143 -0
- package/src/runtime/routes/run-routes.ts +15 -1
- package/src/runtime/run-orchestrator.ts +86 -35
- package/src/schedule/schedule-store.ts +36 -32
- package/src/schedule/scheduler.ts +3 -3
- package/src/security/encrypted-store.ts +5 -7
- package/src/security/oauth2.ts +45 -15
- package/src/security/parental-control-store.ts +183 -0
- package/src/security/secret-allowlist.ts +4 -3
- package/src/security/secret-scanner.ts +5 -5
- package/src/security/secure-keys.ts +1 -1
- package/src/security/token-manager.ts +3 -2
- package/src/services/vercel-deploy.ts +6 -2
- package/src/skills/tool-manifest.ts +3 -3
- package/src/skills/vellum-catalog-remote.ts +75 -16
- package/src/slack/slack-webhook.ts +2 -1
- package/src/swarm/orchestrator.ts +92 -1
- package/src/swarm/router-planner.ts +6 -9
- package/src/swarm/worker-prompts.ts +9 -12
- package/src/tasks/task-compiler.ts +19 -28
- package/src/tasks/task-runner.ts +1 -1
- package/src/tools/assets/materialize.ts +2 -2
- package/src/tools/assets/search.ts +15 -14
- package/src/tools/browser/__tests__/auth-detector.test.ts +1 -0
- package/src/tools/browser/auto-navigate.ts +1 -0
- package/src/tools/browser/browser-execution.ts +10 -1
- package/src/tools/browser/browser-manager.ts +119 -4
- package/src/tools/browser/network-recorder.ts +5 -0
- package/src/tools/calls/call-start.ts +1 -0
- package/src/tools/credentials/broker.ts +11 -2
- package/src/tools/credentials/metadata-store.ts +18 -14
- package/src/tools/credentials/post-connect-hooks.ts +61 -0
- package/src/tools/credentials/vault.ts +49 -23
- package/src/tools/execution-target.ts +11 -1
- package/src/tools/executor.ts +68 -9
- package/src/tools/host-terminal/cli-discover.ts +1 -1
- package/src/tools/network/script-proxy/http-forwarder.ts +1 -1
- package/src/tools/network/script-proxy/mitm-handler.ts +1 -1
- package/src/tools/network/script-proxy/server.ts +1 -1
- package/src/tools/network/script-proxy/session-manager.ts +6 -5
- package/src/tools/network/web-fetch.ts +18 -2
- package/src/tools/network/web-search.ts +8 -4
- package/src/tools/reminder/reminder-store.ts +14 -15
- package/src/tools/schedule/create.ts +1 -0
- package/src/tools/schedule/list.ts +2 -1
- package/src/tools/shared/filesystem/file-ops-service.ts +5 -7
- package/src/tools/skills/skill-script-runner.ts +24 -9
- package/src/tools/skills/skill-tool-factory.ts +1 -0
- package/src/tools/tasks/work-item-enqueue.ts +2 -2
- package/src/tools/terminal/evaluate-typescript.ts +21 -12
- package/src/tools/terminal/parser.ts +50 -0
- package/src/tools/types.ts +2 -0
- package/src/tools/watcher/delete.ts +6 -0
- package/src/tools/weather/service.ts +1 -1
- package/src/twitter/client.ts +190 -24
- package/src/twitter/router.ts +1 -1
- package/src/twitter/session.ts +4 -3
- package/src/util/clipboard.ts +1 -1
- package/src/util/errors.ts +65 -8
- package/src/util/fs.ts +40 -0
- package/src/util/json.ts +10 -0
- package/src/util/log-redact.ts +189 -0
- package/src/util/logger.ts +19 -17
- package/src/util/object.ts +3 -0
- package/src/util/platform.ts +105 -363
- package/src/util/pricing.ts +1 -1
- package/src/util/promise-guard.ts +1 -1
- package/src/util/retry.ts +19 -0
- package/src/util/row-mapper.ts +79 -0
- package/src/util/silently.ts +21 -0
- package/src/watcher/engine.ts +5 -1
- package/src/watcher/provider-types.ts +20 -0
- package/src/watcher/providers/github.ts +156 -0
- package/src/watcher/providers/gmail.ts +1 -0
- package/src/watcher/providers/google-calendar.ts +1 -0
- package/src/watcher/providers/linear.ts +460 -0
- package/src/watcher/providers/slack.ts +1 -0
- package/src/work-items/work-item-runner.ts +1 -1
- package/src/workspace/git-service.ts +1 -1
- package/src/workspace/provider-commit-message-generator.ts +51 -22
- package/src/__tests__/call-bridge.test.ts +0 -517
- package/src/__tests__/session-process-bridge.test.ts +0 -244
- package/src/calls/call-bridge.ts +0 -168
- package/src/config/vellum-skills/google-oauth-setup/SKILL.md +0 -199
|
@@ -1,15 +1,16 @@
|
|
|
1
|
-
import Anthropic from '@anthropic-ai/sdk';
|
|
2
1
|
import { and, eq, sql } from 'drizzle-orm';
|
|
3
2
|
import { v4 as uuid } from 'uuid';
|
|
4
3
|
import { getConfig } from '../config/loader.js';
|
|
5
4
|
import type { MemoryExtractionConfig } from '../config/types.js';
|
|
6
5
|
import { getLogger } from '../util/logger.js';
|
|
7
6
|
import { truncate } from '../util/truncate.js';
|
|
7
|
+
import { getConfiguredProvider, createTimeout, extractToolUse, userMessage } from '../providers/provider-send-message.js';
|
|
8
8
|
import { computeMemoryFingerprint } from './fingerprint.js';
|
|
9
9
|
import { enqueueMemoryJob } from './jobs-store.js';
|
|
10
10
|
import { extractTextFromStoredMessageContent } from './message-content.js';
|
|
11
11
|
import { getDb } from './db.js';
|
|
12
12
|
import { memoryItemConflicts, memoryItems, memoryItemSources, messages } from './schema.js';
|
|
13
|
+
import { clampUnitInterval } from './validation.js';
|
|
13
14
|
|
|
14
15
|
const log = getLogger('memory-items-extractor');
|
|
15
16
|
|
|
@@ -114,108 +115,105 @@ async function extractItemsWithLLM(
|
|
|
114
115
|
extractionConfig: MemoryExtractionConfig,
|
|
115
116
|
scopeId: string,
|
|
116
117
|
): Promise<ExtractedItem[]> {
|
|
117
|
-
const
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
log.debug('No Anthropic API key available for LLM extraction, falling back to pattern-based');
|
|
118
|
+
const provider = getConfiguredProvider();
|
|
119
|
+
if (!provider) {
|
|
120
|
+
log.debug('Configured provider unavailable for LLM extraction, falling back to pattern-based');
|
|
121
121
|
return extractItemsPatternBased(text, scopeId);
|
|
122
122
|
}
|
|
123
123
|
|
|
124
124
|
try {
|
|
125
|
-
const
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
type: 'object' as const,
|
|
137
|
-
properties: {
|
|
138
|
-
items: {
|
|
139
|
-
type: 'array',
|
|
125
|
+
const { signal, cleanup } = createTimeout(15000);
|
|
126
|
+
|
|
127
|
+
try {
|
|
128
|
+
const response = await provider.sendMessage(
|
|
129
|
+
[userMessage(text)],
|
|
130
|
+
[{
|
|
131
|
+
name: 'store_memory_items',
|
|
132
|
+
description: 'Store extracted memory items from the message',
|
|
133
|
+
input_schema: {
|
|
134
|
+
type: 'object' as const,
|
|
135
|
+
properties: {
|
|
140
136
|
items: {
|
|
141
|
-
type: '
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
137
|
+
type: 'array',
|
|
138
|
+
items: {
|
|
139
|
+
type: 'object',
|
|
140
|
+
properties: {
|
|
141
|
+
kind: {
|
|
142
|
+
type: 'string',
|
|
143
|
+
enum: [...VALID_KINDS],
|
|
144
|
+
description: 'Category of memory item',
|
|
145
|
+
},
|
|
146
|
+
subject: {
|
|
147
|
+
type: 'string',
|
|
148
|
+
description: 'Short label (2-8 words) for what this is about',
|
|
149
|
+
},
|
|
150
|
+
statement: {
|
|
151
|
+
type: 'string',
|
|
152
|
+
description: 'Full factual statement to remember (1-2 sentences)',
|
|
153
|
+
},
|
|
154
|
+
confidence: {
|
|
155
|
+
type: 'number',
|
|
156
|
+
description: 'Confidence that this is accurate (0.0-1.0)',
|
|
157
|
+
},
|
|
158
|
+
importance: {
|
|
159
|
+
type: 'number',
|
|
160
|
+
description: 'How valuable this is to remember (0.0-1.0)',
|
|
161
|
+
},
|
|
163
162
|
},
|
|
163
|
+
required: ['kind', 'subject', 'statement', 'confidence', 'importance'],
|
|
164
164
|
},
|
|
165
|
-
required: ['kind', 'subject', 'statement', 'confidence', 'importance'],
|
|
166
165
|
},
|
|
167
166
|
},
|
|
167
|
+
required: ['items'],
|
|
168
|
+
},
|
|
169
|
+
}],
|
|
170
|
+
EXTRACTION_SYSTEM_PROMPT,
|
|
171
|
+
{
|
|
172
|
+
config: {
|
|
173
|
+
model: extractionConfig.model,
|
|
174
|
+
max_tokens: 1024,
|
|
175
|
+
tool_choice: { type: 'tool' as const, name: 'store_memory_items' },
|
|
168
176
|
},
|
|
169
|
-
|
|
177
|
+
signal,
|
|
170
178
|
},
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
179
|
+
);
|
|
180
|
+
cleanup();
|
|
181
|
+
|
|
182
|
+
const toolBlock = extractToolUse(response);
|
|
183
|
+
if (!toolBlock) {
|
|
184
|
+
log.warn('No tool_use block in LLM extraction response, falling back to pattern-based');
|
|
185
|
+
return extractItemsPatternBased(text, scopeId);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
const input = toolBlock.input as { items?: LLMExtractedItem[] };
|
|
189
|
+
if (!Array.isArray(input.items)) {
|
|
190
|
+
log.warn('Invalid items in LLM extraction response, falling back to pattern-based');
|
|
191
|
+
return extractItemsPatternBased(text, scopeId);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const items: ExtractedItem[] = [];
|
|
195
|
+
for (const raw of input.items) {
|
|
196
|
+
if (!VALID_KINDS.has(raw.kind)) continue;
|
|
197
|
+
if (!raw.subject || !raw.statement) continue;
|
|
198
|
+
const subject = truncate(String(raw.subject), 80, '');
|
|
199
|
+
const statement = truncate(String(raw.statement), 500, '');
|
|
200
|
+
const confidence = clampUnitInterval(parseScore(raw.confidence, 0.5));
|
|
201
|
+
const importance = clampUnitInterval(parseScore(raw.importance, 0.5));
|
|
202
|
+
const fingerprint = computeMemoryFingerprint(scopeId, raw.kind, subject, statement);
|
|
203
|
+
items.push({
|
|
204
|
+
kind: raw.kind as MemoryItemKind,
|
|
205
|
+
subject,
|
|
206
|
+
statement,
|
|
207
|
+
confidence,
|
|
208
|
+
importance,
|
|
209
|
+
fingerprint,
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
return deduplicateItems(items);
|
|
214
|
+
} finally {
|
|
215
|
+
cleanup();
|
|
191
216
|
}
|
|
192
|
-
|
|
193
|
-
const input = toolBlock.input as { items?: LLMExtractedItem[] };
|
|
194
|
-
if (!Array.isArray(input.items)) {
|
|
195
|
-
log.warn('Invalid items in LLM extraction response, falling back to pattern-based');
|
|
196
|
-
return extractItemsPatternBased(text, scopeId);
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
const items: ExtractedItem[] = [];
|
|
200
|
-
for (const raw of input.items) {
|
|
201
|
-
if (!VALID_KINDS.has(raw.kind)) continue;
|
|
202
|
-
if (!raw.subject || !raw.statement) continue;
|
|
203
|
-
const subject = truncate(String(raw.subject), 80, '');
|
|
204
|
-
const statement = truncate(String(raw.statement), 500, '');
|
|
205
|
-
const confidence = clamp(parseScore(raw.confidence, 0.5), 0, 1);
|
|
206
|
-
const importance = clamp(parseScore(raw.importance, 0.5), 0, 1);
|
|
207
|
-
const fingerprint = computeMemoryFingerprint(scopeId, raw.kind, subject, statement);
|
|
208
|
-
items.push({
|
|
209
|
-
kind: raw.kind as MemoryItemKind,
|
|
210
|
-
subject,
|
|
211
|
-
statement,
|
|
212
|
-
confidence,
|
|
213
|
-
importance,
|
|
214
|
-
fingerprint,
|
|
215
|
-
});
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
return deduplicateItems(items);
|
|
219
217
|
} catch (err) {
|
|
220
218
|
const message = err instanceof Error ? err.message : String(err);
|
|
221
219
|
log.warn({ err: message }, 'LLM extraction failed, falling back to pattern-based');
|
|
@@ -287,8 +285,8 @@ export async function extractAndUpsertMemoryItemsForMessage(messageId: string, s
|
|
|
287
285
|
db.update(memoryItems)
|
|
288
286
|
.set({
|
|
289
287
|
status: effectiveStatus,
|
|
290
|
-
confidence: Math.max(existing.confidence, item.confidence),
|
|
291
|
-
importance: Math.max(existing.importance ?? 0, item.importance),
|
|
288
|
+
confidence: clampUnitInterval(Math.max(existing.confidence, item.confidence)),
|
|
289
|
+
importance: clampUnitInterval(Math.max(existing.importance ?? 0, item.importance)),
|
|
292
290
|
lastSeenAt: Math.max(existing.lastSeenAt, seenAt),
|
|
293
291
|
verificationState: promotedState,
|
|
294
292
|
})
|
|
@@ -441,10 +439,6 @@ function parseScore(value: unknown, fallback: number): number {
|
|
|
441
439
|
return Number.isFinite(n) ? n : fallback;
|
|
442
440
|
}
|
|
443
441
|
|
|
444
|
-
function clamp(value: number, min: number, max: number): number {
|
|
445
|
-
return Math.min(max, Math.max(min, value));
|
|
446
|
-
}
|
|
447
|
-
|
|
448
442
|
/** Returns true if the given memory item is the candidate in an unresolved conflict. */
|
|
449
443
|
function hasPendingConflict(itemId: string): boolean {
|
|
450
444
|
const db = getDb();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { eq } from 'drizzle-orm';
|
|
2
2
|
import { getLogger } from '../../util/logger.js';
|
|
3
|
-
import { getDb } from '../db.js';
|
|
3
|
+
import { getDb, rawExec } from '../db.js';
|
|
4
4
|
import { enqueueMemoryJob, type MemoryJob } from '../jobs-store.js';
|
|
5
5
|
import { asString, BackendUnavailableError } from '../job-utils.js';
|
|
6
6
|
import { getQdrantClient } from '../qdrant-client.js';
|
|
@@ -10,8 +10,8 @@ const log = getLogger('memory-jobs-worker');
|
|
|
10
10
|
|
|
11
11
|
export function rebuildIndexJob(): void {
|
|
12
12
|
const db = getDb();
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
rawExec(/*sql*/ `DELETE FROM memory_segment_fts`);
|
|
14
|
+
rawExec(/*sql*/ `
|
|
15
15
|
INSERT INTO memory_segment_fts(segment_id, text)
|
|
16
16
|
SELECT id, text FROM memory_segments
|
|
17
17
|
`);
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { getLogger } from '../../util/logger.js';
|
|
2
|
+
import { asString } from '../job-utils.js';
|
|
3
|
+
import { getMediaAssetById, updateMediaAssetStatus } from '../media-store.js';
|
|
4
|
+
import type { MemoryJob } from '../jobs-store.js';
|
|
5
|
+
import {
|
|
6
|
+
runPipeline,
|
|
7
|
+
type PipelineStageName,
|
|
8
|
+
type StageHandler,
|
|
9
|
+
} from '../../config/bundled-skills/media-processing/services/processing-pipeline.js';
|
|
10
|
+
import { preprocessForAsset } from '../../config/bundled-skills/media-processing/tools/extract-keyframes.js';
|
|
11
|
+
import { mapSegmentsForAsset } from '../../config/bundled-skills/media-processing/tools/analyze-keyframes.js';
|
|
12
|
+
import { reduceForAsset } from '../../config/bundled-skills/media-processing/tools/query-media-events.js';
|
|
13
|
+
|
|
14
|
+
const log = getLogger('media-processing-job');
|
|
15
|
+
|
|
16
|
+
export async function mediaProcessingJob(job: MemoryJob): Promise<void> {
|
|
17
|
+
const mediaAssetId = asString(job.payload.mediaAssetId);
|
|
18
|
+
if (!mediaAssetId) {
|
|
19
|
+
log.warn({ jobId: job.id }, 'Missing mediaAssetId in job payload');
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const asset = getMediaAssetById(mediaAssetId);
|
|
24
|
+
if (!asset) {
|
|
25
|
+
log.warn({ jobId: job.id, mediaAssetId }, 'Media asset not found');
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
if (asset.mediaType !== 'video') {
|
|
30
|
+
log.info(
|
|
31
|
+
{ assetId: mediaAssetId, mediaType: asset.mediaType },
|
|
32
|
+
'Skipping media processing pipeline — only video assets are supported',
|
|
33
|
+
);
|
|
34
|
+
updateMediaAssetStatus(mediaAssetId, 'indexed');
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const handlers: Record<PipelineStageName, StageHandler> = {
|
|
39
|
+
preprocess: { execute: async (assetId, onProgress) => { await preprocessForAsset(assetId, {}, onProgress); } },
|
|
40
|
+
map: { execute: async (assetId, onProgress) => { await mapSegmentsForAsset(assetId, {
|
|
41
|
+
systemPrompt: 'Describe what you see in these video frames. For each frame, note: subjects present, actions occurring, scene context, and any text visible.',
|
|
42
|
+
outputSchema: { type: 'object', properties: { frames: { type: 'array', items: { type: 'object', properties: { timestamp: { type: 'number' }, subjects: { type: 'array', items: { type: 'string' } }, actions: { type: 'array', items: { type: 'string' } }, scene: { type: 'string' }, text: { type: 'string' } } } } } }
|
|
43
|
+
}, onProgress); } },
|
|
44
|
+
reduce: { execute: async (assetId, onProgress) => { await reduceForAsset(assetId, {
|
|
45
|
+
systemPrompt: 'Summarize the video content based on the structured observations.',
|
|
46
|
+
}, onProgress); } },
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
const result = await runPipeline(mediaAssetId, handlers, {
|
|
50
|
+
onProgress: (msg) => log.info({ mediaAssetId }, msg),
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
log.info(
|
|
54
|
+
{
|
|
55
|
+
mediaAssetId,
|
|
56
|
+
completedStages: result.completedStages,
|
|
57
|
+
failedStage: result.failedStage,
|
|
58
|
+
cancelled: result.cancelled,
|
|
59
|
+
},
|
|
60
|
+
'Media processing pipeline finished',
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
if (result.failedStage) {
|
|
64
|
+
throw new Error(`Media processing failed at stage ${result.failedStage}: ${result.failureReason}`);
|
|
65
|
+
}
|
|
66
|
+
if (result.cancelled) {
|
|
67
|
+
throw new Error(`Media processing cancelled for asset ${mediaAssetId}`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import Anthropic from '@anthropic-ai/sdk';
|
|
2
1
|
import { and, desc, eq, gte, isNull, lt } from 'drizzle-orm';
|
|
3
2
|
import { v4 as uuid } from 'uuid';
|
|
4
3
|
import type { AssistantConfig } from '../../config/types.js';
|
|
5
4
|
import { estimateTextTokens } from '../../context/token-estimator.js';
|
|
6
5
|
import { getLogger } from '../../util/logger.js';
|
|
6
|
+
import { getConfiguredProvider, createTimeout, extractText, userMessage } from '../../providers/provider-send-message.js';
|
|
7
7
|
import { getConversationMemoryScopeId } from '../conversation-store.js';
|
|
8
8
|
import { getDb } from '../db.js';
|
|
9
9
|
import { enqueueMemoryJob, type MemoryJob } from '../jobs-store.js';
|
|
@@ -230,9 +230,9 @@ async function summarizeWithLLM(
|
|
|
230
230
|
return buildFallbackSummary(existingSummary, newContent, label);
|
|
231
231
|
}
|
|
232
232
|
|
|
233
|
-
const
|
|
234
|
-
if (!
|
|
235
|
-
log.debug({ label }, '
|
|
233
|
+
const provider = getConfiguredProvider();
|
|
234
|
+
if (!provider) {
|
|
235
|
+
log.debug({ label }, 'Configured provider unavailable for summarization, using fallback');
|
|
236
236
|
return buildFallbackSummary(existingSummary, newContent, label);
|
|
237
237
|
}
|
|
238
238
|
|
|
@@ -247,30 +247,36 @@ async function summarizeWithLLM(
|
|
|
247
247
|
userParts.push('### New Data', newContent);
|
|
248
248
|
|
|
249
249
|
try {
|
|
250
|
-
const
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
const textBlock = response.content.find((b) => b.type === 'text');
|
|
264
|
-
if (textBlock && textBlock.type === 'text' && textBlock.text.trim().length > 0) {
|
|
265
|
-
log.debug(
|
|
266
|
-
{ label, inputTokens: response.usage.input_tokens, outputTokens: response.usage.output_tokens },
|
|
267
|
-
'LLM summarization completed',
|
|
250
|
+
const { signal, cleanup } = createTimeout(SUMMARY_LLM_TIMEOUT_MS);
|
|
251
|
+
try {
|
|
252
|
+
const response = await provider.sendMessage(
|
|
253
|
+
[userMessage(userParts.join('\n'))],
|
|
254
|
+
undefined,
|
|
255
|
+
systemPrompt,
|
|
256
|
+
{
|
|
257
|
+
config: {
|
|
258
|
+
model: summarizationConfig.model,
|
|
259
|
+
max_tokens: SUMMARY_MAX_TOKENS,
|
|
260
|
+
},
|
|
261
|
+
signal,
|
|
262
|
+
},
|
|
268
263
|
);
|
|
269
|
-
|
|
270
|
-
}
|
|
264
|
+
cleanup();
|
|
271
265
|
|
|
272
|
-
|
|
273
|
-
|
|
266
|
+
const text = extractText(response);
|
|
267
|
+
if (text.length > 0) {
|
|
268
|
+
log.debug(
|
|
269
|
+
{ label, inputTokens: response.usage.inputTokens, outputTokens: response.usage.outputTokens },
|
|
270
|
+
'LLM summarization completed',
|
|
271
|
+
);
|
|
272
|
+
return text;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
log.warn({ label }, 'LLM summarization returned empty text, using fallback');
|
|
276
|
+
return buildFallbackSummary(existingSummary, newContent, label);
|
|
277
|
+
} finally {
|
|
278
|
+
cleanup();
|
|
279
|
+
}
|
|
274
280
|
} catch (err) {
|
|
275
281
|
const message = err instanceof Error ? err.message : String(err);
|
|
276
282
|
log.warn({ err: message, label }, 'LLM summarization failed, using fallback');
|
package/src/memory/job-utils.ts
CHANGED
|
@@ -6,18 +6,11 @@ import { getDb } from './db.js';
|
|
|
6
6
|
import { getQdrantClient } from './qdrant-client.js';
|
|
7
7
|
import { memoryEmbeddings } from './schema.js';
|
|
8
8
|
import type { AssistantConfig } from '../config/types.js';
|
|
9
|
+
import { BackendUnavailableError } from '../util/errors.js';
|
|
9
10
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
// ── Sentinel error ─────────────────────────────────────────────────
|
|
11
|
+
export { BackendUnavailableError };
|
|
13
12
|
|
|
14
|
-
|
|
15
|
-
export class BackendUnavailableError extends Error {
|
|
16
|
-
constructor(reason: string) {
|
|
17
|
-
super(reason);
|
|
18
|
-
this.name = 'BackendUnavailableError';
|
|
19
|
-
}
|
|
20
|
-
}
|
|
13
|
+
const log = getLogger('memory-jobs-worker');
|
|
21
14
|
|
|
22
15
|
// ── Error classification for LLM / API errors ─────────────────────
|
|
23
16
|
|
package/src/memory/jobs-store.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { and, asc, eq, lte, notInArray, inArray } from 'drizzle-orm';
|
|
2
2
|
import { v4 as uuid } from 'uuid';
|
|
3
|
-
import { getDb } from './db.js';
|
|
3
|
+
import { getDb, rawGet, rawAll } from './db.js';
|
|
4
4
|
import { memoryJobs } from './schema.js';
|
|
5
5
|
import { truncate } from '../util/truncate.js';
|
|
6
6
|
|
|
@@ -20,7 +20,8 @@ export type MemoryJobType =
|
|
|
20
20
|
| 'build_conversation_summary'
|
|
21
21
|
| 'backfill'
|
|
22
22
|
| 'rebuild_index'
|
|
23
|
-
| 'delete_qdrant_vectors'
|
|
23
|
+
| 'delete_qdrant_vectors'
|
|
24
|
+
| 'media_processing';
|
|
24
25
|
|
|
25
26
|
const EMBED_JOB_TYPES: MemoryJobType[] = ['embed_segment', 'embed_item', 'embed_summary'];
|
|
26
27
|
|
|
@@ -109,9 +110,8 @@ export function enqueueResolvePendingConflictsForMessageJob(
|
|
|
109
110
|
throw new Error('enqueueResolvePendingConflictsForMessageJob requires a non-empty messageId');
|
|
110
111
|
}
|
|
111
112
|
const normalizedScopeId = scopeId.trim() || 'default';
|
|
112
|
-
// Dedup check always uses root db since tx doesn't expose
|
|
113
|
-
const
|
|
114
|
-
const existing = raw.query(`
|
|
113
|
+
// Dedup check always uses root db since tx doesn't expose raw client
|
|
114
|
+
const existing = rawGet<{ id: string }>(`
|
|
115
115
|
SELECT id
|
|
116
116
|
FROM memory_jobs
|
|
117
117
|
WHERE type = 'resolve_pending_conflicts_for_message'
|
|
@@ -120,7 +120,7 @@ export function enqueueResolvePendingConflictsForMessageJob(
|
|
|
120
120
|
AND COALESCE(json_extract(payload, '$.scopeId'), 'default') = ?
|
|
121
121
|
ORDER BY created_at ASC
|
|
122
122
|
LIMIT 1
|
|
123
|
-
|
|
123
|
+
`, normalizedMessageId, normalizedScopeId);
|
|
124
124
|
if (existing?.id) return existing.id;
|
|
125
125
|
|
|
126
126
|
return enqueueMemoryJob('resolve_pending_conflicts_for_message', {
|
|
@@ -365,13 +365,11 @@ export function resetRunningJobsToPending(): number {
|
|
|
365
365
|
}
|
|
366
366
|
|
|
367
367
|
export function getMemoryJobCounts(): Record<string, number> {
|
|
368
|
-
const
|
|
369
|
-
const raw = (db as unknown as { $client: { query: (q: string) => { all: () => unknown[] } } }).$client;
|
|
370
|
-
const rows = raw.query(`
|
|
368
|
+
const rows = rawAll<{ status: string; c: number }>(`
|
|
371
369
|
SELECT status, COUNT(*) AS c
|
|
372
370
|
FROM memory_jobs
|
|
373
371
|
GROUP BY status
|
|
374
|
-
`)
|
|
372
|
+
`);
|
|
375
373
|
const counts: Record<string, number> = { pending: 0, running: 0, completed: 0, failed: 0 };
|
|
376
374
|
for (const row of rows) {
|
|
377
375
|
counts[row.status] = row.c;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { AssistantConfig } from '../config/types.js';
|
|
2
2
|
import { getConfig } from '../config/loader.js';
|
|
3
3
|
import { getLogger } from '../util/logger.js';
|
|
4
|
-
import {
|
|
4
|
+
import { rawRun } from './db.js';
|
|
5
5
|
import {
|
|
6
6
|
claimMemoryJobs,
|
|
7
7
|
completeMemoryJob,
|
|
@@ -29,6 +29,7 @@ import { checkContradictionsJob, cleanupStaleSupersededItemsJob } from './job-ha
|
|
|
29
29
|
import { buildConversationSummaryJob, buildGlobalSummaryJob } from './job-handlers/summarization.js';
|
|
30
30
|
import { backfillJob, backfillEntityRelationsJob } from './job-handlers/backfill.js';
|
|
31
31
|
import { rebuildIndexJob, deleteQdrantVectorsJob } from './job-handlers/index-maintenance.js';
|
|
32
|
+
import { mediaProcessingJob } from './job-handlers/media-processing.js';
|
|
32
33
|
|
|
33
34
|
// Re-export public utilities consumed by tests and other modules
|
|
34
35
|
export { currentWeekWindow } from './job-utils.js';
|
|
@@ -111,38 +112,56 @@ export async function runMemoryJobsOnce(
|
|
|
111
112
|
let processed = 0;
|
|
112
113
|
const typeGroups = [...jobsByType.values()];
|
|
113
114
|
|
|
114
|
-
// Run type groups concurrently (up to workerConcurrency
|
|
115
|
-
//
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
} catch (handlerErr) {
|
|
131
|
-
log.error({ err: handlerErr, jobId: job.id, type: job.type }, 'handleJobError itself threw, job left in running status');
|
|
132
|
-
}
|
|
133
|
-
}
|
|
115
|
+
// Run type groups concurrently using a task pool (up to workerConcurrency
|
|
116
|
+
// active at a time). Unlike the old wave approach, a new group starts as
|
|
117
|
+
// soon as any slot frees up — no waiting for an entire wave to finish.
|
|
118
|
+
const processGroup = async (group: MemoryJob[]): Promise<number> => {
|
|
119
|
+
let groupProcessed = 0;
|
|
120
|
+
for (const job of group) {
|
|
121
|
+
try {
|
|
122
|
+
await processJob(job, config);
|
|
123
|
+
completeMemoryJob(job.id);
|
|
124
|
+
bumpMemoryVersion();
|
|
125
|
+
groupProcessed += 1;
|
|
126
|
+
} catch (err) {
|
|
127
|
+
try {
|
|
128
|
+
handleJobError(job, err);
|
|
129
|
+
} catch (handlerErr) {
|
|
130
|
+
log.error({ err: handlerErr, jobId: job.id, type: job.type }, 'handleJobError itself threw, job left in running status');
|
|
134
131
|
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return groupProcessed;
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
if (typeGroups.length <= concurrency) {
|
|
138
|
+
// Fast path: all groups fit within the concurrency limit
|
|
139
|
+
const results = await Promise.allSettled(typeGroups.map(processGroup));
|
|
140
|
+
for (const result of results) {
|
|
139
141
|
if (result.status === 'fulfilled') {
|
|
140
142
|
processed += result.value;
|
|
143
|
+
} else {
|
|
144
|
+
log.error({ err: result.reason }, 'Memory job group rejected unexpectedly — jobs in this batch may have been dropped');
|
|
141
145
|
}
|
|
142
|
-
// Errors within groups are already handled per-job above;
|
|
143
|
-
// a rejected group promise would only come from an unexpected
|
|
144
|
-
// error in the loop itself, which is unlikely.
|
|
145
146
|
}
|
|
147
|
+
} else {
|
|
148
|
+
// Task pool: maintain `concurrency` in-flight groups at all times
|
|
149
|
+
let nextIdx = 0;
|
|
150
|
+
|
|
151
|
+
const startNext = (): Promise<void> | undefined => {
|
|
152
|
+
if (nextIdx >= typeGroups.length) return undefined;
|
|
153
|
+
const group = typeGroups[nextIdx++]!;
|
|
154
|
+
return processGroup(group).then(
|
|
155
|
+
(count) => { processed += count; },
|
|
156
|
+
(err) => { log.error({ err }, 'Memory job group rejected unexpectedly — jobs in this batch may have been dropped'); },
|
|
157
|
+
).then(() => startNext());
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
const workers = Array.from(
|
|
161
|
+
{ length: Math.min(concurrency, typeGroups.length) },
|
|
162
|
+
() => startNext()!,
|
|
163
|
+
);
|
|
164
|
+
await Promise.all(workers);
|
|
146
165
|
}
|
|
147
166
|
if (enableScheduledCleanup) {
|
|
148
167
|
maybeEnqueueScheduledCleanupJobs(config);
|
|
@@ -229,6 +248,9 @@ async function processJob(job: MemoryJob, config: AssistantConfig): Promise<void
|
|
|
229
248
|
case 'delete_qdrant_vectors':
|
|
230
249
|
await deleteQdrantVectorsJob(job);
|
|
231
250
|
return;
|
|
251
|
+
case 'media_processing':
|
|
252
|
+
await mediaProcessingJob(job);
|
|
253
|
+
return;
|
|
232
254
|
default:
|
|
233
255
|
throw new Error(`Unknown memory job type: ${(job as { type: string }).type}`);
|
|
234
256
|
}
|
|
@@ -292,16 +314,13 @@ export function sweepStaleItems(config: AssistantConfig): number {
|
|
|
292
314
|
if (now - lastStaleSweepMs < STALE_SWEEP_INTERVAL_MS) return 0;
|
|
293
315
|
lastStaleSweepMs = now;
|
|
294
316
|
|
|
295
|
-
const db = getDb();
|
|
296
|
-
const raw = (db as unknown as { $client: { query: (q: string) => { run: (...params: unknown[]) => { changes: number } } } }).$client;
|
|
297
|
-
|
|
298
317
|
let totalMarked = 0;
|
|
299
318
|
for (const [kind, maxAgeDays] of Object.entries(freshness.maxAgeDays)) {
|
|
300
319
|
if (maxAgeDays <= 0) continue;
|
|
301
320
|
// Mark invalid if: past 2x window, no access in the shield period, and not already invalid
|
|
302
321
|
const cutoffMs = now - maxAgeDays * 2 * 86_400_000;
|
|
303
322
|
const shieldCutoffMs = now - freshness.reinforcementShieldDays * 86_400_000;
|
|
304
|
-
const
|
|
323
|
+
const changes = rawRun(`
|
|
305
324
|
UPDATE memory_items
|
|
306
325
|
SET invalid_at = ?
|
|
307
326
|
WHERE kind = ?
|
|
@@ -309,10 +328,10 @@ export function sweepStaleItems(config: AssistantConfig): number {
|
|
|
309
328
|
AND invalid_at IS NULL
|
|
310
329
|
AND last_seen_at < ?
|
|
311
330
|
AND (access_count = 0 OR COALESCE(last_used_at, 0) < ?)
|
|
312
|
-
|
|
313
|
-
if (
|
|
314
|
-
log.info({ kind, marked:
|
|
315
|
-
totalMarked +=
|
|
331
|
+
`, now, kind, cutoffMs, shieldCutoffMs);
|
|
332
|
+
if (changes > 0) {
|
|
333
|
+
log.info({ kind, marked: changes, cutoffMs }, 'Marked stale memory items as invalid');
|
|
334
|
+
totalMarked += changes;
|
|
316
335
|
}
|
|
317
336
|
}
|
|
318
337
|
return totalMarked;
|