@vellumai/assistant 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -0
- package/ARCHITECTURE.md +68 -15
- package/Dockerfile +2 -2
- package/bun.lock +6 -2
- package/docker-entrypoint.sh +42 -1
- package/docs/architecture/integrations.md +1 -1
- package/docs/architecture/memory.md +21 -24
- package/node_modules/@vellumai/ces-contracts/src/handles.ts +7 -9
- package/openapi.yaml +539 -4
- package/package.json +5 -1
- package/src/__tests__/anthropic-provider.test.ts +160 -95
- package/src/__tests__/app-dir-path-guard.test.ts +1 -0
- package/src/__tests__/app-executors.test.ts +47 -1
- package/src/__tests__/app-source-watcher.test.ts +159 -0
- package/src/__tests__/assistant-event-hub.test.ts +30 -0
- package/src/__tests__/checker.test.ts +138 -172
- package/src/__tests__/cli-command-risk-guard.test.ts +1 -1
- package/src/__tests__/config-schema.test.ts +5 -0
- package/src/__tests__/context-overflow-approval.test.ts +5 -5
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +4 -6
- package/src/__tests__/conversation-agent-loop.test.ts +4 -51
- package/src/__tests__/conversation-analysis-routes.test.ts +169 -0
- package/src/__tests__/conversation-directories-parse.test.ts +105 -0
- package/src/__tests__/conversation-history-web-search.test.ts +1 -1
- package/src/__tests__/conversation-runtime-assembly.test.ts +653 -832
- package/src/__tests__/conversation-runtime-workspace.test.ts +1 -93
- package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +17 -4
- package/src/__tests__/conversation-wipe.test.ts +2 -6
- package/src/__tests__/conversation-workspace-cache-state.test.ts +6 -12
- package/src/__tests__/conversation-workspace-injection.test.ts +25 -26
- package/src/__tests__/conversation-workspace-tool-tracking.test.ts +1 -1
- package/src/__tests__/copy-composer-tc-templates.test.ts +335 -0
- package/src/__tests__/credential-execution-approval-bridge.test.ts +0 -2
- package/src/__tests__/date-context.test.ts +76 -210
- package/src/__tests__/db-schedule-syntax-migration.test.ts +16 -1
- package/src/__tests__/file-list-tool.test.ts +219 -0
- package/src/__tests__/first-greeting.test.ts +1 -1
- package/src/__tests__/heartbeat-service.test.ts +180 -3
- package/src/__tests__/identity-routes.test.ts +328 -0
- package/src/__tests__/init-feature-flag-overrides.test.ts +167 -0
- package/src/__tests__/injection-block.test.ts +24 -0
- package/src/__tests__/inline-command-runner.test.ts +7 -5
- package/src/__tests__/install-skill-routing.test.ts +7 -6
- package/src/__tests__/jobs-store-qdrant-breaker.test.ts +15 -14
- package/src/__tests__/list-messages-tool-merge.test.ts +300 -0
- package/src/__tests__/llm-context-normalization.test.ts +18 -18
- package/src/__tests__/llm-context-route-provider.test.ts +101 -0
- package/src/__tests__/llm-request-log-turn-query.test.ts +162 -0
- package/src/__tests__/log-export-workspace.test.ts +257 -100
- package/src/__tests__/managed-credential-catalog-cli.test.ts +12 -14
- package/src/__tests__/mcp-abort-signal.test.ts +5 -0
- package/src/__tests__/mcp-client-auth.test.ts +5 -0
- package/src/__tests__/memory-recall-log-store.test.ts +132 -0
- package/src/__tests__/migration-export-streaming.test.ts +304 -0
- package/src/__tests__/migration-import-commit-http.test.ts +11 -10
- package/src/__tests__/mock-fetch.ts +87 -0
- package/src/__tests__/navigate-settings-tab.test.ts +14 -1
- package/src/__tests__/notification-broadcaster.test.ts +65 -0
- package/src/__tests__/notification-decision-recipient-context.test.ts +282 -0
- package/src/__tests__/onboarding-template-contract.test.ts +63 -14
- package/src/__tests__/parser.test.ts +32 -0
- package/src/__tests__/permission-checker-host-gate.test.ts +452 -0
- package/src/__tests__/permission-controls-v2-flag.test.ts +55 -0
- package/src/__tests__/permission-mode-sse.test.ts +418 -0
- package/src/__tests__/permission-mode-store.test.ts +277 -0
- package/src/__tests__/permission-mode.test.ts +101 -0
- package/src/__tests__/pkb-autoinject.test.ts +96 -0
- package/src/__tests__/platform-bash-auto-approve.test.ts +359 -0
- package/src/__tests__/profiler-routes.test.ts +502 -0
- package/src/__tests__/profiler-run-store.test.ts +441 -0
- package/src/__tests__/proxy-approval-callback.test.ts +4 -75
- package/src/__tests__/registry.test.ts +1 -1
- package/src/__tests__/require-fresh-approval.test.ts +0 -2
- package/src/__tests__/sandbox-diagnostics.test.ts +1 -32
- package/src/__tests__/sandbox-host-parity.test.ts +5 -4
- package/src/__tests__/scheduler-reuse-conversation.test.ts +368 -0
- package/src/__tests__/scrub-corrupted-image-attachments.test.ts +278 -0
- package/src/__tests__/search-skills-unified.test.ts +4 -3
- package/src/__tests__/send-endpoint-busy.test.ts +42 -3
- package/src/__tests__/set-permission-mode.test.ts +274 -0
- package/src/__tests__/skill-load-feature-flag.test.ts +12 -0
- package/src/__tests__/skill-memory.test.ts +2 -783
- package/src/__tests__/strip-memory-injections.test.ts +187 -0
- package/src/__tests__/subagent-detail.test.ts +84 -0
- package/src/__tests__/subagent-disposal.test.ts +308 -0
- package/src/__tests__/subagent-manager-notify.test.ts +19 -10
- package/src/__tests__/subagent-notify-parent.test.ts +390 -0
- package/src/__tests__/subagent-role-registry.test.ts +108 -0
- package/src/__tests__/subagent-tool-filtering.test.ts +71 -0
- package/src/__tests__/subagent-tools.test.ts +464 -4
- package/src/__tests__/system-prompt-ask-mode.test.ts +139 -0
- package/src/__tests__/task-memory-cleanup.test.ts +12 -12
- package/src/__tests__/terminal-sandbox.test.ts +1 -1
- package/src/__tests__/terminal-tools.test.ts +16 -29
- package/src/__tests__/test-preload.ts +18 -0
- package/src/__tests__/tool-domain-event-publisher.test.ts +0 -1
- package/src/__tests__/tool-executor-lifecycle-events.test.ts +1 -8
- package/src/__tests__/tool-executor.test.ts +4 -27
- package/src/__tests__/tool-side-effects-slack-dm.test.ts +1 -0
- package/src/__tests__/top-level-renderer.test.ts +10 -13
- package/src/__tests__/transport-hints-queue.test.ts +77 -0
- package/src/__tests__/trust-store.test.ts +4 -4
- package/src/__tests__/trusted-contact-lifecycle-notifications.test.ts +116 -2
- package/src/__tests__/workspace-migration-028-recover-conversations-from-disk-view.test.ts +387 -0
- package/src/__tests__/workspace-migration-030-seed-pkb-autoinject.test.ts +168 -0
- package/src/__tests__/workspace-policy.test.ts +2 -7
- package/src/agent/loop.ts +6 -29
- package/src/approvals/guardian-request-resolvers.ts +24 -0
- package/src/avatar/traits-png-sync.ts +3 -3
- package/src/channels/types.ts +5 -0
- package/src/cli/__tests__/run-assistant-command.ts +56 -0
- package/src/cli/__tests__/unknown-command.test.ts +33 -0
- package/src/cli/commands/__tests__/email-download.test.ts +245 -0
- package/src/cli/commands/__tests__/email-list.test.ts +192 -0
- package/src/cli/commands/__tests__/email-register.test.ts +186 -0
- package/src/cli/commands/__tests__/email-send.test.ts +291 -0
- package/src/cli/commands/__tests__/email-status.test.ts +181 -0
- package/src/cli/commands/__tests__/email-unregister.test.ts +139 -0
- package/src/cli/commands/__tests__/routes.test.ts +562 -0
- package/src/cli/commands/conversations.ts +1 -8
- package/src/cli/commands/default-action.ts +68 -1
- package/src/cli/commands/email.ts +584 -835
- package/src/cli/commands/memory.ts +1 -34
- package/src/cli/commands/notifications.ts +7 -2
- package/src/cli/commands/oauth/__tests__/connect.test.ts +27 -0
- package/src/cli/commands/oauth/connect.ts +25 -5
- package/src/cli/commands/platform/__tests__/connect.test.ts +1 -1
- package/src/cli/commands/platform/__tests__/disconnect.test.ts +1 -1
- package/src/cli/commands/platform/__tests__/status.test.ts +1 -1
- package/src/cli/commands/routes.ts +396 -0
- package/src/cli/commands/skills.ts +130 -20
- package/src/cli/program.ts +11 -2
- package/src/cli.ts +1 -120
- package/src/config/assistant-feature-flags.ts +59 -55
- package/src/config/bundled-skills/app-builder/SKILL.md +91 -5
- package/src/config/bundled-skills/gmail/SKILL.md +13 -8
- package/src/config/bundled-skills/gmail/TOOLS.json +1 -1
- package/src/config/bundled-skills/gmail/tools/gmail-sender-digest.ts +2 -1
- package/src/config/bundled-skills/messaging/SKILL.md +7 -0
- package/src/config/bundled-skills/schedule/SKILL.md +22 -2
- package/src/config/bundled-skills/schedule/TOOLS.json +8 -0
- package/src/config/bundled-skills/settings/TOOLS.json +1 -1
- package/src/config/bundled-skills/settings/tools/avatar-get.ts +3 -13
- package/src/config/bundled-skills/settings/tools/avatar-remove.ts +2 -4
- package/src/config/bundled-skills/settings/tools/avatar-update.ts +5 -2
- package/src/config/bundled-skills/settings/tools/navigate-settings-tab.ts +8 -3
- package/src/config/bundled-skills/slack/SKILL.md +2 -0
- package/src/config/bundled-skills/subagent/SKILL.md +43 -3
- package/src/config/bundled-skills/subagent/TOOLS.json +29 -4
- package/src/config/env-registry.ts +63 -0
- package/src/config/feature-flag-registry.json +17 -1
- package/src/config/schema.ts +8 -0
- package/src/config/schemas/filing.ts +51 -0
- package/src/config/schemas/heartbeat.ts +15 -12
- package/src/config/schemas/memory-lifecycle.ts +12 -0
- package/src/config/schemas/security.ts +14 -0
- package/src/config/schemas/services.ts +8 -0
- package/src/credential-execution/approval-bridge.ts +0 -1
- package/src/credential-execution/managed-catalog.ts +3 -7
- package/src/daemon/app-source-watcher.ts +93 -0
- package/src/daemon/config-watcher.ts +85 -3
- package/src/daemon/context-overflow-approval.ts +0 -1
- package/src/daemon/conversation-agent-loop-handlers.ts +20 -0
- package/src/daemon/conversation-agent-loop.ts +179 -65
- package/src/daemon/conversation-attachments.ts +0 -1
- package/src/daemon/conversation-history.ts +4 -19
- package/src/daemon/conversation-lifecycle.ts +8 -14
- package/src/daemon/conversation-messaging.ts +3 -0
- package/src/daemon/conversation-process.ts +30 -8
- package/src/daemon/conversation-queue-manager.ts +8 -0
- package/src/daemon/conversation-runtime-assembly.ts +359 -308
- package/src/daemon/conversation-surfaces.ts +65 -0
- package/src/daemon/conversation-tool-setup.ts +44 -17
- package/src/daemon/conversation-workspace.ts +1 -2
- package/src/daemon/conversation.ts +19 -3
- package/src/daemon/date-context.ts +26 -53
- package/src/daemon/first-greeting.ts +1 -1
- package/src/daemon/handlers/conversations.ts +5 -7
- package/src/daemon/handlers/shared.test.ts +143 -0
- package/src/daemon/handlers/shared.ts +70 -5
- package/src/daemon/handlers/skills.ts +11 -18
- package/src/daemon/lifecycle.ts +220 -158
- package/src/daemon/message-types/conversations.ts +29 -6
- package/src/daemon/message-types/messages.ts +9 -2
- package/src/daemon/message-types/notifications.ts +12 -0
- package/src/daemon/message-types/schedules.ts +1 -0
- package/src/daemon/message-types/settings.ts +18 -0
- package/src/daemon/profiler-run-store.ts +557 -0
- package/src/daemon/server.ts +87 -10
- package/src/daemon/shutdown-handlers.ts +5 -0
- package/src/daemon/tool-side-effects.ts +23 -3
- package/src/daemon/transport-hints.ts +33 -0
- package/src/export/transcript-formatter.ts +148 -0
- package/src/filing/filing-service.ts +228 -0
- package/src/heartbeat/heartbeat-service.ts +96 -7
- package/src/index.ts +1 -1
- package/src/mcp/client.ts +6 -0
- package/src/mcp/mcp-oauth-provider.ts +149 -27
- package/src/memory/admin.ts +33 -32
- package/src/memory/app-store.ts +69 -0
- package/src/memory/conversation-bootstrap.ts +1 -1
- package/src/memory/conversation-crud.ts +151 -117
- package/src/memory/conversation-directories.ts +39 -0
- package/src/memory/conversation-group-migration.ts +66 -6
- package/src/memory/conversation-queries.ts +58 -12
- package/src/memory/conversation-title-service.ts +1 -0
- package/src/memory/db-init.ts +182 -376
- package/src/memory/embedding-local.ts +1 -1
- package/src/memory/graph/bootstrap.ts +75 -66
- package/src/memory/graph/capability-seed.ts +167 -17
- package/src/memory/graph/consolidation.ts +38 -4
- package/src/memory/graph/conversation-graph-memory.ts +133 -104
- package/src/memory/graph/extraction-job.ts +9 -4
- package/src/memory/graph/extraction.ts +66 -23
- package/src/memory/graph/graph-memory-state-store.ts +37 -0
- package/src/memory/graph/graph-search.ts +29 -15
- package/src/memory/graph/injection.ts +38 -8
- package/src/memory/graph/inspect.ts +12 -3
- package/src/memory/graph/retriever.ts +365 -262
- package/src/memory/graph/store.test.ts +48 -0
- package/src/memory/graph/store.ts +150 -11
- package/src/memory/graph/tool-handlers.ts +84 -209
- package/src/memory/graph/tools.ts +8 -52
- package/src/memory/graph/types.ts +24 -0
- package/src/memory/group-crud.ts +25 -9
- package/src/memory/job-handlers/cleanup.ts +44 -1
- package/src/memory/jobs-store.ts +70 -60
- package/src/memory/jobs-worker.ts +44 -28
- package/src/memory/llm-request-log-store.ts +96 -12
- package/src/memory/memory-recall-log-store.ts +49 -5
- package/src/memory/migrations/203-drop-memory-items-tables.ts +33 -1
- package/src/memory/migrations/206-memory-graph-node-edits.ts +19 -0
- package/src/memory/migrations/206-scrub-corrupted-image-attachments.ts +131 -0
- package/src/memory/migrations/207-conversation-graph-memory-state.ts +20 -0
- package/src/memory/migrations/208-conversations-last-message-at.ts +35 -0
- package/src/memory/migrations/209-strip-thinking-from-consolidated.ts +85 -0
- package/src/memory/migrations/210-schedule-reuse-conversation.ts +13 -0
- package/src/memory/migrations/211-memory-recall-logs-query-context.ts +21 -0
- package/src/memory/migrations/212-llm-request-logs-created-at-index.ts +19 -0
- package/src/memory/migrations/index.ts +8 -0
- package/src/memory/migrations/registry.ts +8 -0
- package/src/memory/schema/conversations.ts +14 -0
- package/src/memory/schema/infrastructure.ts +8 -1
- package/src/memory/schema/memory-core.ts +0 -51
- package/src/memory/schema/memory-graph.ts +15 -0
- package/src/memory/task-memory-cleanup.ts +30 -11
- package/src/messaging/provider.ts +1 -1
- package/src/notifications/broadcaster.ts +6 -0
- package/src/notifications/conversation-pairing.ts +12 -4
- package/src/notifications/copy-composer.ts +86 -0
- package/src/notifications/decision-engine.ts +35 -0
- package/src/notifications/emit-signal.ts +14 -0
- package/src/notifications/signal.ts +11 -0
- package/src/oauth/platform-connection.test.ts +2 -2
- package/src/oauth/seed-providers.ts +1 -0
- package/src/permissions/checker.ts +15 -4
- package/src/permissions/defaults.ts +7 -8
- package/src/permissions/permission-mode-store.ts +180 -0
- package/src/permissions/permission-mode.ts +31 -0
- package/src/permissions/prompter.ts +0 -2
- package/src/permissions/workspace-policy.ts +9 -0
- package/src/platform/client.ts +1 -1
- package/src/prompts/system-prompt.ts +59 -7
- package/src/prompts/templates/BOOTSTRAP-REFERENCE.md +100 -0
- package/src/prompts/templates/BOOTSTRAP.md +76 -162
- package/src/prompts/templates/HEARTBEAT.md +3 -1
- package/src/prompts/templates/SOUL.md +30 -9
- package/src/prompts/templates/UPDATES.md +8 -0
- package/src/providers/anthropic/client.ts +107 -219
- package/src/runtime/assistant-event-hub.ts +22 -0
- package/src/runtime/auth/route-policy.ts +23 -0
- package/src/runtime/auth/token-service.ts +8 -0
- package/src/runtime/http-server.ts +32 -2
- package/src/runtime/http-types.ts +12 -1
- package/src/runtime/migrations/vbundle-builder.ts +389 -3
- package/src/runtime/migrations/vbundle-importer.ts +8 -6
- package/src/runtime/routes/__tests__/user-route-dispatcher.test.ts +378 -0
- package/src/runtime/routes/app-management-routes.ts +1 -11
- package/src/runtime/routes/approval-strategies/guardian-callback-strategy.ts +26 -0
- package/src/runtime/routes/archive-utils.ts +29 -0
- package/src/runtime/routes/avatar-routes.ts +2 -9
- package/src/runtime/routes/btw-routes.ts +14 -1
- package/src/runtime/routes/conversation-analysis-routes.ts +185 -0
- package/src/runtime/routes/conversation-management-routes.ts +1 -14
- package/src/runtime/routes/conversation-query-routes.ts +49 -3
- package/src/runtime/routes/conversation-routes.ts +270 -44
- package/src/runtime/routes/group-routes.ts +22 -8
- package/src/runtime/routes/heartbeat-routes.ts +4 -10
- package/src/runtime/routes/identity-routes.ts +53 -18
- package/src/runtime/routes/llm-context-normalization.ts +14 -10
- package/src/runtime/routes/log-export/AGENTS.md +104 -0
- package/src/runtime/routes/log-export/__tests__/workspace-allowlist-error-contract.test.ts +103 -0
- package/src/runtime/routes/log-export/__tests__/workspace-allowlist.test.ts +716 -0
- package/src/runtime/routes/log-export/workspace-allowlist.ts +458 -0
- package/src/runtime/routes/log-export-routes.ts +41 -278
- package/src/runtime/routes/memory-item-routes.test.ts +168 -233
- package/src/runtime/routes/migration-routes.ts +18 -7
- package/src/runtime/routes/profiler-routes.ts +350 -0
- package/src/runtime/routes/schedule-routes.ts +27 -12
- package/src/runtime/routes/settings-routes.ts +95 -8
- package/src/runtime/routes/subagents-routes.ts +28 -7
- package/src/runtime/routes/user-route-dispatcher.ts +223 -0
- package/src/runtime/routes/user-routes.ts +41 -0
- package/src/runtime/routes/workspace-routes.ts +0 -1
- package/src/schedule/schedule-store.ts +30 -0
- package/src/schedule/scheduler.ts +45 -18
- package/src/skills/catalog-install.ts +10 -2
- package/src/skills/inline-command-runner.ts +12 -14
- package/src/skills/managed-store.ts +2 -2
- package/src/skills/skill-memory.ts +1 -293
- package/src/subagent/index.ts +13 -3
- package/src/subagent/manager.ts +308 -29
- package/src/subagent/types.ts +68 -0
- package/src/tasks/task-runner.ts +4 -4
- package/src/tools/apps/executors.ts +29 -4
- package/src/tools/filesystem/list.ts +93 -0
- package/src/tools/permission-checker.ts +78 -18
- package/src/tools/registry.ts +4 -0
- package/src/tools/schedule/create.ts +3 -0
- package/src/tools/schedule/list.ts +1 -0
- package/src/tools/schedule/update.ts +6 -0
- package/src/tools/secret-detection-handler.ts +0 -1
- package/src/tools/shared/filesystem/errors.ts +5 -0
- package/src/tools/shared/filesystem/file-ops-service.ts +90 -2
- package/src/tools/shared/filesystem/types.ts +17 -0
- package/src/tools/shared/shell-output.ts +31 -2
- package/src/tools/skills/sandbox-runner.ts +3 -6
- package/src/tools/subagent/abort.ts +12 -2
- package/src/tools/subagent/message.ts +9 -2
- package/src/tools/subagent/notify-parent.ts +79 -0
- package/src/tools/subagent/read.ts +29 -8
- package/src/tools/subagent/resolve.ts +21 -0
- package/src/tools/subagent/spawn.ts +2 -0
- package/src/tools/subagent/status.ts +11 -1
- package/src/tools/system/avatar-generator.ts +3 -3
- package/src/tools/system/register.ts +23 -0
- package/src/tools/system/set-permission-mode.ts +103 -0
- package/src/tools/terminal/parser.ts +30 -5
- package/src/tools/terminal/safe-env.ts +16 -1
- package/src/tools/terminal/sandbox-diagnostics.ts +4 -4
- package/src/tools/terminal/sandbox.ts +4 -1
- package/src/tools/terminal/shell.ts +3 -5
- package/src/tools/tool-manifest.ts +6 -0
- package/src/tools/types.ts +2 -3
- package/src/util/logger.ts +1 -1
- package/src/util/platform.ts +50 -17
- package/src/watcher/provider-types.ts +1 -1
- package/src/workspace/migrations/023-move-config-files-to-workspace.ts +2 -2
- package/src/workspace/migrations/024-move-runtime-files-to-workspace.ts +2 -2
- package/src/workspace/migrations/028-recover-conversations-from-disk-view.ts +270 -0
- package/src/workspace/migrations/029-seed-pkb.ts +85 -0
- package/src/workspace/migrations/030-seed-pkb-autoinject.ts +73 -0
- package/src/workspace/migrations/registry.ts +6 -0
- package/src/workspace/top-level-renderer.ts +5 -9
- package/src/__tests__/cli-memory.test.ts +0 -377
- package/src/__tests__/clipboard.test.ts +0 -88
- package/src/cli/cli-memory.ts +0 -179
- package/src/util/clipboard.ts +0 -34
|
@@ -33,7 +33,7 @@ export const graphRecallDefinition: ToolDefinition = {
|
|
|
33
33
|
type: "string",
|
|
34
34
|
enum: ["memory", "archive"],
|
|
35
35
|
description:
|
|
36
|
-
'"memory" searches the living memory graph using semantic similarity (default). "archive" searches raw conversation transcripts using keyword matching
|
|
36
|
+
'"memory" searches the living memory graph using semantic similarity (default). "archive" searches raw conversation transcripts using keyword matching. Supports FTS5 syntax: use "quoted phrases" for exact matching, AND/OR for boolean logic, NEAR(word1 word2, N) for proximity. Without operators, all keywords must appear (implicit AND). Short words like "I" and "a" are ignored. Prefer "memory" for conceptual/emotional queries, "archive" for finding specific wording.',
|
|
37
37
|
},
|
|
38
38
|
filters: {
|
|
39
39
|
type: "object",
|
|
@@ -74,68 +74,24 @@ export const graphRecallDefinition: ToolDefinition = {
|
|
|
74
74
|
};
|
|
75
75
|
|
|
76
76
|
/**
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
* learns something worth remembering mid-conversation, it calls this
|
|
82
|
-
* tool rather than waiting for end-of-conversation extraction.
|
|
77
|
+
* Save a fact to the personal knowledge base. The fact is appended to
|
|
78
|
+
* buffer.md (immediately available in the next conversation) and the
|
|
79
|
+
* daily archive (permanent date-indexed record). Filing into topic
|
|
80
|
+
* files happens during the periodic filing job.
|
|
83
81
|
*/
|
|
84
82
|
export const graphRememberDefinition: ToolDefinition = {
|
|
85
83
|
name: "remember",
|
|
86
84
|
description:
|
|
87
|
-
"Save,
|
|
85
|
+
"Save a fact to your knowledge base. Call this AGGRESSIVELY — every preference, location, name, date, habit, opinion, health detail, plan, relationship fact, routine, or commitment. The bar is: if you wouldn't want to ask about it again, remember it. Examples: 'Prefers UberEats over DoorDash', 'Lives in NYC, from Texas', 'Takes 45mg nicotine daily, tapering', 'Girlfriend Yen is in Texas', 'Watches vampire show Saturday nights', 'NYU Summit April 10-11'. Call this multiple times per conversation — it's cheap (one line appended to a file). Don't wait until the end. Don't batch. Every new fact, immediately. Remembering too much is infinitely better than forgetting something that mattered.",
|
|
88
86
|
input_schema: {
|
|
89
87
|
type: "object",
|
|
90
88
|
properties: {
|
|
91
|
-
op: {
|
|
92
|
-
type: "string",
|
|
93
|
-
enum: ["save", "update", "delete"],
|
|
94
|
-
description: "The operation to perform",
|
|
95
|
-
},
|
|
96
|
-
memory_id: {
|
|
97
|
-
type: "string",
|
|
98
|
-
description: "ID of existing memory (required for update/delete)",
|
|
99
|
-
},
|
|
100
89
|
content: {
|
|
101
90
|
type: "string",
|
|
102
91
|
description:
|
|
103
|
-
"
|
|
104
|
-
},
|
|
105
|
-
type: {
|
|
106
|
-
type: "string",
|
|
107
|
-
enum: [
|
|
108
|
-
"episodic",
|
|
109
|
-
"semantic",
|
|
110
|
-
"procedural",
|
|
111
|
-
"emotional",
|
|
112
|
-
"prospective",
|
|
113
|
-
"behavioral",
|
|
114
|
-
"narrative",
|
|
115
|
-
"shared",
|
|
116
|
-
],
|
|
117
|
-
description: "Category of memory (required for save)",
|
|
118
|
-
},
|
|
119
|
-
significance: {
|
|
120
|
-
type: "number",
|
|
121
|
-
description:
|
|
122
|
-
"How important is this? 0-1. Mundane: 0.2-0.4, important: 0.5-0.7, life events: 0.8-1.0 (optional, defaults to 0.5)",
|
|
123
|
-
},
|
|
124
|
-
emotional_charge: {
|
|
125
|
-
type: "object",
|
|
126
|
-
description: "Emotional context (optional)",
|
|
127
|
-
properties: {
|
|
128
|
-
valence: {
|
|
129
|
-
type: "number",
|
|
130
|
-
description: "Positive vs negative (-1 to 1)",
|
|
131
|
-
},
|
|
132
|
-
intensity: {
|
|
133
|
-
type: "number",
|
|
134
|
-
description: "How strong the feeling (0 to 1)",
|
|
135
|
-
},
|
|
136
|
-
},
|
|
92
|
+
"The fact to remember. Write naturally — a preference, a detail, a commitment, a plan. No need to categorize.",
|
|
137
93
|
},
|
|
138
94
|
},
|
|
139
|
-
required: ["
|
|
95
|
+
required: ["content"],
|
|
140
96
|
},
|
|
141
97
|
};
|
|
@@ -253,6 +253,30 @@ export interface ScoredNode {
|
|
|
253
253
|
};
|
|
254
254
|
}
|
|
255
255
|
|
|
256
|
+
// ---------------------------------------------------------------------------
|
|
257
|
+
// Retrieval Metrics
|
|
258
|
+
// ---------------------------------------------------------------------------
|
|
259
|
+
|
|
260
|
+
export interface RetrievalMetrics {
|
|
261
|
+
semanticHits: number;
|
|
262
|
+
mergedCount: number;
|
|
263
|
+
selectedCount: number;
|
|
264
|
+
tier1Count: number;
|
|
265
|
+
tier2Count: number;
|
|
266
|
+
hybridSearchLatencyMs: number;
|
|
267
|
+
sparseVectorUsed: boolean;
|
|
268
|
+
embeddingProvider: string | null;
|
|
269
|
+
embeddingModel: string | null;
|
|
270
|
+
queryContext: string | null;
|
|
271
|
+
topCandidates: Array<{
|
|
272
|
+
nodeId: string;
|
|
273
|
+
type: string;
|
|
274
|
+
score: number;
|
|
275
|
+
semanticSimilarity: number;
|
|
276
|
+
recencyBoost: number;
|
|
277
|
+
}>;
|
|
278
|
+
}
|
|
279
|
+
|
|
256
280
|
// ---------------------------------------------------------------------------
|
|
257
281
|
// Results
|
|
258
282
|
// ---------------------------------------------------------------------------
|
package/src/memory/group-crud.ts
CHANGED
|
@@ -33,7 +33,7 @@ export function listGroups(): ConversationGroupRow[] {
|
|
|
33
33
|
created_at: number;
|
|
34
34
|
updated_at: number;
|
|
35
35
|
}>(
|
|
36
|
-
"SELECT id, name, sort_position, is_system_group, created_at, updated_at FROM conversation_groups WHERE id
|
|
36
|
+
"SELECT id, name, sort_position, is_system_group, created_at, updated_at FROM conversation_groups WHERE id NOT IN ('_backfill_complete', '_backfill_all_complete', '_sort_shift_complete') ORDER BY sort_position ASC",
|
|
37
37
|
);
|
|
38
38
|
return rows.map((r) => ({
|
|
39
39
|
id: r.id,
|
|
@@ -79,8 +79,8 @@ export function getGroup(groupId: string): ConversationGroupRow | null {
|
|
|
79
79
|
|
|
80
80
|
/**
|
|
81
81
|
* Create a custom group. Server assigns sort_position as max(custom) + 1.
|
|
82
|
-
* System groups occupy positions 0 (pinned
|
|
83
|
-
* First custom group gets position
|
|
82
|
+
* System groups occupy positions 0–3 (pinned, scheduled, background, all).
|
|
83
|
+
* First custom group gets position 4. Fallback ?? 3 ensures 3 + 1 = 4 when
|
|
84
84
|
* no custom groups exist.
|
|
85
85
|
*/
|
|
86
86
|
export function createGroup(name: string): ConversationGroupRow {
|
|
@@ -88,7 +88,7 @@ export function createGroup(name: string): ConversationGroupRow {
|
|
|
88
88
|
const maxPos =
|
|
89
89
|
rawGet<{ max: number | null }>(
|
|
90
90
|
"SELECT MAX(sort_position) as max FROM conversation_groups WHERE is_system_group = 0",
|
|
91
|
-
)?.max ??
|
|
91
|
+
)?.max ?? 3;
|
|
92
92
|
const sortPosition = maxPos + 1;
|
|
93
93
|
const id = uuid();
|
|
94
94
|
const now = Math.floor(Date.now() / 1000);
|
|
@@ -153,13 +153,16 @@ export function updateGroup(
|
|
|
153
153
|
// Delete
|
|
154
154
|
// ---------------------------------------------------------------------------
|
|
155
155
|
|
|
156
|
-
//
|
|
157
|
-
//
|
|
158
|
-
//
|
|
159
|
-
//
|
|
160
|
-
// non-existent group.
|
|
156
|
+
// Reassign conversations to system:all before deleting the group so they
|
|
157
|
+
// don't end up with NULL group_id (which would violate the system:all
|
|
158
|
+
// invariant). The FK ON DELETE SET NULL would otherwise leave NULLs that
|
|
159
|
+
// the one-time backfill won't re-fix.
|
|
161
160
|
export function deleteGroup(groupId: string): boolean {
|
|
162
161
|
ensureGroupMigration();
|
|
162
|
+
rawRun(
|
|
163
|
+
"UPDATE conversations SET group_id = 'system:all' WHERE group_id = ?",
|
|
164
|
+
groupId,
|
|
165
|
+
);
|
|
163
166
|
rawRun("DELETE FROM conversation_groups WHERE id = ?", groupId);
|
|
164
167
|
return true;
|
|
165
168
|
}
|
|
@@ -176,6 +179,19 @@ export function reorderGroups(
|
|
|
176
179
|
rawExec("BEGIN");
|
|
177
180
|
try {
|
|
178
181
|
for (const update of updates) {
|
|
182
|
+
// Look up the group first — skip unknown/stale IDs and system groups
|
|
183
|
+
const group = rawGet<{ id: string; is_system_group: number }>(
|
|
184
|
+
"SELECT id, is_system_group FROM conversation_groups WHERE id = ?",
|
|
185
|
+
update.groupId,
|
|
186
|
+
);
|
|
187
|
+
if (!group) continue;
|
|
188
|
+
if (group.is_system_group === 1) continue;
|
|
189
|
+
|
|
190
|
+
if (update.sortPosition < 4) {
|
|
191
|
+
throw new Error(
|
|
192
|
+
`Custom group sort_position must be >= 4 (got ${update.sortPosition} for ${update.groupId})`,
|
|
193
|
+
);
|
|
194
|
+
}
|
|
179
195
|
rawRun(
|
|
180
196
|
"UPDATE conversation_groups SET sort_position = ?, updated_at = ? WHERE id = ?",
|
|
181
197
|
update.sortPosition,
|
|
@@ -1,11 +1,54 @@
|
|
|
1
1
|
import type { AssistantConfig } from "../../config/types.js";
|
|
2
2
|
import { getLogger } from "../../util/logger.js";
|
|
3
|
-
import { getDb, rawAll, rawRun } from "../db.js";
|
|
3
|
+
import { getDb, rawAll, rawChanges, rawRun } from "../db.js";
|
|
4
4
|
import { enqueueMemoryJob, type MemoryJob } from "../jobs-store.js";
|
|
5
5
|
|
|
6
6
|
const log = getLogger("memory-jobs-worker");
|
|
7
7
|
|
|
8
8
|
const PRUNE_BATCH_LIMIT = 100;
|
|
9
|
+
const PRUNE_LOG_BATCH_LIMIT = 1000;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Delete LLM request/response logs older than the configured retention period.
|
|
13
|
+
* Processes in batches to avoid long DB locks and excessive WAL growth.
|
|
14
|
+
* Re-enqueues itself if more rows remain.
|
|
15
|
+
*/
|
|
16
|
+
export function pruneOldLlmRequestLogsJob(
|
|
17
|
+
job: MemoryJob,
|
|
18
|
+
config: AssistantConfig,
|
|
19
|
+
): void {
|
|
20
|
+
const retentionMs =
|
|
21
|
+
typeof job.payload.retentionMs === "number" &&
|
|
22
|
+
Number.isFinite(job.payload.retentionMs) &&
|
|
23
|
+
job.payload.retentionMs >= 0
|
|
24
|
+
? job.payload.retentionMs
|
|
25
|
+
: config.memory.cleanup.llmRequestLogRetentionMs;
|
|
26
|
+
|
|
27
|
+
// 0 means disabled
|
|
28
|
+
if (retentionMs === 0) return;
|
|
29
|
+
|
|
30
|
+
const cutoffMs = Date.now() - retentionMs;
|
|
31
|
+
|
|
32
|
+
rawRun(
|
|
33
|
+
`DELETE FROM llm_request_logs WHERE rowid IN (SELECT rowid FROM llm_request_logs WHERE created_at < ? LIMIT ?)`,
|
|
34
|
+
cutoffMs,
|
|
35
|
+
PRUNE_LOG_BATCH_LIMIT,
|
|
36
|
+
);
|
|
37
|
+
const deleted = rawChanges();
|
|
38
|
+
|
|
39
|
+
if (deleted >= PRUNE_LOG_BATCH_LIMIT) {
|
|
40
|
+
enqueueMemoryJob("prune_old_llm_request_logs", { retentionMs });
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
log.info(
|
|
44
|
+
{
|
|
45
|
+
deleted,
|
|
46
|
+
retentionMs,
|
|
47
|
+
cutoffMs,
|
|
48
|
+
},
|
|
49
|
+
"Pruned old LLM request logs",
|
|
50
|
+
);
|
|
51
|
+
}
|
|
9
52
|
|
|
10
53
|
/**
|
|
11
54
|
* Delete conversations that have had no activity (updatedAt) for longer than
|
package/src/memory/jobs-store.ts
CHANGED
|
@@ -14,16 +14,9 @@ const log = getLogger("memory-jobs-store");
|
|
|
14
14
|
|
|
15
15
|
export type MemoryJobType =
|
|
16
16
|
| "embed_segment"
|
|
17
|
-
| "embed_item"
|
|
18
17
|
| "embed_summary"
|
|
19
|
-
| "extract_items"
|
|
20
|
-
| "extract_entities"
|
|
21
|
-
| "batch_extract"
|
|
22
|
-
| "cleanup_stale_superseded_items" // legacy compat — silently dropped by worker (memory_items table dropped)
|
|
23
18
|
| "prune_old_conversations"
|
|
24
|
-
| "
|
|
25
|
-
| "refresh_weekly_summary"
|
|
26
|
-
| "refresh_monthly_summary"
|
|
19
|
+
| "prune_old_llm_request_logs"
|
|
27
20
|
| "build_conversation_summary"
|
|
28
21
|
| "backfill"
|
|
29
22
|
| "rebuild_index"
|
|
@@ -32,7 +25,6 @@ export type MemoryJobType =
|
|
|
32
25
|
| "embed_media"
|
|
33
26
|
| "embed_attachment"
|
|
34
27
|
| "generate_conversation_starters"
|
|
35
|
-
| "journal_carry_forward"
|
|
36
28
|
| "embed_graph_node"
|
|
37
29
|
| "graph_extract"
|
|
38
30
|
| "graph_decay"
|
|
@@ -40,13 +32,10 @@ export type MemoryJobType =
|
|
|
40
32
|
| "graph_pattern_scan"
|
|
41
33
|
| "graph_narrative_refine"
|
|
42
34
|
| "graph_trigger_embed"
|
|
43
|
-
| "graph_bootstrap"
|
|
44
|
-
| "generate_capability_cards" // legacy compat — silently dropped by worker (capability cards removed)
|
|
45
|
-
| "generate_thread_starters"; // legacy compat — silently dropped by worker (renamed to generate_conversation_starters)
|
|
35
|
+
| "graph_bootstrap";
|
|
46
36
|
|
|
47
37
|
const EMBED_JOB_TYPES: MemoryJobType[] = [
|
|
48
38
|
"embed_segment",
|
|
49
|
-
"embed_item",
|
|
50
39
|
"embed_summary",
|
|
51
40
|
"embed_media",
|
|
52
41
|
"embed_attachment",
|
|
@@ -73,10 +62,10 @@ export function enqueueMemoryJob(
|
|
|
73
62
|
payload: Record<string, unknown>,
|
|
74
63
|
runAfter = Date.now(),
|
|
75
64
|
dbOverride?: Parameters<ReturnType<typeof getDb>["transaction"]>[0] extends (
|
|
76
|
-
tx: infer T
|
|
65
|
+
tx: infer T,
|
|
77
66
|
) => unknown
|
|
78
67
|
? T
|
|
79
|
-
: never
|
|
68
|
+
: never,
|
|
80
69
|
): string {
|
|
81
70
|
const db = dbOverride ?? getDb();
|
|
82
71
|
const id = uuid();
|
|
@@ -111,10 +100,10 @@ export function upsertDebouncedJob(
|
|
|
111
100
|
payload: { conversationId: string },
|
|
112
101
|
runAfter: number,
|
|
113
102
|
dbOverride?: Parameters<ReturnType<typeof getDb>["transaction"]>[0] extends (
|
|
114
|
-
tx: infer T
|
|
103
|
+
tx: infer T,
|
|
115
104
|
) => unknown
|
|
116
105
|
? T
|
|
117
|
-
: never
|
|
106
|
+
: never,
|
|
118
107
|
): void {
|
|
119
108
|
const db = dbOverride ?? getDb();
|
|
120
109
|
const existing = db
|
|
@@ -124,8 +113,8 @@ export function upsertDebouncedJob(
|
|
|
124
113
|
and(
|
|
125
114
|
eq(memoryJobs.type, type),
|
|
126
115
|
eq(memoryJobs.status, "pending"),
|
|
127
|
-
sql`json_extract(${memoryJobs.payload}, '$.conversationId') = ${payload.conversationId}
|
|
128
|
-
)
|
|
116
|
+
sql`json_extract(${memoryJobs.payload}, '$.conversationId') = ${payload.conversationId}`,
|
|
117
|
+
),
|
|
129
118
|
)
|
|
130
119
|
.get();
|
|
131
120
|
if (existing) {
|
|
@@ -138,32 +127,6 @@ export function upsertDebouncedJob(
|
|
|
138
127
|
}
|
|
139
128
|
}
|
|
140
129
|
|
|
141
|
-
/**
|
|
142
|
-
* Check whether a pending or running `journal_carry_forward` job already
|
|
143
|
-
* exists for the given filename + userSlug. Used to prevent duplicate
|
|
144
|
-
* enqueues while a carry-forward job is still in flight.
|
|
145
|
-
*/
|
|
146
|
-
export function hasActiveCarryForwardJob(
|
|
147
|
-
filename: string,
|
|
148
|
-
userSlug: string
|
|
149
|
-
): boolean {
|
|
150
|
-
const db = getDb();
|
|
151
|
-
return (
|
|
152
|
-
db
|
|
153
|
-
.select({ id: memoryJobs.id })
|
|
154
|
-
.from(memoryJobs)
|
|
155
|
-
.where(
|
|
156
|
-
and(
|
|
157
|
-
eq(memoryJobs.type, "journal_carry_forward"),
|
|
158
|
-
inArray(memoryJobs.status, ["pending", "running"]),
|
|
159
|
-
sql`json_extract(${memoryJobs.payload}, '$.filename') = ${filename}`,
|
|
160
|
-
sql`json_extract(${memoryJobs.payload}, '$.userSlug') = ${userSlug}`
|
|
161
|
-
)
|
|
162
|
-
)
|
|
163
|
-
.get() != null
|
|
164
|
-
);
|
|
165
|
-
}
|
|
166
|
-
|
|
167
130
|
/**
|
|
168
131
|
* Check whether a pending or running job of the given type already exists.
|
|
169
132
|
* Used to prevent duplicate enqueues for long-running maintenance jobs.
|
|
@@ -177,15 +140,62 @@ export function hasActiveJobOfType(type: MemoryJobType): boolean {
|
|
|
177
140
|
.where(
|
|
178
141
|
and(
|
|
179
142
|
eq(memoryJobs.type, type),
|
|
180
|
-
inArray(memoryJobs.status, ["pending", "running"])
|
|
181
|
-
)
|
|
143
|
+
inArray(memoryJobs.status, ["pending", "running"]),
|
|
144
|
+
),
|
|
182
145
|
)
|
|
183
146
|
.get() != null
|
|
184
147
|
);
|
|
185
148
|
}
|
|
186
149
|
|
|
150
|
+
export function enqueuePruneOldLlmRequestLogsJob(retentionMs?: number): string {
|
|
151
|
+
const db = getDb();
|
|
152
|
+
const existing = db
|
|
153
|
+
.select()
|
|
154
|
+
.from(memoryJobs)
|
|
155
|
+
.where(
|
|
156
|
+
and(
|
|
157
|
+
eq(memoryJobs.type, "prune_old_llm_request_logs"),
|
|
158
|
+
inArray(memoryJobs.status, ["pending", "running"]),
|
|
159
|
+
),
|
|
160
|
+
)
|
|
161
|
+
.orderBy(asc(memoryJobs.createdAt))
|
|
162
|
+
.get();
|
|
163
|
+
if (existing) {
|
|
164
|
+
if (
|
|
165
|
+
existing.status === "pending" &&
|
|
166
|
+
typeof retentionMs === "number" &&
|
|
167
|
+
Number.isFinite(retentionMs) &&
|
|
168
|
+
retentionMs >= 0
|
|
169
|
+
) {
|
|
170
|
+
let payload: Record<string, unknown> = {};
|
|
171
|
+
try {
|
|
172
|
+
payload = JSON.parse(existing.payload) as Record<string, unknown>;
|
|
173
|
+
} catch {
|
|
174
|
+
payload = {};
|
|
175
|
+
}
|
|
176
|
+
if (payload.retentionMs !== retentionMs) {
|
|
177
|
+
db.update(memoryJobs)
|
|
178
|
+
.set({
|
|
179
|
+
payload: JSON.stringify({ ...payload, retentionMs }),
|
|
180
|
+
updatedAt: Date.now(),
|
|
181
|
+
})
|
|
182
|
+
.where(eq(memoryJobs.id, existing.id))
|
|
183
|
+
.run();
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return existing.id;
|
|
187
|
+
}
|
|
188
|
+
const payload =
|
|
189
|
+
typeof retentionMs === "number" &&
|
|
190
|
+
Number.isFinite(retentionMs) &&
|
|
191
|
+
retentionMs >= 0
|
|
192
|
+
? { retentionMs }
|
|
193
|
+
: {};
|
|
194
|
+
return enqueueMemoryJob("prune_old_llm_request_logs", payload);
|
|
195
|
+
}
|
|
196
|
+
|
|
187
197
|
export function enqueuePruneOldConversationsJob(
|
|
188
|
-
retentionDays?: number
|
|
198
|
+
retentionDays?: number,
|
|
189
199
|
): string {
|
|
190
200
|
const db = getDb();
|
|
191
201
|
const existing = db
|
|
@@ -194,8 +204,8 @@ export function enqueuePruneOldConversationsJob(
|
|
|
194
204
|
.where(
|
|
195
205
|
and(
|
|
196
206
|
eq(memoryJobs.type, "prune_old_conversations"),
|
|
197
|
-
inArray(memoryJobs.status, ["pending", "running"])
|
|
198
|
-
)
|
|
207
|
+
inArray(memoryJobs.status, ["pending", "running"]),
|
|
208
|
+
),
|
|
199
209
|
)
|
|
200
210
|
.orderBy(asc(memoryJobs.createdAt))
|
|
201
211
|
.get();
|
|
@@ -239,7 +249,7 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
|
|
|
239
249
|
const now = Date.now();
|
|
240
250
|
const pendingFilter = and(
|
|
241
251
|
eq(memoryJobs.status, "pending"),
|
|
242
|
-
lte(memoryJobs.runAfter, now)
|
|
252
|
+
lte(memoryJobs.runAfter, now),
|
|
243
253
|
);
|
|
244
254
|
|
|
245
255
|
// Claim non-embed jobs first, then fill remaining slots with embed jobs.
|
|
@@ -268,7 +278,7 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
|
|
|
268
278
|
}
|
|
269
279
|
if (probeAllowed && remainingSlots > 0) {
|
|
270
280
|
log.debug(
|
|
271
|
-
"Allowing 1 embed probe job — Qdrant circuit breaker cooldown elapsed"
|
|
281
|
+
"Allowing 1 embed probe job — Qdrant circuit breaker cooldown elapsed",
|
|
272
282
|
);
|
|
273
283
|
}
|
|
274
284
|
|
|
@@ -298,7 +308,7 @@ export function claimMemoryJobs(limit: number): MemoryJob[] {
|
|
|
298
308
|
status: "running",
|
|
299
309
|
startedAt: now,
|
|
300
310
|
updatedAt: now,
|
|
301
|
-
})
|
|
311
|
+
}),
|
|
302
312
|
);
|
|
303
313
|
}
|
|
304
314
|
return claimed;
|
|
@@ -343,7 +353,7 @@ export function deferMemoryJob(id: string): "deferred" | "failed" {
|
|
|
343
353
|
if (deferrals >= MAX_DEFERRALS) {
|
|
344
354
|
log.error(
|
|
345
355
|
{ jobId: id, type: row.type, deferrals },
|
|
346
|
-
"Job exceeded max deferrals, marking as failed"
|
|
356
|
+
"Job exceeded max deferrals, marking as failed",
|
|
347
357
|
);
|
|
348
358
|
db.update(memoryJobs)
|
|
349
359
|
.set({
|
|
@@ -362,14 +372,14 @@ export function deferMemoryJob(id: string): "deferred" | "failed" {
|
|
|
362
372
|
if (DEFERRAL_WARN_MILESTONES.includes(deferrals)) {
|
|
363
373
|
log.warn(
|
|
364
374
|
{ jobId: id, type: row.type, deferrals, max: MAX_DEFERRALS },
|
|
365
|
-
"Job approaching max deferral limit"
|
|
375
|
+
"Job approaching max deferral limit",
|
|
366
376
|
);
|
|
367
377
|
}
|
|
368
378
|
|
|
369
379
|
// Exponential backoff: 30s, 60s, 120s, ... capped at 5 minutes
|
|
370
380
|
const delay = Math.min(
|
|
371
381
|
DEFER_BASE_DELAY_MS * Math.pow(2, Math.min(deferrals - 1, 10)),
|
|
372
|
-
DEFER_MAX_DELAY_MS
|
|
382
|
+
DEFER_MAX_DELAY_MS,
|
|
373
383
|
);
|
|
374
384
|
db.update(memoryJobs)
|
|
375
385
|
.set({
|
|
@@ -386,7 +396,7 @@ export function deferMemoryJob(id: string): "deferred" | "failed" {
|
|
|
386
396
|
export function failMemoryJob(
|
|
387
397
|
id: string,
|
|
388
398
|
error: string,
|
|
389
|
-
options?: { retryDelayMs?: number; maxAttempts?: number }
|
|
399
|
+
options?: { retryDelayMs?: number; maxAttempts?: number },
|
|
390
400
|
): void {
|
|
391
401
|
const retryDelayMs = options?.retryDelayMs ?? 30_000;
|
|
392
402
|
const maxAttempts = options?.maxAttempts ?? 5;
|
|
@@ -443,7 +453,7 @@ export function failStalledJobs(timeoutMs: number): number {
|
|
|
443
453
|
AND started_at IS NOT NULL
|
|
444
454
|
AND started_at < ?
|
|
445
455
|
`,
|
|
446
|
-
cutoff
|
|
456
|
+
cutoff,
|
|
447
457
|
);
|
|
448
458
|
if (stalled.length === 0) return 0;
|
|
449
459
|
|
|
@@ -454,14 +464,14 @@ export function failStalledJobs(timeoutMs: number): number {
|
|
|
454
464
|
status: "failed",
|
|
455
465
|
updatedAt: now,
|
|
456
466
|
lastError: `Job timed out after ${Math.round(
|
|
457
|
-
timeoutMs / 60_000
|
|
467
|
+
timeoutMs / 60_000,
|
|
458
468
|
)} minutes`,
|
|
459
469
|
})
|
|
460
470
|
.where(and(eq(memoryJobs.id, row.id), eq(memoryJobs.status, "running")))
|
|
461
471
|
.run();
|
|
462
472
|
log.warn(
|
|
463
473
|
{ jobId: row.id, type: row.type, timeoutMs },
|
|
464
|
-
"Failed stalled memory job due to timeout"
|
|
474
|
+
"Failed stalled memory job due to timeout",
|
|
465
475
|
);
|
|
466
476
|
}
|
|
467
477
|
return stalled.length;
|
|
@@ -2,6 +2,7 @@ import { getConfig } from "../config/loader.js";
|
|
|
2
2
|
import type { AssistantConfig } from "../config/types.js";
|
|
3
3
|
import { getLogger } from "../util/logger.js";
|
|
4
4
|
import { getMemoryCheckpoint, setMemoryCheckpoint } from "./checkpoints.js";
|
|
5
|
+
import { bootstrapFromHistory } from "./graph/bootstrap.js";
|
|
5
6
|
import { runConsolidation } from "./graph/consolidation.js";
|
|
6
7
|
import { runDecayTick } from "./graph/decay.js";
|
|
7
8
|
import { graphExtractJob } from "./graph/extraction-job.js";
|
|
@@ -12,7 +13,10 @@ import {
|
|
|
12
13
|
import { runNarrativeRefinement } from "./graph/narrative.js";
|
|
13
14
|
import { runPatternScan } from "./graph/pattern-scan.js";
|
|
14
15
|
import { backfillJob } from "./job-handlers/backfill.js";
|
|
15
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
pruneOldConversationsJob,
|
|
18
|
+
pruneOldLlmRequestLogsJob,
|
|
19
|
+
} from "./job-handlers/cleanup.js";
|
|
16
20
|
import { generateConversationStartersJob } from "./job-handlers/conversation-starters.js";
|
|
17
21
|
// ── Per-job-type handlers ──────────────────────────────────────────
|
|
18
22
|
import {
|
|
@@ -39,6 +43,7 @@ import {
|
|
|
39
43
|
deferMemoryJob,
|
|
40
44
|
enqueueMemoryJob,
|
|
41
45
|
enqueuePruneOldConversationsJob,
|
|
46
|
+
enqueuePruneOldLlmRequestLogsJob,
|
|
42
47
|
failMemoryJob,
|
|
43
48
|
failStalledJobs,
|
|
44
49
|
type MemoryJob,
|
|
@@ -49,6 +54,24 @@ import { QdrantCircuitOpenError } from "./qdrant-circuit-breaker.js";
|
|
|
49
54
|
|
|
50
55
|
const log = getLogger("memory-jobs-worker");
|
|
51
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Job types whose handlers have been removed. Existing rows may still sit in
|
|
59
|
+
* the database — the worker completes them silently instead of throwing.
|
|
60
|
+
*/
|
|
61
|
+
const LEGACY_JOB_TYPES = new Set([
|
|
62
|
+
"embed_item",
|
|
63
|
+
"extract_items",
|
|
64
|
+
"batch_extract",
|
|
65
|
+
"extract_entities",
|
|
66
|
+
"cleanup_stale_superseded_items",
|
|
67
|
+
"backfill_entity_relations",
|
|
68
|
+
"refresh_weekly_summary",
|
|
69
|
+
"refresh_monthly_summary",
|
|
70
|
+
"journal_carry_forward",
|
|
71
|
+
"generate_capability_cards",
|
|
72
|
+
"generate_thread_starters",
|
|
73
|
+
]);
|
|
74
|
+
|
|
52
75
|
export const POLL_INTERVAL_MIN_MS = 1_500;
|
|
53
76
|
export const POLL_INTERVAL_MAX_MS = 30_000;
|
|
54
77
|
|
|
@@ -355,32 +378,21 @@ async function processJob(
|
|
|
355
378
|
case "embed_segment":
|
|
356
379
|
await embedSegmentJob(job, config);
|
|
357
380
|
return;
|
|
358
|
-
case "embed_item":
|
|
359
|
-
case "extract_items":
|
|
360
|
-
case "batch_extract":
|
|
361
|
-
case "extract_entities":
|
|
362
|
-
case "cleanup_stale_superseded_items":
|
|
363
|
-
// Old extraction pipeline replaced by memory graph — silently drop
|
|
364
|
-
return;
|
|
365
381
|
case "embed_summary":
|
|
366
382
|
await embedSummaryJob(job, config);
|
|
367
383
|
return;
|
|
368
384
|
case "prune_old_conversations":
|
|
369
385
|
pruneOldConversationsJob(job, config);
|
|
370
386
|
return;
|
|
387
|
+
case "prune_old_llm_request_logs":
|
|
388
|
+
pruneOldLlmRequestLogsJob(job, config);
|
|
389
|
+
return;
|
|
371
390
|
case "build_conversation_summary":
|
|
372
391
|
await buildConversationSummaryJob(job, config);
|
|
373
392
|
return;
|
|
374
393
|
case "backfill":
|
|
375
394
|
await backfillJob(job, config);
|
|
376
395
|
return;
|
|
377
|
-
case "backfill_entity_relations":
|
|
378
|
-
// Entity relation backfill has been removed — silently drop legacy jobs
|
|
379
|
-
return;
|
|
380
|
-
case "refresh_weekly_summary":
|
|
381
|
-
case "refresh_monthly_summary":
|
|
382
|
-
// Global summary rollups have been removed — silently drop legacy jobs
|
|
383
|
-
return;
|
|
384
396
|
case "rebuild_index":
|
|
385
397
|
await rebuildIndexJob();
|
|
386
398
|
return;
|
|
@@ -396,9 +408,6 @@ async function processJob(
|
|
|
396
408
|
case "embed_attachment":
|
|
397
409
|
await embedAttachmentJob(job, config);
|
|
398
410
|
return;
|
|
399
|
-
case "journal_carry_forward":
|
|
400
|
-
// Journal carry-forward replaced by graph extraction — silently drop
|
|
401
|
-
return;
|
|
402
411
|
case "embed_graph_node":
|
|
403
412
|
await embedGraphNodeJob(job, config);
|
|
404
413
|
return;
|
|
@@ -423,16 +432,18 @@ async function processJob(
|
|
|
423
432
|
case "generate_conversation_starters":
|
|
424
433
|
await generateConversationStartersJob(job);
|
|
425
434
|
return;
|
|
426
|
-
case "
|
|
427
|
-
|
|
428
|
-
return;
|
|
429
|
-
case "generate_thread_starters":
|
|
430
|
-
// Thread starters renamed to conversation starters — silently drop legacy jobs
|
|
435
|
+
case "graph_bootstrap":
|
|
436
|
+
await bootstrapFromHistory();
|
|
431
437
|
return;
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
)
|
|
438
|
+
|
|
439
|
+
default: {
|
|
440
|
+
const rawType = (job as { type: string }).type;
|
|
441
|
+
if (LEGACY_JOB_TYPES.has(rawType)) {
|
|
442
|
+
log.debug({ jobId: job.id, type: rawType }, "Dropping legacy job");
|
|
443
|
+
return;
|
|
444
|
+
}
|
|
445
|
+
throw new Error(`Unknown memory job type: ${rawType}`);
|
|
446
|
+
}
|
|
436
447
|
}
|
|
437
448
|
}
|
|
438
449
|
|
|
@@ -462,12 +473,18 @@ export function maybeEnqueueScheduledCleanupJobs(
|
|
|
462
473
|
cleanup.conversationRetentionDays > 0
|
|
463
474
|
? enqueuePruneOldConversationsJob(cleanup.conversationRetentionDays)
|
|
464
475
|
: null;
|
|
476
|
+
const pruneLlmRequestLogsJobId =
|
|
477
|
+
cleanup.llmRequestLogRetentionMs > 0
|
|
478
|
+
? enqueuePruneOldLlmRequestLogsJob(cleanup.llmRequestLogRetentionMs)
|
|
479
|
+
: null;
|
|
465
480
|
lastScheduledCleanupEnqueueMs = nowMs;
|
|
466
481
|
log.debug(
|
|
467
482
|
{
|
|
468
483
|
pruneConversationsJobId,
|
|
484
|
+
pruneLlmRequestLogsJobId,
|
|
469
485
|
enqueueIntervalMs: cleanup.enqueueIntervalMs,
|
|
470
486
|
conversationRetentionDays: cleanup.conversationRetentionDays,
|
|
487
|
+
llmRequestLogRetentionMs: cleanup.llmRequestLogRetentionMs,
|
|
471
488
|
},
|
|
472
489
|
"Enqueued scheduled memory cleanup jobs",
|
|
473
490
|
);
|
|
@@ -529,4 +546,3 @@ function maybeEnqueueGraphMaintenanceJobs(nowMs = Date.now()): void {
|
|
|
529
546
|
}
|
|
530
547
|
}
|
|
531
548
|
}
|
|
532
|
-
|