@vellumai/assistant 0.3.15 → 0.3.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +142 -0
- package/Dockerfile +1 -1
- package/README.md +5 -5
- package/docs/architecture/http-token-refresh.md +252 -0
- package/docs/architecture/memory.md +5 -4
- package/docs/architecture/scheduling.md +4 -88
- package/docs/runbook-trusted-contacts.md +283 -0
- package/docs/trusted-contact-access.md +247 -0
- package/package.json +1 -1
- package/scripts/ipc/check-swift-decoder-drift.ts +2 -0
- package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +2 -6
- package/src/__tests__/access-request-decision.test.ts +331 -0
- package/src/__tests__/asset-materialize-tool.test.ts +7 -7
- package/src/__tests__/asset-search-tool.test.ts +15 -15
- package/src/__tests__/attachments-store.test.ts +13 -13
- package/src/__tests__/call-controller.test.ts +150 -4
- package/src/__tests__/call-conversation-messages.test.ts +2 -2
- package/src/__tests__/call-pointer-messages.test.ts +28 -0
- package/src/__tests__/call-start-guardian-guard.test.ts +93 -0
- package/src/__tests__/channel-approval-routes.test.ts +108 -12
- package/src/__tests__/channel-guardian.test.ts +16 -14
- package/src/__tests__/checker.test.ts +24 -0
- package/src/__tests__/computer-use-skill-manifest-regression.test.ts +2 -2
- package/src/__tests__/config-watcher.test.ts +358 -0
- package/src/__tests__/conversation-pairing.test.ts +24 -24
- package/src/__tests__/conversation-store.test.ts +36 -36
- package/src/__tests__/date-context.test.ts +179 -1
- package/src/__tests__/db-migration-rollback.test.ts +4 -7
- package/src/__tests__/deterministic-verification-control-plane.test.ts +5 -5
- package/src/__tests__/emit-signal-routing-intent.test.ts +179 -0
- package/src/__tests__/gateway-only-guard.test.ts +188 -0
- package/src/__tests__/guardian-action-conversation-turn.test.ts +451 -0
- package/src/__tests__/guardian-action-copy-generator.test.ts +197 -0
- package/src/__tests__/guardian-action-followup-executor.test.ts +379 -0
- package/src/__tests__/guardian-action-followup-store.test.ts +376 -0
- package/src/__tests__/guardian-action-late-reply.test.ts +294 -0
- package/src/__tests__/guardian-action-no-hardcoded-copy.test.ts +71 -0
- package/src/__tests__/guardian-action-sweep.test.ts +9 -9
- package/src/__tests__/guardian-outbound-http.test.ts +194 -2
- package/src/__tests__/guardian-verification-intent-routing.test.ts +179 -0
- package/src/__tests__/guardian-verify-setup-skill-regression.test.ts +141 -0
- package/src/__tests__/handlers-telegram-config.test.ts +6 -6
- package/src/__tests__/hooks-runner.test.ts +13 -4
- package/src/__tests__/ingress-routes-http.test.ts +443 -0
- package/src/__tests__/intent-routing.test.ts +14 -0
- package/src/__tests__/ipc-snapshot.test.ts +2 -5
- package/src/__tests__/media-reuse-story.e2e.test.ts +7 -7
- package/src/__tests__/memory-regressions.test.ts +16 -12
- package/src/__tests__/non-member-access-request.test.ts +282 -0
- package/src/__tests__/notification-decision-strategy.test.ts +136 -0
- package/src/__tests__/notification-routing-intent.test.ts +11 -1
- package/src/__tests__/notification-thread-candidates.test.ts +166 -0
- package/src/__tests__/recording-intent.test.ts +1 -0
- package/src/__tests__/recording-state-machine.test.ts +328 -17
- package/src/__tests__/registry.test.ts +17 -8
- package/src/__tests__/relay-server.test.ts +105 -0
- package/src/__tests__/reminder.test.ts +13 -0
- package/src/__tests__/runtime-attachment-metadata.test.ts +4 -4
- package/src/__tests__/scheduler-recurrence.test.ts +50 -0
- package/src/__tests__/server-history-render.test.ts +8 -8
- package/src/__tests__/session-agent-loop.test.ts +1 -0
- package/src/__tests__/session-runtime-assembly.test.ts +49 -0
- package/src/__tests__/session-skill-tools.test.ts +1 -0
- package/src/__tests__/skill-projection.benchmark.test.ts +11 -3
- package/src/__tests__/slack-channel-config.test.ts +230 -0
- package/src/__tests__/subagent-manager-notify.test.ts +4 -4
- package/src/__tests__/swarm-session-integration.test.ts +2 -2
- package/src/__tests__/system-prompt.test.ts +43 -0
- package/src/__tests__/task-management-tools.test.ts +3 -3
- package/src/__tests__/task-tools.test.ts +3 -3
- package/src/__tests__/trust-store.test.ts +17 -1
- package/src/__tests__/trusted-contact-lifecycle-notifications.test.ts +491 -0
- package/src/__tests__/trusted-contact-multichannel.test.ts +409 -0
- package/src/__tests__/trusted-contact-verification.test.ts +360 -0
- package/src/__tests__/update-bulletin-format.test.ts +119 -0
- package/src/__tests__/update-bulletin-state.test.ts +129 -0
- package/src/__tests__/update-bulletin.test.ts +260 -0
- package/src/__tests__/update-template-contract.test.ts +29 -0
- package/src/agent/loop.ts +2 -2
- package/src/amazon/client.ts +2 -3
- package/src/calls/call-controller.ts +115 -34
- package/src/calls/call-conversation-messages.ts +2 -2
- package/src/calls/call-domain.ts +10 -3
- package/src/calls/call-pointer-messages.ts +17 -5
- package/src/calls/guardian-action-sweep.ts +77 -36
- package/src/calls/relay-server.ts +51 -12
- package/src/calls/twilio-routes.ts +3 -1
- package/src/calls/types.ts +1 -1
- package/src/calls/voice-session-bridge.ts +4 -4
- package/src/cli/core-commands.ts +3 -3
- package/src/cli/map.ts +8 -5
- package/src/config/bundled-skills/phone-calls/SKILL.md +16 -1
- package/src/config/bundled-skills/tasks/SKILL.md +1 -1
- package/src/config/bundled-skills/tasks/TOOLS.json +4 -4
- package/src/config/bundled-skills/time-based-actions/SKILL.md +11 -1
- package/src/config/computer-use-prompt.ts +1 -0
- package/src/config/core-schema.ts +16 -0
- package/src/config/env-registry.ts +1 -0
- package/src/config/env.ts +16 -1
- package/src/config/memory-schema.ts +5 -0
- package/src/config/schema.ts +4 -0
- package/src/config/system-prompt.ts +69 -2
- package/src/config/templates/BOOTSTRAP.md +1 -1
- package/src/config/templates/IDENTITY.md +8 -4
- package/src/config/templates/SOUL.md +14 -0
- package/src/config/templates/UPDATES.md +16 -0
- package/src/config/templates/USER.md +5 -1
- package/src/config/types.ts +1 -0
- package/src/config/update-bulletin-format.ts +52 -0
- package/src/config/update-bulletin-state.ts +49 -0
- package/src/config/update-bulletin.ts +82 -0
- package/src/config/vellum-skills/catalog.json +6 -0
- package/src/config/vellum-skills/chatgpt-import/tools/chatgpt-import.ts +1 -1
- package/src/config/vellum-skills/guardian-verify-setup/SKILL.md +44 -10
- package/src/config/vellum-skills/telegram-setup/SKILL.md +4 -4
- package/src/config/vellum-skills/trusted-contacts/SKILL.md +147 -0
- package/src/config/vellum-skills/twilio-setup/SKILL.md +2 -2
- package/src/context/window-manager.ts +43 -3
- package/src/daemon/config-watcher.ts +1 -0
- package/src/daemon/connection-policy.ts +21 -1
- package/src/daemon/daemon-control.ts +164 -7
- package/src/daemon/date-context.ts +174 -1
- package/src/daemon/guardian-action-generators.ts +175 -0
- package/src/daemon/guardian-verification-intent.ts +120 -0
- package/src/daemon/handlers/apps.ts +1 -3
- package/src/daemon/handlers/config-channels.ts +2 -2
- package/src/daemon/handlers/config-heartbeat.ts +1 -1
- package/src/daemon/handlers/config-inbox.ts +55 -159
- package/src/daemon/handlers/config-ingress.ts +1 -1
- package/src/daemon/handlers/config-integrations.ts +1 -1
- package/src/daemon/handlers/config-platform.ts +1 -1
- package/src/daemon/handlers/config-scheduling.ts +2 -2
- package/src/daemon/handlers/config-slack-channel.ts +190 -0
- package/src/daemon/handlers/config-telegram.ts +1 -1
- package/src/daemon/handlers/config-twilio.ts +1 -1
- package/src/daemon/handlers/config-voice.ts +100 -0
- package/src/daemon/handlers/config.ts +3 -0
- package/src/daemon/handlers/misc.ts +83 -5
- package/src/daemon/handlers/navigate-settings.ts +27 -0
- package/src/daemon/handlers/recording.ts +270 -144
- package/src/daemon/handlers/sessions.ts +100 -17
- package/src/daemon/handlers/subagents.ts +3 -3
- package/src/daemon/handlers/work-items.ts +10 -7
- package/src/daemon/ipc-contract/integrations.ts +9 -1
- package/src/daemon/ipc-contract/messages.ts +4 -0
- package/src/daemon/ipc-contract/sessions.ts +1 -1
- package/src/daemon/ipc-contract/settings.ts +26 -0
- package/src/daemon/ipc-contract/shared.ts +2 -0
- package/src/daemon/ipc-contract/work-items.ts +1 -7
- package/src/daemon/ipc-contract-inventory.json +5 -1
- package/src/daemon/ipc-contract.ts +5 -1
- package/src/daemon/lifecycle.ts +306 -266
- package/src/daemon/recording-intent.ts +0 -41
- package/src/daemon/response-tier.ts +2 -2
- package/src/daemon/server.ts +6 -6
- package/src/daemon/session-agent-loop-handlers.ts +34 -9
- package/src/daemon/session-agent-loop.ts +15 -8
- package/src/daemon/session-history.ts +3 -2
- package/src/daemon/session-media-retry.ts +3 -0
- package/src/daemon/session-messaging.ts +38 -4
- package/src/daemon/session-notifiers.ts +2 -2
- package/src/daemon/session-process.ts +256 -23
- package/src/daemon/session-queue-manager.ts +2 -0
- package/src/daemon/session-runtime-assembly.ts +39 -0
- package/src/daemon/session-skill-tools.ts +13 -4
- package/src/daemon/session-tool-setup.ts +5 -6
- package/src/daemon/session.ts +19 -8
- package/src/daemon/tls-certs.ts +55 -13
- package/src/daemon/tool-side-effects.ts +13 -5
- package/src/gallery/default-gallery.ts +32 -9
- package/src/influencer/client.ts +2 -1
- package/src/memory/channel-delivery-store.ts +37 -567
- package/src/memory/channel-guardian-store.ts +66 -1317
- package/src/memory/conflict-store.ts +4 -4
- package/src/memory/conversation-attention-store.ts +0 -3
- package/src/memory/conversation-crud.ts +668 -0
- package/src/memory/conversation-queries.ts +361 -0
- package/src/memory/conversation-store.ts +45 -983
- package/src/memory/db-connection.ts +3 -0
- package/src/memory/db-init.ts +25 -0
- package/src/memory/delivery-channels.ts +175 -0
- package/src/memory/delivery-crud.ts +211 -0
- package/src/memory/delivery-status.ts +199 -0
- package/src/memory/embedding-backend.ts +70 -4
- package/src/memory/embedding-local.ts +12 -2
- package/src/memory/entity-extractor.ts +3 -8
- package/src/memory/fts-reconciler.ts +121 -0
- package/src/memory/guardian-action-store.ts +366 -3
- package/src/memory/guardian-approvals.ts +569 -0
- package/src/memory/guardian-bindings.ts +130 -0
- package/src/memory/guardian-rate-limits.ts +196 -0
- package/src/memory/guardian-verification.ts +520 -0
- package/src/memory/job-handlers/index-maintenance.ts +2 -1
- package/src/memory/job-utils.ts +8 -5
- package/src/memory/jobs-store.ts +66 -6
- package/src/memory/jobs-worker.ts +23 -1
- package/src/memory/migrations/030-guardian-action-followup.ts +21 -0
- package/src/memory/migrations/030-guardian-verification-purpose.ts +17 -0
- package/src/memory/migrations/031-conversations-thread-type-index.ts +5 -0
- package/src/memory/migrations/100-core-tables.ts +1 -1
- package/src/memory/migrations/101-watchers-and-logs.ts +4 -0
- package/src/memory/migrations/108-tasks-and-work-items.ts +1 -1
- package/src/memory/migrations/112-assistant-inbox.ts +1 -1
- package/src/memory/migrations/113-late-migrations.ts +1 -1
- package/src/memory/migrations/116-messages-fts.ts +13 -0
- package/src/memory/migrations/119-schema-indexes-and-columns.ts +37 -0
- package/src/memory/migrations/120-fk-cascade-rebuilds.ts +161 -0
- package/src/memory/migrations/index.ts +8 -3
- package/src/memory/migrations/validate-migration-state.ts +114 -15
- package/src/memory/qdrant-circuit-breaker.ts +105 -0
- package/src/memory/retriever.ts +46 -13
- package/src/memory/schema-migration.ts +3 -0
- package/src/memory/schema.ts +25 -7
- package/src/memory/search/semantic.ts +8 -90
- package/src/notifications/README.md +1 -1
- package/src/notifications/broadcaster.ts +20 -2
- package/src/notifications/conversation-pairing.ts +3 -3
- package/src/notifications/decision-engine.ts +173 -8
- package/src/notifications/deliveries-store.ts +27 -8
- package/src/notifications/preferences-store.ts +7 -7
- package/src/notifications/thread-candidates.ts +234 -0
- package/src/notifications/types.ts +18 -0
- package/src/permissions/defaults.ts +11 -1
- package/src/permissions/prompter.ts +17 -0
- package/src/permissions/trust-store.ts +2 -0
- package/src/providers/failover.ts +19 -0
- package/src/providers/registry.ts +46 -1
- package/src/runtime/approval-message-composer.ts +1 -1
- package/src/runtime/channel-guardian-service.ts +15 -3
- package/src/runtime/channel-retry-sweep.ts +7 -2
- package/src/runtime/guardian-action-conversation-turn.ts +85 -0
- package/src/runtime/guardian-action-followup-executor.ts +301 -0
- package/src/runtime/guardian-action-message-composer.ts +245 -0
- package/src/runtime/guardian-outbound-actions.ts +26 -6
- package/src/runtime/guardian-verification-templates.ts +15 -9
- package/src/runtime/http-errors.ts +93 -0
- package/src/runtime/http-server.ts +133 -44
- package/src/runtime/http-types.ts +53 -0
- package/src/runtime/ingress-service.ts +237 -0
- package/src/runtime/middleware/error-handler.ts +4 -3
- package/src/runtime/middleware/rate-limiter.ts +160 -0
- package/src/runtime/middleware/request-logger.ts +71 -0
- package/src/runtime/middleware/twilio-validation.ts +7 -6
- package/src/runtime/pending-interactions.ts +12 -0
- package/src/runtime/routes/access-request-decision.ts +215 -0
- package/src/runtime/routes/app-routes.ts +25 -18
- package/src/runtime/routes/approval-routes.ts +18 -47
- package/src/runtime/routes/attachment-routes.ts +15 -41
- package/src/runtime/routes/call-routes.ts +20 -20
- package/src/runtime/routes/channel-delivery-routes.ts +6 -5
- package/src/runtime/routes/contact-routes.ts +4 -9
- package/src/runtime/routes/conversation-attention-routes.ts +2 -1
- package/src/runtime/routes/conversation-routes.ts +26 -57
- package/src/runtime/routes/debug-routes.ts +71 -0
- package/src/runtime/routes/events-routes.ts +3 -2
- package/src/runtime/routes/guardian-approval-interception.ts +221 -0
- package/src/runtime/routes/identity-routes.ts +14 -10
- package/src/runtime/routes/inbound-conversation.ts +3 -2
- package/src/runtime/routes/inbound-message-handler.ts +527 -62
- package/src/runtime/routes/ingress-routes.ts +174 -0
- package/src/runtime/routes/integration-routes.ts +78 -16
- package/src/runtime/routes/pairing-routes.ts +11 -10
- package/src/runtime/routes/secret-routes.ts +10 -18
- package/src/runtime/verification-rate-limiter.ts +83 -0
- package/src/schedule/schedule-store.ts +13 -1
- package/src/schedule/scheduler.ts +1 -1
- package/src/security/secret-ingress.ts +5 -2
- package/src/security/secret-scanner.ts +72 -6
- package/src/subagent/manager.ts +6 -4
- package/src/swarm/plan-validator.ts +4 -1
- package/src/tasks/task-runner.ts +3 -1
- package/src/tools/browser/api-map.ts +9 -6
- package/src/tools/calls/call-start.ts +20 -0
- package/src/tools/executor.ts +50 -568
- package/src/tools/permission-checker.ts +272 -0
- package/src/tools/registry.ts +14 -6
- package/src/tools/reminder/reminder-store.ts +7 -7
- package/src/tools/reminder/reminder.ts +6 -3
- package/src/tools/secret-detection-handler.ts +301 -0
- package/src/tools/subagent/message.ts +1 -1
- package/src/tools/system/voice-config.ts +62 -0
- package/src/tools/tasks/index.ts +3 -3
- package/src/tools/tasks/work-item-list.ts +3 -3
- package/src/tools/tasks/work-item-update.ts +4 -5
- package/src/tools/tool-approval-handler.ts +192 -0
- package/src/tools/tool-manifest.ts +2 -0
- package/src/watcher/watcher-store.ts +9 -9
- package/src/work-items/work-item-runner.ts +9 -6
- /package/src/memory/migrations/{026-embeddings-nullable-vector-json.ts → 026a-embeddings-nullable-vector-json.ts} +0 -0
- /package/src/memory/migrations/{027-guardian-bootstrap-token.ts → 027a-guardian-bootstrap-token.ts} +0 -0
|
@@ -1,18 +1,103 @@
|
|
|
1
|
+
import { IntegrityError } from '../../util/errors.js';
|
|
1
2
|
import { getLogger } from '../../util/logger.js';
|
|
3
|
+
import { getDbPath } from '../../util/platform.js';
|
|
2
4
|
import { type DrizzleDb,getSqliteFrom } from '../db-connection.js';
|
|
3
5
|
import { MIGRATION_REGISTRY, type MigrationValidationResult } from './registry.js';
|
|
4
6
|
|
|
5
7
|
const log = getLogger('memory-db');
|
|
6
8
|
|
|
9
|
+
/**
|
|
10
|
+
* Recover from crashed migrations before the migration runner executes.
|
|
11
|
+
*
|
|
12
|
+
* Scans memory_checkpoints for entries with value 'started' — these represent
|
|
13
|
+
* migrations that began but never completed (e.g., due to a process crash).
|
|
14
|
+
* Deletes the stalled checkpoint so the migration can re-run from scratch on
|
|
15
|
+
* this startup. Each migration's own idempotency guards (DDL IF NOT EXISTS,
|
|
16
|
+
* transactional rollback) ensure re-running is safe.
|
|
17
|
+
*
|
|
18
|
+
* Call this BEFORE running migrations so that stalled checkpoints don't block
|
|
19
|
+
* re-execution.
|
|
20
|
+
*/
|
|
21
|
+
export function recoverCrashedMigrations(database: DrizzleDb): string[] {
|
|
22
|
+
const raw = getSqliteFrom(database);
|
|
23
|
+
|
|
24
|
+
let rows: Array<{ key: string; value: string }>;
|
|
25
|
+
try {
|
|
26
|
+
rows = raw.query(`SELECT key, value FROM memory_checkpoints`).all() as Array<{ key: string; value: string }>;
|
|
27
|
+
} catch {
|
|
28
|
+
return [];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const crashed = rows.filter((r) => r.value === 'started').map((r) => r.key);
|
|
32
|
+
if (crashed.length === 0) return [];
|
|
33
|
+
|
|
34
|
+
log.error(
|
|
35
|
+
{ crashed },
|
|
36
|
+
[
|
|
37
|
+
'╔══════════════════════════════════════════════════════════════╗',
|
|
38
|
+
'║ CRASHED MIGRATIONS DETECTED — AUTO-RECOVERING ║',
|
|
39
|
+
'╚══════════════════════════════════════════════════════════════╝',
|
|
40
|
+
'',
|
|
41
|
+
`The following migrations started but never completed: ${crashed.join(', ')}`,
|
|
42
|
+
'',
|
|
43
|
+
'Clearing stalled checkpoints so they can be retried on this startup.',
|
|
44
|
+
'If retries continue to fail, manually inspect the database:',
|
|
45
|
+
` sqlite3 ${getDbPath()} "SELECT * FROM memory_checkpoints"`,
|
|
46
|
+
].join('\n'),
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
for (const key of crashed) {
|
|
50
|
+
raw.query(`DELETE FROM memory_checkpoints WHERE key = ?`).run(key);
|
|
51
|
+
log.info({ key }, `Cleared stalled checkpoint "${key}" — migration will re-run`);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return crashed;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Wrap a migration function with crash-recovery bookkeeping.
|
|
59
|
+
*
|
|
60
|
+
* Writes a 'started' checkpoint before executing the migration body, then
|
|
61
|
+
* overwrites it with the completion value on success. If the process crashes
|
|
62
|
+
* between the start marker and completion, recoverCrashedMigrations (which
|
|
63
|
+
* runs before all migrations) will detect and clear it on the next startup.
|
|
64
|
+
*
|
|
65
|
+
* The migrationFn receives the raw SQLite database and should perform its
|
|
66
|
+
* own transaction management internally.
|
|
67
|
+
*/
|
|
68
|
+
export function withCrashRecovery(
|
|
69
|
+
database: DrizzleDb,
|
|
70
|
+
checkpointKey: string,
|
|
71
|
+
migrationFn: () => void,
|
|
72
|
+
): void {
|
|
73
|
+
const raw = getSqliteFrom(database);
|
|
74
|
+
|
|
75
|
+
const existing = raw.query(
|
|
76
|
+
`SELECT value FROM memory_checkpoints WHERE key = ?`,
|
|
77
|
+
).get(checkpointKey) as { value: string } | null;
|
|
78
|
+
if (existing && existing.value !== 'started') return;
|
|
79
|
+
|
|
80
|
+
raw.query(
|
|
81
|
+
`INSERT OR REPLACE INTO memory_checkpoints (key, value, updated_at) VALUES (?, 'started', ?)`,
|
|
82
|
+
).run(checkpointKey, Date.now());
|
|
83
|
+
|
|
84
|
+
migrationFn();
|
|
85
|
+
|
|
86
|
+
raw.query(
|
|
87
|
+
`UPDATE memory_checkpoints SET value = '1', updated_at = ? WHERE key = ?`,
|
|
88
|
+
).run(Date.now(), checkpointKey);
|
|
89
|
+
}
|
|
90
|
+
|
|
7
91
|
/**
|
|
8
92
|
* Validate the applied migration state against the registry at startup.
|
|
9
93
|
*
|
|
10
|
-
* Logs
|
|
11
|
-
*
|
|
12
|
-
* missing from the checkpoints table (dependency ordering violation).
|
|
94
|
+
* Logs a prominent error when a migration started but never completed (crash
|
|
95
|
+
* detected) — startup continues so the migration can be retried.
|
|
13
96
|
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
97
|
+
* Throws an IntegrityError when a migration was applied but a declared
|
|
98
|
+
* prerequisite is missing from the checkpoints table (dependency ordering
|
|
99
|
+
* violation). This blocks daemon startup to prevent running with an
|
|
100
|
+
* inconsistent database schema.
|
|
16
101
|
*
|
|
17
102
|
* Call this AFTER all DDL and migration functions have run so that the final
|
|
18
103
|
* state is inspected.
|
|
@@ -28,15 +113,23 @@ export function validateMigrationState(database: DrizzleDb): MigrationValidation
|
|
|
28
113
|
return { crashed: [], dependencyViolations: [] };
|
|
29
114
|
}
|
|
30
115
|
|
|
31
|
-
//
|
|
32
|
-
// migration
|
|
33
|
-
// The migration will re-run on the next startup (its own idempotency guard
|
|
34
|
-
// will determine safety), but we surface a warning for visibility.
|
|
116
|
+
// Any remaining 'started' checkpoints after recovery + migration execution
|
|
117
|
+
// indicate a migration that was retried but failed again.
|
|
35
118
|
const crashed = rows.filter((r) => r.value === 'started').map((r) => r.key);
|
|
36
119
|
if (crashed.length > 0) {
|
|
37
|
-
log.
|
|
120
|
+
log.error(
|
|
38
121
|
{ crashed },
|
|
39
|
-
|
|
122
|
+
[
|
|
123
|
+
'╔══════════════════════════════════════════════════════════════╗',
|
|
124
|
+
'║ MIGRATIONS STILL INCOMPLETE AFTER RETRY ║',
|
|
125
|
+
'╚══════════════════════════════════════════════════════════════╝',
|
|
126
|
+
'',
|
|
127
|
+
`The following migrations were retried but still did not complete: ${crashed.join(', ')}`,
|
|
128
|
+
'',
|
|
129
|
+
'Manual intervention is required. Inspect the database and resolve:',
|
|
130
|
+
` sqlite3 ${getDbPath()} "DELETE FROM memory_checkpoints WHERE key = '<migration_key>'"`,
|
|
131
|
+
'Then restart the daemon.',
|
|
132
|
+
].join('\n'),
|
|
40
133
|
);
|
|
41
134
|
}
|
|
42
135
|
|
|
@@ -56,14 +149,20 @@ export function validateMigrationState(database: DrizzleDb): MigrationValidation
|
|
|
56
149
|
|
|
57
150
|
for (const dep of entry.dependsOn) {
|
|
58
151
|
if (!completed.has(dep)) {
|
|
59
|
-
log.error(
|
|
60
|
-
{ migration: entry.key, missingDependency: dep, version: entry.version },
|
|
61
|
-
'Migration dependency violation: this migration is marked complete but its declared prerequisite has no checkpoint — database schema may be inconsistent',
|
|
62
|
-
);
|
|
63
152
|
dependencyViolations.push({ migration: entry.key, missingDependency: dep });
|
|
64
153
|
}
|
|
65
154
|
}
|
|
66
155
|
}
|
|
67
156
|
|
|
157
|
+
if (dependencyViolations.length > 0) {
|
|
158
|
+
const details = dependencyViolations
|
|
159
|
+
.map((v) => ` - "${v.migration}" requires "${v.missingDependency}" but it has no checkpoint`)
|
|
160
|
+
.join('\n');
|
|
161
|
+
throw new IntegrityError(
|
|
162
|
+
`Migration dependency violations detected — database schema may be inconsistent:\n${details}\n` +
|
|
163
|
+
'The daemon cannot start safely. Inspect the database and re-run missing migrations.',
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
|
|
68
167
|
return { crashed, dependencyViolations };
|
|
69
168
|
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { getLogger } from '../util/logger.js';
|
|
2
|
+
|
|
3
|
+
const log = getLogger('qdrant-circuit-breaker');
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Circuit breaker for Qdrant operations.
|
|
7
|
+
*
|
|
8
|
+
* After FAILURE_THRESHOLD consecutive failures, the circuit opens and
|
|
9
|
+
* all calls fail-fast without hitting Qdrant. After COOLDOWN_MS, one
|
|
10
|
+
* probe request is allowed through (half-open). If the probe succeeds,
|
|
11
|
+
* the circuit closes; if it fails, the circuit re-opens.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const FAILURE_THRESHOLD = 5;
|
|
15
|
+
const COOLDOWN_MS = 30_000;
|
|
16
|
+
|
|
17
|
+
type BreakerState = 'closed' | 'open' | 'half-open';
|
|
18
|
+
|
|
19
|
+
let breakerState: BreakerState = 'closed';
|
|
20
|
+
let consecutiveFailures = 0;
|
|
21
|
+
let openedAt = 0;
|
|
22
|
+
let halfOpenProbeInFlight = false;
|
|
23
|
+
|
|
24
|
+
export class QdrantCircuitOpenError extends Error {
|
|
25
|
+
constructor() {
|
|
26
|
+
super('Qdrant circuit breaker open');
|
|
27
|
+
this.name = 'QdrantCircuitOpenError';
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function allows(): boolean {
|
|
32
|
+
if (breakerState === 'closed') return true;
|
|
33
|
+
if (breakerState === 'open') {
|
|
34
|
+
if (Date.now() - openedAt >= COOLDOWN_MS) {
|
|
35
|
+
breakerState = 'half-open';
|
|
36
|
+
halfOpenProbeInFlight = true;
|
|
37
|
+
log.info('Qdrant circuit breaker entering half-open state — allowing probe request');
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
// half-open: only allow through if no probe is already in flight
|
|
43
|
+
if (halfOpenProbeInFlight) return false;
|
|
44
|
+
halfOpenProbeInFlight = true;
|
|
45
|
+
return true;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function recordSuccess(): void {
|
|
49
|
+
if (breakerState !== 'closed') {
|
|
50
|
+
log.info({ previousFailures: consecutiveFailures }, 'Qdrant circuit breaker closed — operation succeeded');
|
|
51
|
+
}
|
|
52
|
+
consecutiveFailures = 0;
|
|
53
|
+
breakerState = 'closed';
|
|
54
|
+
openedAt = 0;
|
|
55
|
+
halfOpenProbeInFlight = false;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function recordFailure(): void {
|
|
59
|
+
consecutiveFailures++;
|
|
60
|
+
halfOpenProbeInFlight = false;
|
|
61
|
+
if (consecutiveFailures >= FAILURE_THRESHOLD) {
|
|
62
|
+
breakerState = 'open';
|
|
63
|
+
openedAt = Date.now();
|
|
64
|
+
log.warn(
|
|
65
|
+
{ consecutiveFailures, cooldownMs: COOLDOWN_MS },
|
|
66
|
+
'Qdrant circuit breaker opened — Qdrant operations disabled until probe succeeds',
|
|
67
|
+
);
|
|
68
|
+
} else if (breakerState === 'half-open') {
|
|
69
|
+
breakerState = 'open';
|
|
70
|
+
openedAt = Date.now();
|
|
71
|
+
log.warn('Qdrant circuit breaker re-opened — half-open probe failed');
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Execute a Qdrant operation through the circuit breaker.
|
|
77
|
+
* Throws QdrantCircuitOpenError if the circuit is open.
|
|
78
|
+
* Re-throws the original error on failure after recording it.
|
|
79
|
+
*/
|
|
80
|
+
export async function withQdrantBreaker<T>(fn: () => Promise<T>): Promise<T> {
|
|
81
|
+
if (!allows()) {
|
|
82
|
+
throw new QdrantCircuitOpenError();
|
|
83
|
+
}
|
|
84
|
+
try {
|
|
85
|
+
const result = await fn();
|
|
86
|
+
recordSuccess();
|
|
87
|
+
return result;
|
|
88
|
+
} catch (err) {
|
|
89
|
+
recordFailure();
|
|
90
|
+
throw err;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** @internal Test-only: reset circuit breaker state */
|
|
95
|
+
export function _resetQdrantBreaker(): void {
|
|
96
|
+
breakerState = 'closed';
|
|
97
|
+
consecutiveFailures = 0;
|
|
98
|
+
openedAt = 0;
|
|
99
|
+
halfOpenProbeInFlight = false;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** @internal Test-only: get breaker state */
|
|
103
|
+
export function _getQdrantBreakerState(): { state: BreakerState; consecutiveFailures: number } {
|
|
104
|
+
return { state: breakerState, consecutiveFailures };
|
|
105
|
+
}
|
package/src/memory/retriever.ts
CHANGED
|
@@ -139,6 +139,8 @@ async function collectAndMergeCandidates(
|
|
|
139
139
|
// A per-call scopePolicyOverride takes precedence over the global policy.
|
|
140
140
|
const scopeIds = buildScopeFilter(scopeId, scopePolicy, opts?.scopePolicyOverride);
|
|
141
141
|
|
|
142
|
+
let semanticSearchFailed = false;
|
|
143
|
+
|
|
142
144
|
// -- Phase 1: cheap local searches (always run) --
|
|
143
145
|
const lexical = lexicalSearch(query, config.memory.retrieval.lexicalTopK, excludeMessageIds, scopeIds);
|
|
144
146
|
|
|
@@ -220,8 +222,7 @@ async function collectAndMergeCandidates(
|
|
|
220
222
|
|
|
221
223
|
// -- Early termination check --
|
|
222
224
|
// If cheap sources already produced enough high-relevance candidates,
|
|
223
|
-
// skip
|
|
224
|
-
// relation traversal to reduce recall latency.
|
|
225
|
+
// skip semantic and entity search entirely.
|
|
225
226
|
//
|
|
226
227
|
// Deduplicate before counting: lexical and recency can return the same
|
|
227
228
|
// segment (common when recent messages match the query), so checking raw
|
|
@@ -248,12 +249,8 @@ async function collectAndMergeCandidates(
|
|
|
248
249
|
&& cheapCandidates.length >= etConfig.minCandidates
|
|
249
250
|
&& cheapCandidates.filter((c) => c.lexical >= etConfig.confidenceThreshold).length >= etConfig.minHighConfidence;
|
|
250
251
|
|
|
251
|
-
// -- Phase 2:
|
|
252
|
-
// Semantic search (async Qdrant network call) and entity search (sync
|
|
253
|
-
// SQLite graph traversal) are independent. Start the network call first,
|
|
254
|
-
// run the sync work while it's in flight, then await the result.
|
|
252
|
+
// -- Phase 2: entity search + await semantic (skipped on early termination) --
|
|
255
253
|
let semantic: Candidate[] = [];
|
|
256
|
-
let semanticSearchFailed = false;
|
|
257
254
|
let entity: Candidate[] = [];
|
|
258
255
|
let candidateDepths: Map<string, number> | undefined;
|
|
259
256
|
let relationSeedEntityCount = 0;
|
|
@@ -262,6 +259,8 @@ async function collectAndMergeCandidates(
|
|
|
262
259
|
let relationExpandedItemCount = 0;
|
|
263
260
|
|
|
264
261
|
if (!canTerminateEarly) {
|
|
262
|
+
// Start semantic search now that we know early termination won't apply.
|
|
263
|
+
// The network round-trip overlaps with entity search below.
|
|
265
264
|
const semanticPromise = queryVector
|
|
266
265
|
? semanticSearch(queryVector, opts?.provider ?? 'unknown', opts?.model ?? 'unknown', config.memory.retrieval.semanticTopK, excludeMessageIds, scopeIds)
|
|
267
266
|
.catch((err): Candidate[] => {
|
|
@@ -275,6 +274,8 @@ async function collectAndMergeCandidates(
|
|
|
275
274
|
})
|
|
276
275
|
: null;
|
|
277
276
|
|
|
277
|
+
// Entity search is synchronous — run it while the semantic promise
|
|
278
|
+
// is in flight.
|
|
278
279
|
if (config.memory.entity.enabled) {
|
|
279
280
|
const entitySearchResult = entitySearch(
|
|
280
281
|
query,
|
|
@@ -469,10 +470,39 @@ function formatRecallResult(
|
|
|
469
470
|
1,
|
|
470
471
|
Math.floor(options?.maxInjectTokensOverride ?? config.memory.retrieval.maxInjectTokens),
|
|
471
472
|
);
|
|
472
|
-
|
|
473
|
+
|
|
474
|
+
// Reserve token budget for the degradation notice so it doesn't push
|
|
475
|
+
// injected text over maxInjectTokens when appended after trimming.
|
|
476
|
+
const degradationNotice = collected.semanticSearchFailed
|
|
477
|
+
? '[Note: Semantic search is currently unavailable. Memory recall is limited to lexical and recency matching — results may be incomplete or miss semantically relevant memories.]'
|
|
478
|
+
: undefined;
|
|
479
|
+
const noticeOnlyTokenCost = degradationNotice
|
|
480
|
+
? estimateTextTokens(degradationNotice)
|
|
481
|
+
: 0;
|
|
482
|
+
// +2 for '\n\n' separator — only needed when candidates are also present
|
|
483
|
+
const noticeTokenCost = noticeOnlyTokenCost + (degradationNotice ? 2 : 0);
|
|
484
|
+
// When the notice alone exceeds the budget, skip it entirely so
|
|
485
|
+
// injectedText never exceeds maxInjectTokens.
|
|
486
|
+
const budgetForNotice = noticeTokenCost <= maxInjectTokens;
|
|
487
|
+
const candidateBudget = budgetForNotice ? maxInjectTokens - noticeTokenCost : maxInjectTokens;
|
|
488
|
+
|
|
489
|
+
const selected = trimToTokenBudget(merged, candidateBudget, config.memory.retrieval.injectionFormat);
|
|
473
490
|
markItemUsage(selected);
|
|
474
491
|
|
|
475
|
-
|
|
492
|
+
let injectedText = buildInjectedText(selected, config.memory.retrieval.injectionFormat);
|
|
493
|
+
|
|
494
|
+
// Show the notice if it fits: when candidates are present the separator
|
|
495
|
+
// cost was already reserved; when no candidates were selected, the notice
|
|
496
|
+
// alone (without separator) may still fit even if the full cost didn't.
|
|
497
|
+
const canShowNotice = degradationNotice && (
|
|
498
|
+
budgetForNotice || (selected.length === 0 && noticeOnlyTokenCost <= maxInjectTokens)
|
|
499
|
+
);
|
|
500
|
+
if (canShowNotice) {
|
|
501
|
+
injectedText = injectedText.length > 0
|
|
502
|
+
? injectedText + '\n\n' + degradationNotice
|
|
503
|
+
: degradationNotice;
|
|
504
|
+
}
|
|
505
|
+
|
|
476
506
|
const topCandidates: MemoryRecallCandiateDebug[] = selected.slice(0, 10).map((c) => ({
|
|
477
507
|
key: c.key,
|
|
478
508
|
type: c.type,
|
|
@@ -720,14 +750,17 @@ export function injectMemoryRecallAsSeparateMessage<T extends { role: 'user' | '
|
|
|
720
750
|
): T[] {
|
|
721
751
|
if (memoryRecallText.trim().length === 0) return messages;
|
|
722
752
|
if (messages.length === 0) return messages;
|
|
753
|
+
// These synthetic messages satisfy the structural constraint T extends { role; content }
|
|
754
|
+
// but may lack extra fields present on T. In practice T is always Message which has
|
|
755
|
+
// only role and content, so the cast is safe.
|
|
723
756
|
const contextMessage = {
|
|
724
757
|
role: 'user' as const,
|
|
725
|
-
content: [{ type: 'text', text: memoryRecallText }],
|
|
726
|
-
} as
|
|
758
|
+
content: [{ type: 'text' as const, text: memoryRecallText }],
|
|
759
|
+
} as T;
|
|
727
760
|
const ackMessage = {
|
|
728
761
|
role: 'assistant' as const,
|
|
729
|
-
content: [{ type: 'text', text: MEMORY_CONTEXT_ACK }],
|
|
730
|
-
} as
|
|
762
|
+
content: [{ type: 'text' as const, text: MEMORY_CONTEXT_ACK }],
|
|
763
|
+
} as T;
|
|
731
764
|
return [
|
|
732
765
|
...messages.slice(0, -1),
|
|
733
766
|
contextMessage,
|
|
@@ -24,9 +24,12 @@ export {
|
|
|
24
24
|
migrateRemainingTableIndexes,
|
|
25
25
|
migrateReminderRoutingIntent,
|
|
26
26
|
migrateRemoveAssistantIdColumns,
|
|
27
|
+
migrateSchemaIndexesAndColumns,
|
|
27
28
|
migrateToolInvocationsFk,
|
|
28
29
|
MIGRATION_REGISTRY,
|
|
29
30
|
type MigrationRegistryEntry,
|
|
30
31
|
type MigrationValidationResult,
|
|
32
|
+
recoverCrashedMigrations,
|
|
31
33
|
validateMigrationState,
|
|
34
|
+
withCrashRecovery,
|
|
32
35
|
} from './migrations/index.js';
|
package/src/memory/schema.ts
CHANGED
|
@@ -17,13 +17,16 @@ export const conversations = sqliteTable('conversations', {
|
|
|
17
17
|
originChannel: text('origin_channel'),
|
|
18
18
|
originInterface: text('origin_interface'),
|
|
19
19
|
isAutoTitle: integer('is_auto_title').notNull().default(1),
|
|
20
|
-
})
|
|
20
|
+
}, (table) => [
|
|
21
|
+
index('idx_conversations_updated_at').on(table.updatedAt),
|
|
22
|
+
index('idx_conversations_thread_type').on(table.threadType),
|
|
23
|
+
]);
|
|
21
24
|
|
|
22
25
|
export const messages = sqliteTable('messages', {
|
|
23
26
|
id: text('id').primaryKey(),
|
|
24
27
|
conversationId: text('conversation_id')
|
|
25
28
|
.notNull()
|
|
26
|
-
.references(() => conversations.id),
|
|
29
|
+
.references(() => conversations.id, { onDelete: 'cascade' }),
|
|
27
30
|
role: text('role').notNull(),
|
|
28
31
|
content: text('content').notNull(),
|
|
29
32
|
createdAt: integer('created_at').notNull(),
|
|
@@ -166,6 +169,7 @@ export const memoryJobs = sqliteTable('memory_jobs', {
|
|
|
166
169
|
deferrals: integer('deferrals').notNull().default(0),
|
|
167
170
|
runAfter: integer('run_after').notNull(),
|
|
168
171
|
lastError: text('last_error'),
|
|
172
|
+
startedAt: integer('started_at'),
|
|
169
173
|
createdAt: integer('created_at').notNull(),
|
|
170
174
|
updatedAt: integer('updated_at').notNull(),
|
|
171
175
|
});
|
|
@@ -416,7 +420,7 @@ export const taskRuns = sqliteTable('task_runs', {
|
|
|
416
420
|
id: text('id').primaryKey(),
|
|
417
421
|
taskId: text('task_id')
|
|
418
422
|
.notNull()
|
|
419
|
-
.references(() => tasks.id),
|
|
423
|
+
.references(() => tasks.id, { onDelete: 'cascade' }),
|
|
420
424
|
conversationId: text('conversation_id'),
|
|
421
425
|
status: text('status').notNull().default('pending'),
|
|
422
426
|
startedAt: integer('started_at'),
|
|
@@ -540,7 +544,9 @@ export const llmUsageEvents = sqliteTable('llm_usage_events', {
|
|
|
540
544
|
estimatedCostUsd: real('estimated_cost_usd'),
|
|
541
545
|
pricingStatus: text('pricing_status').notNull(),
|
|
542
546
|
metadataJson: text('metadata_json'),
|
|
543
|
-
})
|
|
547
|
+
}, (table) => [
|
|
548
|
+
index('idx_llm_usage_events_conversation_id').on(table.conversationId),
|
|
549
|
+
]);
|
|
544
550
|
|
|
545
551
|
// ── Call Sessions (outgoing AI phone calls) ──────────────────────────
|
|
546
552
|
|
|
@@ -566,7 +572,9 @@ export const callSessions = sqliteTable('call_sessions', {
|
|
|
566
572
|
lastError: text('last_error'),
|
|
567
573
|
createdAt: integer('created_at').notNull(),
|
|
568
574
|
updatedAt: integer('updated_at').notNull(),
|
|
569
|
-
})
|
|
575
|
+
}, (table) => [
|
|
576
|
+
index('idx_call_sessions_status').on(table.status),
|
|
577
|
+
]);
|
|
570
578
|
|
|
571
579
|
export const callEvents = sqliteTable('call_events', {
|
|
572
580
|
id: text('id').primaryKey(),
|
|
@@ -660,6 +668,8 @@ export const channelGuardianVerificationChallenges = sqliteTable('channel_guardi
|
|
|
660
668
|
// Session configuration
|
|
661
669
|
codeDigits: integer('code_digits').default(6),
|
|
662
670
|
maxAttempts: integer('max_attempts').default(3),
|
|
671
|
+
// Distinguishes guardian verification from trusted contact verification
|
|
672
|
+
verificationPurpose: text('verification_purpose').default('guardian'),
|
|
663
673
|
// Telegram bootstrap deep-link token hash
|
|
664
674
|
bootstrapTokenHash: text('bootstrap_token_hash'),
|
|
665
675
|
createdAt: integer('created_at').notNull(),
|
|
@@ -827,6 +837,12 @@ export const guardianActionRequests = sqliteTable('guardian_action_requests', {
|
|
|
827
837
|
answeredByExternalUserId: text('answered_by_external_user_id'),
|
|
828
838
|
answeredAt: integer('answered_at'),
|
|
829
839
|
expiresAt: integer('expires_at').notNull(),
|
|
840
|
+
expiredReason: text('expired_reason'), // call_timeout | sweep_timeout | cancelled
|
|
841
|
+
followupState: text('followup_state').notNull().default('none'), // none | awaiting_guardian_choice | dispatching | completed | declined | failed
|
|
842
|
+
lateAnswerText: text('late_answer_text'),
|
|
843
|
+
lateAnsweredAt: integer('late_answered_at'),
|
|
844
|
+
followupAction: text('followup_action'), // call_back | message_back | decline
|
|
845
|
+
followupCompletedAt: integer('followup_completed_at'),
|
|
830
846
|
createdAt: integer('created_at').notNull(),
|
|
831
847
|
updatedAt: integer('updated_at').notNull(),
|
|
832
848
|
});
|
|
@@ -881,7 +897,7 @@ export const assistantIngressMembers = sqliteTable('assistant_ingress_members',
|
|
|
881
897
|
status: text('status').notNull().default('pending'),
|
|
882
898
|
policy: text('policy').notNull().default('allow'),
|
|
883
899
|
inviteId: text('invite_id')
|
|
884
|
-
.references(() => assistantIngressInvites.id),
|
|
900
|
+
.references(() => assistantIngressInvites.id, { onDelete: 'cascade' }),
|
|
885
901
|
createdBySessionId: text('created_by_session_id'),
|
|
886
902
|
revokedReason: text('revoked_reason'),
|
|
887
903
|
blockedReason: text('blocked_reason'),
|
|
@@ -1007,7 +1023,9 @@ export const notificationDeliveries = sqliteTable('notification_deliveries', {
|
|
|
1007
1023
|
clientDeliveryAt: integer('client_delivery_at'),
|
|
1008
1024
|
createdAt: integer('created_at').notNull(),
|
|
1009
1025
|
updatedAt: integer('updated_at').notNull(),
|
|
1010
|
-
})
|
|
1026
|
+
}, (table) => [
|
|
1027
|
+
uniqueIndex('idx_notification_deliveries_decision_channel').on(table.notificationDecisionId, table.channel),
|
|
1028
|
+
]);
|
|
1011
1029
|
|
|
1012
1030
|
// ── Conversation Attention ───────────────────────────────────────────
|
|
1013
1031
|
|
|
@@ -2,6 +2,7 @@ import { inArray } from 'drizzle-orm';
|
|
|
2
2
|
|
|
3
3
|
import { getLogger } from '../../util/logger.js';
|
|
4
4
|
import { getDb } from '../db.js';
|
|
5
|
+
import { withQdrantBreaker, _resetQdrantBreaker, _getQdrantBreakerState } from '../qdrant-circuit-breaker.js';
|
|
5
6
|
import type { QdrantSearchResult } from '../qdrant-client.js';
|
|
6
7
|
import { getQdrantClient } from '../qdrant-client.js';
|
|
7
8
|
import {
|
|
@@ -13,82 +14,10 @@ import {
|
|
|
13
14
|
import { computeRecencyScore } from './ranking.js';
|
|
14
15
|
import type { Candidate } from './types.js';
|
|
15
16
|
|
|
16
|
-
const log = getLogger('
|
|
17
|
+
const log = getLogger('semantic-search');
|
|
17
18
|
|
|
18
|
-
//
|
|
19
|
-
|
|
20
|
-
// requests (open state). After COOLDOWN_MS, allow a single probe
|
|
21
|
-
// request (half-open). If the probe succeeds, close the circuit; if it
|
|
22
|
-
// fails, re-open and restart the cooldown.
|
|
23
|
-
|
|
24
|
-
const FAILURE_THRESHOLD = 5;
|
|
25
|
-
const COOLDOWN_MS = 60_000;
|
|
26
|
-
|
|
27
|
-
type BreakerState = 'closed' | 'open' | 'half-open';
|
|
28
|
-
|
|
29
|
-
let breakerState: BreakerState = 'closed';
|
|
30
|
-
let consecutiveFailures = 0;
|
|
31
|
-
let openedAt = 0;
|
|
32
|
-
// Ensures only one request passes through during half-open state
|
|
33
|
-
let halfOpenProbeInFlight = false;
|
|
34
|
-
|
|
35
|
-
function qdrantBreakerAllows(): boolean {
|
|
36
|
-
if (breakerState === 'closed') return true;
|
|
37
|
-
if (breakerState === 'open') {
|
|
38
|
-
if (Date.now() - openedAt >= COOLDOWN_MS) {
|
|
39
|
-
breakerState = 'half-open';
|
|
40
|
-
halfOpenProbeInFlight = true;
|
|
41
|
-
log.info('Qdrant circuit breaker entering half-open state — allowing probe request');
|
|
42
|
-
return true;
|
|
43
|
-
}
|
|
44
|
-
return false;
|
|
45
|
-
}
|
|
46
|
-
// half-open: only allow through if no probe is already in flight
|
|
47
|
-
if (halfOpenProbeInFlight) return false;
|
|
48
|
-
halfOpenProbeInFlight = true;
|
|
49
|
-
return true;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
function qdrantBreakerRecordSuccess(): void {
|
|
53
|
-
if (breakerState !== 'closed') {
|
|
54
|
-
log.info({ previousFailures: consecutiveFailures }, 'Qdrant circuit breaker closed — search succeeded');
|
|
55
|
-
}
|
|
56
|
-
consecutiveFailures = 0;
|
|
57
|
-
breakerState = 'closed';
|
|
58
|
-
openedAt = 0;
|
|
59
|
-
halfOpenProbeInFlight = false;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
function qdrantBreakerRecordFailure(): void {
|
|
63
|
-
consecutiveFailures++;
|
|
64
|
-
halfOpenProbeInFlight = false;
|
|
65
|
-
if (consecutiveFailures >= FAILURE_THRESHOLD) {
|
|
66
|
-
breakerState = 'open';
|
|
67
|
-
openedAt = Date.now();
|
|
68
|
-
log.warn(
|
|
69
|
-
{ consecutiveFailures, cooldownMs: COOLDOWN_MS },
|
|
70
|
-
'Qdrant circuit breaker opened — semantic search disabled until probe succeeds',
|
|
71
|
-
);
|
|
72
|
-
} else if (breakerState === 'half-open') {
|
|
73
|
-
// Probe failed — re-open
|
|
74
|
-
breakerState = 'open';
|
|
75
|
-
openedAt = Date.now();
|
|
76
|
-
log.warn('Qdrant circuit breaker re-opened — half-open probe failed');
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/** @internal Test-only: reset circuit breaker state */
|
|
81
|
-
export function _resetQdrantBreaker(): void {
|
|
82
|
-
breakerState = 'closed';
|
|
83
|
-
consecutiveFailures = 0;
|
|
84
|
-
openedAt = 0;
|
|
85
|
-
halfOpenProbeInFlight = false;
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
/** @internal Test-only: get breaker state */
|
|
89
|
-
export function _getQdrantBreakerState(): { state: BreakerState; consecutiveFailures: number } {
|
|
90
|
-
return { state: breakerState, consecutiveFailures };
|
|
91
|
-
}
|
|
19
|
+
// Re-export for tests that depend on these from this module
|
|
20
|
+
export { _resetQdrantBreaker, _getQdrantBreakerState };
|
|
92
21
|
|
|
93
22
|
export async function semanticSearch(
|
|
94
23
|
queryVector: number[],
|
|
@@ -100,31 +29,20 @@ export async function semanticSearch(
|
|
|
100
29
|
): Promise<Candidate[]> {
|
|
101
30
|
if (limit <= 0) return [];
|
|
102
31
|
|
|
103
|
-
// Circuit breaker: throw so the caller's .catch() marks the result as degraded
|
|
104
|
-
if (!qdrantBreakerAllows()) {
|
|
105
|
-
log.debug('Qdrant circuit breaker open — skipping semantic search');
|
|
106
|
-
throw new Error('Qdrant circuit breaker open');
|
|
107
|
-
}
|
|
108
|
-
|
|
109
32
|
const qdrant = getQdrantClient();
|
|
110
33
|
|
|
111
34
|
// Overfetch to account for items filtered out post-query (invalidated, excluded, etc.)
|
|
112
35
|
// Use 3x when exclusions are active to ensure enough results survive filtering
|
|
113
36
|
const overfetchMultiplier = excludedMessageIds.length > 0 ? 3 : 2;
|
|
114
37
|
const fetchLimit = limit * overfetchMultiplier;
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
results = await qdrant.searchWithFilter(
|
|
38
|
+
const results: QdrantSearchResult[] = await withQdrantBreaker(() =>
|
|
39
|
+
qdrant.searchWithFilter(
|
|
118
40
|
queryVector,
|
|
119
41
|
fetchLimit,
|
|
120
42
|
['item', 'summary', 'segment'],
|
|
121
43
|
excludedMessageIds,
|
|
122
|
-
)
|
|
123
|
-
|
|
124
|
-
} catch (err) {
|
|
125
|
-
qdrantBreakerRecordFailure();
|
|
126
|
-
throw err;
|
|
127
|
-
}
|
|
44
|
+
),
|
|
45
|
+
);
|
|
128
46
|
|
|
129
47
|
const db = getDb();
|
|
130
48
|
|
|
@@ -190,7 +190,7 @@ Reminder fires (scheduler)
|
|
|
190
190
|
The `enforceRoutingIntent()` function in `decision-engine.ts` runs after the LLM produces its channel selection but before deterministic checks. It overrides the decision's `selectedChannels` based on the routing intent:
|
|
191
191
|
|
|
192
192
|
- **`all_channels`**: Replaces `selectedChannels` with all connected channels (from `getConnectedChannels()`).
|
|
193
|
-
- **`multi_channel`**: If the LLM selected fewer than 2 channels but 2+ are connected, expands `selectedChannels` to
|
|
193
|
+
- **`multi_channel`**: If the LLM selected fewer than 2 channels but 2+ are connected, expands `selectedChannels` to at least two connected channels.
|
|
194
194
|
- **`single_channel`**: No override -- the LLM's selection stands.
|
|
195
195
|
|
|
196
196
|
When enforcement changes the decision, the updated channel selection is re-persisted to the `notification_decisions` table so the stored decision matches what was actually dispatched. The `reasoningSummary` is annotated with the enforcement action (e.g. `[routing_intent=all_channels enforced: vellum, telegram, sms]`).
|
|
@@ -14,7 +14,7 @@ import { v4 as uuid } from 'uuid';
|
|
|
14
14
|
import { getLogger } from '../util/logger.js';
|
|
15
15
|
import { pairDeliveryWithConversation } from './conversation-pairing.js';
|
|
16
16
|
import { composeFallbackCopy } from './copy-composer.js';
|
|
17
|
-
import { createDelivery, updateDeliveryStatus } from './deliveries-store.js';
|
|
17
|
+
import { createDelivery, findDeliveryByDecisionAndChannel, updateDeliveryStatus } from './deliveries-store.js';
|
|
18
18
|
import { resolveDestinations } from './destination-resolver.js';
|
|
19
19
|
import type { NotificationSignal } from './signal.js';
|
|
20
20
|
import type {
|
|
@@ -126,7 +126,7 @@ export class NotificationBroadcaster {
|
|
|
126
126
|
}
|
|
127
127
|
|
|
128
128
|
// Pair the delivery with a conversation before sending
|
|
129
|
-
const pairing = pairDeliveryWithConversation(signal, channel, copy);
|
|
129
|
+
const pairing = await pairDeliveryWithConversation(signal, channel, copy);
|
|
130
130
|
|
|
131
131
|
// For the vellum channel, merge the conversationId into deep-link metadata
|
|
132
132
|
// so the macOS/iOS client can navigate directly to the notification thread.
|
|
@@ -188,6 +188,24 @@ export class NotificationBroadcaster {
|
|
|
188
188
|
|
|
189
189
|
try {
|
|
190
190
|
if (hasPersistedDecision) {
|
|
191
|
+
const existingDelivery = findDeliveryByDecisionAndChannel(persistedDecisionId, channel);
|
|
192
|
+
if (existingDelivery) {
|
|
193
|
+
log.info(
|
|
194
|
+
{ channel, signalId: signal.signalId, existingDeliveryId: existingDelivery.id },
|
|
195
|
+
'Delivery already exists for this decision+channel — skipping duplicate',
|
|
196
|
+
);
|
|
197
|
+
results.push({
|
|
198
|
+
channel,
|
|
199
|
+
destination: destinationLabel,
|
|
200
|
+
status: 'skipped',
|
|
201
|
+
errorMessage: 'Duplicate delivery skipped',
|
|
202
|
+
conversationId: existingDelivery.conversationId ?? undefined,
|
|
203
|
+
messageId: existingDelivery.messageId ?? undefined,
|
|
204
|
+
conversationStrategy: existingDelivery.conversationStrategy ?? undefined,
|
|
205
|
+
});
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
|
|
191
209
|
createDelivery({
|
|
192
210
|
id: deliveryId,
|
|
193
211
|
notificationDecisionId: persistedDecisionId,
|