@vellumai/assistant 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +109 -0
- package/docs/architecture/memory.md +105 -0
- package/docs/skills.md +100 -0
- package/package.json +1 -1
- package/src/__tests__/archive-recall.test.ts +560 -0
- package/src/__tests__/conversation-agent-loop-overflow.test.ts +7 -0
- package/src/__tests__/conversation-agent-loop.test.ts +7 -0
- package/src/__tests__/conversation-clear-safety.test.ts +259 -0
- package/src/__tests__/conversation-memory-dirty-tail.test.ts +150 -0
- package/src/__tests__/conversation-provider-retry-repair.test.ts +7 -0
- package/src/__tests__/conversation-switch-memory-reduction.test.ts +474 -0
- package/src/__tests__/conversation-wipe.test.ts +226 -0
- package/src/__tests__/db-memory-archive-migration.test.ts +372 -0
- package/src/__tests__/db-memory-brief-state-migration.test.ts +213 -0
- package/src/__tests__/db-memory-reducer-checkpoints.test.ts +273 -0
- package/src/__tests__/db-schedule-syntax-migration.test.ts +3 -0
- package/src/__tests__/inline-command-runner.test.ts +311 -0
- package/src/__tests__/inline-skill-authoring-guard.test.ts +220 -0
- package/src/__tests__/inline-skill-load-permissions.test.ts +435 -0
- package/src/__tests__/list-messages-attachments.test.ts +96 -0
- package/src/__tests__/memory-brief-open-loops.test.ts +530 -0
- package/src/__tests__/memory-brief-time.test.ts +285 -0
- package/src/__tests__/memory-brief-wrapper.test.ts +311 -0
- package/src/__tests__/memory-chunk-archive.test.ts +400 -0
- package/src/__tests__/memory-chunk-dual-write.test.ts +453 -0
- package/src/__tests__/memory-episode-archive.test.ts +370 -0
- package/src/__tests__/memory-episode-dual-write.test.ts +626 -0
- package/src/__tests__/memory-observation-archive.test.ts +375 -0
- package/src/__tests__/memory-observation-dual-write.test.ts +318 -0
- package/src/__tests__/memory-recall-quality.test.ts +2 -2
- package/src/__tests__/memory-reducer-job.test.ts +538 -0
- package/src/__tests__/memory-reducer-scheduling.test.ts +473 -0
- package/src/__tests__/memory-reducer-store.test.ts +728 -0
- package/src/__tests__/memory-reducer-types.test.ts +707 -0
- package/src/__tests__/memory-reducer.test.ts +704 -0
- package/src/__tests__/memory-regressions.test.ts +30 -8
- package/src/__tests__/memory-simplified-config.test.ts +281 -0
- package/src/__tests__/parse-identity-fields.test.ts +129 -0
- package/src/__tests__/simplified-memory-e2e.test.ts +666 -0
- package/src/__tests__/simplified-memory-runtime.test.ts +616 -0
- package/src/__tests__/skill-load-inline-command.test.ts +598 -0
- package/src/__tests__/skill-load-inline-includes.test.ts +644 -0
- package/src/__tests__/skills-inline-command-expansions.test.ts +301 -0
- package/src/__tests__/skills-transitive-hash.test.ts +333 -0
- package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +320 -0
- package/src/__tests__/workspace-migration-backfill-installation-id.test.ts +4 -4
- package/src/cli/commands/conversations.ts +18 -0
- package/src/config/bundled-skills/app-builder/SKILL.md +8 -8
- package/src/config/bundled-skills/schedule/TOOLS.json +8 -0
- package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
- package/src/config/bundled-skills/skill-management/TOOLS.json +2 -2
- package/src/config/feature-flag-registry.json +16 -0
- package/src/config/raw-config-utils.ts +28 -0
- package/src/config/schema.ts +12 -0
- package/src/config/schemas/memory-simplified.ts +101 -0
- package/src/config/schemas/memory.ts +4 -0
- package/src/config/skills.ts +50 -4
- package/src/daemon/conversation-agent-loop-handlers.ts +8 -3
- package/src/daemon/conversation-agent-loop.ts +71 -1
- package/src/daemon/conversation-lifecycle.ts +11 -1
- package/src/daemon/conversation-memory.ts +117 -0
- package/src/daemon/conversation-runtime-assembly.ts +3 -1
- package/src/daemon/conversation-surfaces.ts +31 -8
- package/src/daemon/conversation.ts +40 -23
- package/src/daemon/handlers/config-embeddings.ts +10 -2
- package/src/daemon/handlers/config-model.ts +0 -9
- package/src/daemon/handlers/conversations.ts +11 -0
- package/src/daemon/handlers/identity.ts +12 -1
- package/src/daemon/lifecycle.ts +52 -1
- package/src/daemon/message-types/conversations.ts +0 -1
- package/src/daemon/server.ts +1 -1
- package/src/followups/followup-store.ts +47 -1
- package/src/memory/archive-recall.ts +516 -0
- package/src/memory/archive-store.ts +400 -0
- package/src/memory/brief-formatting.ts +33 -0
- package/src/memory/brief-open-loops.ts +266 -0
- package/src/memory/brief-time.ts +162 -0
- package/src/memory/brief.ts +75 -0
- package/src/memory/conversation-crud.ts +455 -101
- package/src/memory/conversation-key-store.ts +33 -4
- package/src/memory/db-init.ts +16 -0
- package/src/memory/indexer.ts +106 -15
- package/src/memory/job-handlers/backfill-simplified-memory.ts +462 -0
- package/src/memory/job-handlers/conversation-starters.ts +9 -3
- package/src/memory/job-handlers/embedding.test.ts +1 -0
- package/src/memory/job-handlers/embedding.ts +83 -0
- package/src/memory/job-handlers/reduce-conversation-memory.ts +229 -0
- package/src/memory/job-utils.ts +1 -1
- package/src/memory/jobs-store.ts +8 -0
- package/src/memory/jobs-worker.ts +20 -0
- package/src/memory/migrations/036-normalize-phone-identities.ts +49 -14
- package/src/memory/migrations/135-backfill-contact-interaction-stats.ts +9 -1
- package/src/memory/migrations/141-rename-verification-table.ts +8 -0
- package/src/memory/migrations/142-rename-verification-session-id-column.ts +7 -2
- package/src/memory/migrations/174-rename-thread-starters-table.ts +8 -0
- package/src/memory/migrations/185-memory-brief-state.ts +52 -0
- package/src/memory/migrations/186-memory-archive.ts +109 -0
- package/src/memory/migrations/187-memory-reducer-checkpoints.ts +19 -0
- package/src/memory/migrations/188-schedule-quiet-flag.ts +13 -0
- package/src/memory/migrations/index.ts +4 -0
- package/src/memory/qdrant-client.ts +23 -4
- package/src/memory/reducer-scheduler.ts +242 -0
- package/src/memory/reducer-store.ts +271 -0
- package/src/memory/reducer-types.ts +106 -0
- package/src/memory/reducer.ts +467 -0
- package/src/memory/schema/conversations.ts +3 -0
- package/src/memory/schema/index.ts +2 -0
- package/src/memory/schema/infrastructure.ts +1 -0
- package/src/memory/schema/memory-archive.ts +121 -0
- package/src/memory/schema/memory-brief.ts +55 -0
- package/src/memory/search/semantic.ts +17 -4
- package/src/oauth/oauth-store.ts +3 -1
- package/src/permissions/checker.ts +89 -6
- package/src/permissions/defaults.ts +14 -0
- package/src/runtime/auth/route-policy.ts +10 -1
- package/src/runtime/routes/conversation-management-routes.ts +94 -2
- package/src/runtime/routes/conversation-query-routes.ts +7 -0
- package/src/runtime/routes/conversation-routes.ts +52 -5
- package/src/runtime/routes/guardian-bootstrap-routes.ts +19 -7
- package/src/runtime/routes/identity-routes.ts +2 -35
- package/src/runtime/routes/llm-context-normalization.ts +14 -1
- package/src/runtime/routes/memory-item-routes.ts +90 -5
- package/src/runtime/routes/secret-routes.ts +3 -0
- package/src/runtime/routes/surface-action-routes.ts +68 -1
- package/src/schedule/schedule-store.ts +28 -0
- package/src/schedule/scheduler.ts +6 -2
- package/src/skills/inline-command-expansions.ts +204 -0
- package/src/skills/inline-command-render.ts +127 -0
- package/src/skills/inline-command-runner.ts +242 -0
- package/src/skills/transitive-version-hash.ts +88 -0
- package/src/tasks/task-store.ts +43 -1
- package/src/telemetry/usage-telemetry-reporter.ts +1 -1
- package/src/tools/filesystem/edit.ts +6 -1
- package/src/tools/filesystem/read.ts +6 -1
- package/src/tools/filesystem/write.ts +6 -1
- package/src/tools/memory/handlers.ts +129 -1
- package/src/tools/permission-checker.ts +8 -1
- package/src/tools/schedule/create.ts +3 -0
- package/src/tools/schedule/list.ts +5 -1
- package/src/tools/schedule/update.ts +6 -0
- package/src/tools/skills/load.ts +140 -6
- package/src/util/platform.ts +18 -0
- package/src/workspace/migrations/{002-backfill-installation-id.ts → 011-backfill-installation-id.ts} +1 -1
- package/src/workspace/migrations/registry.ts +1 -1
|
@@ -45,14 +45,20 @@ import { enqueueMemoryJob } from "./jobs-store.js";
|
|
|
45
45
|
import {
|
|
46
46
|
channelInboundEvents,
|
|
47
47
|
conversations,
|
|
48
|
+
conversationStarters,
|
|
48
49
|
llmRequestLogs,
|
|
50
|
+
memoryChunks,
|
|
49
51
|
memoryEmbeddings,
|
|
52
|
+
memoryEpisodes,
|
|
50
53
|
memoryItems,
|
|
51
54
|
memoryItemSources,
|
|
55
|
+
memoryObservations,
|
|
52
56
|
memorySegments,
|
|
53
57
|
memorySummaries,
|
|
54
58
|
messageAttachments,
|
|
55
59
|
messages,
|
|
60
|
+
openLoops,
|
|
61
|
+
timeContexts,
|
|
56
62
|
toolInvocations,
|
|
57
63
|
} from "./schema.js";
|
|
58
64
|
import { cancelPendingJobsForConversation } from "./task-memory-cleanup.js";
|
|
@@ -171,6 +177,9 @@ export interface ConversationRow {
|
|
|
171
177
|
forkParentMessageId: string | null;
|
|
172
178
|
isAutoTitle: number;
|
|
173
179
|
scheduleJobId: string | null;
|
|
180
|
+
memoryReducedThroughMessageId: string | null;
|
|
181
|
+
memoryDirtyTailSinceMessageId: string | null;
|
|
182
|
+
memoryLastReducedAt: number | null;
|
|
174
183
|
}
|
|
175
184
|
|
|
176
185
|
export const parseConversation = createRowMapper<
|
|
@@ -196,6 +205,9 @@ export const parseConversation = createRowMapper<
|
|
|
196
205
|
forkParentMessageId: "forkParentMessageId",
|
|
197
206
|
isAutoTitle: "isAutoTitle",
|
|
198
207
|
scheduleJobId: "scheduleJobId",
|
|
208
|
+
memoryReducedThroughMessageId: "memoryReducedThroughMessageId",
|
|
209
|
+
memoryDirtyTailSinceMessageId: "memoryDirtyTailSinceMessageId",
|
|
210
|
+
memoryLastReducedAt: "memoryLastReducedAt",
|
|
199
211
|
});
|
|
200
212
|
|
|
201
213
|
export interface MessageRow {
|
|
@@ -375,127 +387,153 @@ export function forkConversation(params: {
|
|
|
375
387
|
: ([] as MessageRow[]);
|
|
376
388
|
const forkParentMessageId = messagesToCopy.at(-1)?.id ?? null;
|
|
377
389
|
const forkTitle = `${sourceConversation.title ?? "Untitled"} (Fork)`;
|
|
378
|
-
const forkedConversation = createConversation({
|
|
379
|
-
title: forkTitle,
|
|
380
|
-
conversationType: "standard",
|
|
381
|
-
});
|
|
382
390
|
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
391
|
+
// Collect disk-sync work to run after the transaction commits.
|
|
392
|
+
const diskSyncQueue: Array<{
|
|
393
|
+
conversationId: string;
|
|
394
|
+
messageId: string;
|
|
395
|
+
createdAt: number;
|
|
396
|
+
}> = [];
|
|
397
|
+
|
|
398
|
+
// Wrap all DB mutations in a single transaction so a mid-flight failure
|
|
399
|
+
// rolls back cleanly instead of leaving a partial fork. Helper functions
|
|
400
|
+
// (linkAttachmentToMessage, relinkAttachments, seedForkedConversationAttention)
|
|
401
|
+
// use the same underlying bun:sqlite connection, so their writes participate
|
|
402
|
+
// in this transaction automatically.
|
|
403
|
+
const forkedConversation = db.transaction(() => {
|
|
404
|
+
const fc = createConversation({
|
|
405
|
+
title: forkTitle,
|
|
406
|
+
conversationType: "standard",
|
|
407
|
+
});
|
|
399
408
|
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
metadata: cloneForkMessageMetadata(message.metadata, message.id),
|
|
409
|
+
db.update(conversations)
|
|
410
|
+
.set({
|
|
411
|
+
forkParentConversationId: sourceConversation.id,
|
|
412
|
+
forkParentMessageId,
|
|
413
|
+
contextSummary: preserveSourceCompactionState
|
|
414
|
+
? sourceConversation.contextSummary
|
|
415
|
+
: null,
|
|
416
|
+
contextCompactedMessageCount: preserveSourceCompactionState
|
|
417
|
+
? sourceConversation.contextCompactedMessageCount
|
|
418
|
+
: 0,
|
|
419
|
+
contextCompactedAt: preserveSourceCompactionState
|
|
420
|
+
? sourceConversation.contextCompactedAt
|
|
421
|
+
: null,
|
|
414
422
|
})
|
|
423
|
+
.where(eq(conversations.id, fc.id))
|
|
415
424
|
.run();
|
|
416
|
-
forkedMessageIds.set(message.id, forkedMessageId);
|
|
417
|
-
|
|
418
|
-
if (message.role === "assistant") {
|
|
419
|
-
latestForkedAssistant = {
|
|
420
|
-
messageId: forkedMessageId,
|
|
421
|
-
messageAt: message.createdAt,
|
|
422
|
-
};
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
|
|
426
|
-
const attachmentIdMap = new Map<string, string>();
|
|
427
|
-
for (const message of messagesToCopy) {
|
|
428
|
-
const forkedMessageId = forkedMessageIds.get(message.id);
|
|
429
|
-
if (!forkedMessageId) continue;
|
|
430
425
|
|
|
431
|
-
const
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
.from(messageAttachments)
|
|
437
|
-
.where(eq(messageAttachments.messageId, message.id))
|
|
438
|
-
.orderBy(messageAttachments.position)
|
|
439
|
-
.all();
|
|
440
|
-
const uncachedAttachmentLinks = attachmentLinks.filter(
|
|
441
|
-
(link) => !attachmentIdMap.has(link.attachmentId),
|
|
442
|
-
);
|
|
443
|
-
const stagingMessageId = uncachedAttachmentLinks.length > 0 ? uuid() : null;
|
|
426
|
+
const forkedMessageIds = new Map<string, string>();
|
|
427
|
+
let latestForkedAssistant: {
|
|
428
|
+
messageId: string;
|
|
429
|
+
messageAt: number;
|
|
430
|
+
} | null = null;
|
|
444
431
|
|
|
445
|
-
|
|
432
|
+
for (const message of messagesToCopy) {
|
|
433
|
+
const forkedMessageId = uuid();
|
|
446
434
|
db.insert(messages)
|
|
447
435
|
.values({
|
|
448
|
-
id:
|
|
449
|
-
conversationId:
|
|
436
|
+
id: forkedMessageId,
|
|
437
|
+
conversationId: fc.id,
|
|
450
438
|
role: message.role,
|
|
451
|
-
content:
|
|
439
|
+
content: message.content,
|
|
452
440
|
createdAt: message.createdAt,
|
|
453
|
-
metadata:
|
|
441
|
+
metadata: cloneForkMessageMetadata(message.metadata, message.id),
|
|
454
442
|
})
|
|
455
443
|
.run();
|
|
444
|
+
forkedMessageIds.set(message.id, forkedMessageId);
|
|
445
|
+
|
|
446
|
+
if (message.role === "assistant") {
|
|
447
|
+
latestForkedAssistant = {
|
|
448
|
+
messageId: forkedMessageId,
|
|
449
|
+
messageAt: message.createdAt,
|
|
450
|
+
};
|
|
451
|
+
}
|
|
456
452
|
}
|
|
457
453
|
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
454
|
+
const attachmentIdMap = new Map<string, string>();
|
|
455
|
+
for (const message of messagesToCopy) {
|
|
456
|
+
const forkedMessageId = forkedMessageIds.get(message.id);
|
|
457
|
+
if (!forkedMessageId) continue;
|
|
458
|
+
|
|
459
|
+
const attachmentLinks = db
|
|
460
|
+
.select({
|
|
461
|
+
attachmentId: messageAttachments.attachmentId,
|
|
462
|
+
position: messageAttachments.position,
|
|
463
|
+
})
|
|
464
|
+
.from(messageAttachments)
|
|
465
|
+
.where(eq(messageAttachments.messageId, message.id))
|
|
466
|
+
.orderBy(messageAttachments.position)
|
|
467
|
+
.all();
|
|
468
|
+
const uncachedAttachmentLinks = attachmentLinks.filter(
|
|
469
|
+
(link) => !attachmentIdMap.has(link.attachmentId),
|
|
470
|
+
);
|
|
471
|
+
const stagingMessageId =
|
|
472
|
+
uncachedAttachmentLinks.length > 0 ? uuid() : null;
|
|
473
|
+
|
|
474
|
+
if (stagingMessageId) {
|
|
475
|
+
db.insert(messages)
|
|
462
476
|
.values({
|
|
463
|
-
id:
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
createdAt:
|
|
477
|
+
id: stagingMessageId,
|
|
478
|
+
conversationId: fc.id,
|
|
479
|
+
role: message.role,
|
|
480
|
+
content: "",
|
|
481
|
+
createdAt: message.createdAt,
|
|
482
|
+
metadata: null,
|
|
468
483
|
})
|
|
469
484
|
.run();
|
|
470
|
-
continue;
|
|
471
485
|
}
|
|
472
486
|
|
|
473
|
-
const
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
487
|
+
for (const link of attachmentLinks) {
|
|
488
|
+
const cachedAttachmentId = attachmentIdMap.get(link.attachmentId);
|
|
489
|
+
if (cachedAttachmentId) {
|
|
490
|
+
db.insert(messageAttachments)
|
|
491
|
+
.values({
|
|
492
|
+
id: uuid(),
|
|
493
|
+
messageId: forkedMessageId,
|
|
494
|
+
attachmentId: cachedAttachmentId,
|
|
495
|
+
position: link.position,
|
|
496
|
+
createdAt: Date.now(),
|
|
497
|
+
})
|
|
498
|
+
.run();
|
|
499
|
+
continue;
|
|
500
|
+
}
|
|
480
501
|
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
502
|
+
const scopedAttachmentId = linkAttachmentToMessage(
|
|
503
|
+
stagingMessageId ?? forkedMessageId,
|
|
504
|
+
link.attachmentId,
|
|
505
|
+
link.position,
|
|
506
|
+
);
|
|
507
|
+
attachmentIdMap.set(link.attachmentId, scopedAttachmentId);
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
if (stagingMessageId) {
|
|
511
|
+
relinkAttachments([stagingMessageId], forkedMessageId);
|
|
512
|
+
db.delete(messages).where(eq(messages.id, stagingMessageId)).run();
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
diskSyncQueue.push({
|
|
516
|
+
conversationId: fc.id,
|
|
517
|
+
messageId: forkedMessageId,
|
|
518
|
+
createdAt: fc.createdAt,
|
|
519
|
+
});
|
|
484
520
|
}
|
|
485
521
|
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
);
|
|
491
|
-
}
|
|
522
|
+
seedForkedConversationAttention({
|
|
523
|
+
conversationId: fc.id,
|
|
524
|
+
latestAssistantMessageId: latestForkedAssistant?.messageId ?? null,
|
|
525
|
+
latestAssistantMessageAt: latestForkedAssistant?.messageAt ?? null,
|
|
526
|
+
});
|
|
492
527
|
|
|
493
|
-
|
|
494
|
-
conversationId: forkedConversation.id,
|
|
495
|
-
latestAssistantMessageId: latestForkedAssistant?.messageId ?? null,
|
|
496
|
-
latestAssistantMessageAt: latestForkedAssistant?.messageAt ?? null,
|
|
528
|
+
return fc;
|
|
497
529
|
});
|
|
498
530
|
|
|
531
|
+
// Disk-view sync runs after commit — file I/O is idempotent and
|
|
532
|
+
// conversation deletion cleans up orphaned directories.
|
|
533
|
+
for (const entry of diskSyncQueue) {
|
|
534
|
+
syncMessageToDisk(entry.conversationId, entry.messageId, entry.createdAt);
|
|
535
|
+
}
|
|
536
|
+
|
|
499
537
|
const persistedFork = getConversation(forkedConversation.id);
|
|
500
538
|
if (!persistedFork) {
|
|
501
539
|
throw new Error(
|
|
@@ -513,12 +551,21 @@ export function forkConversation(params: {
|
|
|
513
551
|
*/
|
|
514
552
|
export function deleteConversation(id: string): DeletedMemoryIds {
|
|
515
553
|
const db = getDb();
|
|
516
|
-
const result: DeletedMemoryIds = {
|
|
554
|
+
const result: DeletedMemoryIds = {
|
|
555
|
+
segmentIds: [],
|
|
556
|
+
orphanedItemIds: [],
|
|
557
|
+
deletedSummaryIds: [],
|
|
558
|
+
deletedObservationIds: [],
|
|
559
|
+
deletedChunkIds: [],
|
|
560
|
+
deletedEpisodeIds: [],
|
|
561
|
+
};
|
|
517
562
|
|
|
518
563
|
// Capture createdAt before the transaction deletes the row — needed to
|
|
519
564
|
// resolve the conversation's disk-view directory path after deletion.
|
|
520
565
|
const convBeforeDelete = getConversation(id);
|
|
521
566
|
const createdAtForDiskCleanup = convBeforeDelete?.createdAt;
|
|
567
|
+
const memoryScopeId = convBeforeDelete?.memoryScopeId;
|
|
568
|
+
const isPrivateScope = memoryScopeId?.startsWith("private:") ?? false;
|
|
522
569
|
|
|
523
570
|
db.transaction((tx) => {
|
|
524
571
|
// Collect all message IDs for this conversation.
|
|
@@ -607,6 +654,134 @@ export function deleteConversation(id: string): DeletedMemoryIds {
|
|
|
607
654
|
.run();
|
|
608
655
|
}
|
|
609
656
|
|
|
657
|
+
if (isPrivateScope && memoryScopeId) {
|
|
658
|
+
// Sweep remaining memory items with this private scopeId.
|
|
659
|
+
const scopeItems = tx
|
|
660
|
+
.select({ id: memoryItems.id })
|
|
661
|
+
.from(memoryItems)
|
|
662
|
+
.where(eq(memoryItems.scopeId, memoryScopeId))
|
|
663
|
+
.all();
|
|
664
|
+
const alreadyDeleted = new Set(result.orphanedItemIds);
|
|
665
|
+
const scopeItemIds = scopeItems
|
|
666
|
+
.map((r) => r.id)
|
|
667
|
+
.filter((id) => !alreadyDeleted.has(id));
|
|
668
|
+
|
|
669
|
+
if (scopeItemIds.length > 0) {
|
|
670
|
+
tx.delete(memoryEmbeddings)
|
|
671
|
+
.where(
|
|
672
|
+
and(
|
|
673
|
+
eq(memoryEmbeddings.targetType, "item"),
|
|
674
|
+
inArray(memoryEmbeddings.targetId, scopeItemIds),
|
|
675
|
+
),
|
|
676
|
+
)
|
|
677
|
+
.run();
|
|
678
|
+
tx.delete(memoryItemSources)
|
|
679
|
+
.where(inArray(memoryItemSources.memoryItemId, scopeItemIds))
|
|
680
|
+
.run();
|
|
681
|
+
tx.delete(memoryItems)
|
|
682
|
+
.where(inArray(memoryItems.id, scopeItemIds))
|
|
683
|
+
.run();
|
|
684
|
+
result.orphanedItemIds.push(...scopeItemIds);
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
// Sweep memory summaries with this private scopeId.
|
|
688
|
+
const scopeSummaries = tx
|
|
689
|
+
.select({ id: memorySummaries.id })
|
|
690
|
+
.from(memorySummaries)
|
|
691
|
+
.where(eq(memorySummaries.scopeId, memoryScopeId))
|
|
692
|
+
.all();
|
|
693
|
+
const scopeSummaryIds = scopeSummaries.map((r) => r.id);
|
|
694
|
+
|
|
695
|
+
if (scopeSummaryIds.length > 0) {
|
|
696
|
+
tx.delete(memoryEmbeddings)
|
|
697
|
+
.where(
|
|
698
|
+
and(
|
|
699
|
+
eq(memoryEmbeddings.targetType, "summary"),
|
|
700
|
+
inArray(memoryEmbeddings.targetId, scopeSummaryIds),
|
|
701
|
+
),
|
|
702
|
+
)
|
|
703
|
+
.run();
|
|
704
|
+
tx.delete(memorySummaries)
|
|
705
|
+
.where(inArray(memorySummaries.id, scopeSummaryIds))
|
|
706
|
+
.run();
|
|
707
|
+
result.deletedSummaryIds.push(...scopeSummaryIds);
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
// Sweep conversation starters with this private scopeId.
|
|
711
|
+
tx.delete(conversationStarters)
|
|
712
|
+
.where(eq(conversationStarters.scopeId, memoryScopeId))
|
|
713
|
+
.run();
|
|
714
|
+
|
|
715
|
+
// Sweep brief-state tables scoped to this private conversation.
|
|
716
|
+
tx.delete(timeContexts)
|
|
717
|
+
.where(eq(timeContexts.scopeId, memoryScopeId))
|
|
718
|
+
.run();
|
|
719
|
+
tx.delete(openLoops).where(eq(openLoops.scopeId, memoryScopeId)).run();
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
// Collect archive table IDs before the cascade delete removes them.
|
|
723
|
+
// Observations and episodes reference conversations with ON DELETE CASCADE,
|
|
724
|
+
// and chunks cascade from observations.
|
|
725
|
+
const observationRows = tx
|
|
726
|
+
.select({ id: memoryObservations.id })
|
|
727
|
+
.from(memoryObservations)
|
|
728
|
+
.where(eq(memoryObservations.conversationId, id))
|
|
729
|
+
.all();
|
|
730
|
+
const observationIds = observationRows.map((r) => r.id);
|
|
731
|
+
|
|
732
|
+
if (observationIds.length > 0) {
|
|
733
|
+
// Collect chunk IDs before observations cascade-delete them.
|
|
734
|
+
const chunkRows = tx
|
|
735
|
+
.select({ id: memoryChunks.id })
|
|
736
|
+
.from(memoryChunks)
|
|
737
|
+
.where(inArray(memoryChunks.observationId, observationIds))
|
|
738
|
+
.all();
|
|
739
|
+
const chunkIds = chunkRows.map((r) => r.id);
|
|
740
|
+
|
|
741
|
+
// Clean up embeddings for chunks.
|
|
742
|
+
if (chunkIds.length > 0) {
|
|
743
|
+
tx.delete(memoryEmbeddings)
|
|
744
|
+
.where(
|
|
745
|
+
and(
|
|
746
|
+
eq(memoryEmbeddings.targetType, "chunk"),
|
|
747
|
+
inArray(memoryEmbeddings.targetId, chunkIds),
|
|
748
|
+
),
|
|
749
|
+
)
|
|
750
|
+
.run();
|
|
751
|
+
result.deletedChunkIds.push(...chunkIds);
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
// Clean up embeddings for observations.
|
|
755
|
+
tx.delete(memoryEmbeddings)
|
|
756
|
+
.where(
|
|
757
|
+
and(
|
|
758
|
+
eq(memoryEmbeddings.targetType, "observation"),
|
|
759
|
+
inArray(memoryEmbeddings.targetId, observationIds),
|
|
760
|
+
),
|
|
761
|
+
)
|
|
762
|
+
.run();
|
|
763
|
+
result.deletedObservationIds.push(...observationIds);
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
const episodeRows = tx
|
|
767
|
+
.select({ id: memoryEpisodes.id })
|
|
768
|
+
.from(memoryEpisodes)
|
|
769
|
+
.where(eq(memoryEpisodes.conversationId, id))
|
|
770
|
+
.all();
|
|
771
|
+
const episodeIds = episodeRows.map((r) => r.id);
|
|
772
|
+
|
|
773
|
+
if (episodeIds.length > 0) {
|
|
774
|
+
tx.delete(memoryEmbeddings)
|
|
775
|
+
.where(
|
|
776
|
+
and(
|
|
777
|
+
eq(memoryEmbeddings.targetType, "episode"),
|
|
778
|
+
inArray(memoryEmbeddings.targetId, episodeIds),
|
|
779
|
+
),
|
|
780
|
+
)
|
|
781
|
+
.run();
|
|
782
|
+
result.deletedEpisodeIds.push(...episodeIds);
|
|
783
|
+
}
|
|
784
|
+
|
|
610
785
|
tx.delete(conversations).where(eq(conversations.id, id)).run();
|
|
611
786
|
});
|
|
612
787
|
|
|
@@ -799,7 +974,10 @@ export function wipeConversation(id: string): WipeConversationResult {
|
|
|
799
974
|
return {
|
|
800
975
|
...deletedMemoryIds,
|
|
801
976
|
unsupersededItemIds,
|
|
802
|
-
deletedSummaryIds
|
|
977
|
+
deletedSummaryIds: [
|
|
978
|
+
...deletedSummaryIds,
|
|
979
|
+
...deletedMemoryIds.deletedSummaryIds,
|
|
980
|
+
],
|
|
803
981
|
cancelledJobCount,
|
|
804
982
|
};
|
|
805
983
|
}
|
|
@@ -821,16 +999,34 @@ export function purgePrivateConversations(): {
|
|
|
821
999
|
.all();
|
|
822
1000
|
|
|
823
1001
|
if (privateConvs.length === 0) {
|
|
824
|
-
return {
|
|
1002
|
+
return {
|
|
1003
|
+
count: 0,
|
|
1004
|
+
deletedMemory: {
|
|
1005
|
+
segmentIds: [],
|
|
1006
|
+
orphanedItemIds: [],
|
|
1007
|
+
deletedSummaryIds: [],
|
|
1008
|
+
deletedObservationIds: [],
|
|
1009
|
+
deletedChunkIds: [],
|
|
1010
|
+
deletedEpisodeIds: [],
|
|
1011
|
+
},
|
|
1012
|
+
};
|
|
825
1013
|
}
|
|
826
1014
|
|
|
827
1015
|
const allSegmentIds: string[] = [];
|
|
828
1016
|
const allOrphanedItemIds: string[] = [];
|
|
1017
|
+
const allDeletedSummaryIds: string[] = [];
|
|
1018
|
+
const allDeletedObservationIds: string[] = [];
|
|
1019
|
+
const allDeletedChunkIds: string[] = [];
|
|
1020
|
+
const allDeletedEpisodeIds: string[] = [];
|
|
829
1021
|
|
|
830
1022
|
for (const conv of privateConvs) {
|
|
831
1023
|
const deleted = deleteConversation(conv.id);
|
|
832
1024
|
allSegmentIds.push(...deleted.segmentIds);
|
|
833
1025
|
allOrphanedItemIds.push(...deleted.orphanedItemIds);
|
|
1026
|
+
allDeletedSummaryIds.push(...deleted.deletedSummaryIds);
|
|
1027
|
+
allDeletedObservationIds.push(...deleted.deletedObservationIds);
|
|
1028
|
+
allDeletedChunkIds.push(...deleted.deletedChunkIds);
|
|
1029
|
+
allDeletedEpisodeIds.push(...deleted.deletedEpisodeIds);
|
|
834
1030
|
}
|
|
835
1031
|
|
|
836
1032
|
return {
|
|
@@ -838,6 +1034,10 @@ export function purgePrivateConversations(): {
|
|
|
838
1034
|
deletedMemory: {
|
|
839
1035
|
segmentIds: allSegmentIds,
|
|
840
1036
|
orphanedItemIds: allOrphanedItemIds,
|
|
1037
|
+
deletedSummaryIds: allDeletedSummaryIds,
|
|
1038
|
+
deletedObservationIds: allDeletedObservationIds,
|
|
1039
|
+
deletedChunkIds: allDeletedChunkIds,
|
|
1040
|
+
deletedEpisodeIds: allDeletedEpisodeIds,
|
|
841
1041
|
},
|
|
842
1042
|
};
|
|
843
1043
|
}
|
|
@@ -920,6 +1120,13 @@ export async function addMessage(
|
|
|
920
1120
|
throw err;
|
|
921
1121
|
}
|
|
922
1122
|
}
|
|
1123
|
+
|
|
1124
|
+
// Mark the conversation dirty for delayed memory reduction. This runs
|
|
1125
|
+
// after the insert transaction succeeds so the reducer knows which
|
|
1126
|
+
// conversations have unprocessed messages. The helper preserves the
|
|
1127
|
+
// earliest unreduced boundary (no-op when already dirty).
|
|
1128
|
+
markConversationMemoryDirty(conversationId, messageId);
|
|
1129
|
+
|
|
923
1130
|
const message = {
|
|
924
1131
|
id: messageId,
|
|
925
1132
|
conversationId,
|
|
@@ -1108,6 +1315,15 @@ export function clearAll(): { conversations: number; messages: number } {
|
|
|
1108
1315
|
rawExec("DELETE FROM messages");
|
|
1109
1316
|
rawExec("DELETE FROM conversations");
|
|
1110
1317
|
|
|
1318
|
+
// Record audit event — lifecycle_events is NOT deleted by clearAll(),
|
|
1319
|
+
// so this survives the wipe and provides a permanent trail.
|
|
1320
|
+
rawRun(
|
|
1321
|
+
`INSERT INTO lifecycle_events (id, event_name, created_at) VALUES (?, ?, ?)`,
|
|
1322
|
+
uuid(),
|
|
1323
|
+
"conversations_clear_all",
|
|
1324
|
+
Date.now(),
|
|
1325
|
+
);
|
|
1326
|
+
|
|
1111
1327
|
// Rebuild corrupted FTS tables and restore triggers after all base-table
|
|
1112
1328
|
// DELETEs have completed. Dropping the virtual table clears the corruption,
|
|
1113
1329
|
// and recreating it + triggers means subsequent writes maintain FTS
|
|
@@ -1214,11 +1430,14 @@ export function deleteLastExchange(conversationId: string): number {
|
|
|
1214
1430
|
export interface DeletedMemoryIds {
|
|
1215
1431
|
segmentIds: string[];
|
|
1216
1432
|
orphanedItemIds: string[];
|
|
1433
|
+
deletedSummaryIds: string[];
|
|
1434
|
+
deletedObservationIds: string[];
|
|
1435
|
+
deletedChunkIds: string[];
|
|
1436
|
+
deletedEpisodeIds: string[];
|
|
1217
1437
|
}
|
|
1218
1438
|
|
|
1219
1439
|
export interface WipeConversationResult extends DeletedMemoryIds {
|
|
1220
1440
|
unsupersededItemIds: string[];
|
|
1221
|
-
deletedSummaryIds: string[];
|
|
1222
1441
|
cancelledJobCount: number;
|
|
1223
1442
|
}
|
|
1224
1443
|
|
|
@@ -1284,7 +1503,14 @@ export function relinkAttachments(
|
|
|
1284
1503
|
*/
|
|
1285
1504
|
export function deleteMessageById(messageId: string): DeletedMemoryIds {
|
|
1286
1505
|
const db = getDb();
|
|
1287
|
-
const result: DeletedMemoryIds = {
|
|
1506
|
+
const result: DeletedMemoryIds = {
|
|
1507
|
+
segmentIds: [],
|
|
1508
|
+
orphanedItemIds: [],
|
|
1509
|
+
deletedSummaryIds: [],
|
|
1510
|
+
deletedObservationIds: [],
|
|
1511
|
+
deletedChunkIds: [],
|
|
1512
|
+
deletedEpisodeIds: [],
|
|
1513
|
+
};
|
|
1288
1514
|
|
|
1289
1515
|
// Collect attachment IDs linked to this message before cascade-delete
|
|
1290
1516
|
// so we can scope orphan cleanup to only those candidates.
|
|
@@ -1372,6 +1598,134 @@ export function deleteMessageById(messageId: string): DeletedMemoryIds {
|
|
|
1372
1598
|
return result;
|
|
1373
1599
|
}
|
|
1374
1600
|
|
|
1601
|
+
/**
|
|
1602
|
+
* Mark a conversation as having unreduced messages starting from the given
|
|
1603
|
+
* message. Sets `memoryDirtyTailSinceMessageId` only when it is currently
|
|
1604
|
+
* null so the earliest unreduced boundary is preserved across multiple
|
|
1605
|
+
* messages — later messages must not clobber the original dirty marker.
|
|
1606
|
+
*
|
|
1607
|
+
* Also upserts a pending `reduce_conversation_memory` job scheduled at
|
|
1608
|
+
* `now + idleDelayMs`. If a pending job for this conversation already exists,
|
|
1609
|
+
* its `runAfter` is pushed forward (rescheduled) so the reducer waits for
|
|
1610
|
+
* the full idle window after the *latest* message — avoiding premature runs
|
|
1611
|
+
* while the user is still actively typing.
|
|
1612
|
+
*/
|
|
1613
|
+
export function markConversationMemoryDirty(
|
|
1614
|
+
conversationId: string,
|
|
1615
|
+
messageId: string,
|
|
1616
|
+
): void {
|
|
1617
|
+
const db = getDb();
|
|
1618
|
+
db.update(conversations)
|
|
1619
|
+
.set({ memoryDirtyTailSinceMessageId: messageId })
|
|
1620
|
+
.where(
|
|
1621
|
+
and(
|
|
1622
|
+
eq(conversations.id, conversationId),
|
|
1623
|
+
isNull(conversations.memoryDirtyTailSinceMessageId),
|
|
1624
|
+
),
|
|
1625
|
+
)
|
|
1626
|
+
.run();
|
|
1627
|
+
|
|
1628
|
+
// Schedule (or reschedule) a deferred reducer job for this conversation.
|
|
1629
|
+
scheduleReducerJob(conversationId);
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
/**
|
|
1633
|
+
* Upsert a pending `reduce_conversation_memory` job for the given
|
|
1634
|
+
* conversation, scheduled `idleDelayMs` from now. If one already exists in
|
|
1635
|
+
* pending state, its `runAfter` is pushed forward to restart the idle timer.
|
|
1636
|
+
* This ensures exactly one pending reducer job per conversation — new
|
|
1637
|
+
* messages reschedule rather than duplicate.
|
|
1638
|
+
*/
|
|
1639
|
+
export function scheduleReducerJob(
|
|
1640
|
+
conversationId: string,
|
|
1641
|
+
runAfter?: number,
|
|
1642
|
+
): void {
|
|
1643
|
+
const idleDelayMs = getReducerIdleDelayMs();
|
|
1644
|
+
const scheduledAt = runAfter ?? Date.now() + idleDelayMs;
|
|
1645
|
+
|
|
1646
|
+
const existing = rawGet<{ id: string; status: string }>(
|
|
1647
|
+
`SELECT id, status FROM memory_jobs
|
|
1648
|
+
WHERE type = 'reduce_conversation_memory'
|
|
1649
|
+
AND json_extract(payload, '$.conversationId') = ?
|
|
1650
|
+
AND status = 'pending'
|
|
1651
|
+
LIMIT 1`,
|
|
1652
|
+
conversationId,
|
|
1653
|
+
);
|
|
1654
|
+
|
|
1655
|
+
if (existing) {
|
|
1656
|
+
// Reschedule: push runAfter forward so the idle timer resets.
|
|
1657
|
+
rawRun(
|
|
1658
|
+
`UPDATE memory_jobs SET run_after = ?, updated_at = ? WHERE id = ?`,
|
|
1659
|
+
scheduledAt,
|
|
1660
|
+
Date.now(),
|
|
1661
|
+
existing.id,
|
|
1662
|
+
);
|
|
1663
|
+
} else {
|
|
1664
|
+
enqueueMemoryJob(
|
|
1665
|
+
"reduce_conversation_memory",
|
|
1666
|
+
{ conversationId },
|
|
1667
|
+
scheduledAt,
|
|
1668
|
+
);
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
/**
|
|
1673
|
+
* Startup sweep: find conversations that are marked dirty and whose tail
|
|
1674
|
+
* message is already older than the idle delay. For these conversations the
|
|
1675
|
+
* reducer should have run but didn't (daemon was down). Enqueue immediate
|
|
1676
|
+
* reducer jobs for each so they are processed on the next worker tick.
|
|
1677
|
+
*
|
|
1678
|
+
* Conversations whose tail is still within the idle window are skipped —
|
|
1679
|
+
* the normal `markConversationMemoryDirty` path will schedule them when
|
|
1680
|
+
* new messages arrive (or on the next conversation interaction).
|
|
1681
|
+
*
|
|
1682
|
+
* Returns the number of jobs enqueued.
|
|
1683
|
+
*/
|
|
1684
|
+
export function sweepStaleReducerJobs(): number {
|
|
1685
|
+
const idleDelayMs = getReducerIdleDelayMs();
|
|
1686
|
+
const cutoff = Date.now() - idleDelayMs;
|
|
1687
|
+
|
|
1688
|
+
// Find dirty conversations whose latest message is older than the idle
|
|
1689
|
+
// window AND that don't already have a pending reducer job.
|
|
1690
|
+
const stale = rawAll<{ conversationId: string }>(
|
|
1691
|
+
`SELECT c.id AS conversationId
|
|
1692
|
+
FROM conversations c
|
|
1693
|
+
WHERE c.memory_dirty_tail_since_message_id IS NOT NULL
|
|
1694
|
+
AND NOT EXISTS (
|
|
1695
|
+
SELECT 1 FROM memory_jobs mj
|
|
1696
|
+
WHERE mj.type = 'reduce_conversation_memory'
|
|
1697
|
+
AND json_extract(mj.payload, '$.conversationId') = c.id
|
|
1698
|
+
AND mj.status IN ('pending', 'running')
|
|
1699
|
+
)
|
|
1700
|
+
AND (
|
|
1701
|
+
SELECT MAX(m.created_at) FROM messages m
|
|
1702
|
+
WHERE m.conversation_id = c.id
|
|
1703
|
+
) <= ?`,
|
|
1704
|
+
cutoff,
|
|
1705
|
+
);
|
|
1706
|
+
|
|
1707
|
+
for (const { conversationId } of stale) {
|
|
1708
|
+
enqueueMemoryJob("reduce_conversation_memory", { conversationId });
|
|
1709
|
+
}
|
|
1710
|
+
|
|
1711
|
+
return stale.length;
|
|
1712
|
+
}
|
|
1713
|
+
|
|
1714
|
+
function getReducerIdleDelayMs(): number {
|
|
1715
|
+
// Some test suites mock getConfig() with partial objects; fall back to the
|
|
1716
|
+
// schema default so reducer scheduling stays stable outside full config load.
|
|
1717
|
+
const config = getConfig() as {
|
|
1718
|
+
memory?: {
|
|
1719
|
+
simplified?: {
|
|
1720
|
+
reducer?: {
|
|
1721
|
+
idleDelayMs?: number;
|
|
1722
|
+
};
|
|
1723
|
+
};
|
|
1724
|
+
};
|
|
1725
|
+
};
|
|
1726
|
+
return config.memory?.simplified?.reducer?.idleDelayMs ?? 30_000;
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1375
1729
|
export function setConversationOriginChannelIfUnset(
|
|
1376
1730
|
conversationId: string,
|
|
1377
1731
|
channel: ChannelId,
|