@vellumai/assistant 0.5.2 → 0.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/ARCHITECTURE.md +109 -0
  2. package/docs/architecture/memory.md +105 -0
  3. package/docs/skills.md +100 -0
  4. package/package.json +1 -1
  5. package/src/__tests__/archive-recall.test.ts +560 -0
  6. package/src/__tests__/conversation-agent-loop-overflow.test.ts +7 -0
  7. package/src/__tests__/conversation-agent-loop.test.ts +7 -0
  8. package/src/__tests__/conversation-clear-safety.test.ts +259 -0
  9. package/src/__tests__/conversation-memory-dirty-tail.test.ts +150 -0
  10. package/src/__tests__/conversation-provider-retry-repair.test.ts +7 -0
  11. package/src/__tests__/conversation-switch-memory-reduction.test.ts +474 -0
  12. package/src/__tests__/conversation-wipe.test.ts +226 -0
  13. package/src/__tests__/db-memory-archive-migration.test.ts +372 -0
  14. package/src/__tests__/db-memory-brief-state-migration.test.ts +213 -0
  15. package/src/__tests__/db-memory-reducer-checkpoints.test.ts +273 -0
  16. package/src/__tests__/db-schedule-syntax-migration.test.ts +3 -0
  17. package/src/__tests__/inline-command-runner.test.ts +311 -0
  18. package/src/__tests__/inline-skill-authoring-guard.test.ts +220 -0
  19. package/src/__tests__/inline-skill-load-permissions.test.ts +435 -0
  20. package/src/__tests__/list-messages-attachments.test.ts +96 -0
  21. package/src/__tests__/memory-brief-open-loops.test.ts +530 -0
  22. package/src/__tests__/memory-brief-time.test.ts +285 -0
  23. package/src/__tests__/memory-brief-wrapper.test.ts +311 -0
  24. package/src/__tests__/memory-chunk-archive.test.ts +400 -0
  25. package/src/__tests__/memory-chunk-dual-write.test.ts +453 -0
  26. package/src/__tests__/memory-episode-archive.test.ts +370 -0
  27. package/src/__tests__/memory-episode-dual-write.test.ts +626 -0
  28. package/src/__tests__/memory-observation-archive.test.ts +375 -0
  29. package/src/__tests__/memory-observation-dual-write.test.ts +318 -0
  30. package/src/__tests__/memory-recall-quality.test.ts +2 -2
  31. package/src/__tests__/memory-reducer-job.test.ts +538 -0
  32. package/src/__tests__/memory-reducer-scheduling.test.ts +473 -0
  33. package/src/__tests__/memory-reducer-store.test.ts +728 -0
  34. package/src/__tests__/memory-reducer-types.test.ts +707 -0
  35. package/src/__tests__/memory-reducer.test.ts +704 -0
  36. package/src/__tests__/memory-regressions.test.ts +30 -8
  37. package/src/__tests__/memory-simplified-config.test.ts +281 -0
  38. package/src/__tests__/parse-identity-fields.test.ts +129 -0
  39. package/src/__tests__/simplified-memory-e2e.test.ts +666 -0
  40. package/src/__tests__/simplified-memory-runtime.test.ts +616 -0
  41. package/src/__tests__/skill-load-inline-command.test.ts +598 -0
  42. package/src/__tests__/skill-load-inline-includes.test.ts +644 -0
  43. package/src/__tests__/skills-inline-command-expansions.test.ts +301 -0
  44. package/src/__tests__/skills-transitive-hash.test.ts +333 -0
  45. package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +320 -0
  46. package/src/__tests__/workspace-migration-backfill-installation-id.test.ts +4 -4
  47. package/src/cli/commands/conversations.ts +18 -0
  48. package/src/config/bundled-skills/app-builder/SKILL.md +8 -8
  49. package/src/config/bundled-skills/schedule/TOOLS.json +8 -0
  50. package/src/config/bundled-skills/skill-management/SKILL.md +1 -1
  51. package/src/config/bundled-skills/skill-management/TOOLS.json +2 -2
  52. package/src/config/feature-flag-registry.json +16 -0
  53. package/src/config/raw-config-utils.ts +28 -0
  54. package/src/config/schema.ts +12 -0
  55. package/src/config/schemas/memory-simplified.ts +101 -0
  56. package/src/config/schemas/memory.ts +4 -0
  57. package/src/config/skills.ts +50 -4
  58. package/src/daemon/conversation-agent-loop-handlers.ts +8 -3
  59. package/src/daemon/conversation-agent-loop.ts +71 -1
  60. package/src/daemon/conversation-lifecycle.ts +11 -1
  61. package/src/daemon/conversation-memory.ts +117 -0
  62. package/src/daemon/conversation-runtime-assembly.ts +3 -1
  63. package/src/daemon/conversation-surfaces.ts +31 -8
  64. package/src/daemon/conversation.ts +40 -23
  65. package/src/daemon/handlers/config-embeddings.ts +10 -2
  66. package/src/daemon/handlers/config-model.ts +0 -9
  67. package/src/daemon/handlers/conversations.ts +11 -0
  68. package/src/daemon/handlers/identity.ts +12 -1
  69. package/src/daemon/lifecycle.ts +52 -1
  70. package/src/daemon/message-types/conversations.ts +0 -1
  71. package/src/daemon/server.ts +1 -1
  72. package/src/followups/followup-store.ts +47 -1
  73. package/src/memory/archive-recall.ts +516 -0
  74. package/src/memory/archive-store.ts +400 -0
  75. package/src/memory/brief-formatting.ts +33 -0
  76. package/src/memory/brief-open-loops.ts +266 -0
  77. package/src/memory/brief-time.ts +162 -0
  78. package/src/memory/brief.ts +75 -0
  79. package/src/memory/conversation-crud.ts +455 -101
  80. package/src/memory/conversation-key-store.ts +33 -4
  81. package/src/memory/db-init.ts +16 -0
  82. package/src/memory/indexer.ts +106 -15
  83. package/src/memory/job-handlers/backfill-simplified-memory.ts +462 -0
  84. package/src/memory/job-handlers/conversation-starters.ts +9 -3
  85. package/src/memory/job-handlers/embedding.test.ts +1 -0
  86. package/src/memory/job-handlers/embedding.ts +83 -0
  87. package/src/memory/job-handlers/reduce-conversation-memory.ts +229 -0
  88. package/src/memory/job-utils.ts +1 -1
  89. package/src/memory/jobs-store.ts +8 -0
  90. package/src/memory/jobs-worker.ts +20 -0
  91. package/src/memory/migrations/036-normalize-phone-identities.ts +49 -14
  92. package/src/memory/migrations/135-backfill-contact-interaction-stats.ts +9 -1
  93. package/src/memory/migrations/141-rename-verification-table.ts +8 -0
  94. package/src/memory/migrations/142-rename-verification-session-id-column.ts +7 -2
  95. package/src/memory/migrations/174-rename-thread-starters-table.ts +8 -0
  96. package/src/memory/migrations/185-memory-brief-state.ts +52 -0
  97. package/src/memory/migrations/186-memory-archive.ts +109 -0
  98. package/src/memory/migrations/187-memory-reducer-checkpoints.ts +19 -0
  99. package/src/memory/migrations/188-schedule-quiet-flag.ts +13 -0
  100. package/src/memory/migrations/index.ts +4 -0
  101. package/src/memory/qdrant-client.ts +23 -4
  102. package/src/memory/reducer-scheduler.ts +242 -0
  103. package/src/memory/reducer-store.ts +271 -0
  104. package/src/memory/reducer-types.ts +106 -0
  105. package/src/memory/reducer.ts +467 -0
  106. package/src/memory/schema/conversations.ts +3 -0
  107. package/src/memory/schema/index.ts +2 -0
  108. package/src/memory/schema/infrastructure.ts +1 -0
  109. package/src/memory/schema/memory-archive.ts +121 -0
  110. package/src/memory/schema/memory-brief.ts +55 -0
  111. package/src/memory/search/semantic.ts +17 -4
  112. package/src/oauth/oauth-store.ts +3 -1
  113. package/src/permissions/checker.ts +89 -6
  114. package/src/permissions/defaults.ts +14 -0
  115. package/src/runtime/auth/route-policy.ts +10 -1
  116. package/src/runtime/routes/conversation-management-routes.ts +94 -2
  117. package/src/runtime/routes/conversation-query-routes.ts +7 -0
  118. package/src/runtime/routes/conversation-routes.ts +52 -5
  119. package/src/runtime/routes/guardian-bootstrap-routes.ts +19 -7
  120. package/src/runtime/routes/identity-routes.ts +2 -35
  121. package/src/runtime/routes/llm-context-normalization.ts +14 -1
  122. package/src/runtime/routes/memory-item-routes.ts +90 -5
  123. package/src/runtime/routes/secret-routes.ts +3 -0
  124. package/src/runtime/routes/surface-action-routes.ts +68 -1
  125. package/src/schedule/schedule-store.ts +28 -0
  126. package/src/schedule/scheduler.ts +6 -2
  127. package/src/skills/inline-command-expansions.ts +204 -0
  128. package/src/skills/inline-command-render.ts +127 -0
  129. package/src/skills/inline-command-runner.ts +242 -0
  130. package/src/skills/transitive-version-hash.ts +88 -0
  131. package/src/tasks/task-store.ts +43 -1
  132. package/src/telemetry/usage-telemetry-reporter.ts +1 -1
  133. package/src/tools/filesystem/edit.ts +6 -1
  134. package/src/tools/filesystem/read.ts +6 -1
  135. package/src/tools/filesystem/write.ts +6 -1
  136. package/src/tools/memory/handlers.ts +129 -1
  137. package/src/tools/permission-checker.ts +8 -1
  138. package/src/tools/schedule/create.ts +3 -0
  139. package/src/tools/schedule/list.ts +5 -1
  140. package/src/tools/schedule/update.ts +6 -0
  141. package/src/tools/skills/load.ts +140 -6
  142. package/src/util/platform.ts +18 -0
  143. package/src/workspace/migrations/{002-backfill-installation-id.ts → 011-backfill-installation-id.ts} +1 -1
  144. package/src/workspace/migrations/registry.ts +1 -1
@@ -45,14 +45,20 @@ import { enqueueMemoryJob } from "./jobs-store.js";
45
45
  import {
46
46
  channelInboundEvents,
47
47
  conversations,
48
+ conversationStarters,
48
49
  llmRequestLogs,
50
+ memoryChunks,
49
51
  memoryEmbeddings,
52
+ memoryEpisodes,
50
53
  memoryItems,
51
54
  memoryItemSources,
55
+ memoryObservations,
52
56
  memorySegments,
53
57
  memorySummaries,
54
58
  messageAttachments,
55
59
  messages,
60
+ openLoops,
61
+ timeContexts,
56
62
  toolInvocations,
57
63
  } from "./schema.js";
58
64
  import { cancelPendingJobsForConversation } from "./task-memory-cleanup.js";
@@ -171,6 +177,9 @@ export interface ConversationRow {
171
177
  forkParentMessageId: string | null;
172
178
  isAutoTitle: number;
173
179
  scheduleJobId: string | null;
180
+ memoryReducedThroughMessageId: string | null;
181
+ memoryDirtyTailSinceMessageId: string | null;
182
+ memoryLastReducedAt: number | null;
174
183
  }
175
184
 
176
185
  export const parseConversation = createRowMapper<
@@ -196,6 +205,9 @@ export const parseConversation = createRowMapper<
196
205
  forkParentMessageId: "forkParentMessageId",
197
206
  isAutoTitle: "isAutoTitle",
198
207
  scheduleJobId: "scheduleJobId",
208
+ memoryReducedThroughMessageId: "memoryReducedThroughMessageId",
209
+ memoryDirtyTailSinceMessageId: "memoryDirtyTailSinceMessageId",
210
+ memoryLastReducedAt: "memoryLastReducedAt",
199
211
  });
200
212
 
201
213
  export interface MessageRow {
@@ -375,127 +387,153 @@ export function forkConversation(params: {
375
387
  : ([] as MessageRow[]);
376
388
  const forkParentMessageId = messagesToCopy.at(-1)?.id ?? null;
377
389
  const forkTitle = `${sourceConversation.title ?? "Untitled"} (Fork)`;
378
- const forkedConversation = createConversation({
379
- title: forkTitle,
380
- conversationType: "standard",
381
- });
382
390
 
383
- db.update(conversations)
384
- .set({
385
- forkParentConversationId: sourceConversation.id,
386
- forkParentMessageId,
387
- contextSummary: preserveSourceCompactionState
388
- ? sourceConversation.contextSummary
389
- : null,
390
- contextCompactedMessageCount: preserveSourceCompactionState
391
- ? sourceConversation.contextCompactedMessageCount
392
- : 0,
393
- contextCompactedAt: preserveSourceCompactionState
394
- ? sourceConversation.contextCompactedAt
395
- : null,
396
- })
397
- .where(eq(conversations.id, forkedConversation.id))
398
- .run();
391
+ // Collect disk-sync work to run after the transaction commits.
392
+ const diskSyncQueue: Array<{
393
+ conversationId: string;
394
+ messageId: string;
395
+ createdAt: number;
396
+ }> = [];
397
+
398
+ // Wrap all DB mutations in a single transaction so a mid-flight failure
399
+ // rolls back cleanly instead of leaving a partial fork. Helper functions
400
+ // (linkAttachmentToMessage, relinkAttachments, seedForkedConversationAttention)
401
+ // use the same underlying bun:sqlite connection, so their writes participate
402
+ // in this transaction automatically.
403
+ const forkedConversation = db.transaction(() => {
404
+ const fc = createConversation({
405
+ title: forkTitle,
406
+ conversationType: "standard",
407
+ });
399
408
 
400
- const forkedMessageIds = new Map<string, string>();
401
- let latestForkedAssistant: { messageId: string; messageAt: number } | null =
402
- null;
403
-
404
- for (const message of messagesToCopy) {
405
- const forkedMessageId = uuid();
406
- db.insert(messages)
407
- .values({
408
- id: forkedMessageId,
409
- conversationId: forkedConversation.id,
410
- role: message.role,
411
- content: message.content,
412
- createdAt: message.createdAt,
413
- metadata: cloneForkMessageMetadata(message.metadata, message.id),
409
+ db.update(conversations)
410
+ .set({
411
+ forkParentConversationId: sourceConversation.id,
412
+ forkParentMessageId,
413
+ contextSummary: preserveSourceCompactionState
414
+ ? sourceConversation.contextSummary
415
+ : null,
416
+ contextCompactedMessageCount: preserveSourceCompactionState
417
+ ? sourceConversation.contextCompactedMessageCount
418
+ : 0,
419
+ contextCompactedAt: preserveSourceCompactionState
420
+ ? sourceConversation.contextCompactedAt
421
+ : null,
414
422
  })
423
+ .where(eq(conversations.id, fc.id))
415
424
  .run();
416
- forkedMessageIds.set(message.id, forkedMessageId);
417
-
418
- if (message.role === "assistant") {
419
- latestForkedAssistant = {
420
- messageId: forkedMessageId,
421
- messageAt: message.createdAt,
422
- };
423
- }
424
- }
425
-
426
- const attachmentIdMap = new Map<string, string>();
427
- for (const message of messagesToCopy) {
428
- const forkedMessageId = forkedMessageIds.get(message.id);
429
- if (!forkedMessageId) continue;
430
425
 
431
- const attachmentLinks = db
432
- .select({
433
- attachmentId: messageAttachments.attachmentId,
434
- position: messageAttachments.position,
435
- })
436
- .from(messageAttachments)
437
- .where(eq(messageAttachments.messageId, message.id))
438
- .orderBy(messageAttachments.position)
439
- .all();
440
- const uncachedAttachmentLinks = attachmentLinks.filter(
441
- (link) => !attachmentIdMap.has(link.attachmentId),
442
- );
443
- const stagingMessageId = uncachedAttachmentLinks.length > 0 ? uuid() : null;
426
+ const forkedMessageIds = new Map<string, string>();
427
+ let latestForkedAssistant: {
428
+ messageId: string;
429
+ messageAt: number;
430
+ } | null = null;
444
431
 
445
- if (stagingMessageId) {
432
+ for (const message of messagesToCopy) {
433
+ const forkedMessageId = uuid();
446
434
  db.insert(messages)
447
435
  .values({
448
- id: stagingMessageId,
449
- conversationId: forkedConversation.id,
436
+ id: forkedMessageId,
437
+ conversationId: fc.id,
450
438
  role: message.role,
451
- content: "",
439
+ content: message.content,
452
440
  createdAt: message.createdAt,
453
- metadata: null,
441
+ metadata: cloneForkMessageMetadata(message.metadata, message.id),
454
442
  })
455
443
  .run();
444
+ forkedMessageIds.set(message.id, forkedMessageId);
445
+
446
+ if (message.role === "assistant") {
447
+ latestForkedAssistant = {
448
+ messageId: forkedMessageId,
449
+ messageAt: message.createdAt,
450
+ };
451
+ }
456
452
  }
457
453
 
458
- for (const link of attachmentLinks) {
459
- const cachedAttachmentId = attachmentIdMap.get(link.attachmentId);
460
- if (cachedAttachmentId) {
461
- db.insert(messageAttachments)
454
+ const attachmentIdMap = new Map<string, string>();
455
+ for (const message of messagesToCopy) {
456
+ const forkedMessageId = forkedMessageIds.get(message.id);
457
+ if (!forkedMessageId) continue;
458
+
459
+ const attachmentLinks = db
460
+ .select({
461
+ attachmentId: messageAttachments.attachmentId,
462
+ position: messageAttachments.position,
463
+ })
464
+ .from(messageAttachments)
465
+ .where(eq(messageAttachments.messageId, message.id))
466
+ .orderBy(messageAttachments.position)
467
+ .all();
468
+ const uncachedAttachmentLinks = attachmentLinks.filter(
469
+ (link) => !attachmentIdMap.has(link.attachmentId),
470
+ );
471
+ const stagingMessageId =
472
+ uncachedAttachmentLinks.length > 0 ? uuid() : null;
473
+
474
+ if (stagingMessageId) {
475
+ db.insert(messages)
462
476
  .values({
463
- id: uuid(),
464
- messageId: forkedMessageId,
465
- attachmentId: cachedAttachmentId,
466
- position: link.position,
467
- createdAt: Date.now(),
477
+ id: stagingMessageId,
478
+ conversationId: fc.id,
479
+ role: message.role,
480
+ content: "",
481
+ createdAt: message.createdAt,
482
+ metadata: null,
468
483
  })
469
484
  .run();
470
- continue;
471
485
  }
472
486
 
473
- const scopedAttachmentId = linkAttachmentToMessage(
474
- stagingMessageId ?? forkedMessageId,
475
- link.attachmentId,
476
- link.position,
477
- );
478
- attachmentIdMap.set(link.attachmentId, scopedAttachmentId);
479
- }
487
+ for (const link of attachmentLinks) {
488
+ const cachedAttachmentId = attachmentIdMap.get(link.attachmentId);
489
+ if (cachedAttachmentId) {
490
+ db.insert(messageAttachments)
491
+ .values({
492
+ id: uuid(),
493
+ messageId: forkedMessageId,
494
+ attachmentId: cachedAttachmentId,
495
+ position: link.position,
496
+ createdAt: Date.now(),
497
+ })
498
+ .run();
499
+ continue;
500
+ }
480
501
 
481
- if (stagingMessageId) {
482
- relinkAttachments([stagingMessageId], forkedMessageId);
483
- db.delete(messages).where(eq(messages.id, stagingMessageId)).run();
502
+ const scopedAttachmentId = linkAttachmentToMessage(
503
+ stagingMessageId ?? forkedMessageId,
504
+ link.attachmentId,
505
+ link.position,
506
+ );
507
+ attachmentIdMap.set(link.attachmentId, scopedAttachmentId);
508
+ }
509
+
510
+ if (stagingMessageId) {
511
+ relinkAttachments([stagingMessageId], forkedMessageId);
512
+ db.delete(messages).where(eq(messages.id, stagingMessageId)).run();
513
+ }
514
+
515
+ diskSyncQueue.push({
516
+ conversationId: fc.id,
517
+ messageId: forkedMessageId,
518
+ createdAt: fc.createdAt,
519
+ });
484
520
  }
485
521
 
486
- syncMessageToDisk(
487
- forkedConversation.id,
488
- forkedMessageId,
489
- forkedConversation.createdAt,
490
- );
491
- }
522
+ seedForkedConversationAttention({
523
+ conversationId: fc.id,
524
+ latestAssistantMessageId: latestForkedAssistant?.messageId ?? null,
525
+ latestAssistantMessageAt: latestForkedAssistant?.messageAt ?? null,
526
+ });
492
527
 
493
- seedForkedConversationAttention({
494
- conversationId: forkedConversation.id,
495
- latestAssistantMessageId: latestForkedAssistant?.messageId ?? null,
496
- latestAssistantMessageAt: latestForkedAssistant?.messageAt ?? null,
528
+ return fc;
497
529
  });
498
530
 
531
+ // Disk-view sync runs after commit — file I/O is idempotent and
532
+ // conversation deletion cleans up orphaned directories.
533
+ for (const entry of diskSyncQueue) {
534
+ syncMessageToDisk(entry.conversationId, entry.messageId, entry.createdAt);
535
+ }
536
+
499
537
  const persistedFork = getConversation(forkedConversation.id);
500
538
  if (!persistedFork) {
501
539
  throw new Error(
@@ -513,12 +551,21 @@ export function forkConversation(params: {
513
551
  */
514
552
  export function deleteConversation(id: string): DeletedMemoryIds {
515
553
  const db = getDb();
516
- const result: DeletedMemoryIds = { segmentIds: [], orphanedItemIds: [] };
554
+ const result: DeletedMemoryIds = {
555
+ segmentIds: [],
556
+ orphanedItemIds: [],
557
+ deletedSummaryIds: [],
558
+ deletedObservationIds: [],
559
+ deletedChunkIds: [],
560
+ deletedEpisodeIds: [],
561
+ };
517
562
 
518
563
  // Capture createdAt before the transaction deletes the row — needed to
519
564
  // resolve the conversation's disk-view directory path after deletion.
520
565
  const convBeforeDelete = getConversation(id);
521
566
  const createdAtForDiskCleanup = convBeforeDelete?.createdAt;
567
+ const memoryScopeId = convBeforeDelete?.memoryScopeId;
568
+ const isPrivateScope = memoryScopeId?.startsWith("private:") ?? false;
522
569
 
523
570
  db.transaction((tx) => {
524
571
  // Collect all message IDs for this conversation.
@@ -607,6 +654,134 @@ export function deleteConversation(id: string): DeletedMemoryIds {
607
654
  .run();
608
655
  }
609
656
 
657
+ if (isPrivateScope && memoryScopeId) {
658
+ // Sweep remaining memory items with this private scopeId.
659
+ const scopeItems = tx
660
+ .select({ id: memoryItems.id })
661
+ .from(memoryItems)
662
+ .where(eq(memoryItems.scopeId, memoryScopeId))
663
+ .all();
664
+ const alreadyDeleted = new Set(result.orphanedItemIds);
665
+ const scopeItemIds = scopeItems
666
+ .map((r) => r.id)
667
+ .filter((id) => !alreadyDeleted.has(id));
668
+
669
+ if (scopeItemIds.length > 0) {
670
+ tx.delete(memoryEmbeddings)
671
+ .where(
672
+ and(
673
+ eq(memoryEmbeddings.targetType, "item"),
674
+ inArray(memoryEmbeddings.targetId, scopeItemIds),
675
+ ),
676
+ )
677
+ .run();
678
+ tx.delete(memoryItemSources)
679
+ .where(inArray(memoryItemSources.memoryItemId, scopeItemIds))
680
+ .run();
681
+ tx.delete(memoryItems)
682
+ .where(inArray(memoryItems.id, scopeItemIds))
683
+ .run();
684
+ result.orphanedItemIds.push(...scopeItemIds);
685
+ }
686
+
687
+ // Sweep memory summaries with this private scopeId.
688
+ const scopeSummaries = tx
689
+ .select({ id: memorySummaries.id })
690
+ .from(memorySummaries)
691
+ .where(eq(memorySummaries.scopeId, memoryScopeId))
692
+ .all();
693
+ const scopeSummaryIds = scopeSummaries.map((r) => r.id);
694
+
695
+ if (scopeSummaryIds.length > 0) {
696
+ tx.delete(memoryEmbeddings)
697
+ .where(
698
+ and(
699
+ eq(memoryEmbeddings.targetType, "summary"),
700
+ inArray(memoryEmbeddings.targetId, scopeSummaryIds),
701
+ ),
702
+ )
703
+ .run();
704
+ tx.delete(memorySummaries)
705
+ .where(inArray(memorySummaries.id, scopeSummaryIds))
706
+ .run();
707
+ result.deletedSummaryIds.push(...scopeSummaryIds);
708
+ }
709
+
710
+ // Sweep conversation starters with this private scopeId.
711
+ tx.delete(conversationStarters)
712
+ .where(eq(conversationStarters.scopeId, memoryScopeId))
713
+ .run();
714
+
715
+ // Sweep brief-state tables scoped to this private conversation.
716
+ tx.delete(timeContexts)
717
+ .where(eq(timeContexts.scopeId, memoryScopeId))
718
+ .run();
719
+ tx.delete(openLoops).where(eq(openLoops.scopeId, memoryScopeId)).run();
720
+ }
721
+
722
+ // Collect archive table IDs before the cascade delete removes them.
723
+ // Observations and episodes reference conversations with ON DELETE CASCADE,
724
+ // and chunks cascade from observations.
725
+ const observationRows = tx
726
+ .select({ id: memoryObservations.id })
727
+ .from(memoryObservations)
728
+ .where(eq(memoryObservations.conversationId, id))
729
+ .all();
730
+ const observationIds = observationRows.map((r) => r.id);
731
+
732
+ if (observationIds.length > 0) {
733
+ // Collect chunk IDs before observations cascade-delete them.
734
+ const chunkRows = tx
735
+ .select({ id: memoryChunks.id })
736
+ .from(memoryChunks)
737
+ .where(inArray(memoryChunks.observationId, observationIds))
738
+ .all();
739
+ const chunkIds = chunkRows.map((r) => r.id);
740
+
741
+ // Clean up embeddings for chunks.
742
+ if (chunkIds.length > 0) {
743
+ tx.delete(memoryEmbeddings)
744
+ .where(
745
+ and(
746
+ eq(memoryEmbeddings.targetType, "chunk"),
747
+ inArray(memoryEmbeddings.targetId, chunkIds),
748
+ ),
749
+ )
750
+ .run();
751
+ result.deletedChunkIds.push(...chunkIds);
752
+ }
753
+
754
+ // Clean up embeddings for observations.
755
+ tx.delete(memoryEmbeddings)
756
+ .where(
757
+ and(
758
+ eq(memoryEmbeddings.targetType, "observation"),
759
+ inArray(memoryEmbeddings.targetId, observationIds),
760
+ ),
761
+ )
762
+ .run();
763
+ result.deletedObservationIds.push(...observationIds);
764
+ }
765
+
766
+ const episodeRows = tx
767
+ .select({ id: memoryEpisodes.id })
768
+ .from(memoryEpisodes)
769
+ .where(eq(memoryEpisodes.conversationId, id))
770
+ .all();
771
+ const episodeIds = episodeRows.map((r) => r.id);
772
+
773
+ if (episodeIds.length > 0) {
774
+ tx.delete(memoryEmbeddings)
775
+ .where(
776
+ and(
777
+ eq(memoryEmbeddings.targetType, "episode"),
778
+ inArray(memoryEmbeddings.targetId, episodeIds),
779
+ ),
780
+ )
781
+ .run();
782
+ result.deletedEpisodeIds.push(...episodeIds);
783
+ }
784
+
610
785
  tx.delete(conversations).where(eq(conversations.id, id)).run();
611
786
  });
612
787
 
@@ -799,7 +974,10 @@ export function wipeConversation(id: string): WipeConversationResult {
799
974
  return {
800
975
  ...deletedMemoryIds,
801
976
  unsupersededItemIds,
802
- deletedSummaryIds,
977
+ deletedSummaryIds: [
978
+ ...deletedSummaryIds,
979
+ ...deletedMemoryIds.deletedSummaryIds,
980
+ ],
803
981
  cancelledJobCount,
804
982
  };
805
983
  }
@@ -821,16 +999,34 @@ export function purgePrivateConversations(): {
821
999
  .all();
822
1000
 
823
1001
  if (privateConvs.length === 0) {
824
- return { count: 0, deletedMemory: { segmentIds: [], orphanedItemIds: [] } };
1002
+ return {
1003
+ count: 0,
1004
+ deletedMemory: {
1005
+ segmentIds: [],
1006
+ orphanedItemIds: [],
1007
+ deletedSummaryIds: [],
1008
+ deletedObservationIds: [],
1009
+ deletedChunkIds: [],
1010
+ deletedEpisodeIds: [],
1011
+ },
1012
+ };
825
1013
  }
826
1014
 
827
1015
  const allSegmentIds: string[] = [];
828
1016
  const allOrphanedItemIds: string[] = [];
1017
+ const allDeletedSummaryIds: string[] = [];
1018
+ const allDeletedObservationIds: string[] = [];
1019
+ const allDeletedChunkIds: string[] = [];
1020
+ const allDeletedEpisodeIds: string[] = [];
829
1021
 
830
1022
  for (const conv of privateConvs) {
831
1023
  const deleted = deleteConversation(conv.id);
832
1024
  allSegmentIds.push(...deleted.segmentIds);
833
1025
  allOrphanedItemIds.push(...deleted.orphanedItemIds);
1026
+ allDeletedSummaryIds.push(...deleted.deletedSummaryIds);
1027
+ allDeletedObservationIds.push(...deleted.deletedObservationIds);
1028
+ allDeletedChunkIds.push(...deleted.deletedChunkIds);
1029
+ allDeletedEpisodeIds.push(...deleted.deletedEpisodeIds);
834
1030
  }
835
1031
 
836
1032
  return {
@@ -838,6 +1034,10 @@ export function purgePrivateConversations(): {
838
1034
  deletedMemory: {
839
1035
  segmentIds: allSegmentIds,
840
1036
  orphanedItemIds: allOrphanedItemIds,
1037
+ deletedSummaryIds: allDeletedSummaryIds,
1038
+ deletedObservationIds: allDeletedObservationIds,
1039
+ deletedChunkIds: allDeletedChunkIds,
1040
+ deletedEpisodeIds: allDeletedEpisodeIds,
841
1041
  },
842
1042
  };
843
1043
  }
@@ -920,6 +1120,13 @@ export async function addMessage(
920
1120
  throw err;
921
1121
  }
922
1122
  }
1123
+
1124
+ // Mark the conversation dirty for delayed memory reduction. This runs
1125
+ // after the insert transaction succeeds so the reducer knows which
1126
+ // conversations have unprocessed messages. The helper preserves the
1127
+ // earliest unreduced boundary (no-op when already dirty).
1128
+ markConversationMemoryDirty(conversationId, messageId);
1129
+
923
1130
  const message = {
924
1131
  id: messageId,
925
1132
  conversationId,
@@ -1108,6 +1315,15 @@ export function clearAll(): { conversations: number; messages: number } {
1108
1315
  rawExec("DELETE FROM messages");
1109
1316
  rawExec("DELETE FROM conversations");
1110
1317
 
1318
+ // Record audit event — lifecycle_events is NOT deleted by clearAll(),
1319
+ // so this survives the wipe and provides a permanent trail.
1320
+ rawRun(
1321
+ `INSERT INTO lifecycle_events (id, event_name, created_at) VALUES (?, ?, ?)`,
1322
+ uuid(),
1323
+ "conversations_clear_all",
1324
+ Date.now(),
1325
+ );
1326
+
1111
1327
  // Rebuild corrupted FTS tables and restore triggers after all base-table
1112
1328
  // DELETEs have completed. Dropping the virtual table clears the corruption,
1113
1329
  // and recreating it + triggers means subsequent writes maintain FTS
@@ -1214,11 +1430,14 @@ export function deleteLastExchange(conversationId: string): number {
1214
1430
  export interface DeletedMemoryIds {
1215
1431
  segmentIds: string[];
1216
1432
  orphanedItemIds: string[];
1433
+ deletedSummaryIds: string[];
1434
+ deletedObservationIds: string[];
1435
+ deletedChunkIds: string[];
1436
+ deletedEpisodeIds: string[];
1217
1437
  }
1218
1438
 
1219
1439
  export interface WipeConversationResult extends DeletedMemoryIds {
1220
1440
  unsupersededItemIds: string[];
1221
- deletedSummaryIds: string[];
1222
1441
  cancelledJobCount: number;
1223
1442
  }
1224
1443
 
@@ -1284,7 +1503,14 @@ export function relinkAttachments(
1284
1503
  */
1285
1504
  export function deleteMessageById(messageId: string): DeletedMemoryIds {
1286
1505
  const db = getDb();
1287
- const result: DeletedMemoryIds = { segmentIds: [], orphanedItemIds: [] };
1506
+ const result: DeletedMemoryIds = {
1507
+ segmentIds: [],
1508
+ orphanedItemIds: [],
1509
+ deletedSummaryIds: [],
1510
+ deletedObservationIds: [],
1511
+ deletedChunkIds: [],
1512
+ deletedEpisodeIds: [],
1513
+ };
1288
1514
 
1289
1515
  // Collect attachment IDs linked to this message before cascade-delete
1290
1516
  // so we can scope orphan cleanup to only those candidates.
@@ -1372,6 +1598,134 @@ export function deleteMessageById(messageId: string): DeletedMemoryIds {
1372
1598
  return result;
1373
1599
  }
1374
1600
 
1601
+ /**
1602
+ * Mark a conversation as having unreduced messages starting from the given
1603
+ * message. Sets `memoryDirtyTailSinceMessageId` only when it is currently
1604
+ * null so the earliest unreduced boundary is preserved across multiple
1605
+ * messages — later messages must not clobber the original dirty marker.
1606
+ *
1607
+ * Also upserts a pending `reduce_conversation_memory` job scheduled at
1608
+ * `now + idleDelayMs`. If a pending job for this conversation already exists,
1609
+ * its `runAfter` is pushed forward (rescheduled) so the reducer waits for
1610
+ * the full idle window after the *latest* message — avoiding premature runs
1611
+ * while the user is still actively typing.
1612
+ */
1613
+ export function markConversationMemoryDirty(
1614
+ conversationId: string,
1615
+ messageId: string,
1616
+ ): void {
1617
+ const db = getDb();
1618
+ db.update(conversations)
1619
+ .set({ memoryDirtyTailSinceMessageId: messageId })
1620
+ .where(
1621
+ and(
1622
+ eq(conversations.id, conversationId),
1623
+ isNull(conversations.memoryDirtyTailSinceMessageId),
1624
+ ),
1625
+ )
1626
+ .run();
1627
+
1628
+ // Schedule (or reschedule) a deferred reducer job for this conversation.
1629
+ scheduleReducerJob(conversationId);
1630
+ }
1631
+
1632
+ /**
1633
+ * Upsert a pending `reduce_conversation_memory` job for the given
1634
+ * conversation, scheduled `idleDelayMs` from now. If one already exists in
1635
+ * pending state, its `runAfter` is pushed forward to restart the idle timer.
1636
+ * This ensures exactly one pending reducer job per conversation — new
1637
+ * messages reschedule rather than duplicate.
1638
+ */
1639
+ export function scheduleReducerJob(
1640
+ conversationId: string,
1641
+ runAfter?: number,
1642
+ ): void {
1643
+ const idleDelayMs = getReducerIdleDelayMs();
1644
+ const scheduledAt = runAfter ?? Date.now() + idleDelayMs;
1645
+
1646
+ const existing = rawGet<{ id: string; status: string }>(
1647
+ `SELECT id, status FROM memory_jobs
1648
+ WHERE type = 'reduce_conversation_memory'
1649
+ AND json_extract(payload, '$.conversationId') = ?
1650
+ AND status = 'pending'
1651
+ LIMIT 1`,
1652
+ conversationId,
1653
+ );
1654
+
1655
+ if (existing) {
1656
+ // Reschedule: push runAfter forward so the idle timer resets.
1657
+ rawRun(
1658
+ `UPDATE memory_jobs SET run_after = ?, updated_at = ? WHERE id = ?`,
1659
+ scheduledAt,
1660
+ Date.now(),
1661
+ existing.id,
1662
+ );
1663
+ } else {
1664
+ enqueueMemoryJob(
1665
+ "reduce_conversation_memory",
1666
+ { conversationId },
1667
+ scheduledAt,
1668
+ );
1669
+ }
1670
+ }
1671
+
1672
+ /**
1673
+ * Startup sweep: find conversations that are marked dirty and whose tail
1674
+ * message is already older than the idle delay. For these conversations the
1675
+ * reducer should have run but didn't (daemon was down). Enqueue immediate
1676
+ * reducer jobs for each so they are processed on the next worker tick.
1677
+ *
1678
+ * Conversations whose tail is still within the idle window are skipped —
1679
+ * the normal `markConversationMemoryDirty` path will schedule them when
1680
+ * new messages arrive (or on the next conversation interaction).
1681
+ *
1682
+ * Returns the number of jobs enqueued.
1683
+ */
1684
+ export function sweepStaleReducerJobs(): number {
1685
+ const idleDelayMs = getReducerIdleDelayMs();
1686
+ const cutoff = Date.now() - idleDelayMs;
1687
+
1688
+ // Find dirty conversations whose latest message is older than the idle
1689
+ // window AND that don't already have a pending reducer job.
1690
+ const stale = rawAll<{ conversationId: string }>(
1691
+ `SELECT c.id AS conversationId
1692
+ FROM conversations c
1693
+ WHERE c.memory_dirty_tail_since_message_id IS NOT NULL
1694
+ AND NOT EXISTS (
1695
+ SELECT 1 FROM memory_jobs mj
1696
+ WHERE mj.type = 'reduce_conversation_memory'
1697
+ AND json_extract(mj.payload, '$.conversationId') = c.id
1698
+ AND mj.status IN ('pending', 'running')
1699
+ )
1700
+ AND (
1701
+ SELECT MAX(m.created_at) FROM messages m
1702
+ WHERE m.conversation_id = c.id
1703
+ ) <= ?`,
1704
+ cutoff,
1705
+ );
1706
+
1707
+ for (const { conversationId } of stale) {
1708
+ enqueueMemoryJob("reduce_conversation_memory", { conversationId });
1709
+ }
1710
+
1711
+ return stale.length;
1712
+ }
1713
+
1714
+ function getReducerIdleDelayMs(): number {
1715
+ // Some test suites mock getConfig() with partial objects; fall back to the
1716
+ // schema default so reducer scheduling stays stable outside full config load.
1717
+ const config = getConfig() as {
1718
+ memory?: {
1719
+ simplified?: {
1720
+ reducer?: {
1721
+ idleDelayMs?: number;
1722
+ };
1723
+ };
1724
+ };
1725
+ };
1726
+ return config.memory?.simplified?.reducer?.idleDelayMs ?? 30_000;
1727
+ }
1728
+
1375
1729
  export function setConversationOriginChannelIfUnset(
1376
1730
  conversationId: string,
1377
1731
  channel: ChannelId,