@vellumai/assistant 0.5.4 → 0.5.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/Dockerfile +17 -27
  2. package/node_modules/@vellumai/ces-contracts/src/index.ts +1 -0
  3. package/node_modules/@vellumai/ces-contracts/src/trust-rules.ts +42 -0
  4. package/package.json +1 -1
  5. package/src/__tests__/actor-token-service.test.ts +113 -0
  6. package/src/__tests__/config-schema.test.ts +2 -2
  7. package/src/__tests__/context-window-manager.test.ts +78 -0
  8. package/src/__tests__/conversation-title-service.test.ts +30 -1
  9. package/src/__tests__/credential-security-invariants.test.ts +2 -0
  10. package/src/__tests__/docker-signing-key-bootstrap.test.ts +207 -0
  11. package/src/__tests__/memory-regressions.test.ts +8 -30
  12. package/src/__tests__/openai-whisper.test.ts +93 -0
  13. package/src/__tests__/require-fresh-approval.test.ts +4 -0
  14. package/src/__tests__/slack-messaging-token-resolution.test.ts +319 -0
  15. package/src/__tests__/tool-executor-lifecycle-events.test.ts +4 -0
  16. package/src/__tests__/tool-executor.test.ts +4 -0
  17. package/src/__tests__/volume-security-guard.test.ts +155 -0
  18. package/src/cli/commands/conversations.ts +0 -18
  19. package/src/config/bundled-skills/messaging/tools/shared.ts +1 -0
  20. package/src/config/bundled-skills/transcribe/tools/transcribe-media.ts +16 -37
  21. package/src/config/env-registry.ts +9 -0
  22. package/src/config/env.ts +8 -2
  23. package/src/config/feature-flag-registry.json +8 -8
  24. package/src/config/schema.ts +0 -12
  25. package/src/config/schemas/memory.ts +0 -4
  26. package/src/config/schemas/platform.ts +1 -1
  27. package/src/config/schemas/security.ts +4 -0
  28. package/src/context/window-manager.ts +53 -2
  29. package/src/credential-execution/managed-catalog.ts +5 -15
  30. package/src/daemon/conversation-agent-loop.ts +0 -60
  31. package/src/daemon/conversation-memory.ts +0 -117
  32. package/src/daemon/conversation-runtime-assembly.ts +0 -2
  33. package/src/daemon/daemon-control.ts +7 -0
  34. package/src/daemon/handlers/conversations.ts +0 -11
  35. package/src/daemon/lifecycle.ts +10 -47
  36. package/src/daemon/providers-setup.ts +2 -1
  37. package/src/followups/followup-store.ts +5 -2
  38. package/src/hooks/manager.ts +7 -0
  39. package/src/instrument.ts +33 -1
  40. package/src/memory/conversation-crud.ts +0 -236
  41. package/src/memory/conversation-title-service.ts +26 -10
  42. package/src/memory/db-init.ts +5 -13
  43. package/src/memory/embedding-local.ts +11 -5
  44. package/src/memory/indexer.ts +15 -106
  45. package/src/memory/job-handlers/conversation-starters.ts +24 -36
  46. package/src/memory/job-handlers/embedding.ts +0 -79
  47. package/src/memory/job-utils.ts +1 -1
  48. package/src/memory/jobs-store.ts +0 -8
  49. package/src/memory/jobs-worker.ts +0 -20
  50. package/src/memory/migrations/189-drop-simplified-memory.ts +42 -0
  51. package/src/memory/migrations/index.ts +1 -3
  52. package/src/memory/qdrant-client.ts +4 -6
  53. package/src/memory/schema/conversations.ts +0 -3
  54. package/src/memory/schema/index.ts +0 -2
  55. package/src/messaging/draft-store.ts +2 -2
  56. package/src/messaging/provider.ts +9 -0
  57. package/src/messaging/providers/slack/adapter.ts +29 -2
  58. package/src/oauth/connection-resolver.test.ts +22 -18
  59. package/src/oauth/connection-resolver.ts +92 -7
  60. package/src/oauth/platform-connection.test.ts +78 -69
  61. package/src/oauth/platform-connection.ts +12 -19
  62. package/src/permissions/defaults.ts +3 -3
  63. package/src/permissions/trust-client.ts +332 -0
  64. package/src/permissions/trust-store-interface.ts +105 -0
  65. package/src/permissions/trust-store.ts +531 -39
  66. package/src/platform/client.test.ts +148 -0
  67. package/src/platform/client.ts +71 -0
  68. package/src/providers/speech-to-text/openai-whisper.test.ts +190 -0
  69. package/src/providers/speech-to-text/openai-whisper.ts +68 -0
  70. package/src/providers/speech-to-text/resolve.ts +9 -0
  71. package/src/providers/speech-to-text/types.ts +17 -0
  72. package/src/runtime/auth/route-policy.ts +14 -0
  73. package/src/runtime/auth/token-service.ts +133 -0
  74. package/src/runtime/http-server.ts +4 -2
  75. package/src/runtime/routes/conversation-management-routes.ts +0 -36
  76. package/src/runtime/routes/conversation-query-routes.ts +44 -2
  77. package/src/runtime/routes/conversation-routes.ts +2 -1
  78. package/src/runtime/routes/inbound-message-handler.ts +27 -3
  79. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +16 -1
  80. package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +287 -0
  81. package/src/runtime/routes/inbound-stages/transcribe-audio.ts +122 -0
  82. package/src/runtime/routes/log-export-routes.ts +1 -0
  83. package/src/runtime/routes/memory-item-routes.test.ts +221 -3
  84. package/src/runtime/routes/memory-item-routes.ts +124 -2
  85. package/src/runtime/routes/secret-routes.ts +4 -1
  86. package/src/runtime/routes/upgrade-broadcast-routes.ts +151 -0
  87. package/src/schedule/schedule-store.ts +0 -21
  88. package/src/security/ces-credential-client.ts +173 -0
  89. package/src/security/secure-keys.ts +65 -22
  90. package/src/signals/bash.ts +3 -0
  91. package/src/signals/cancel.ts +3 -0
  92. package/src/signals/confirm.ts +3 -0
  93. package/src/signals/conversation-undo.ts +3 -0
  94. package/src/signals/event-stream.ts +7 -0
  95. package/src/signals/shotgun.ts +3 -0
  96. package/src/signals/trust-rule.ts +3 -0
  97. package/src/skills/inline-command-render.ts +5 -1
  98. package/src/skills/inline-command-runner.ts +30 -2
  99. package/src/telemetry/usage-telemetry-reporter.test.ts +23 -36
  100. package/src/telemetry/usage-telemetry-reporter.ts +21 -19
  101. package/src/tools/memory/handlers.ts +1 -129
  102. package/src/tools/permission-checker.ts +18 -0
  103. package/src/tools/skills/load.ts +9 -2
  104. package/src/util/device-id.ts +70 -7
  105. package/src/util/logger.ts +35 -9
  106. package/src/util/platform.ts +29 -5
  107. package/src/util/xml.ts +8 -0
  108. package/src/workspace/heartbeat-service.ts +5 -24
  109. package/src/workspace/migrations/migrate-to-workspace-volume.ts +113 -0
  110. package/src/workspace/migrations/registry.ts +2 -0
  111. package/src/__tests__/archive-recall.test.ts +0 -560
  112. package/src/__tests__/conversation-memory-dirty-tail.test.ts +0 -150
  113. package/src/__tests__/conversation-switch-memory-reduction.test.ts +0 -474
  114. package/src/__tests__/db-memory-archive-migration.test.ts +0 -372
  115. package/src/__tests__/db-memory-brief-state-migration.test.ts +0 -213
  116. package/src/__tests__/db-memory-reducer-checkpoints.test.ts +0 -273
  117. package/src/__tests__/memory-brief-open-loops.test.ts +0 -530
  118. package/src/__tests__/memory-brief-time.test.ts +0 -285
  119. package/src/__tests__/memory-brief-wrapper.test.ts +0 -311
  120. package/src/__tests__/memory-chunk-archive.test.ts +0 -400
  121. package/src/__tests__/memory-chunk-dual-write.test.ts +0 -453
  122. package/src/__tests__/memory-episode-archive.test.ts +0 -370
  123. package/src/__tests__/memory-episode-dual-write.test.ts +0 -626
  124. package/src/__tests__/memory-observation-archive.test.ts +0 -375
  125. package/src/__tests__/memory-observation-dual-write.test.ts +0 -318
  126. package/src/__tests__/memory-reducer-job.test.ts +0 -538
  127. package/src/__tests__/memory-reducer-scheduling.test.ts +0 -473
  128. package/src/__tests__/memory-reducer-store.test.ts +0 -728
  129. package/src/__tests__/memory-reducer-types.test.ts +0 -707
  130. package/src/__tests__/memory-reducer.test.ts +0 -704
  131. package/src/__tests__/memory-simplified-config.test.ts +0 -281
  132. package/src/__tests__/simplified-memory-e2e.test.ts +0 -666
  133. package/src/__tests__/simplified-memory-runtime.test.ts +0 -616
  134. package/src/config/schemas/memory-simplified.ts +0 -101
  135. package/src/memory/archive-recall.ts +0 -516
  136. package/src/memory/archive-store.ts +0 -400
  137. package/src/memory/brief-formatting.ts +0 -33
  138. package/src/memory/brief-open-loops.ts +0 -266
  139. package/src/memory/brief-time.ts +0 -162
  140. package/src/memory/brief.ts +0 -75
  141. package/src/memory/job-handlers/backfill-simplified-memory.ts +0 -462
  142. package/src/memory/job-handlers/reduce-conversation-memory.ts +0 -229
  143. package/src/memory/migrations/185-memory-brief-state.ts +0 -52
  144. package/src/memory/migrations/186-memory-archive.ts +0 -109
  145. package/src/memory/migrations/187-memory-reducer-checkpoints.ts +0 -19
  146. package/src/memory/reducer-scheduler.ts +0 -242
  147. package/src/memory/reducer-store.ts +0 -271
  148. package/src/memory/reducer-types.ts +0 -106
  149. package/src/memory/reducer.ts +0 -467
  150. package/src/memory/schema/memory-archive.ts +0 -121
  151. package/src/memory/schema/memory-brief.ts +0 -55
@@ -47,18 +47,13 @@ import {
47
47
  conversations,
48
48
  conversationStarters,
49
49
  llmRequestLogs,
50
- memoryChunks,
51
50
  memoryEmbeddings,
52
- memoryEpisodes,
53
51
  memoryItems,
54
52
  memoryItemSources,
55
- memoryObservations,
56
53
  memorySegments,
57
54
  memorySummaries,
58
55
  messageAttachments,
59
56
  messages,
60
- openLoops,
61
- timeContexts,
62
57
  toolInvocations,
63
58
  } from "./schema.js";
64
59
  import { cancelPendingJobsForConversation } from "./task-memory-cleanup.js";
@@ -177,9 +172,6 @@ export interface ConversationRow {
177
172
  forkParentMessageId: string | null;
178
173
  isAutoTitle: number;
179
174
  scheduleJobId: string | null;
180
- memoryReducedThroughMessageId: string | null;
181
- memoryDirtyTailSinceMessageId: string | null;
182
- memoryLastReducedAt: number | null;
183
175
  }
184
176
 
185
177
  export const parseConversation = createRowMapper<
@@ -205,9 +197,6 @@ export const parseConversation = createRowMapper<
205
197
  forkParentMessageId: "forkParentMessageId",
206
198
  isAutoTitle: "isAutoTitle",
207
199
  scheduleJobId: "scheduleJobId",
208
- memoryReducedThroughMessageId: "memoryReducedThroughMessageId",
209
- memoryDirtyTailSinceMessageId: "memoryDirtyTailSinceMessageId",
210
- memoryLastReducedAt: "memoryLastReducedAt",
211
200
  });
212
201
 
213
202
  export interface MessageRow {
@@ -555,9 +544,6 @@ export function deleteConversation(id: string): DeletedMemoryIds {
555
544
  segmentIds: [],
556
545
  orphanedItemIds: [],
557
546
  deletedSummaryIds: [],
558
- deletedObservationIds: [],
559
- deletedChunkIds: [],
560
- deletedEpisodeIds: [],
561
547
  };
562
548
 
563
549
  // Capture createdAt before the transaction deletes the row — needed to
@@ -711,75 +697,6 @@ export function deleteConversation(id: string): DeletedMemoryIds {
711
697
  tx.delete(conversationStarters)
712
698
  .where(eq(conversationStarters.scopeId, memoryScopeId))
713
699
  .run();
714
-
715
- // Sweep brief-state tables scoped to this private conversation.
716
- tx.delete(timeContexts)
717
- .where(eq(timeContexts.scopeId, memoryScopeId))
718
- .run();
719
- tx.delete(openLoops).where(eq(openLoops.scopeId, memoryScopeId)).run();
720
- }
721
-
722
- // Collect archive table IDs before the cascade delete removes them.
723
- // Observations and episodes reference conversations with ON DELETE CASCADE,
724
- // and chunks cascade from observations.
725
- const observationRows = tx
726
- .select({ id: memoryObservations.id })
727
- .from(memoryObservations)
728
- .where(eq(memoryObservations.conversationId, id))
729
- .all();
730
- const observationIds = observationRows.map((r) => r.id);
731
-
732
- if (observationIds.length > 0) {
733
- // Collect chunk IDs before observations cascade-delete them.
734
- const chunkRows = tx
735
- .select({ id: memoryChunks.id })
736
- .from(memoryChunks)
737
- .where(inArray(memoryChunks.observationId, observationIds))
738
- .all();
739
- const chunkIds = chunkRows.map((r) => r.id);
740
-
741
- // Clean up embeddings for chunks.
742
- if (chunkIds.length > 0) {
743
- tx.delete(memoryEmbeddings)
744
- .where(
745
- and(
746
- eq(memoryEmbeddings.targetType, "chunk"),
747
- inArray(memoryEmbeddings.targetId, chunkIds),
748
- ),
749
- )
750
- .run();
751
- result.deletedChunkIds.push(...chunkIds);
752
- }
753
-
754
- // Clean up embeddings for observations.
755
- tx.delete(memoryEmbeddings)
756
- .where(
757
- and(
758
- eq(memoryEmbeddings.targetType, "observation"),
759
- inArray(memoryEmbeddings.targetId, observationIds),
760
- ),
761
- )
762
- .run();
763
- result.deletedObservationIds.push(...observationIds);
764
- }
765
-
766
- const episodeRows = tx
767
- .select({ id: memoryEpisodes.id })
768
- .from(memoryEpisodes)
769
- .where(eq(memoryEpisodes.conversationId, id))
770
- .all();
771
- const episodeIds = episodeRows.map((r) => r.id);
772
-
773
- if (episodeIds.length > 0) {
774
- tx.delete(memoryEmbeddings)
775
- .where(
776
- and(
777
- eq(memoryEmbeddings.targetType, "episode"),
778
- inArray(memoryEmbeddings.targetId, episodeIds),
779
- ),
780
- )
781
- .run();
782
- result.deletedEpisodeIds.push(...episodeIds);
783
700
  }
784
701
 
785
702
  tx.delete(conversations).where(eq(conversations.id, id)).run();
@@ -1005,9 +922,6 @@ export function purgePrivateConversations(): {
1005
922
  segmentIds: [],
1006
923
  orphanedItemIds: [],
1007
924
  deletedSummaryIds: [],
1008
- deletedObservationIds: [],
1009
- deletedChunkIds: [],
1010
- deletedEpisodeIds: [],
1011
925
  },
1012
926
  };
1013
927
  }
@@ -1015,18 +929,12 @@ export function purgePrivateConversations(): {
1015
929
  const allSegmentIds: string[] = [];
1016
930
  const allOrphanedItemIds: string[] = [];
1017
931
  const allDeletedSummaryIds: string[] = [];
1018
- const allDeletedObservationIds: string[] = [];
1019
- const allDeletedChunkIds: string[] = [];
1020
- const allDeletedEpisodeIds: string[] = [];
1021
932
 
1022
933
  for (const conv of privateConvs) {
1023
934
  const deleted = deleteConversation(conv.id);
1024
935
  allSegmentIds.push(...deleted.segmentIds);
1025
936
  allOrphanedItemIds.push(...deleted.orphanedItemIds);
1026
937
  allDeletedSummaryIds.push(...deleted.deletedSummaryIds);
1027
- allDeletedObservationIds.push(...deleted.deletedObservationIds);
1028
- allDeletedChunkIds.push(...deleted.deletedChunkIds);
1029
- allDeletedEpisodeIds.push(...deleted.deletedEpisodeIds);
1030
938
  }
1031
939
 
1032
940
  return {
@@ -1035,9 +943,6 @@ export function purgePrivateConversations(): {
1035
943
  segmentIds: allSegmentIds,
1036
944
  orphanedItemIds: allOrphanedItemIds,
1037
945
  deletedSummaryIds: allDeletedSummaryIds,
1038
- deletedObservationIds: allDeletedObservationIds,
1039
- deletedChunkIds: allDeletedChunkIds,
1040
- deletedEpisodeIds: allDeletedEpisodeIds,
1041
946
  },
1042
947
  };
1043
948
  }
@@ -1120,13 +1025,6 @@ export async function addMessage(
1120
1025
  throw err;
1121
1026
  }
1122
1027
  }
1123
-
1124
- // Mark the conversation dirty for delayed memory reduction. This runs
1125
- // after the insert transaction succeeds so the reducer knows which
1126
- // conversations have unprocessed messages. The helper preserves the
1127
- // earliest unreduced boundary (no-op when already dirty).
1128
- markConversationMemoryDirty(conversationId, messageId);
1129
-
1130
1028
  const message = {
1131
1029
  id: messageId,
1132
1030
  conversationId,
@@ -1431,9 +1329,6 @@ export interface DeletedMemoryIds {
1431
1329
  segmentIds: string[];
1432
1330
  orphanedItemIds: string[];
1433
1331
  deletedSummaryIds: string[];
1434
- deletedObservationIds: string[];
1435
- deletedChunkIds: string[];
1436
- deletedEpisodeIds: string[];
1437
1332
  }
1438
1333
 
1439
1334
  export interface WipeConversationResult extends DeletedMemoryIds {
@@ -1507,9 +1402,6 @@ export function deleteMessageById(messageId: string): DeletedMemoryIds {
1507
1402
  segmentIds: [],
1508
1403
  orphanedItemIds: [],
1509
1404
  deletedSummaryIds: [],
1510
- deletedObservationIds: [],
1511
- deletedChunkIds: [],
1512
- deletedEpisodeIds: [],
1513
1405
  };
1514
1406
 
1515
1407
  // Collect attachment IDs linked to this message before cascade-delete
@@ -1598,134 +1490,6 @@ export function deleteMessageById(messageId: string): DeletedMemoryIds {
1598
1490
  return result;
1599
1491
  }
1600
1492
 
1601
- /**
1602
- * Mark a conversation as having unreduced messages starting from the given
1603
- * message. Sets `memoryDirtyTailSinceMessageId` only when it is currently
1604
- * null so the earliest unreduced boundary is preserved across multiple
1605
- * messages — later messages must not clobber the original dirty marker.
1606
- *
1607
- * Also upserts a pending `reduce_conversation_memory` job scheduled at
1608
- * `now + idleDelayMs`. If a pending job for this conversation already exists,
1609
- * its `runAfter` is pushed forward (rescheduled) so the reducer waits for
1610
- * the full idle window after the *latest* message — avoiding premature runs
1611
- * while the user is still actively typing.
1612
- */
1613
- export function markConversationMemoryDirty(
1614
- conversationId: string,
1615
- messageId: string,
1616
- ): void {
1617
- const db = getDb();
1618
- db.update(conversations)
1619
- .set({ memoryDirtyTailSinceMessageId: messageId })
1620
- .where(
1621
- and(
1622
- eq(conversations.id, conversationId),
1623
- isNull(conversations.memoryDirtyTailSinceMessageId),
1624
- ),
1625
- )
1626
- .run();
1627
-
1628
- // Schedule (or reschedule) a deferred reducer job for this conversation.
1629
- scheduleReducerJob(conversationId);
1630
- }
1631
-
1632
- /**
1633
- * Upsert a pending `reduce_conversation_memory` job for the given
1634
- * conversation, scheduled `idleDelayMs` from now. If one already exists in
1635
- * pending state, its `runAfter` is pushed forward to restart the idle timer.
1636
- * This ensures exactly one pending reducer job per conversation — new
1637
- * messages reschedule rather than duplicate.
1638
- */
1639
- export function scheduleReducerJob(
1640
- conversationId: string,
1641
- runAfter?: number,
1642
- ): void {
1643
- const idleDelayMs = getReducerIdleDelayMs();
1644
- const scheduledAt = runAfter ?? Date.now() + idleDelayMs;
1645
-
1646
- const existing = rawGet<{ id: string; status: string }>(
1647
- `SELECT id, status FROM memory_jobs
1648
- WHERE type = 'reduce_conversation_memory'
1649
- AND json_extract(payload, '$.conversationId') = ?
1650
- AND status = 'pending'
1651
- LIMIT 1`,
1652
- conversationId,
1653
- );
1654
-
1655
- if (existing) {
1656
- // Reschedule: push runAfter forward so the idle timer resets.
1657
- rawRun(
1658
- `UPDATE memory_jobs SET run_after = ?, updated_at = ? WHERE id = ?`,
1659
- scheduledAt,
1660
- Date.now(),
1661
- existing.id,
1662
- );
1663
- } else {
1664
- enqueueMemoryJob(
1665
- "reduce_conversation_memory",
1666
- { conversationId },
1667
- scheduledAt,
1668
- );
1669
- }
1670
- }
1671
-
1672
- /**
1673
- * Startup sweep: find conversations that are marked dirty and whose tail
1674
- * message is already older than the idle delay. For these conversations the
1675
- * reducer should have run but didn't (daemon was down). Enqueue immediate
1676
- * reducer jobs for each so they are processed on the next worker tick.
1677
- *
1678
- * Conversations whose tail is still within the idle window are skipped —
1679
- * the normal `markConversationMemoryDirty` path will schedule them when
1680
- * new messages arrive (or on the next conversation interaction).
1681
- *
1682
- * Returns the number of jobs enqueued.
1683
- */
1684
- export function sweepStaleReducerJobs(): number {
1685
- const idleDelayMs = getReducerIdleDelayMs();
1686
- const cutoff = Date.now() - idleDelayMs;
1687
-
1688
- // Find dirty conversations whose latest message is older than the idle
1689
- // window AND that don't already have a pending reducer job.
1690
- const stale = rawAll<{ conversationId: string }>(
1691
- `SELECT c.id AS conversationId
1692
- FROM conversations c
1693
- WHERE c.memory_dirty_tail_since_message_id IS NOT NULL
1694
- AND NOT EXISTS (
1695
- SELECT 1 FROM memory_jobs mj
1696
- WHERE mj.type = 'reduce_conversation_memory'
1697
- AND json_extract(mj.payload, '$.conversationId') = c.id
1698
- AND mj.status IN ('pending', 'running')
1699
- )
1700
- AND (
1701
- SELECT MAX(m.created_at) FROM messages m
1702
- WHERE m.conversation_id = c.id
1703
- ) <= ?`,
1704
- cutoff,
1705
- );
1706
-
1707
- for (const { conversationId } of stale) {
1708
- enqueueMemoryJob("reduce_conversation_memory", { conversationId });
1709
- }
1710
-
1711
- return stale.length;
1712
- }
1713
-
1714
- function getReducerIdleDelayMs(): number {
1715
- // Some test suites mock getConfig() with partial objects; fall back to the
1716
- // schema default so reducer scheduling stays stable outside full config load.
1717
- const config = getConfig() as {
1718
- memory?: {
1719
- simplified?: {
1720
- reducer?: {
1721
- idleDelayMs?: number;
1722
- };
1723
- };
1724
- };
1725
- };
1726
- return config.memory?.simplified?.reducer?.idleDelayMs ?? 30_000;
1727
- }
1728
-
1729
1493
  export function setConversationOriginChannelIfUnset(
1730
1494
  conversationId: string,
1731
1495
  channel: ChannelId,
@@ -133,6 +133,8 @@ export async function generateAndPersistConversationTitle(
133
133
  const result = await runBtwSidechain({
134
134
  content: prompt,
135
135
  provider,
136
+ systemPrompt: buildTitleSystemPrompt(),
137
+ tools: [],
136
138
  maxTokens: config.daemon.titleGenerationMaxTokens,
137
139
  modelIntent: "latency-optimized",
138
140
  signal,
@@ -236,6 +238,8 @@ export async function regenerateConversationTitle(
236
238
  const result = await runBtwSidechain({
237
239
  content: prompt,
238
240
  provider,
241
+ systemPrompt: buildTitleSystemPrompt(),
242
+ tools: [],
239
243
  maxTokens: config.daemon.titleGenerationMaxTokens,
240
244
  modelIntent: "latency-optimized",
241
245
  signal,
@@ -277,14 +281,30 @@ export function queueRegenerateConversationTitle(
277
281
 
278
282
  // ── Internal helpers ─────────────────────────────────────────────────
279
283
 
284
+ /**
285
+ * Dedicated system prompt for title generation. Replaces the default
286
+ * assistant system prompt that btw-sidechain would otherwise inject,
287
+ * which caused the model to respond to the conversation content instead
288
+ * of titling it.
289
+ */
290
+ function buildTitleSystemPrompt(): string {
291
+ return [
292
+ "You generate short conversation titles. Output ONLY the title text — no explanation, no quotes, no markdown, no preamble.",
293
+ "",
294
+ "Rules:",
295
+ "- Maximum 5 words and 40 characters",
296
+ "- Summarize the TOPIC the user is asking about",
297
+ "- Do NOT respond to the conversation content",
298
+ "- Do NOT assess feasibility or comment on capabilities",
299
+ ].join("\n");
300
+ }
301
+
280
302
  function buildTitlePrompt(
281
303
  context?: TitleContext,
282
304
  userMessage?: string,
283
305
  assistantResponse?: string,
284
306
  ): string {
285
- const parts: string[] = [
286
- "Generate a very short title summarizing the TOPIC of this conversation. Rules: at most 5 words, at most 40 characters, no quotes, no markdown formatting. IMPORTANT: Summarize what the user is asking about — do NOT respond to the message, do NOT assess feasibility, and do NOT comment on your own capabilities.",
287
- ];
307
+ const parts: string[] = [];
288
308
 
289
309
  if (context) {
290
310
  const hints: string[] = [];
@@ -295,12 +315,12 @@ function buildTitlePrompt(
295
315
  if (context.metadataHints?.length)
296
316
  hints.push(`Hints: ${context.metadataHints.join(", ")}`);
297
317
  if (hints.length > 0) {
298
- parts.push("", "Metadata:", ...hints);
318
+ parts.push("Metadata:", ...hints, "");
299
319
  }
300
320
  }
301
321
 
302
322
  if (userMessage) {
303
- parts.push("", `User: ${truncate(userMessage, 200, "")}`);
323
+ parts.push(`User: ${truncate(userMessage, 200, "")}`);
304
324
  }
305
325
  if (assistantResponse) {
306
326
  parts.push(`Assistant: ${truncate(assistantResponse, 200, "")}`);
@@ -339,11 +359,7 @@ function deriveFallbackTitle(context?: TitleContext): string | null {
339
359
  }
340
360
 
341
361
  function buildRegenerationPrompt(recentMessages: MessageRow[]): string {
342
- const parts: string[] = [
343
- "Generate a very short title summarizing the TOPIC of this conversation based on the recent messages below. Rules: at most 5 words, at most 40 characters, no quotes, no markdown formatting. IMPORTANT: Summarize what the user is asking about — do NOT respond to the messages, do NOT assess feasibility, and do NOT comment on your own capabilities.",
344
- "",
345
- "Recent messages:",
346
- ];
362
+ const parts: string[] = ["Recent messages:"];
347
363
 
348
364
  for (const msg of recentMessages) {
349
365
  const role = msg.role === "user" ? "User" : "Assistant";
@@ -66,6 +66,7 @@ import {
66
66
  migrateDropMemorySegmentFts,
67
67
  migrateDropOrphanedMediaTables,
68
68
  migrateDropRemindersTable,
69
+ migrateDropSimplifiedMemory,
69
70
  migrateDropUsageCompositeIndexes,
70
71
  migrateFkCascadeRebuilds,
71
72
  migrateGuardianActionFollowup,
@@ -82,10 +83,7 @@ import {
82
83
  migrateInviteContactId,
83
84
  migrateLlmRequestLogMessageId,
84
85
  migrateLlmRequestLogProvider,
85
- migrateMemoryArchiveTables,
86
- migrateMemoryBriefState,
87
86
  migrateMemoryItemSupersession,
88
- migrateMemoryReducerCheckpoints,
89
87
  migrateMessagesFtsBackfill,
90
88
  migrateNormalizePhoneIdentities,
91
89
  migrateNotificationDeliveryThreadDecision,
@@ -488,18 +486,12 @@ export function initializeDb(): void {
488
486
  // 84. Add nullable conversation fork lineage columns and parent lookup index
489
487
  migrateConversationForkLineage(database);
490
488
 
491
- // 85. Memory brief state tables (time_contexts, open_loops) for simplified memory system
492
- migrateMemoryBriefState(database);
493
-
494
- // 86. Memory archive tables (observations, chunks, episodes) for simplified memory v1
495
- migrateMemoryArchiveTables(database);
496
-
497
- // 87. Add memory reducer checkpoint columns to conversations
498
- migrateMemoryReducerCheckpoints(database);
499
-
500
- // 88. Add quiet flag to schedule jobs
489
+ // 85. Add quiet flag to schedule jobs
501
490
  migrateScheduleQuietFlag(database);
502
491
 
492
+ // 86. Drop simplified-memory tables and reducer checkpoint columns
493
+ migrateDropSimplifiedMemory(database);
494
+
503
495
  validateMigrationState(database);
504
496
 
505
497
  if (process.env.BUN_TEST === "1") {
@@ -1,6 +1,7 @@
1
1
  import { existsSync, unlinkSync, writeFileSync } from "node:fs";
2
2
  import { join } from "node:path";
3
3
 
4
+ import { getIsContainerized } from "../config/env-registry.js";
4
5
  import { getLogger } from "../util/logger.js";
5
6
  import { getEmbeddingModelsDir, getRootDir } from "../util/platform.js";
6
7
  import { PromiseGuard } from "../util/promise-guard.js";
@@ -353,12 +354,17 @@ export class LocalEmbeddingBackend implements EmbeddingBackend {
353
354
 
354
355
  private static readonly PID_FILENAME = "embed-worker.pid";
355
356
 
357
+ /** PID files are process-local state — store in /tmp when containerized to keep shared volumes clean. */
358
+ private getPidFilePath(): string {
359
+ if (getIsContainerized()) {
360
+ return join("/tmp", LocalEmbeddingBackend.PID_FILENAME);
361
+ }
362
+ return join(getRootDir(), LocalEmbeddingBackend.PID_FILENAME);
363
+ }
364
+
356
365
  private writePidFile(pid: number): void {
357
366
  try {
358
- writeFileSync(
359
- join(getRootDir(), LocalEmbeddingBackend.PID_FILENAME),
360
- String(pid),
361
- );
367
+ writeFileSync(this.getPidFilePath(), String(pid));
362
368
  } catch {
363
369
  // Best-effort — doesn't affect functionality
364
370
  }
@@ -366,7 +372,7 @@ export class LocalEmbeddingBackend implements EmbeddingBackend {
366
372
 
367
373
  private removePidFile(): void {
368
374
  try {
369
- unlinkSync(join(getRootDir(), LocalEmbeddingBackend.PID_FILENAME));
375
+ unlinkSync(this.getPidFilePath());
370
376
  } catch {
371
377
  // Best-effort
372
378
  }
@@ -5,7 +5,6 @@ import { getConfig } from "../config/loader.js";
5
5
  import type { MemoryConfig } from "../config/types.js";
6
6
  import type { TrustClass } from "../runtime/actor-trust-resolver.js";
7
7
  import { getLogger } from "../util/logger.js";
8
- import { computeChunkContentHash } from "./archive-store.js";
9
8
  import { getDb } from "./db.js";
10
9
  import { selectedBackendSupportsMultimodal } from "./embedding-backend.js";
11
10
  import { enqueueMemoryJob } from "./jobs-store.js";
@@ -13,7 +12,7 @@ import {
13
12
  extractMediaBlockMeta,
14
13
  extractTextFromStoredMessageContent,
15
14
  } from "./message-content.js";
16
- import { memoryChunks, memoryObservations, memorySegments } from "./schema.js";
15
+ import { memorySegments } from "./schema.js";
17
16
  import { segmentText } from "./segmenter.js";
18
17
 
19
18
  const log = getLogger("memory-indexer");
@@ -54,12 +53,7 @@ export async function indexMessageNow(
54
53
  input.provenanceTrustClass === undefined;
55
54
 
56
55
  const text = extractTextFromStoredMessageContent(input.content);
57
- const hasText = text.length > 0;
58
- const candidateMediaMeta = extractMediaBlockMeta(input.content).filter(
59
- (b) => b.type === "image",
60
- );
61
- const hasMedia = candidateMediaMeta.length > 0;
62
- if (!hasText && !hasMedia) {
56
+ if (text.length === 0) {
63
57
  enqueueMemoryJob("build_conversation_summary", {
64
58
  conversationId: input.conversationId,
65
59
  });
@@ -68,13 +62,11 @@ export async function indexMessageNow(
68
62
 
69
63
  const db = getDb();
70
64
  const now = Date.now();
71
- const segments = hasText
72
- ? segmentText(
73
- text,
74
- config.segmentation.targetTokens,
75
- config.segmentation.overlapTokens,
76
- )
77
- : [];
65
+ const segments = segmentText(
66
+ text,
67
+ config.segmentation.targetTokens,
68
+ config.segmentation.overlapTokens,
69
+ );
78
70
  const shouldExtract =
79
71
  input.role === "user" ||
80
72
  (input.role === "assistant" && config.extraction.extractFromAssistant);
@@ -84,6 +76,9 @@ export async function indexMessageNow(
84
76
  // overhead for messages on non-multimodal backends.
85
77
  // selectedBackendSupportsMultimodal requires async key resolution, so we
86
78
  // skip it entirely for text-only messages.
79
+ const candidateMediaMeta = extractMediaBlockMeta(input.content).filter(
80
+ (b) => b.type === "image",
81
+ );
87
82
  const mediaBlocks =
88
83
  candidateMediaMeta.length > 0 &&
89
84
  (await selectedBackendSupportsMultimodal(getConfig()))
@@ -93,10 +88,7 @@ export async function indexMessageNow(
93
88
  // Wrap all segment inserts and job enqueues in a single transaction so they
94
89
  // either all succeed or all roll back, preventing partial/orphaned state.
95
90
  let skippedEmbedJobs = 0;
96
- let skippedChunkEmbedJobs = 0;
97
- const scopeId = input.scopeId ?? "default";
98
91
  db.transaction((tx) => {
99
- // ── Legacy segment path (kept intact for parallel validation) ───
100
92
  for (const segment of segments) {
101
93
  const segmentId = buildSegmentId(input.messageId, segment.segmentIndex);
102
94
  const hash = createHash("sha256").update(segment.text).digest("hex");
@@ -117,7 +109,7 @@ export async function indexMessageNow(
117
109
  segmentIndex: segment.segmentIndex,
118
110
  text: segment.text,
119
111
  tokenEstimate: segment.tokenEstimate,
120
- scopeId,
112
+ scopeId: input.scopeId ?? "default",
121
113
  contentHash: hash,
122
114
  createdAt: input.createdAt,
123
115
  updatedAt: now,
@@ -127,7 +119,7 @@ export async function indexMessageNow(
127
119
  set: {
128
120
  text: segment.text,
129
121
  tokenEstimate: segment.tokenEstimate,
130
- scopeId,
122
+ scopeId: input.scopeId ?? "default",
131
123
  contentHash: hash,
132
124
  updatedAt: now,
133
125
  },
@@ -141,65 +133,6 @@ export async function indexMessageNow(
141
133
  }
142
134
  }
143
135
 
144
- // ── Archive chunk dual-write (mirrors segment boundaries) ──────
145
- // Create a single observation per message, then create one chunk per
146
- // segment using the same segmentation boundaries. Chunks are
147
- // deduplicated by (scopeId, contentHash) via onConflictDoNothing so
148
- // unchanged content does not enqueue duplicate embed_chunk jobs.
149
- const observationId = buildObservationId(input.messageId);
150
- tx.insert(memoryObservations)
151
- .values({
152
- id: observationId,
153
- scopeId,
154
- conversationId: input.conversationId,
155
- messageId: input.messageId,
156
- role: input.role,
157
- content: hasText ? text : input.content,
158
- modality: hasMedia ? "multimodal" : "text",
159
- source: null,
160
- createdAt: input.createdAt,
161
- })
162
- .onConflictDoNothing({ target: memoryObservations.id })
163
- .run();
164
-
165
- for (const segment of segments) {
166
- const chunkId = buildChunkId(input.messageId, segment.segmentIndex);
167
- const chunkHash = computeChunkContentHash(scopeId, segment.text);
168
-
169
- // Check if this chunk already exists with the same content hash
170
- const existingChunk = tx
171
- .select({ contentHash: memoryChunks.contentHash })
172
- .from(memoryChunks)
173
- .where(eq(memoryChunks.id, chunkId))
174
- .get();
175
-
176
- tx.insert(memoryChunks)
177
- .values({
178
- id: chunkId,
179
- scopeId,
180
- observationId,
181
- content: segment.text,
182
- tokenEstimate: segment.tokenEstimate,
183
- contentHash: chunkHash,
184
- createdAt: input.createdAt,
185
- })
186
- .onConflictDoUpdate({
187
- target: memoryChunks.id,
188
- set: {
189
- content: segment.text,
190
- tokenEstimate: segment.tokenEstimate,
191
- contentHash: chunkHash,
192
- },
193
- })
194
- .run();
195
-
196
- if (existingChunk?.contentHash === chunkHash) {
197
- skippedChunkEmbedJobs++;
198
- } else {
199
- enqueueMemoryJob("embed_chunk", { chunkId, scopeId }, Date.now(), tx);
200
- }
201
- }
202
-
203
136
  // Enqueue embed_attachment jobs for image content blocks when the
204
137
  // embedding provider supports multimodal (Gemini only).
205
138
  for (const block of mediaBlocks) {
@@ -214,7 +147,7 @@ export async function indexMessageNow(
214
147
  if (shouldExtract && isTrustedActor && !input.automated) {
215
148
  enqueueMemoryJob(
216
149
  "extract_items",
217
- { messageId: input.messageId, scopeId },
150
+ { messageId: input.messageId, scopeId: input.scopeId ?? "default" },
218
151
  Date.now(),
219
152
  tx,
220
153
  );
@@ -233,12 +166,6 @@ export async function indexMessageNow(
233
166
  );
234
167
  }
235
168
 
236
- if (skippedChunkEmbedJobs > 0) {
237
- log.debug(
238
- `Skipped ${skippedChunkEmbedJobs}/${segments.length} embed_chunk jobs (content unchanged)`,
239
- );
240
- }
241
-
242
169
  if (!isTrustedActor && shouldExtract) {
243
170
  log.info(
244
171
  `Skipping extraction jobs for untrusted actor (trustClass=${input.provenanceTrustClass})`,
@@ -250,11 +177,9 @@ export async function indexMessageNow(
250
177
  }
251
178
 
252
179
  const extractionGated = !isTrustedActor || !!input.automated;
253
- const segmentEmbedJobs = segments.length - skippedEmbedJobs;
254
- const chunkEmbedJobs = segments.length - skippedChunkEmbedJobs;
255
180
  const enqueuedJobs =
256
- segmentEmbedJobs +
257
- chunkEmbedJobs +
181
+ segments.length -
182
+ skippedEmbedJobs +
258
183
  mediaBlocks.length +
259
184
  (shouldExtract && !extractionGated ? 2 : 1);
260
185
  return {
@@ -288,19 +213,3 @@ export function getRecentSegmentsForConversation(
288
213
  function buildSegmentId(messageId: string, segmentIndex: number): string {
289
214
  return `${messageId}:${segmentIndex}`;
290
215
  }
291
-
292
- /**
293
- * Deterministic observation ID derived from the messageId so repeated
294
- * indexer runs for the same message converge on the same observation row.
295
- */
296
- function buildObservationId(messageId: string): string {
297
- return `obs:${messageId}`;
298
- }
299
-
300
- /**
301
- * Deterministic chunk ID derived from the messageId and segment index so
302
- * the dual-write path mirrors the legacy segment identity scheme exactly.
303
- */
304
- function buildChunkId(messageId: string, segmentIndex: number): string {
305
- return `chunk:${messageId}:${segmentIndex}`;
306
- }