@wfm-cn/dsh-storage 0.0.0-stage → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/LICENSE +21 -0
  3. package/README.md +137 -2
  4. package/cordis.patch.yml +3 -0
  5. package/lib/backends/database.js +578 -0
  6. package/lib/backfill.js +331 -0
  7. package/lib/client/index.js +79 -0
  8. package/lib/client/panel.js +299 -0
  9. package/lib/client/toast.js +66 -0
  10. package/lib/client.js +1 -0
  11. package/lib/compaction.js +108 -0
  12. package/lib/config.js +171 -0
  13. package/lib/ddl.js +323 -0
  14. package/lib/generated/mysql/browser.js +17 -0
  15. package/lib/generated/mysql/client.js +34 -0
  16. package/lib/generated/mysql/commonInputTypes.js +10 -0
  17. package/lib/generated/mysql/enums.js +10 -0
  18. package/lib/generated/mysql/internal/class.js +49 -0
  19. package/lib/generated/mysql/internal/prismaNamespace.js +146 -0
  20. package/lib/generated/mysql/internal/prismaNamespaceBrowser.js +117 -0
  21. package/lib/generated/mysql/models/AiChatHistory.js +1 -0
  22. package/lib/generated/mysql/models/AiMessage.js +1 -0
  23. package/lib/generated/mysql/models.js +1 -0
  24. package/lib/generated/postgresql/browser.js +17 -0
  25. package/lib/generated/postgresql/client.js +34 -0
  26. package/lib/generated/postgresql/commonInputTypes.js +10 -0
  27. package/lib/generated/postgresql/enums.js +10 -0
  28. package/lib/generated/postgresql/internal/class.js +49 -0
  29. package/lib/generated/postgresql/internal/prismaNamespace.js +131 -0
  30. package/lib/generated/postgresql/internal/prismaNamespaceBrowser.js +102 -0
  31. package/lib/generated/postgresql/models/AiChatHistory.js +1 -0
  32. package/lib/generated/postgresql/models/AiMessage.js +1 -0
  33. package/lib/generated/postgresql/models.js +1 -0
  34. package/lib/generated/sqlite/browser.js +17 -0
  35. package/lib/generated/sqlite/client.js +34 -0
  36. package/lib/generated/sqlite/commonInputTypes.js +10 -0
  37. package/lib/generated/sqlite/enums.js +10 -0
  38. package/lib/generated/sqlite/internal/class.js +49 -0
  39. package/lib/generated/sqlite/internal/prismaNamespace.js +128 -0
  40. package/lib/generated/sqlite/internal/prismaNamespaceBrowser.js +99 -0
  41. package/lib/generated/sqlite/models/AiChatHistory.js +1 -0
  42. package/lib/generated/sqlite/models/AiMessage.js +1 -0
  43. package/lib/generated/sqlite/models.js +1 -0
  44. package/lib/generated/sqlserver/browser.js +17 -0
  45. package/lib/generated/sqlserver/client.js +34 -0
  46. package/lib/generated/sqlserver/commonInputTypes.js +10 -0
  47. package/lib/generated/sqlserver/enums.js +10 -0
  48. package/lib/generated/sqlserver/internal/class.js +49 -0
  49. package/lib/generated/sqlserver/internal/prismaNamespace.js +119 -0
  50. package/lib/generated/sqlserver/internal/prismaNamespaceBrowser.js +90 -0
  51. package/lib/generated/sqlserver/models/AiChatHistory.js +1 -0
  52. package/lib/generated/sqlserver/models/AiMessage.js +1 -0
  53. package/lib/generated/sqlserver/models.js +1 -0
  54. package/lib/health.js +180 -0
  55. package/lib/index.js +681 -0
  56. package/lib/memory-tools.js +365 -0
  57. package/lib/notify.js +56 -0
  58. package/lib/projector.js +191 -0
  59. package/lib/status-route.js +172 -0
  60. package/lib/types/backends/database.d.ts +78 -0
  61. package/lib/types/backfill.d.ts +67 -0
  62. package/lib/types/client/index.d.ts +12 -0
  63. package/lib/types/client/panel.d.ts +79 -0
  64. package/lib/types/client/toast.d.ts +38 -0
  65. package/lib/types/compaction.d.ts +3 -0
  66. package/lib/types/config.d.ts +52 -0
  67. package/lib/types/ddl.d.ts +49 -0
  68. package/lib/types/generated/mysql/browser.d.ts +14 -0
  69. package/lib/types/generated/mysql/client.d.ts +33 -0
  70. package/lib/types/generated/mysql/commonInputTypes.d.ts +396 -0
  71. package/lib/types/generated/mysql/enums.d.ts +1 -0
  72. package/lib/types/generated/mysql/internal/class.d.ts +143 -0
  73. package/lib/types/generated/mysql/internal/prismaNamespace.d.ts +709 -0
  74. package/lib/types/generated/mysql/internal/prismaNamespaceBrowser.d.ts +112 -0
  75. package/lib/types/generated/mysql/models/AiChatHistory.d.ts +1258 -0
  76. package/lib/types/generated/mysql/models/AiMessage.d.ts +1359 -0
  77. package/lib/types/generated/mysql/models.d.ts +3 -0
  78. package/lib/types/generated/postgresql/browser.d.ts +14 -0
  79. package/lib/types/generated/postgresql/client.d.ts +33 -0
  80. package/lib/types/generated/postgresql/commonInputTypes.d.ts +392 -0
  81. package/lib/types/generated/postgresql/enums.d.ts +1 -0
  82. package/lib/types/generated/postgresql/internal/class.d.ts +143 -0
  83. package/lib/types/generated/postgresql/internal/prismaNamespace.d.ts +728 -0
  84. package/lib/types/generated/postgresql/internal/prismaNamespaceBrowser.d.ts +95 -0
  85. package/lib/types/generated/postgresql/models/AiChatHistory.d.ts +1373 -0
  86. package/lib/types/generated/postgresql/models/AiMessage.d.ts +1496 -0
  87. package/lib/types/generated/postgresql/models.d.ts +3 -0
  88. package/lib/types/generated/sqlite/browser.d.ts +14 -0
  89. package/lib/types/generated/sqlite/client.d.ts +33 -0
  90. package/lib/types/generated/sqlite/commonInputTypes.d.ts +373 -0
  91. package/lib/types/generated/sqlite/enums.d.ts +1 -0
  92. package/lib/types/generated/sqlite/internal/class.d.ts +143 -0
  93. package/lib/types/generated/sqlite/internal/prismaNamespace.d.ts +705 -0
  94. package/lib/types/generated/sqlite/internal/prismaNamespaceBrowser.d.ts +92 -0
  95. package/lib/types/generated/sqlite/models/AiChatHistory.d.ts +1371 -0
  96. package/lib/types/generated/sqlite/models/AiMessage.d.ts +1493 -0
  97. package/lib/types/generated/sqlite/models.d.ts +3 -0
  98. package/lib/types/generated/sqlserver/browser.d.ts +14 -0
  99. package/lib/types/generated/sqlserver/client.d.ts +33 -0
  100. package/lib/types/generated/sqlserver/commonInputTypes.d.ts +333 -0
  101. package/lib/types/generated/sqlserver/enums.d.ts +1 -0
  102. package/lib/types/generated/sqlserver/internal/class.d.ts +143 -0
  103. package/lib/types/generated/sqlserver/internal/prismaNamespace.d.ts +669 -0
  104. package/lib/types/generated/sqlserver/internal/prismaNamespaceBrowser.d.ts +80 -0
  105. package/lib/types/generated/sqlserver/models/AiChatHistory.d.ts +1257 -0
  106. package/lib/types/generated/sqlserver/models/AiMessage.d.ts +1375 -0
  107. package/lib/types/generated/sqlserver/models.d.ts +3 -0
  108. package/lib/types/health.d.ts +52 -0
  109. package/lib/types/index.d.ts +91 -0
  110. package/lib/types/memory-tools.d.ts +168 -0
  111. package/lib/types/notify.d.ts +35 -0
  112. package/lib/types/projector.d.ts +51 -0
  113. package/lib/types/status-route.d.ts +48 -0
  114. package/lib/types/types.d.ts +58 -0
  115. package/lib/types.js +8 -0
  116. package/package.json +127 -4
  117. package/preview.png +0 -0
  118. package/prisma/schema.mysql.prisma +78 -0
  119. package/prisma/schema.postgresql.prisma +70 -0
  120. package/prisma/schema.sqlite.prisma +70 -0
  121. package/prisma/schema.sqlserver.prisma +76 -0
package/lib/index.js ADDED
@@ -0,0 +1,681 @@
1
+ /**
2
+ * dsh-storage — mirrors the dsh session event stream into the ai_messages /
3
+ * ai_chat_histories tables (MySQL/PostgreSQL/SQLite/SQL Server).
4
+ *
5
+ * Design: dsh's own session persistence (JSONL/SQLite) stays authoritative.
6
+ * This plugin taps `session/event` (the same seam the built-in persistence
7
+ * coordinator uses) and projects events into MessageRow / SessionRow shaped
8
+ * after the source project's ai_messages / ai_chat_histories tables.
9
+ *
10
+ * A2A task state (task metadata in Redis, workspace archives in GCS) is NOT
11
+ * here — that belongs to the A2A layer, see dsh-a2a's TaskStore backends.
12
+ *
13
+ * Resume caveat (docs/subsystems/persistence.md): constructor seeds do not
14
+ * emit events. On agent resume, only events >= `session.firstLiveSeq` fire
15
+ * on this stream — historical rows must come from the database itself, not
16
+ * from replaying the tap.
17
+ *
18
+ * @module dsh-storage
19
+ */
20
+ import Schema from '@deepseek-ai/schemastery';
21
+ import { existsSync } from 'node:fs';
22
+ import { join } from 'node:path';
23
+ import { DatabaseBackend } from './backends/database.js';
24
+ import { backfillAll, LAST_USAGE_KEY } from './backfill.js';
25
+ import { compressionSnapshot } from './compaction.js';
26
+ import { configToUrl, defaultConfigPath, dshHome, loadConfig, defaultDevice, saveConfig as saveConfigFile, tzOffsetMinutes, watchConfig, } from './config.js';
27
+ import { createTables, ensureDeviceColumn } from './ddl.js';
28
+ import { ConfigError, HealthMonitor, probeDatabase, testConnection, } from './health.js';
29
+ import { drainToasts, wireNotifications } from './notify.js';
30
+ import { registerMemoryTools } from './memory-tools.js';
31
+ import { mergeToolResult, projectEvent, toolResultCallId, usageSampleOf } from './projector.js';
32
+ import { registerStatusRoute } from './status-route.js';
33
+ export const name = 'dsh-storage';
34
+ // Event-tap only: no hard service dependency. Add 'sessionPersistence' to
35
+ // `inject` if you want fail-fast when persistence is not composed.
36
+ export const inject = [];
37
+ export const Config = Schema.object({
38
+ enabled: Schema.boolean().default(false),
39
+ database: Schema.object({
40
+ enabled: Schema.boolean().default(false),
41
+ provider: Schema.union(['mysql', 'postgresql', 'sqlite', 'sqlserver']).default('sqlite'),
42
+ url: Schema.string().role('secret').default(''),
43
+ timezone: Schema.string().default('+08:00'),
44
+ }),
45
+ });
46
+ /** Metadata key for our usage bookkeeping — namespaced so other writers' fields never collide with it. */
47
+ const TOKEN_BASELINE_KEY = 'dsh-storage:token-baseline';
48
+ const ARCHIVE_IN_PROGRESS = 'dsh-storage:archive-in-progress';
49
+ /**
50
+ * The patched @prisma/adapter-mariadb (pnpm patch) reads this global when
51
+ * serializing Date args: the stock adapter hard-codes UTC on writes and
52
+ * ignores its `timezone` option, so we hand it the config's offset instead.
53
+ */
54
+ const TZ_OFFSET_GLOBAL_KEY = '__DSH_STORAGE_TZ_OFFSET_MIN__';
55
+ let tzWarningLogged = false;
56
+ /** Publish the config timezone offset for the patched mariadb adapter. */
57
+ function publishTzOffset(tz) {
58
+ const minutes = tzOffsetMinutes(tz ?? '');
59
+ globalThis[TZ_OFFSET_GLOBAL_KEY] = minutes;
60
+ const wellFormed = tz !== undefined && (tz === 'UTC' || tz === 'Z' || /^[+-]\d{2}:\d{2}$/.test(tz));
61
+ if (!wellFormed && !tzWarningLogged) {
62
+ tzWarningLogged = true;
63
+ console.error(`[dsh-storage] invalid timezone ${JSON.stringify(tz ?? '')} — assuming UTC offset 0`);
64
+ }
65
+ }
66
+ /** Shape-guarded restore of our usage sample from arbitrary metadata. */
67
+ function usageSampleOfMetadata(value) {
68
+ const v = value;
69
+ return v &&
70
+ typeof v.key === 'string' &&
71
+ typeof v.input === 'number' &&
72
+ typeof v.output === 'number'
73
+ ? { key: v.key, input: v.input, output: v.output }
74
+ : undefined;
75
+ }
76
+ export function apply(ctx, config) {
77
+ // Lifecycle: this cordis fork has no 'ready'/'dispose' events — startup
78
+ // work goes in ctx.effect() (runs immediately at plugin load; the returned
79
+ // disposer runs on fiber unload at shutdown). Event taps verified against
80
+ // @deepseek-ai/dsh-session@0.1.6-alpha.2: session/event(session, event),
81
+ // session/disposed(session), and the awaited session/flush(session)
82
+ // durability checkpoint. The type-only import above registers the event
83
+ // names; handler params stay `any` because storage also mirrors
84
+ // plugin-owned event types (e.g. session/title) it does not depend on.
85
+ // Health monitor: probes the sync config's database (connect + SELECT 1 +
86
+ // ai_messages existence) with exponential-backoff retries. The probe
87
+ // re-reads $DSH_HOME/dsh-storage.json on every run, so config edits and
88
+ // corruption are picked up without a restart. Provided before the enabled
89
+ // guard so the panel can always query status (state stays 'disabled'
90
+ // while sync is off). enabled:false is handled by stopping the monitor
91
+ // (true 'disabled' state) — only a broken config surfaces as
92
+ // 'config-error'.
93
+ const monitor = new HealthMonitor(async () => {
94
+ const { config: syncConfig, error } = loadConfig();
95
+ if (error)
96
+ throw new ConfigError(`sync config invalid (${error})`);
97
+ await probeDatabase(syncConfig);
98
+ });
99
+ // Toast notifications (Task 6): good→bad / bad→good transitions are
100
+ // recorded host-side and drained by the browser toast portal through the
101
+ // status bridge. Mounted for the plugin's whole lifetime (like the monitor).
102
+ ctx.effect(() => wireNotifications(monitor));
103
+ ctx.effect(() => {
104
+ // The monitor lifecycle follows the sync config alone — always evaluated
105
+ // at plugin load (regardless of enabled flags) and re-evaluated on every
106
+ // config edit via T3's watchConfig (300ms debounce). Without this,
107
+ // starting disabled would leave the status stuck at 'disabled' forever
108
+ // after the user enables sync: enabled:false → monitor stopped with the
109
+ // true 'disabled' state; enabled:true (or a broken config, surfaced as
110
+ // config-error) → probe/recheck.
111
+ let monitorRunning = false;
112
+ const evaluate = () => {
113
+ const { config: syncConfig, error } = loadConfig();
114
+ if (!error && !syncConfig.enabled) {
115
+ monitor.stop();
116
+ monitorRunning = false;
117
+ return;
118
+ }
119
+ if (monitorRunning)
120
+ void monitor.recheck();
121
+ else {
122
+ monitorRunning = true;
123
+ monitor.start();
124
+ }
125
+ };
126
+ evaluate();
127
+ const unwatch = watchConfig(defaultConfigPath(), evaluate);
128
+ return () => {
129
+ unwatch();
130
+ monitor.stop();
131
+ monitorRunning = false;
132
+ };
133
+ });
134
+ const statusService = {
135
+ snapshot() {
136
+ const current = monitor.snapshot();
137
+ // Minimal config-error re-arm: the next snapshot call re-reads the
138
+ // config and recovers once it parses and connects again.
139
+ if (current.state === 'config-error')
140
+ void monitor.recheck();
141
+ return current;
142
+ },
143
+ testConnection(cfg) {
144
+ return testConnection(cfg);
145
+ },
146
+ createTables,
147
+ getConfig: () => loadConfig(),
148
+ saveConfig(cfg) {
149
+ saveConfigFile(cfg);
150
+ return { config: cfg };
151
+ },
152
+ toasts(payload) {
153
+ return drainToasts(payload?.sinceId ?? 0);
154
+ },
155
+ async backfill() {
156
+ const { config: cfg, error } = loadConfig();
157
+ if (error)
158
+ throw new Error(`sync config invalid (${error})`);
159
+ // A one-shot backend from the current config — backfill works even
160
+ // while the live mirror is disabled or its connection was rebuilt.
161
+ const backend = new DatabaseBackend({
162
+ provider: cfg.provider,
163
+ url: configToUrl(cfg),
164
+ timezone: cfg.timezone,
165
+ device: cfg.device,
166
+ });
167
+ await backend.init();
168
+ try {
169
+ return await backfillAll(join(dshHome(), 'sessions'), backend);
170
+ }
171
+ finally {
172
+ await backend.close?.().catch(() => { });
173
+ }
174
+ },
175
+ };
176
+ ctx.provide('dshStorageStatus', statusService);
177
+ // Phase 2: AI memory tools (memory_search / memory_read) — host-side only,
178
+ // registered on the dsh-tools service; a disabled/unconfigured sync config
179
+ // surfaces as a friendly error at call time, never at registration.
180
+ registerMemoryTools(ctx);
181
+ // Browser bridge (Task 5): the settings panel client calls this over the
182
+ // connection generic-RPC channel. Optional services — profiles without a
183
+ // web carrier simply skip the route.
184
+ ctx.effect(() => {
185
+ const dispose = registerStatusRoute(ctx, statusService);
186
+ return () => dispose?.();
187
+ });
188
+ // C1: the mirror derives its effective config from the same source the
189
+ // panel/health/backfill use — $DSH_HOME/dsh-storage.json. The JSON file
190
+ // wins whenever it exists (enabled:false in JSON means OFF, even when a
191
+ // legacy yaml block is enabled); the cordis yaml Config is only a compat
192
+ // fallback for pre-panel installs whose JSON file does not exist yet —
193
+ // exactly the state the switch runbook leaves users in after deleting the
194
+ // hand-written yaml block. Config edits rebuild the mirror without a
195
+ // restart: the watcher disposes the old backends + event tap and
196
+ // reconstructs them from the new file.
197
+ let mirror;
198
+ let activeTarget;
199
+ let desiredTarget;
200
+ let switching = false;
201
+ const settleMirror = () => {
202
+ if (switching || sameMirrorTarget(activeTarget, desiredTarget))
203
+ return;
204
+ if (mirror) {
205
+ // Tear the old mirror down first (drain pending writes, close
206
+ // backends); settleMirror re-runs and wires the next target once the
207
+ // teardown has landed.
208
+ switching = true;
209
+ const old = mirror;
210
+ mirror = undefined;
211
+ activeTarget = undefined;
212
+ void old
213
+ .dispose()
214
+ .catch(() => { })
215
+ .then(() => {
216
+ switching = false;
217
+ settleMirror();
218
+ });
219
+ return;
220
+ }
221
+ if (desiredTarget) {
222
+ mirror = startMirror(ctx, desiredTarget);
223
+ activeTarget = desiredTarget;
224
+ }
225
+ };
226
+ // The archive service outlives mirror rebuilds — it delegates to whichever
227
+ // mirror instance is currently wired.
228
+ ctx.provide('dshStorage', {
229
+ archiveSession(sessionId) {
230
+ return mirror?.archiveSession(sessionId) ?? Promise.resolve(null);
231
+ },
232
+ });
233
+ const evaluateMirror = () => {
234
+ desiredTarget = computeMirrorTarget(config);
235
+ // Keep the adapter's timezone global in sync with the effective config —
236
+ // set at load and re-set whenever the watcher fires with a changed
237
+ // timezone (MirrorTarget equality already includes timezone).
238
+ publishTzOffset(desiredTarget?.timezone ?? config.database?.timezone);
239
+ settleMirror();
240
+ };
241
+ evaluateMirror();
242
+ ctx.effect(() => {
243
+ const unwatch = watchConfig(defaultConfigPath(), evaluateMirror);
244
+ return () => unwatch();
245
+ });
246
+ }
247
+ function sameMirrorTarget(a, b) {
248
+ return (a === b ||
249
+ (a?.provider === b?.provider &&
250
+ a?.url === b?.url &&
251
+ a?.timezone === b?.timezone &&
252
+ a?.device === b?.device));
253
+ }
254
+ /**
255
+ * The mirror's effective target. When ~/.dsh/dsh-storage.json exists it wins
256
+ * outright — a panel-configured install must mirror from the panel's config,
257
+ * and enabled:false there must actually stop mirroring; the yaml Config only
258
+ * drives the mirror when no JSON file exists (legacy pre-panel installs).
259
+ * A present-but-broken JSON file disables the mirror (the health monitor
260
+ * surfaces the parse error as 'config-error').
261
+ */
262
+ function computeMirrorTarget(config) {
263
+ const path = defaultConfigPath();
264
+ if (existsSync(path)) {
265
+ const { config: syncConfig, error } = loadConfig(path);
266
+ if (error || !syncConfig.enabled)
267
+ return undefined;
268
+ return {
269
+ provider: syncConfig.provider,
270
+ url: configToUrl(syncConfig),
271
+ timezone: syncConfig.timezone,
272
+ device: syncConfig.device,
273
+ };
274
+ }
275
+ if (!config.enabled || !config.database.enabled || !config.database.url)
276
+ return undefined;
277
+ return {
278
+ provider: config.database.provider,
279
+ url: config.database.url,
280
+ timezone: config.database.timezone,
281
+ device: defaultDevice(),
282
+ };
283
+ }
284
+ /**
285
+ * Wire one mirror instance against `target`: backend construction, the
286
+ * session/event tap, and durability checkpoints. The projector internals are
287
+ * unchanged from upstream — this factory only owns their lifecycle so the
288
+ * config watcher can dispose and reconstruct everything on config changes.
289
+ */
290
+ function startMirror(ctx, target) {
291
+ const backends = [
292
+ new DatabaseBackend({
293
+ provider: target.provider,
294
+ url: target.url,
295
+ timezone: target.timezone,
296
+ device: target.device,
297
+ }),
298
+ ];
299
+ const sessions = new Map();
300
+ const toolCallMessages = new Map();
301
+ const TOOL_CALL_CACHE_LIMIT = 1000;
302
+ function rememberToolCalls(sessionId, row) {
303
+ if (row?.type !== 'model' || !Array.isArray(row.toolCalls))
304
+ return;
305
+ for (const call of row.toolCalls) {
306
+ if (typeof call?.id !== 'string')
307
+ continue;
308
+ const calls = toolCallMessages.get(sessionId) ?? new Map();
309
+ // Bound each session's cache; Map iteration evicts the oldest insertion first.
310
+ if (!calls.has(call.id) && calls.size >= TOOL_CALL_CACHE_LIMIT) {
311
+ const oldest = calls.keys().next().value;
312
+ if (oldest !== undefined)
313
+ calls.delete(oldest);
314
+ }
315
+ calls.set(call.id, row.id);
316
+ toolCallMessages.set(sessionId, calls);
317
+ }
318
+ }
319
+ async function readToolResultRows(session, sessionId, event) {
320
+ const callId = toolResultCallId(event);
321
+ if (!callId)
322
+ return [];
323
+ let messageId = toolCallMessages.get(sessionId)?.get(callId);
324
+ if (!messageId && typeof session?.deriveMessages === 'function') {
325
+ const messages = session.deriveMessages();
326
+ for (let index = messages.length - 1; index >= 0; index--) {
327
+ const message = messages[index];
328
+ if (message.role === 'assistant' &&
329
+ typeof message.id === 'string' &&
330
+ Array.isArray(message.content) &&
331
+ message.content.some((block) => block?.type === 'tool-call' && block.id === callId)) {
332
+ messageId = message.id;
333
+ break;
334
+ }
335
+ }
336
+ }
337
+ if (messageId) {
338
+ const rows = await backends[0].readMessages(sessionId, messageId);
339
+ if (rows.some((row) => Array.isArray(row.toolCalls) && row.toolCalls.some((call) => call?.id === callId)))
340
+ return rows;
341
+ }
342
+ // Cold resume and eviction fall back to bounded database candidate pages.
343
+ const rows = await backends[0].readToolCallMessages(sessionId, callId);
344
+ for (const row of rows)
345
+ rememberToolCalls(sessionId, row);
346
+ return rows;
347
+ }
348
+ // Row writes are fire-and-forget during the run, but tracked so the
349
+ // session/flush checkpoint and shutdown can drain them — a one-shot
350
+ // (headless) run would otherwise lose the tail events to process exit.
351
+ const pending = new Set();
352
+ function track(task) {
353
+ pending.add(task);
354
+ void task.then(() => pending.delete(task), () => pending.delete(task));
355
+ return task;
356
+ }
357
+ function guard(call) {
358
+ // Mirroring must never break the agent loop: log and swallow.
359
+ return Promise.all(backends.map((backend) => call(backend).catch((error) => console.error(`[dsh-storage] ${backend.name} error:`, error)))).then(() => { });
360
+ }
361
+ // All sessions share init and its retry delay; queued events wait rather than skip the barrier.
362
+ let started;
363
+ let retryAt = 0;
364
+ /** Device-column migration runs once per mirror instance, after init. */
365
+ let migrated = false;
366
+ function ensureStarted() {
367
+ if (!started) {
368
+ const attempt = track((async () => {
369
+ const delay = retryAt - Date.now();
370
+ if (delay > 0)
371
+ await new Promise((resolve) => setTimeout(resolve, delay));
372
+ await Promise.all(backends.map((backend) => backend.init?.()));
373
+ if (!migrated) {
374
+ migrated = true;
375
+ // Cheap information_schema check + ALTER/UPDATE only when the
376
+ // device column is missing (multi-device shared database).
377
+ const result = await ensureDeviceColumn(target, target.device);
378
+ if (result !== 'ok')
379
+ console.error(`[dsh-storage] device column migration: ${result}`);
380
+ }
381
+ })());
382
+ started = attempt;
383
+ void attempt.catch(() => {
384
+ if (started === attempt) {
385
+ retryAt = Date.now() + 1000;
386
+ started = undefined;
387
+ }
388
+ });
389
+ }
390
+ return started;
391
+ }
392
+ function fanout(call) {
393
+ // Event failures reject into the per-session queue, which is the no-throw
394
+ // boundary. Swallowing here would make a partial write look committed.
395
+ return track(ensureStarted().then(() => Promise.all(backends.map(call)).then(() => { })));
396
+ }
397
+ // Per-session serialization: all event processing for one session runs
398
+ // through a promise chain, so rollup writes complete in event order
399
+ // (latest-wins — concurrent absolute snapshots must not land out of order)
400
+ // and the first event of an unknown session seeds its accumulator from the
401
+ // stored row before any write (resume: constructor seeds are not re-emitted,
402
+ // so history must come from the database, not from zero).
403
+ const chains = new Map();
404
+ const toolBatches = new Map();
405
+ function enqueue(sessionId, task) {
406
+ // Backend work checks init inside mirrorEvent; cleanup needs only queue ordering.
407
+ const next = (chains.get(sessionId) ?? Promise.resolve())
408
+ .then(task)
409
+ .catch((error) => console.error('[dsh-storage] mirror task error:', error));
410
+ chains.set(sessionId, next);
411
+ track(next);
412
+ }
413
+ async function seedAccum(sessionId) {
414
+ // "Row not found" (a new session) is the only zero-seed case. A read
415
+ // failure is NOT "not found": fail the task so the next event re-seeds,
416
+ // instead of accumulating from zero and overwriting the stored rollup.
417
+ let readFailure;
418
+ for (const backend of backends) {
419
+ try {
420
+ const row = await backend.readSession?.(sessionId);
421
+ if (row) {
422
+ return {
423
+ messageCount: row.messageCount,
424
+ totalTokens: row.totalTokens,
425
+ title: row.title ?? undefined,
426
+ firstMessageAt: row.firstMessageAt ?? undefined,
427
+ lastMessageAt: row.lastMessageAt ?? undefined,
428
+ // Preserve every existing metadata field; we only own our key.
429
+ // The baseline distinguishes cumulative active totals from a
430
+ // restart's fresh-segment deltas during archive recovery.
431
+ metadata: { [TOKEN_BASELINE_KEY]: row.totalTokens, ...(row.metadata ?? {}) },
432
+ usageSample: usageSampleOfMetadata(row.metadata?.[LAST_USAGE_KEY]),
433
+ };
434
+ }
435
+ }
436
+ catch (error) {
437
+ readFailure ??= error;
438
+ console.error(`[dsh-storage] ${backend.name} error:`, error);
439
+ }
440
+ }
441
+ if (readFailure)
442
+ throw readFailure;
443
+ return { messageCount: 0, totalTokens: 0, metadata: { [TOKEN_BASELINE_KEY]: 0 } };
444
+ }
445
+ void ensureStarted().catch((error) => console.error('[dsh-storage] database error:', error));
446
+ function archiveSession(sessionId) {
447
+ toolBatches.delete(sessionId);
448
+ // Join the same per-session chain as event writes, then discard the old
449
+ // in-memory rollup only after the database transaction commits.
450
+ const archived = (chains.get(sessionId) ?? Promise.resolve())
451
+ .then(() => ensureStarted())
452
+ .then(async () => {
453
+ const accum = sessions.get(sessionId);
454
+ if (accum && !accum.metadata[ARCHIVE_IN_PROGRESS]) {
455
+ // Freeze inside the session queue: all earlier events have now
456
+ // committed, and this first baseline survives every retry until
457
+ // the archive actually succeeds.
458
+ accum.metadata[TOKEN_BASELINE_KEY] = accum.totalTokens;
459
+ accum.metadata[ARCHIVE_IN_PROGRESS] = true;
460
+ }
461
+ const id = await backends[0].archiveSession(sessionId);
462
+ sessions.delete(sessionId);
463
+ toolCallMessages.delete(sessionId);
464
+ return id;
465
+ });
466
+ const settled = archived.then(() => { }, () => { });
467
+ chains.set(sessionId, settled);
468
+ track(settled);
469
+ return archived;
470
+ }
471
+ async function mirrorEvent(session, event, snapshot, batch) {
472
+ await ensureStarted();
473
+ const sessionId = session?.id ?? 'unknown';
474
+ if (snapshot) {
475
+ await fanout((backend) => backend.upsertMessage(snapshot));
476
+ return;
477
+ }
478
+ const stored = sessions.get(sessionId) ?? (await seedAccum(sessionId));
479
+ sessions.set(sessionId, stored);
480
+ // Stage every change on a copy. The live accumulator advances only
481
+ // after the message + rollup transaction commits.
482
+ const accum = { ...stored, metadata: { ...stored.metadata } };
483
+ let row = projectEvent(session, event, sessionId);
484
+ if (row && session.header?.createdAt !== undefined) {
485
+ row.metadata = { ...row.metadata, sessionCreatedAt: session.header.createdAt };
486
+ }
487
+ const storedRows = event?.type === 'tool/result'
488
+ ? await readToolResultRows(session, sessionId, event)
489
+ : row
490
+ ? await backends[0].readMessages(sessionId, row.id)
491
+ : [];
492
+ let processedThrough = batch?.index;
493
+ if (event?.type === 'tool/result') {
494
+ row = mergeToolResult(event, storedRows);
495
+ while (row &&
496
+ batch &&
497
+ processedThrough !== undefined &&
498
+ processedThrough + 1 < batch.events.length) {
499
+ const following = batch.events[processedThrough + 1];
500
+ const callId = toolResultCallId(following);
501
+ if (!callId ||
502
+ !Array.isArray(row.toolCalls) ||
503
+ !row.toolCalls.some((call) => call?.id === callId))
504
+ break;
505
+ row = mergeToolResult(following, [row]);
506
+ processedThrough++;
507
+ }
508
+ }
509
+ const existing = row && storedRows.find((stored) => stored.id === row?.id);
510
+ if (event?.type === 'assistant/message' && row && existing && Array.isArray(row.toolCalls)) {
511
+ row.toolCalls = row.toolCalls.map((call) => {
512
+ const previous = Array.isArray(existing.toolCalls)
513
+ ? existing.toolCalls.find((stored) => stored?.id === call?.id)
514
+ : undefined;
515
+ return previous && 'result' in previous
516
+ ? {
517
+ ...call,
518
+ result: previous.result,
519
+ ...(previous.error ? { error: previous.error } : {}),
520
+ }
521
+ : call;
522
+ });
523
+ }
524
+ if (!row && event?.type === 'tool/result') {
525
+ console.error('[dsh-storage] no matching model tool call for result');
526
+ }
527
+ if (row && !existing && event?.type !== 'tool/result') {
528
+ accum.messageCount += 1;
529
+ accum.firstMessageAt ??= row.createdAt;
530
+ accum.lastMessageAt = row.createdAt;
531
+ }
532
+ // Every settled attempt/message is one model call; retries on the same
533
+ // step are additive rather than replacement snapshots.
534
+ const sampled = usageSampleOf(event);
535
+ let usageDelta = 0;
536
+ if (sampled) {
537
+ const sequence = Number(sampled.key);
538
+ const previous = Number(accum.usageSample?.key);
539
+ const duplicate = Number.isSafeInteger(sequence) && Number.isSafeInteger(previous) && sequence <= previous;
540
+ if (!duplicate) {
541
+ usageDelta = sampled.sample.input + sampled.sample.output;
542
+ accum.usageSample = { key: sampled.key, ...sampled.sample };
543
+ accum.totalTokens += usageDelta;
544
+ }
545
+ }
546
+ // Latest-wins title snapshot (log-only `session/title` event; payload
547
+ // { title, messageSeqs, source } verified against dsh-session-title@0.1.6-alpha.2).
548
+ if (event?.type === 'session/title' && typeof event.data?.title === 'string') {
549
+ accum.title = event.data.title;
550
+ }
551
+ if (row || usageDelta !== 0 || event?.type === 'turn/end' || event?.type === 'session/title') {
552
+ // Merge our bookkeeping into the existing metadata instead of
553
+ // replacing the column — other fields (present or future) survive.
554
+ const metadata = {
555
+ ...accum.metadata,
556
+ ...(accum.usageSample ? { [LAST_USAGE_KEY]: accum.usageSample } : {}),
557
+ };
558
+ const sessionRow = {
559
+ sessionId,
560
+ title: accum.title ?? null,
561
+ messageCount: accum.messageCount,
562
+ totalTokens: accum.totalTokens,
563
+ firstMessageAt: accum.firstMessageAt ?? null,
564
+ lastMessageAt: accum.lastMessageAt ?? null,
565
+ ...(Object.keys(metadata).length > 0 ? { metadata } : {}),
566
+ };
567
+ await fanout((backend) => backend.upsertEvent(row, sessionRow));
568
+ }
569
+ // Advance only after commit so a failed merged write cannot skip later results.
570
+ if (batch && processedThrough !== undefined)
571
+ batch.index = processedThrough;
572
+ sessions.set(sessionId, accum);
573
+ rememberToolCalls(sessionId, row);
574
+ }
575
+ /* eslint-disable @typescript-eslint/no-explicit-any */
576
+ const offEvent = ctx.on('session/event', (session, event) => {
577
+ if (![
578
+ 'user/message',
579
+ 'assistant/message',
580
+ 'assistant/attempt',
581
+ 'tool/result',
582
+ 'turn/end',
583
+ 'session/title',
584
+ 'compaction/end',
585
+ ].includes(event?.type))
586
+ return;
587
+ let snapshot;
588
+ try {
589
+ // Capture the committed surface now; queued writes must not observe later steps.
590
+ snapshot = compressionSnapshot(session, event);
591
+ }
592
+ catch (error) {
593
+ console.error('[dsh-storage] compaction snapshot error:', error);
594
+ return;
595
+ }
596
+ if (event.type === 'compaction/end' && !snapshot)
597
+ return;
598
+ const sessionId = session?.id ?? 'unknown';
599
+ if (event.type === 'tool/result') {
600
+ const pendingBatch = toolBatches.get(sessionId);
601
+ if (pendingBatch && pendingBatch.events.length < 32) {
602
+ pendingBatch.events.push(event);
603
+ return;
604
+ }
605
+ const batch = { events: [event], index: 0 };
606
+ toolBatches.set(sessionId, batch);
607
+ enqueue(sessionId, async () => {
608
+ if (toolBatches.get(sessionId) === batch)
609
+ toolBatches.delete(sessionId);
610
+ for (; batch.index < batch.events.length; batch.index++) {
611
+ try {
612
+ await mirrorEvent(session, batch.events[batch.index], null, batch);
613
+ }
614
+ catch (error) {
615
+ console.error('[dsh-storage] mirror task error:', error);
616
+ }
617
+ }
618
+ });
619
+ }
620
+ else {
621
+ toolBatches.delete(sessionId);
622
+ enqueue(sessionId, () => mirrorEvent(session, event, snapshot));
623
+ }
624
+ });
625
+ const offDisposed = ctx.on('session/disposed', (session) => {
626
+ const sessionId = session?.id ?? 'unknown';
627
+ toolBatches.delete(sessionId);
628
+ // The delete must run inside the session's chain — a synchronous delete
629
+ // here races the queued event tasks (the map may not even have the entry
630
+ // yet at emit time, and queued tasks would keep growing the accumulator
631
+ // the dispose was meant to drop).
632
+ enqueue(sessionId, async () => {
633
+ sessions.delete(sessionId);
634
+ toolCallMessages.delete(sessionId);
635
+ });
636
+ // Delete the chain entry only once it has settled AND is still the same
637
+ // entry — deleting earlier would let a recreated session race the
638
+ // orphaned writes; skipping the delete when a new task has since chained
639
+ // on preserves order.
640
+ const chain = chains.get(sessionId);
641
+ chain?.finally(() => {
642
+ if (chains.get(sessionId) === chain)
643
+ chains.delete(sessionId);
644
+ });
645
+ });
646
+ // Durability checkpoint: wait only for this session. Shutdown still drains
647
+ // the global pending set; an unrelated slow session must not block this one.
648
+ const offFlush = ctx.on('session/flush', async (session) => {
649
+ toolBatches.delete(session?.id ?? 'unknown');
650
+ const chain = chains.get(session?.id ?? 'unknown');
651
+ if (chain) {
652
+ await chain;
653
+ return;
654
+ }
655
+ try {
656
+ await ensureStarted();
657
+ }
658
+ catch (error) {
659
+ console.error('[dsh-storage] database error:', error);
660
+ }
661
+ });
662
+ // Teardown runs both on config-driven rebuilds (dispose()) and fiber unload
663
+ // (the effect below); the stopped flag makes it idempotent so a rebuilt-away
664
+ // mirror is never closed twice.
665
+ let stopped = false;
666
+ const stop = async () => {
667
+ if (stopped)
668
+ return;
669
+ stopped = true;
670
+ offEvent();
671
+ offDisposed();
672
+ offFlush();
673
+ toolBatches.clear();
674
+ await Promise.allSettled([...pending]);
675
+ sessions.clear();
676
+ toolCallMessages.clear();
677
+ await guard(async (backend) => backend.close?.());
678
+ };
679
+ ctx.effect(() => () => stop());
680
+ return { archiveSession, dispose: stop };
681
+ }