@wfm-cn/dsh-storage 0.0.0-stage → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/LICENSE +21 -0
- package/README.md +137 -2
- package/cordis.patch.yml +3 -0
- package/lib/backends/database.js +578 -0
- package/lib/backfill.js +331 -0
- package/lib/client/index.js +79 -0
- package/lib/client/panel.js +299 -0
- package/lib/client/toast.js +66 -0
- package/lib/client.js +1 -0
- package/lib/compaction.js +108 -0
- package/lib/config.js +171 -0
- package/lib/ddl.js +323 -0
- package/lib/generated/mysql/browser.js +17 -0
- package/lib/generated/mysql/client.js +34 -0
- package/lib/generated/mysql/commonInputTypes.js +10 -0
- package/lib/generated/mysql/enums.js +10 -0
- package/lib/generated/mysql/internal/class.js +49 -0
- package/lib/generated/mysql/internal/prismaNamespace.js +146 -0
- package/lib/generated/mysql/internal/prismaNamespaceBrowser.js +117 -0
- package/lib/generated/mysql/models/AiChatHistory.js +1 -0
- package/lib/generated/mysql/models/AiMessage.js +1 -0
- package/lib/generated/mysql/models.js +1 -0
- package/lib/generated/postgresql/browser.js +17 -0
- package/lib/generated/postgresql/client.js +34 -0
- package/lib/generated/postgresql/commonInputTypes.js +10 -0
- package/lib/generated/postgresql/enums.js +10 -0
- package/lib/generated/postgresql/internal/class.js +49 -0
- package/lib/generated/postgresql/internal/prismaNamespace.js +131 -0
- package/lib/generated/postgresql/internal/prismaNamespaceBrowser.js +102 -0
- package/lib/generated/postgresql/models/AiChatHistory.js +1 -0
- package/lib/generated/postgresql/models/AiMessage.js +1 -0
- package/lib/generated/postgresql/models.js +1 -0
- package/lib/generated/sqlite/browser.js +17 -0
- package/lib/generated/sqlite/client.js +34 -0
- package/lib/generated/sqlite/commonInputTypes.js +10 -0
- package/lib/generated/sqlite/enums.js +10 -0
- package/lib/generated/sqlite/internal/class.js +49 -0
- package/lib/generated/sqlite/internal/prismaNamespace.js +128 -0
- package/lib/generated/sqlite/internal/prismaNamespaceBrowser.js +99 -0
- package/lib/generated/sqlite/models/AiChatHistory.js +1 -0
- package/lib/generated/sqlite/models/AiMessage.js +1 -0
- package/lib/generated/sqlite/models.js +1 -0
- package/lib/generated/sqlserver/browser.js +17 -0
- package/lib/generated/sqlserver/client.js +34 -0
- package/lib/generated/sqlserver/commonInputTypes.js +10 -0
- package/lib/generated/sqlserver/enums.js +10 -0
- package/lib/generated/sqlserver/internal/class.js +49 -0
- package/lib/generated/sqlserver/internal/prismaNamespace.js +119 -0
- package/lib/generated/sqlserver/internal/prismaNamespaceBrowser.js +90 -0
- package/lib/generated/sqlserver/models/AiChatHistory.js +1 -0
- package/lib/generated/sqlserver/models/AiMessage.js +1 -0
- package/lib/generated/sqlserver/models.js +1 -0
- package/lib/health.js +180 -0
- package/lib/index.js +681 -0
- package/lib/memory-tools.js +365 -0
- package/lib/notify.js +56 -0
- package/lib/projector.js +191 -0
- package/lib/status-route.js +172 -0
- package/lib/types/backends/database.d.ts +78 -0
- package/lib/types/backfill.d.ts +67 -0
- package/lib/types/client/index.d.ts +12 -0
- package/lib/types/client/panel.d.ts +79 -0
- package/lib/types/client/toast.d.ts +38 -0
- package/lib/types/compaction.d.ts +3 -0
- package/lib/types/config.d.ts +52 -0
- package/lib/types/ddl.d.ts +49 -0
- package/lib/types/generated/mysql/browser.d.ts +14 -0
- package/lib/types/generated/mysql/client.d.ts +33 -0
- package/lib/types/generated/mysql/commonInputTypes.d.ts +396 -0
- package/lib/types/generated/mysql/enums.d.ts +1 -0
- package/lib/types/generated/mysql/internal/class.d.ts +143 -0
- package/lib/types/generated/mysql/internal/prismaNamespace.d.ts +709 -0
- package/lib/types/generated/mysql/internal/prismaNamespaceBrowser.d.ts +112 -0
- package/lib/types/generated/mysql/models/AiChatHistory.d.ts +1258 -0
- package/lib/types/generated/mysql/models/AiMessage.d.ts +1359 -0
- package/lib/types/generated/mysql/models.d.ts +3 -0
- package/lib/types/generated/postgresql/browser.d.ts +14 -0
- package/lib/types/generated/postgresql/client.d.ts +33 -0
- package/lib/types/generated/postgresql/commonInputTypes.d.ts +392 -0
- package/lib/types/generated/postgresql/enums.d.ts +1 -0
- package/lib/types/generated/postgresql/internal/class.d.ts +143 -0
- package/lib/types/generated/postgresql/internal/prismaNamespace.d.ts +728 -0
- package/lib/types/generated/postgresql/internal/prismaNamespaceBrowser.d.ts +95 -0
- package/lib/types/generated/postgresql/models/AiChatHistory.d.ts +1373 -0
- package/lib/types/generated/postgresql/models/AiMessage.d.ts +1496 -0
- package/lib/types/generated/postgresql/models.d.ts +3 -0
- package/lib/types/generated/sqlite/browser.d.ts +14 -0
- package/lib/types/generated/sqlite/client.d.ts +33 -0
- package/lib/types/generated/sqlite/commonInputTypes.d.ts +373 -0
- package/lib/types/generated/sqlite/enums.d.ts +1 -0
- package/lib/types/generated/sqlite/internal/class.d.ts +143 -0
- package/lib/types/generated/sqlite/internal/prismaNamespace.d.ts +705 -0
- package/lib/types/generated/sqlite/internal/prismaNamespaceBrowser.d.ts +92 -0
- package/lib/types/generated/sqlite/models/AiChatHistory.d.ts +1371 -0
- package/lib/types/generated/sqlite/models/AiMessage.d.ts +1493 -0
- package/lib/types/generated/sqlite/models.d.ts +3 -0
- package/lib/types/generated/sqlserver/browser.d.ts +14 -0
- package/lib/types/generated/sqlserver/client.d.ts +33 -0
- package/lib/types/generated/sqlserver/commonInputTypes.d.ts +333 -0
- package/lib/types/generated/sqlserver/enums.d.ts +1 -0
- package/lib/types/generated/sqlserver/internal/class.d.ts +143 -0
- package/lib/types/generated/sqlserver/internal/prismaNamespace.d.ts +669 -0
- package/lib/types/generated/sqlserver/internal/prismaNamespaceBrowser.d.ts +80 -0
- package/lib/types/generated/sqlserver/models/AiChatHistory.d.ts +1257 -0
- package/lib/types/generated/sqlserver/models/AiMessage.d.ts +1375 -0
- package/lib/types/generated/sqlserver/models.d.ts +3 -0
- package/lib/types/health.d.ts +52 -0
- package/lib/types/index.d.ts +91 -0
- package/lib/types/memory-tools.d.ts +168 -0
- package/lib/types/notify.d.ts +35 -0
- package/lib/types/projector.d.ts +51 -0
- package/lib/types/status-route.d.ts +48 -0
- package/lib/types/types.d.ts +58 -0
- package/lib/types.js +8 -0
- package/package.json +127 -4
- package/preview.png +0 -0
- package/prisma/schema.mysql.prisma +78 -0
- package/prisma/schema.postgresql.prisma +70 -0
- package/prisma/schema.sqlite.prisma +70 -0
- package/prisma/schema.sqlserver.prisma +76 -0
package/lib/index.js
ADDED
|
@@ -0,0 +1,681 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-storage — mirrors the dsh session event stream into the ai_messages /
|
|
3
|
+
* ai_chat_histories tables (MySQL/PostgreSQL/SQLite/SQL Server).
|
|
4
|
+
*
|
|
5
|
+
* Design: dsh's own session persistence (JSONL/SQLite) stays authoritative.
|
|
6
|
+
* This plugin taps `session/event` (the same seam the built-in persistence
|
|
7
|
+
* coordinator uses) and projects events into MessageRow / SessionRow shaped
|
|
8
|
+
* after the source project's ai_messages / ai_chat_histories tables.
|
|
9
|
+
*
|
|
10
|
+
* A2A task state (task metadata in Redis, workspace archives in GCS) is NOT
|
|
11
|
+
* here — that belongs to the A2A layer, see dsh-a2a's TaskStore backends.
|
|
12
|
+
*
|
|
13
|
+
* Resume caveat (docs/subsystems/persistence.md): constructor seeds do not
|
|
14
|
+
* emit events. On agent resume, only events >= `session.firstLiveSeq` fire
|
|
15
|
+
* on this stream — historical rows must come from the database itself, not
|
|
16
|
+
* from replaying the tap.
|
|
17
|
+
*
|
|
18
|
+
* @module dsh-storage
|
|
19
|
+
*/
|
|
20
|
+
import Schema from '@deepseek-ai/schemastery';
|
|
21
|
+
import { existsSync } from 'node:fs';
|
|
22
|
+
import { join } from 'node:path';
|
|
23
|
+
import { DatabaseBackend } from './backends/database.js';
|
|
24
|
+
import { backfillAll, LAST_USAGE_KEY } from './backfill.js';
|
|
25
|
+
import { compressionSnapshot } from './compaction.js';
|
|
26
|
+
import { configToUrl, defaultConfigPath, dshHome, loadConfig, defaultDevice, saveConfig as saveConfigFile, tzOffsetMinutes, watchConfig, } from './config.js';
|
|
27
|
+
import { createTables, ensureDeviceColumn } from './ddl.js';
|
|
28
|
+
import { ConfigError, HealthMonitor, probeDatabase, testConnection, } from './health.js';
|
|
29
|
+
import { drainToasts, wireNotifications } from './notify.js';
|
|
30
|
+
import { registerMemoryTools } from './memory-tools.js';
|
|
31
|
+
import { mergeToolResult, projectEvent, toolResultCallId, usageSampleOf } from './projector.js';
|
|
32
|
+
import { registerStatusRoute } from './status-route.js';
|
|
33
|
+
export const name = 'dsh-storage';
|
|
34
|
+
// Event-tap only: no hard service dependency. Add 'sessionPersistence' to
|
|
35
|
+
// `inject` if you want fail-fast when persistence is not composed.
|
|
36
|
+
export const inject = [];
|
|
37
|
+
export const Config = Schema.object({
|
|
38
|
+
enabled: Schema.boolean().default(false),
|
|
39
|
+
database: Schema.object({
|
|
40
|
+
enabled: Schema.boolean().default(false),
|
|
41
|
+
provider: Schema.union(['mysql', 'postgresql', 'sqlite', 'sqlserver']).default('sqlite'),
|
|
42
|
+
url: Schema.string().role('secret').default(''),
|
|
43
|
+
timezone: Schema.string().default('+08:00'),
|
|
44
|
+
}),
|
|
45
|
+
});
|
|
46
|
+
/** Metadata key for our usage bookkeeping — namespaced so other writers' fields never collide with it. */
|
|
47
|
+
const TOKEN_BASELINE_KEY = 'dsh-storage:token-baseline';
|
|
48
|
+
const ARCHIVE_IN_PROGRESS = 'dsh-storage:archive-in-progress';
|
|
49
|
+
/**
|
|
50
|
+
* The patched @prisma/adapter-mariadb (pnpm patch) reads this global when
|
|
51
|
+
* serializing Date args: the stock adapter hard-codes UTC on writes and
|
|
52
|
+
* ignores its `timezone` option, so we hand it the config's offset instead.
|
|
53
|
+
*/
|
|
54
|
+
const TZ_OFFSET_GLOBAL_KEY = '__DSH_STORAGE_TZ_OFFSET_MIN__';
|
|
55
|
+
let tzWarningLogged = false;
|
|
56
|
+
/** Publish the config timezone offset for the patched mariadb adapter. */
|
|
57
|
+
function publishTzOffset(tz) {
|
|
58
|
+
const minutes = tzOffsetMinutes(tz ?? '');
|
|
59
|
+
globalThis[TZ_OFFSET_GLOBAL_KEY] = minutes;
|
|
60
|
+
const wellFormed = tz !== undefined && (tz === 'UTC' || tz === 'Z' || /^[+-]\d{2}:\d{2}$/.test(tz));
|
|
61
|
+
if (!wellFormed && !tzWarningLogged) {
|
|
62
|
+
tzWarningLogged = true;
|
|
63
|
+
console.error(`[dsh-storage] invalid timezone ${JSON.stringify(tz ?? '')} — assuming UTC offset 0`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/** Shape-guarded restore of our usage sample from arbitrary metadata. */
|
|
67
|
+
function usageSampleOfMetadata(value) {
|
|
68
|
+
const v = value;
|
|
69
|
+
return v &&
|
|
70
|
+
typeof v.key === 'string' &&
|
|
71
|
+
typeof v.input === 'number' &&
|
|
72
|
+
typeof v.output === 'number'
|
|
73
|
+
? { key: v.key, input: v.input, output: v.output }
|
|
74
|
+
: undefined;
|
|
75
|
+
}
|
|
76
|
+
export function apply(ctx, config) {
|
|
77
|
+
// Lifecycle: this cordis fork has no 'ready'/'dispose' events — startup
|
|
78
|
+
// work goes in ctx.effect() (runs immediately at plugin load; the returned
|
|
79
|
+
// disposer runs on fiber unload at shutdown). Event taps verified against
|
|
80
|
+
// @deepseek-ai/dsh-session@0.1.6-alpha.2: session/event(session, event),
|
|
81
|
+
// session/disposed(session), and the awaited session/flush(session)
|
|
82
|
+
// durability checkpoint. The type-only import above registers the event
|
|
83
|
+
// names; handler params stay `any` because storage also mirrors
|
|
84
|
+
// plugin-owned event types (e.g. session/title) it does not depend on.
|
|
85
|
+
// Health monitor: probes the sync config's database (connect + SELECT 1 +
|
|
86
|
+
// ai_messages existence) with exponential-backoff retries. The probe
|
|
87
|
+
// re-reads $DSH_HOME/dsh-storage.json on every run, so config edits and
|
|
88
|
+
// corruption are picked up without a restart. Provided before the enabled
|
|
89
|
+
// guard so the panel can always query status (state stays 'disabled'
|
|
90
|
+
// while sync is off). enabled:false is handled by stopping the monitor
|
|
91
|
+
// (true 'disabled' state) — only a broken config surfaces as
|
|
92
|
+
// 'config-error'.
|
|
93
|
+
const monitor = new HealthMonitor(async () => {
|
|
94
|
+
const { config: syncConfig, error } = loadConfig();
|
|
95
|
+
if (error)
|
|
96
|
+
throw new ConfigError(`sync config invalid (${error})`);
|
|
97
|
+
await probeDatabase(syncConfig);
|
|
98
|
+
});
|
|
99
|
+
// Toast notifications (Task 6): good→bad / bad→good transitions are
|
|
100
|
+
// recorded host-side and drained by the browser toast portal through the
|
|
101
|
+
// status bridge. Mounted for the plugin's whole lifetime (like the monitor).
|
|
102
|
+
ctx.effect(() => wireNotifications(monitor));
|
|
103
|
+
ctx.effect(() => {
|
|
104
|
+
// The monitor lifecycle follows the sync config alone — always evaluated
|
|
105
|
+
// at plugin load (regardless of enabled flags) and re-evaluated on every
|
|
106
|
+
// config edit via T3's watchConfig (300ms debounce). Without this,
|
|
107
|
+
// starting disabled would leave the status stuck at 'disabled' forever
|
|
108
|
+
// after the user enables sync: enabled:false → monitor stopped with the
|
|
109
|
+
// true 'disabled' state; enabled:true (or a broken config, surfaced as
|
|
110
|
+
// config-error) → probe/recheck.
|
|
111
|
+
let monitorRunning = false;
|
|
112
|
+
const evaluate = () => {
|
|
113
|
+
const { config: syncConfig, error } = loadConfig();
|
|
114
|
+
if (!error && !syncConfig.enabled) {
|
|
115
|
+
monitor.stop();
|
|
116
|
+
monitorRunning = false;
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
if (monitorRunning)
|
|
120
|
+
void monitor.recheck();
|
|
121
|
+
else {
|
|
122
|
+
monitorRunning = true;
|
|
123
|
+
monitor.start();
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
evaluate();
|
|
127
|
+
const unwatch = watchConfig(defaultConfigPath(), evaluate);
|
|
128
|
+
return () => {
|
|
129
|
+
unwatch();
|
|
130
|
+
monitor.stop();
|
|
131
|
+
monitorRunning = false;
|
|
132
|
+
};
|
|
133
|
+
});
|
|
134
|
+
const statusService = {
|
|
135
|
+
snapshot() {
|
|
136
|
+
const current = monitor.snapshot();
|
|
137
|
+
// Minimal config-error re-arm: the next snapshot call re-reads the
|
|
138
|
+
// config and recovers once it parses and connects again.
|
|
139
|
+
if (current.state === 'config-error')
|
|
140
|
+
void monitor.recheck();
|
|
141
|
+
return current;
|
|
142
|
+
},
|
|
143
|
+
testConnection(cfg) {
|
|
144
|
+
return testConnection(cfg);
|
|
145
|
+
},
|
|
146
|
+
createTables,
|
|
147
|
+
getConfig: () => loadConfig(),
|
|
148
|
+
saveConfig(cfg) {
|
|
149
|
+
saveConfigFile(cfg);
|
|
150
|
+
return { config: cfg };
|
|
151
|
+
},
|
|
152
|
+
toasts(payload) {
|
|
153
|
+
return drainToasts(payload?.sinceId ?? 0);
|
|
154
|
+
},
|
|
155
|
+
async backfill() {
|
|
156
|
+
const { config: cfg, error } = loadConfig();
|
|
157
|
+
if (error)
|
|
158
|
+
throw new Error(`sync config invalid (${error})`);
|
|
159
|
+
// A one-shot backend from the current config — backfill works even
|
|
160
|
+
// while the live mirror is disabled or its connection was rebuilt.
|
|
161
|
+
const backend = new DatabaseBackend({
|
|
162
|
+
provider: cfg.provider,
|
|
163
|
+
url: configToUrl(cfg),
|
|
164
|
+
timezone: cfg.timezone,
|
|
165
|
+
device: cfg.device,
|
|
166
|
+
});
|
|
167
|
+
await backend.init();
|
|
168
|
+
try {
|
|
169
|
+
return await backfillAll(join(dshHome(), 'sessions'), backend);
|
|
170
|
+
}
|
|
171
|
+
finally {
|
|
172
|
+
await backend.close?.().catch(() => { });
|
|
173
|
+
}
|
|
174
|
+
},
|
|
175
|
+
};
|
|
176
|
+
ctx.provide('dshStorageStatus', statusService);
|
|
177
|
+
// Phase 2: AI memory tools (memory_search / memory_read) — host-side only,
|
|
178
|
+
// registered on the dsh-tools service; a disabled/unconfigured sync config
|
|
179
|
+
// surfaces as a friendly error at call time, never at registration.
|
|
180
|
+
registerMemoryTools(ctx);
|
|
181
|
+
// Browser bridge (Task 5): the settings panel client calls this over the
|
|
182
|
+
// connection generic-RPC channel. Optional services — profiles without a
|
|
183
|
+
// web carrier simply skip the route.
|
|
184
|
+
ctx.effect(() => {
|
|
185
|
+
const dispose = registerStatusRoute(ctx, statusService);
|
|
186
|
+
return () => dispose?.();
|
|
187
|
+
});
|
|
188
|
+
// C1: the mirror derives its effective config from the same source the
|
|
189
|
+
// panel/health/backfill use — $DSH_HOME/dsh-storage.json. The JSON file
|
|
190
|
+
// wins whenever it exists (enabled:false in JSON means OFF, even when a
|
|
191
|
+
// legacy yaml block is enabled); the cordis yaml Config is only a compat
|
|
192
|
+
// fallback for pre-panel installs whose JSON file does not exist yet —
|
|
193
|
+
// exactly the state the switch runbook leaves users in after deleting the
|
|
194
|
+
// hand-written yaml block. Config edits rebuild the mirror without a
|
|
195
|
+
// restart: the watcher disposes the old backends + event tap and
|
|
196
|
+
// reconstructs them from the new file.
|
|
197
|
+
let mirror;
|
|
198
|
+
let activeTarget;
|
|
199
|
+
let desiredTarget;
|
|
200
|
+
let switching = false;
|
|
201
|
+
const settleMirror = () => {
|
|
202
|
+
if (switching || sameMirrorTarget(activeTarget, desiredTarget))
|
|
203
|
+
return;
|
|
204
|
+
if (mirror) {
|
|
205
|
+
// Tear the old mirror down first (drain pending writes, close
|
|
206
|
+
// backends); settleMirror re-runs and wires the next target once the
|
|
207
|
+
// teardown has landed.
|
|
208
|
+
switching = true;
|
|
209
|
+
const old = mirror;
|
|
210
|
+
mirror = undefined;
|
|
211
|
+
activeTarget = undefined;
|
|
212
|
+
void old
|
|
213
|
+
.dispose()
|
|
214
|
+
.catch(() => { })
|
|
215
|
+
.then(() => {
|
|
216
|
+
switching = false;
|
|
217
|
+
settleMirror();
|
|
218
|
+
});
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
if (desiredTarget) {
|
|
222
|
+
mirror = startMirror(ctx, desiredTarget);
|
|
223
|
+
activeTarget = desiredTarget;
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
// The archive service outlives mirror rebuilds — it delegates to whichever
|
|
227
|
+
// mirror instance is currently wired.
|
|
228
|
+
ctx.provide('dshStorage', {
|
|
229
|
+
archiveSession(sessionId) {
|
|
230
|
+
return mirror?.archiveSession(sessionId) ?? Promise.resolve(null);
|
|
231
|
+
},
|
|
232
|
+
});
|
|
233
|
+
const evaluateMirror = () => {
|
|
234
|
+
desiredTarget = computeMirrorTarget(config);
|
|
235
|
+
// Keep the adapter's timezone global in sync with the effective config —
|
|
236
|
+
// set at load and re-set whenever the watcher fires with a changed
|
|
237
|
+
// timezone (MirrorTarget equality already includes timezone).
|
|
238
|
+
publishTzOffset(desiredTarget?.timezone ?? config.database?.timezone);
|
|
239
|
+
settleMirror();
|
|
240
|
+
};
|
|
241
|
+
evaluateMirror();
|
|
242
|
+
ctx.effect(() => {
|
|
243
|
+
const unwatch = watchConfig(defaultConfigPath(), evaluateMirror);
|
|
244
|
+
return () => unwatch();
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
function sameMirrorTarget(a, b) {
|
|
248
|
+
return (a === b ||
|
|
249
|
+
(a?.provider === b?.provider &&
|
|
250
|
+
a?.url === b?.url &&
|
|
251
|
+
a?.timezone === b?.timezone &&
|
|
252
|
+
a?.device === b?.device));
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* The mirror's effective target. When ~/.dsh/dsh-storage.json exists it wins
|
|
256
|
+
* outright — a panel-configured install must mirror from the panel's config,
|
|
257
|
+
* and enabled:false there must actually stop mirroring; the yaml Config only
|
|
258
|
+
* drives the mirror when no JSON file exists (legacy pre-panel installs).
|
|
259
|
+
* A present-but-broken JSON file disables the mirror (the health monitor
|
|
260
|
+
* surfaces the parse error as 'config-error').
|
|
261
|
+
*/
|
|
262
|
+
function computeMirrorTarget(config) {
|
|
263
|
+
const path = defaultConfigPath();
|
|
264
|
+
if (existsSync(path)) {
|
|
265
|
+
const { config: syncConfig, error } = loadConfig(path);
|
|
266
|
+
if (error || !syncConfig.enabled)
|
|
267
|
+
return undefined;
|
|
268
|
+
return {
|
|
269
|
+
provider: syncConfig.provider,
|
|
270
|
+
url: configToUrl(syncConfig),
|
|
271
|
+
timezone: syncConfig.timezone,
|
|
272
|
+
device: syncConfig.device,
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
if (!config.enabled || !config.database.enabled || !config.database.url)
|
|
276
|
+
return undefined;
|
|
277
|
+
return {
|
|
278
|
+
provider: config.database.provider,
|
|
279
|
+
url: config.database.url,
|
|
280
|
+
timezone: config.database.timezone,
|
|
281
|
+
device: defaultDevice(),
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Wire one mirror instance against `target`: backend construction, the
|
|
286
|
+
* session/event tap, and durability checkpoints. The projector internals are
|
|
287
|
+
* unchanged from upstream — this factory only owns their lifecycle so the
|
|
288
|
+
* config watcher can dispose and reconstruct everything on config changes.
|
|
289
|
+
*/
|
|
290
|
+
function startMirror(ctx, target) {
|
|
291
|
+
const backends = [
|
|
292
|
+
new DatabaseBackend({
|
|
293
|
+
provider: target.provider,
|
|
294
|
+
url: target.url,
|
|
295
|
+
timezone: target.timezone,
|
|
296
|
+
device: target.device,
|
|
297
|
+
}),
|
|
298
|
+
];
|
|
299
|
+
const sessions = new Map();
|
|
300
|
+
const toolCallMessages = new Map();
|
|
301
|
+
const TOOL_CALL_CACHE_LIMIT = 1000;
|
|
302
|
+
function rememberToolCalls(sessionId, row) {
|
|
303
|
+
if (row?.type !== 'model' || !Array.isArray(row.toolCalls))
|
|
304
|
+
return;
|
|
305
|
+
for (const call of row.toolCalls) {
|
|
306
|
+
if (typeof call?.id !== 'string')
|
|
307
|
+
continue;
|
|
308
|
+
const calls = toolCallMessages.get(sessionId) ?? new Map();
|
|
309
|
+
// Bound each session's cache; Map iteration evicts the oldest insertion first.
|
|
310
|
+
if (!calls.has(call.id) && calls.size >= TOOL_CALL_CACHE_LIMIT) {
|
|
311
|
+
const oldest = calls.keys().next().value;
|
|
312
|
+
if (oldest !== undefined)
|
|
313
|
+
calls.delete(oldest);
|
|
314
|
+
}
|
|
315
|
+
calls.set(call.id, row.id);
|
|
316
|
+
toolCallMessages.set(sessionId, calls);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
async function readToolResultRows(session, sessionId, event) {
|
|
320
|
+
const callId = toolResultCallId(event);
|
|
321
|
+
if (!callId)
|
|
322
|
+
return [];
|
|
323
|
+
let messageId = toolCallMessages.get(sessionId)?.get(callId);
|
|
324
|
+
if (!messageId && typeof session?.deriveMessages === 'function') {
|
|
325
|
+
const messages = session.deriveMessages();
|
|
326
|
+
for (let index = messages.length - 1; index >= 0; index--) {
|
|
327
|
+
const message = messages[index];
|
|
328
|
+
if (message.role === 'assistant' &&
|
|
329
|
+
typeof message.id === 'string' &&
|
|
330
|
+
Array.isArray(message.content) &&
|
|
331
|
+
message.content.some((block) => block?.type === 'tool-call' && block.id === callId)) {
|
|
332
|
+
messageId = message.id;
|
|
333
|
+
break;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
if (messageId) {
|
|
338
|
+
const rows = await backends[0].readMessages(sessionId, messageId);
|
|
339
|
+
if (rows.some((row) => Array.isArray(row.toolCalls) && row.toolCalls.some((call) => call?.id === callId)))
|
|
340
|
+
return rows;
|
|
341
|
+
}
|
|
342
|
+
// Cold resume and eviction fall back to bounded database candidate pages.
|
|
343
|
+
const rows = await backends[0].readToolCallMessages(sessionId, callId);
|
|
344
|
+
for (const row of rows)
|
|
345
|
+
rememberToolCalls(sessionId, row);
|
|
346
|
+
return rows;
|
|
347
|
+
}
|
|
348
|
+
// Row writes are fire-and-forget during the run, but tracked so the
|
|
349
|
+
// session/flush checkpoint and shutdown can drain them — a one-shot
|
|
350
|
+
// (headless) run would otherwise lose the tail events to process exit.
|
|
351
|
+
const pending = new Set();
|
|
352
|
+
function track(task) {
|
|
353
|
+
pending.add(task);
|
|
354
|
+
void task.then(() => pending.delete(task), () => pending.delete(task));
|
|
355
|
+
return task;
|
|
356
|
+
}
|
|
357
|
+
function guard(call) {
|
|
358
|
+
// Mirroring must never break the agent loop: log and swallow.
|
|
359
|
+
return Promise.all(backends.map((backend) => call(backend).catch((error) => console.error(`[dsh-storage] ${backend.name} error:`, error)))).then(() => { });
|
|
360
|
+
}
|
|
361
|
+
// All sessions share init and its retry delay; queued events wait rather than skip the barrier.
|
|
362
|
+
let started;
|
|
363
|
+
let retryAt = 0;
|
|
364
|
+
/** Device-column migration runs once per mirror instance, after init. */
|
|
365
|
+
let migrated = false;
|
|
366
|
+
function ensureStarted() {
|
|
367
|
+
if (!started) {
|
|
368
|
+
const attempt = track((async () => {
|
|
369
|
+
const delay = retryAt - Date.now();
|
|
370
|
+
if (delay > 0)
|
|
371
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
372
|
+
await Promise.all(backends.map((backend) => backend.init?.()));
|
|
373
|
+
if (!migrated) {
|
|
374
|
+
migrated = true;
|
|
375
|
+
// Cheap information_schema check + ALTER/UPDATE only when the
|
|
376
|
+
// device column is missing (multi-device shared database).
|
|
377
|
+
const result = await ensureDeviceColumn(target, target.device);
|
|
378
|
+
if (result !== 'ok')
|
|
379
|
+
console.error(`[dsh-storage] device column migration: ${result}`);
|
|
380
|
+
}
|
|
381
|
+
})());
|
|
382
|
+
started = attempt;
|
|
383
|
+
void attempt.catch(() => {
|
|
384
|
+
if (started === attempt) {
|
|
385
|
+
retryAt = Date.now() + 1000;
|
|
386
|
+
started = undefined;
|
|
387
|
+
}
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
return started;
|
|
391
|
+
}
|
|
392
|
+
function fanout(call) {
|
|
393
|
+
// Event failures reject into the per-session queue, which is the no-throw
|
|
394
|
+
// boundary. Swallowing here would make a partial write look committed.
|
|
395
|
+
return track(ensureStarted().then(() => Promise.all(backends.map(call)).then(() => { })));
|
|
396
|
+
}
|
|
397
|
+
// Per-session serialization: all event processing for one session runs
|
|
398
|
+
// through a promise chain, so rollup writes complete in event order
|
|
399
|
+
// (latest-wins — concurrent absolute snapshots must not land out of order)
|
|
400
|
+
// and the first event of an unknown session seeds its accumulator from the
|
|
401
|
+
// stored row before any write (resume: constructor seeds are not re-emitted,
|
|
402
|
+
// so history must come from the database, not from zero).
|
|
403
|
+
const chains = new Map();
|
|
404
|
+
const toolBatches = new Map();
|
|
405
|
+
function enqueue(sessionId, task) {
|
|
406
|
+
// Backend work checks init inside mirrorEvent; cleanup needs only queue ordering.
|
|
407
|
+
const next = (chains.get(sessionId) ?? Promise.resolve())
|
|
408
|
+
.then(task)
|
|
409
|
+
.catch((error) => console.error('[dsh-storage] mirror task error:', error));
|
|
410
|
+
chains.set(sessionId, next);
|
|
411
|
+
track(next);
|
|
412
|
+
}
|
|
413
|
+
async function seedAccum(sessionId) {
|
|
414
|
+
// "Row not found" (a new session) is the only zero-seed case. A read
|
|
415
|
+
// failure is NOT "not found": fail the task so the next event re-seeds,
|
|
416
|
+
// instead of accumulating from zero and overwriting the stored rollup.
|
|
417
|
+
let readFailure;
|
|
418
|
+
for (const backend of backends) {
|
|
419
|
+
try {
|
|
420
|
+
const row = await backend.readSession?.(sessionId);
|
|
421
|
+
if (row) {
|
|
422
|
+
return {
|
|
423
|
+
messageCount: row.messageCount,
|
|
424
|
+
totalTokens: row.totalTokens,
|
|
425
|
+
title: row.title ?? undefined,
|
|
426
|
+
firstMessageAt: row.firstMessageAt ?? undefined,
|
|
427
|
+
lastMessageAt: row.lastMessageAt ?? undefined,
|
|
428
|
+
// Preserve every existing metadata field; we only own our key.
|
|
429
|
+
// The baseline distinguishes cumulative active totals from a
|
|
430
|
+
// restart's fresh-segment deltas during archive recovery.
|
|
431
|
+
metadata: { [TOKEN_BASELINE_KEY]: row.totalTokens, ...(row.metadata ?? {}) },
|
|
432
|
+
usageSample: usageSampleOfMetadata(row.metadata?.[LAST_USAGE_KEY]),
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
catch (error) {
|
|
437
|
+
readFailure ??= error;
|
|
438
|
+
console.error(`[dsh-storage] ${backend.name} error:`, error);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
if (readFailure)
|
|
442
|
+
throw readFailure;
|
|
443
|
+
return { messageCount: 0, totalTokens: 0, metadata: { [TOKEN_BASELINE_KEY]: 0 } };
|
|
444
|
+
}
|
|
445
|
+
void ensureStarted().catch((error) => console.error('[dsh-storage] database error:', error));
|
|
446
|
+
function archiveSession(sessionId) {
|
|
447
|
+
toolBatches.delete(sessionId);
|
|
448
|
+
// Join the same per-session chain as event writes, then discard the old
|
|
449
|
+
// in-memory rollup only after the database transaction commits.
|
|
450
|
+
const archived = (chains.get(sessionId) ?? Promise.resolve())
|
|
451
|
+
.then(() => ensureStarted())
|
|
452
|
+
.then(async () => {
|
|
453
|
+
const accum = sessions.get(sessionId);
|
|
454
|
+
if (accum && !accum.metadata[ARCHIVE_IN_PROGRESS]) {
|
|
455
|
+
// Freeze inside the session queue: all earlier events have now
|
|
456
|
+
// committed, and this first baseline survives every retry until
|
|
457
|
+
// the archive actually succeeds.
|
|
458
|
+
accum.metadata[TOKEN_BASELINE_KEY] = accum.totalTokens;
|
|
459
|
+
accum.metadata[ARCHIVE_IN_PROGRESS] = true;
|
|
460
|
+
}
|
|
461
|
+
const id = await backends[0].archiveSession(sessionId);
|
|
462
|
+
sessions.delete(sessionId);
|
|
463
|
+
toolCallMessages.delete(sessionId);
|
|
464
|
+
return id;
|
|
465
|
+
});
|
|
466
|
+
const settled = archived.then(() => { }, () => { });
|
|
467
|
+
chains.set(sessionId, settled);
|
|
468
|
+
track(settled);
|
|
469
|
+
return archived;
|
|
470
|
+
}
|
|
471
|
+
async function mirrorEvent(session, event, snapshot, batch) {
|
|
472
|
+
await ensureStarted();
|
|
473
|
+
const sessionId = session?.id ?? 'unknown';
|
|
474
|
+
if (snapshot) {
|
|
475
|
+
await fanout((backend) => backend.upsertMessage(snapshot));
|
|
476
|
+
return;
|
|
477
|
+
}
|
|
478
|
+
const stored = sessions.get(sessionId) ?? (await seedAccum(sessionId));
|
|
479
|
+
sessions.set(sessionId, stored);
|
|
480
|
+
// Stage every change on a copy. The live accumulator advances only
|
|
481
|
+
// after the message + rollup transaction commits.
|
|
482
|
+
const accum = { ...stored, metadata: { ...stored.metadata } };
|
|
483
|
+
let row = projectEvent(session, event, sessionId);
|
|
484
|
+
if (row && session.header?.createdAt !== undefined) {
|
|
485
|
+
row.metadata = { ...row.metadata, sessionCreatedAt: session.header.createdAt };
|
|
486
|
+
}
|
|
487
|
+
const storedRows = event?.type === 'tool/result'
|
|
488
|
+
? await readToolResultRows(session, sessionId, event)
|
|
489
|
+
: row
|
|
490
|
+
? await backends[0].readMessages(sessionId, row.id)
|
|
491
|
+
: [];
|
|
492
|
+
let processedThrough = batch?.index;
|
|
493
|
+
if (event?.type === 'tool/result') {
|
|
494
|
+
row = mergeToolResult(event, storedRows);
|
|
495
|
+
while (row &&
|
|
496
|
+
batch &&
|
|
497
|
+
processedThrough !== undefined &&
|
|
498
|
+
processedThrough + 1 < batch.events.length) {
|
|
499
|
+
const following = batch.events[processedThrough + 1];
|
|
500
|
+
const callId = toolResultCallId(following);
|
|
501
|
+
if (!callId ||
|
|
502
|
+
!Array.isArray(row.toolCalls) ||
|
|
503
|
+
!row.toolCalls.some((call) => call?.id === callId))
|
|
504
|
+
break;
|
|
505
|
+
row = mergeToolResult(following, [row]);
|
|
506
|
+
processedThrough++;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
const existing = row && storedRows.find((stored) => stored.id === row?.id);
|
|
510
|
+
if (event?.type === 'assistant/message' && row && existing && Array.isArray(row.toolCalls)) {
|
|
511
|
+
row.toolCalls = row.toolCalls.map((call) => {
|
|
512
|
+
const previous = Array.isArray(existing.toolCalls)
|
|
513
|
+
? existing.toolCalls.find((stored) => stored?.id === call?.id)
|
|
514
|
+
: undefined;
|
|
515
|
+
return previous && 'result' in previous
|
|
516
|
+
? {
|
|
517
|
+
...call,
|
|
518
|
+
result: previous.result,
|
|
519
|
+
...(previous.error ? { error: previous.error } : {}),
|
|
520
|
+
}
|
|
521
|
+
: call;
|
|
522
|
+
});
|
|
523
|
+
}
|
|
524
|
+
if (!row && event?.type === 'tool/result') {
|
|
525
|
+
console.error('[dsh-storage] no matching model tool call for result');
|
|
526
|
+
}
|
|
527
|
+
if (row && !existing && event?.type !== 'tool/result') {
|
|
528
|
+
accum.messageCount += 1;
|
|
529
|
+
accum.firstMessageAt ??= row.createdAt;
|
|
530
|
+
accum.lastMessageAt = row.createdAt;
|
|
531
|
+
}
|
|
532
|
+
// Every settled attempt/message is one model call; retries on the same
|
|
533
|
+
// step are additive rather than replacement snapshots.
|
|
534
|
+
const sampled = usageSampleOf(event);
|
|
535
|
+
let usageDelta = 0;
|
|
536
|
+
if (sampled) {
|
|
537
|
+
const sequence = Number(sampled.key);
|
|
538
|
+
const previous = Number(accum.usageSample?.key);
|
|
539
|
+
const duplicate = Number.isSafeInteger(sequence) && Number.isSafeInteger(previous) && sequence <= previous;
|
|
540
|
+
if (!duplicate) {
|
|
541
|
+
usageDelta = sampled.sample.input + sampled.sample.output;
|
|
542
|
+
accum.usageSample = { key: sampled.key, ...sampled.sample };
|
|
543
|
+
accum.totalTokens += usageDelta;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
// Latest-wins title snapshot (log-only `session/title` event; payload
|
|
547
|
+
// { title, messageSeqs, source } verified against dsh-session-title@0.1.6-alpha.2).
|
|
548
|
+
if (event?.type === 'session/title' && typeof event.data?.title === 'string') {
|
|
549
|
+
accum.title = event.data.title;
|
|
550
|
+
}
|
|
551
|
+
if (row || usageDelta !== 0 || event?.type === 'turn/end' || event?.type === 'session/title') {
|
|
552
|
+
// Merge our bookkeeping into the existing metadata instead of
|
|
553
|
+
// replacing the column — other fields (present or future) survive.
|
|
554
|
+
const metadata = {
|
|
555
|
+
...accum.metadata,
|
|
556
|
+
...(accum.usageSample ? { [LAST_USAGE_KEY]: accum.usageSample } : {}),
|
|
557
|
+
};
|
|
558
|
+
const sessionRow = {
|
|
559
|
+
sessionId,
|
|
560
|
+
title: accum.title ?? null,
|
|
561
|
+
messageCount: accum.messageCount,
|
|
562
|
+
totalTokens: accum.totalTokens,
|
|
563
|
+
firstMessageAt: accum.firstMessageAt ?? null,
|
|
564
|
+
lastMessageAt: accum.lastMessageAt ?? null,
|
|
565
|
+
...(Object.keys(metadata).length > 0 ? { metadata } : {}),
|
|
566
|
+
};
|
|
567
|
+
await fanout((backend) => backend.upsertEvent(row, sessionRow));
|
|
568
|
+
}
|
|
569
|
+
// Advance only after commit so a failed merged write cannot skip later results.
|
|
570
|
+
if (batch && processedThrough !== undefined)
|
|
571
|
+
batch.index = processedThrough;
|
|
572
|
+
sessions.set(sessionId, accum);
|
|
573
|
+
rememberToolCalls(sessionId, row);
|
|
574
|
+
}
|
|
575
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
576
|
+
const offEvent = ctx.on('session/event', (session, event) => {
|
|
577
|
+
if (![
|
|
578
|
+
'user/message',
|
|
579
|
+
'assistant/message',
|
|
580
|
+
'assistant/attempt',
|
|
581
|
+
'tool/result',
|
|
582
|
+
'turn/end',
|
|
583
|
+
'session/title',
|
|
584
|
+
'compaction/end',
|
|
585
|
+
].includes(event?.type))
|
|
586
|
+
return;
|
|
587
|
+
let snapshot;
|
|
588
|
+
try {
|
|
589
|
+
// Capture the committed surface now; queued writes must not observe later steps.
|
|
590
|
+
snapshot = compressionSnapshot(session, event);
|
|
591
|
+
}
|
|
592
|
+
catch (error) {
|
|
593
|
+
console.error('[dsh-storage] compaction snapshot error:', error);
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
if (event.type === 'compaction/end' && !snapshot)
|
|
597
|
+
return;
|
|
598
|
+
const sessionId = session?.id ?? 'unknown';
|
|
599
|
+
if (event.type === 'tool/result') {
|
|
600
|
+
const pendingBatch = toolBatches.get(sessionId);
|
|
601
|
+
if (pendingBatch && pendingBatch.events.length < 32) {
|
|
602
|
+
pendingBatch.events.push(event);
|
|
603
|
+
return;
|
|
604
|
+
}
|
|
605
|
+
const batch = { events: [event], index: 0 };
|
|
606
|
+
toolBatches.set(sessionId, batch);
|
|
607
|
+
enqueue(sessionId, async () => {
|
|
608
|
+
if (toolBatches.get(sessionId) === batch)
|
|
609
|
+
toolBatches.delete(sessionId);
|
|
610
|
+
for (; batch.index < batch.events.length; batch.index++) {
|
|
611
|
+
try {
|
|
612
|
+
await mirrorEvent(session, batch.events[batch.index], null, batch);
|
|
613
|
+
}
|
|
614
|
+
catch (error) {
|
|
615
|
+
console.error('[dsh-storage] mirror task error:', error);
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
});
|
|
619
|
+
}
|
|
620
|
+
else {
|
|
621
|
+
toolBatches.delete(sessionId);
|
|
622
|
+
enqueue(sessionId, () => mirrorEvent(session, event, snapshot));
|
|
623
|
+
}
|
|
624
|
+
});
|
|
625
|
+
const offDisposed = ctx.on('session/disposed', (session) => {
|
|
626
|
+
const sessionId = session?.id ?? 'unknown';
|
|
627
|
+
toolBatches.delete(sessionId);
|
|
628
|
+
// The delete must run inside the session's chain — a synchronous delete
|
|
629
|
+
// here races the queued event tasks (the map may not even have the entry
|
|
630
|
+
// yet at emit time, and queued tasks would keep growing the accumulator
|
|
631
|
+
// the dispose was meant to drop).
|
|
632
|
+
enqueue(sessionId, async () => {
|
|
633
|
+
sessions.delete(sessionId);
|
|
634
|
+
toolCallMessages.delete(sessionId);
|
|
635
|
+
});
|
|
636
|
+
// Delete the chain entry only once it has settled AND is still the same
|
|
637
|
+
// entry — deleting earlier would let a recreated session race the
|
|
638
|
+
// orphaned writes; skipping the delete when a new task has since chained
|
|
639
|
+
// on preserves order.
|
|
640
|
+
const chain = chains.get(sessionId);
|
|
641
|
+
chain?.finally(() => {
|
|
642
|
+
if (chains.get(sessionId) === chain)
|
|
643
|
+
chains.delete(sessionId);
|
|
644
|
+
});
|
|
645
|
+
});
|
|
646
|
+
// Durability checkpoint: wait only for this session. Shutdown still drains
|
|
647
|
+
// the global pending set; an unrelated slow session must not block this one.
|
|
648
|
+
const offFlush = ctx.on('session/flush', async (session) => {
|
|
649
|
+
toolBatches.delete(session?.id ?? 'unknown');
|
|
650
|
+
const chain = chains.get(session?.id ?? 'unknown');
|
|
651
|
+
if (chain) {
|
|
652
|
+
await chain;
|
|
653
|
+
return;
|
|
654
|
+
}
|
|
655
|
+
try {
|
|
656
|
+
await ensureStarted();
|
|
657
|
+
}
|
|
658
|
+
catch (error) {
|
|
659
|
+
console.error('[dsh-storage] database error:', error);
|
|
660
|
+
}
|
|
661
|
+
});
|
|
662
|
+
// Teardown runs both on config-driven rebuilds (dispose()) and fiber unload
|
|
663
|
+
// (the effect below); the stopped flag makes it idempotent so a rebuilt-away
|
|
664
|
+
// mirror is never closed twice.
|
|
665
|
+
let stopped = false;
|
|
666
|
+
const stop = async () => {
|
|
667
|
+
if (stopped)
|
|
668
|
+
return;
|
|
669
|
+
stopped = true;
|
|
670
|
+
offEvent();
|
|
671
|
+
offDisposed();
|
|
672
|
+
offFlush();
|
|
673
|
+
toolBatches.clear();
|
|
674
|
+
await Promise.allSettled([...pending]);
|
|
675
|
+
sessions.clear();
|
|
676
|
+
toolCallMessages.clear();
|
|
677
|
+
await guard(async (backend) => backend.close?.());
|
|
678
|
+
};
|
|
679
|
+
ctx.effect(() => () => stop());
|
|
680
|
+
return { archiveSession, dispose: stop };
|
|
681
|
+
}
|