om-memory-system 5.5.0 → 5.7.0-next.218

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +1 -1
  2. package/dist/adapters/pi/extension.js +5 -0
  3. package/dist/cli/space-command.d.ts +6 -5
  4. package/dist/cli/space-command.js +154 -13
  5. package/dist/cli/space-transfer-command.d.ts +25 -0
  6. package/dist/cli/space-transfer-command.js +79 -0
  7. package/dist/core/memory-operations.d.ts +12 -2
  8. package/dist/core/memory-operations.js +14 -4
  9. package/dist/core/recall-summary.js +7 -5
  10. package/dist/core/retrieval.js +5 -2
  11. package/dist/core/space-tool.d.ts +9 -4
  12. package/dist/core/space-tool.js +35 -3
  13. package/dist/services/active-space-access.d.ts +6 -0
  14. package/dist/services/active-space-access.js +8 -0
  15. package/dist/services/chat/chat-runtime.js +1 -1
  16. package/dist/services/client.d.ts +6 -16
  17. package/dist/services/client.js +54 -62
  18. package/dist/services/context.d.ts +2 -0
  19. package/dist/services/context.js +12 -1
  20. package/dist/services/linked-search.d.ts +108 -0
  21. package/dist/services/linked-search.js +219 -0
  22. package/dist/services/space-routes.d.ts +2 -0
  23. package/dist/services/space-routes.js +4 -3
  24. package/dist/services/space-transfer-routes.d.ts +10 -0
  25. package/dist/services/space-transfer-routes.js +344 -0
  26. package/dist/services/spaces/active-space.js +14 -7
  27. package/dist/services/spaces/delete.d.ts +2 -0
  28. package/dist/services/spaces/delete.js +4 -0
  29. package/dist/services/spaces/export-space.d.ts +60 -0
  30. package/dist/services/spaces/export-space.js +367 -0
  31. package/dist/services/spaces/identifier-scrub.d.ts +26 -0
  32. package/dist/services/spaces/identifier-scrub.js +40 -0
  33. package/dist/services/spaces/import-space.d.ts +85 -0
  34. package/dist/services/spaces/import-space.js +349 -0
  35. package/dist/services/spaces/operations.d.ts +57 -5
  36. package/dist/services/spaces/operations.js +161 -5
  37. package/dist/services/spaces/read-store-cache.d.ts +13 -0
  38. package/dist/services/spaces/read-store-cache.js +34 -0
  39. package/dist/services/spaces/registry-store.d.ts +4 -0
  40. package/dist/services/spaces/registry-store.js +15 -1
  41. package/dist/services/spaces/registry.d.ts +32 -0
  42. package/dist/services/spaces/registry.js +81 -7
  43. package/dist/services/spaces/space-store.d.ts +50 -0
  44. package/dist/services/spaces/space-store.js +72 -0
  45. package/dist/services/spaces/transfer-service.d.ts +41 -0
  46. package/dist/services/spaces/transfer-service.js +408 -0
  47. package/dist/services/spaces/transfer-types.d.ts +163 -0
  48. package/dist/services/spaces/transfer-types.js +10 -0
  49. package/dist/services/spaces/transfer.d.ts +31 -0
  50. package/dist/services/spaces/transfer.js +276 -0
  51. package/dist/services/turso/connection-manager.d.ts +4 -2
  52. package/dist/services/turso/connection-manager.js +11 -5
  53. package/dist/services/turso/cross-process-write-lock.d.ts +1 -1
  54. package/dist/services/turso/cross-process-write-lock.js +5 -5
  55. package/dist/services/turso/operation-lock.d.ts +2 -2
  56. package/dist/services/turso/operation-lock.js +5 -5
  57. package/dist/services/turso/shard-manager.d.ts +8 -0
  58. package/dist/services/turso/shard-manager.js +23 -22
  59. package/dist/services/turso/vector-search.d.ts +6 -2
  60. package/dist/services/turso/vector-search.js +8 -4
  61. package/dist/services/user-profile/user-profile-manager.d.ts +5 -1
  62. package/dist/services/user-profile/user-profile-manager.js +13 -10
  63. package/dist/services/web-route-modules.d.ts +5 -1
  64. package/dist/services/web-route-modules.js +4 -0
  65. package/dist/services/web-server.js +1 -1
  66. package/dist/shared/api/space-export-schemas.d.ts +159 -0
  67. package/dist/shared/api/space-export-schemas.js +72 -0
  68. package/dist/v2/adapter.d.ts +15 -0
  69. package/dist/v2/adapter.js +5 -0
  70. package/dist/web/assets/index-2BNRLGqe.css +2 -0
  71. package/dist/web/assets/index-C6WnDE6h.js +170 -0
  72. package/dist/web/index.html +2 -2
  73. package/package.json +1 -1
  74. package/skills/omms-memory/SKILL.md +16 -14
  75. package/dist/web/assets/index-Dcvz3EXt.js +0 -170
  76. package/dist/web/assets/index-b8k5Tuei.css +0 -2
@@ -0,0 +1,276 @@
1
+ /**
2
+ * The space transfer engine (design D3). Pure over a `SourceReader` and a
3
+ * `TargetWriter`: no CONFIG import, no files, no host adapters. Steps:
4
+ * select, skip exact duplicates, preview, write per shard, check, remove.
5
+ */
6
+ import { tryExtractScopeFromContainerTag } from "../memory-scope.js";
7
+ import { formatTagsForEmbedding } from "../turso/vector-utils.js";
8
+ import { tagTextKey } from "./transfer-types.js";
9
+ export * from "./transfer-types.js";
10
+ /** Preview examples are for the screen only and never logged (spec). */
11
+ const EXAMPLE_LIMIT = 5;
12
+ const EXAMPLE_TEXT_LIMIT = 160;
13
+ const PERSONAL_GROUP_NAME = "Personal";
14
+ function describeError(error) {
15
+ return error instanceof Error ? error.message : String(error);
16
+ }
17
+ /** Both bounds are inclusive; the project filter matches the container tag exactly. */
18
+ function matchesFilter(row, filter) {
19
+ if (row.createdAt < filter.since || row.createdAt > filter.until)
20
+ return false;
21
+ if (filter.projectTag !== undefined && row.containerTag !== filter.projectTag)
22
+ return false;
23
+ return true;
24
+ }
25
+ /**
26
+ * Apply the filter to whatever the reader returned. Rows whose container tag
27
+ * cannot be parsed carry no scope, so they can be neither written to a scoped
28
+ * shard nor removed: the engine leaves them alone in both spaces.
29
+ */
30
+ function selectRows(rows, filter) {
31
+ return rows.filter((row) => matchesFilter(row, filter) && tryExtractScopeFromContainerTag(row.containerTag) !== null);
32
+ }
33
+ /** Exact duplicates: the same id, or the same container tag and text under another id. */
34
+ function skipDuplicates(rows, targetIds, targetTagTexts) {
35
+ const kept = [];
36
+ let duplicates = 0;
37
+ for (const row of rows) {
38
+ if (targetIds.has(row.id) || targetTagTexts.has(tagTextKey(row.containerTag, row.text))) {
39
+ duplicates += 1;
40
+ }
41
+ else {
42
+ kept.push(row);
43
+ }
44
+ }
45
+ return { kept, duplicates };
46
+ }
47
+ /** Vectors move only when the source provably used the target's model and dimensions. */
48
+ function canReuseVectors(source, target) {
49
+ return (source !== undefined &&
50
+ source.model !== undefined &&
51
+ source.model === target.model &&
52
+ source.dimensions === target.dimensions);
53
+ }
54
+ function groupByScope(rows) {
55
+ const groups = new Map();
56
+ for (const row of rows) {
57
+ const scopeRef = tryExtractScopeFromContainerTag(row.containerTag);
58
+ const key = `${scopeRef.scope}:${scopeRef.hash}`;
59
+ const group = groups.get(key) ?? { scope: scopeRef.scope, scopeHash: scopeRef.hash, rows: [] };
60
+ group.rows.push(row);
61
+ groups.set(key, group);
62
+ }
63
+ return [...groups.values()];
64
+ }
65
+ function splitTags(tags) {
66
+ if (!tags)
67
+ return [];
68
+ return tags
69
+ .split(",")
70
+ .map((tag) => tag.trim())
71
+ .filter(Boolean);
72
+ }
73
+ function requireEmbed(embed) {
74
+ if (!embed) {
75
+ throw new Error("the stored vectors cannot be reused and no embed callback was provided");
76
+ }
77
+ return embed;
78
+ }
79
+ /** Give one row a usable vector: reuse the stored one when allowed, embed otherwise. */
80
+ async function prepareWriteRow(row, reuseStoredVectors, embed) {
81
+ let vector = reuseStoredVectors ? row.vector : undefined;
82
+ let tagsVector = reuseStoredVectors ? row.tagsVector : undefined;
83
+ let embedded = false;
84
+ if (vector === undefined) {
85
+ vector = await requireEmbed(embed)(row.text);
86
+ embedded = true;
87
+ }
88
+ const tags = splitTags(row.tags);
89
+ if (tagsVector === undefined && tags.length > 0) {
90
+ tagsVector = await requireEmbed(embed)(formatTagsForEmbedding(tags));
91
+ embedded = true;
92
+ }
93
+ return {
94
+ row: {
95
+ id: row.id,
96
+ text: row.text,
97
+ containerTag: row.containerTag,
98
+ tags: row.tags,
99
+ type: row.type,
100
+ createdAt: row.createdAt,
101
+ updatedAt: row.updatedAt,
102
+ metadata: row.metadata,
103
+ isPinned: row.isPinned,
104
+ vector,
105
+ tagsVector,
106
+ },
107
+ embedded,
108
+ };
109
+ }
110
+ /**
111
+ * Write one scope group in batches that fit the shard's free room, one
112
+ * transaction per batch, and set the shard's vector count after each batch.
113
+ * Progress lands in `result` and `copiedRows` per batch, so a later failure
114
+ * still reports how far the transfer got; earlier batches stay because a
115
+ * re-run skips them as duplicates.
116
+ */
117
+ async function writeScopeGroup(target, group, maxVectorsPerShard, reuseStoredVectors, embed, result, copiedRows) {
118
+ let pending = group.rows;
119
+ while (pending.length > 0) {
120
+ const shard = await target.getWriteShard(group.scope, group.scopeHash);
121
+ const room = maxVectorsPerShard - shard.vectorCount;
122
+ if (room <= 0) {
123
+ throw new Error(`the target offered shard ${shard.key} with no room left`);
124
+ }
125
+ const batch = pending.slice(0, room);
126
+ const prepared = [];
127
+ for (const row of batch) {
128
+ const writeRow = await prepareWriteRow(row, reuseStoredVectors, embed);
129
+ prepared.push(writeRow.row);
130
+ result.embeddedAgain = result.embeddedAgain || writeRow.embedded;
131
+ }
132
+ await target.insertBatch(shard, prepared);
133
+ await target.setShardCount(shard.key, shard.vectorCount + prepared.length);
134
+ copiedRows.push(...batch);
135
+ result.copied += batch.length;
136
+ pending = pending.slice(batch.length);
137
+ }
138
+ }
139
+ /**
140
+ * Preview a transfer: count, project groups, at most five examples (text cut
141
+ * to 160 characters) and the duplicate count. Reads only; never writes.
142
+ */
143
+ export async function previewTransfer(args) {
144
+ const rows = selectRows(await args.source.readRows(args.filter), args.filter);
145
+ const [targetIds, targetTagTexts] = await Promise.all([
146
+ args.target.loadExistingIds(),
147
+ args.target.loadExistingTagTexts(),
148
+ ]);
149
+ const { kept, duplicates } = skipDuplicates(rows, targetIds, targetTagTexts);
150
+ const groups = new Map();
151
+ for (const row of kept) {
152
+ const scope = tryExtractScopeFromContainerTag(row.containerTag).scope;
153
+ const name = scope === "user" ? PERSONAL_GROUP_NAME : (row.projectName ?? row.containerTag);
154
+ const group = groups.get(row.containerTag) ?? { name, tag: row.containerTag, count: 0 };
155
+ group.count += 1;
156
+ groups.set(row.containerTag, group);
157
+ }
158
+ return {
159
+ matched: rows.length,
160
+ toCopy: kept.length,
161
+ duplicates,
162
+ groups: [...groups.values()],
163
+ examples: kept.slice(0, EXAMPLE_LIMIT).map((row) => ({
164
+ id: row.id,
165
+ text: row.text.slice(0, EXAMPLE_TEXT_LIMIT),
166
+ })),
167
+ };
168
+ }
169
+ /**
170
+ * Run a transfer: select and skip duplicates at apply time, write to the
171
+ * target, and for a move check every copied id before removing it from the
172
+ * source. Failures are reported in the result, never thrown: the counts say
173
+ * how far the transfer got.
174
+ */
175
+ export async function applyTransfer(args) {
176
+ const { source, target, filter, mode } = args;
177
+ const result = {
178
+ ok: false,
179
+ mode,
180
+ copied: 0,
181
+ skipped: 0,
182
+ removed: 0,
183
+ changedFromPreview: 0,
184
+ embeddedAgain: false,
185
+ missingIds: [],
186
+ };
187
+ // Steps 1 and 2: select at apply time, then drop exact duplicates.
188
+ let kept;
189
+ try {
190
+ const rows = selectRows(await source.readRows(filter), filter);
191
+ result.changedFromPreview =
192
+ args.previewMatched === undefined ? 0 : rows.length - args.previewMatched;
193
+ const [targetIds, targetTagTexts] = await Promise.all([
194
+ target.loadExistingIds(),
195
+ target.loadExistingTagTexts(),
196
+ ]);
197
+ const selection = skipDuplicates(rows, targetIds, targetTagTexts);
198
+ kept = selection.kept;
199
+ result.skipped = selection.duplicates;
200
+ }
201
+ catch (error) {
202
+ result.error = {
203
+ step: "select",
204
+ message: `reading the source or the target failed: ${describeError(error)}`,
205
+ };
206
+ return result;
207
+ }
208
+ // Step 4: write per scope group, in batches that fit the shard's free room.
209
+ const copiedRows = [];
210
+ try {
211
+ const [sourceEmbedding, targetEmbedding] = await Promise.all([
212
+ source.getEmbeddingInfo(),
213
+ target.getEmbeddingSettings(),
214
+ ]);
215
+ const reuseStoredVectors = canReuseVectors(sourceEmbedding, targetEmbedding);
216
+ for (const group of groupByScope(kept)) {
217
+ await writeScopeGroup(target, group, args.maxVectorsPerShard, reuseStoredVectors, args.embed, result, copiedRows);
218
+ }
219
+ }
220
+ catch (error) {
221
+ result.error = {
222
+ step: "write",
223
+ message: `writing to the target failed after ${result.copied} memories: ${describeError(error)}`,
224
+ };
225
+ return result;
226
+ }
227
+ if (mode === "copy") {
228
+ result.ok = true;
229
+ return result;
230
+ }
231
+ // Step 5: check that every copied id is in the target.
232
+ const copiedIds = copiedRows.map((row) => row.id);
233
+ try {
234
+ const found = await target.findExistingIds(copiedIds);
235
+ result.missingIds = copiedIds.filter((id) => !found.has(id));
236
+ }
237
+ catch (error) {
238
+ result.error = {
239
+ step: "check",
240
+ message: `reading the copied memories back failed: ${describeError(error)}`,
241
+ };
242
+ return result;
243
+ }
244
+ if (result.missingIds.length > 0) {
245
+ result.error = {
246
+ step: "check",
247
+ message: `${result.missingIds.length} copied memory id(s) missing from the target: ${result.missingIds.join(", ")}`,
248
+ };
249
+ return result;
250
+ }
251
+ // Step 6: remove only the checked ids, one transaction per source shard.
252
+ try {
253
+ const byShard = new Map();
254
+ for (const row of copiedRows) {
255
+ const shardRows = byShard.get(row.shardKey) ?? [];
256
+ shardRows.push(row);
257
+ byShard.set(row.shardKey, shardRows);
258
+ }
259
+ const before = await source.getShardCounts([...byShard.keys()]);
260
+ for (const [shardKey, shardRows] of byShard) {
261
+ await source.deleteFromShard(shardKey, shardRows.map((row) => row.id));
262
+ const beforeCount = before.get(shardKey) ?? shardRows.length;
263
+ await source.setShardCount(shardKey, Math.max(0, beforeCount - shardRows.length));
264
+ result.removed += shardRows.length;
265
+ }
266
+ }
267
+ catch (error) {
268
+ result.error = {
269
+ step: "remove",
270
+ message: `removing from the source failed after ${result.removed} memories: ${describeError(error)}`,
271
+ };
272
+ return result;
273
+ }
274
+ result.ok = true;
275
+ return result;
276
+ }
@@ -3,12 +3,14 @@ import { TursoDb } from "./turso-db.js";
3
3
  import { resettable } from "../spaces/store-handles.js";
4
4
  export { resettable };
5
5
  export declare class TursoConnectionManager {
6
- private readonly clientFactory;
7
6
  private readonly connections;
8
7
  private readonly pending;
9
8
  private readonly closingConnections;
10
9
  private closingPromise;
11
- constructor(clientFactory?: typeof createClient);
10
+ private readonly clientFactory;
11
+ /** Roots whose subfolders this manager may open. Default: the two CONFIG roots. */
12
+ private readonly allowedRoots;
13
+ constructor(clientFactory?: typeof createClient, allowedRoots?: () => string[]);
12
14
  getConnection(dbPath: string): Promise<TursoDb>;
13
15
  closeConnection(dbPath: string): Promise<void>;
14
16
  closeAll(): Promise<void>;
@@ -17,21 +17,27 @@ function insideRoot(root, dbPath) {
17
17
  relativePath.startsWith(`..${sep}`) ||
18
18
  isAbsolute(relativePath));
19
19
  }
20
- function assertPathInsideStorage(dbPath) {
20
+ /** The default allowed roots: the active storage path plus the global skills path. */
21
+ function defaultAllowedRoots() {
21
22
  // skills.db stays in the global storage path when a project moves storagePath.
22
- const roots = [CONFIG.storagePath, CONFIG.skillStoragePath].filter((root) => root !== undefined);
23
+ return [CONFIG.storagePath, CONFIG.skillStoragePath].filter((root) => root !== undefined);
24
+ }
25
+ function assertPathInsideStorage(dbPath, roots) {
23
26
  if (!roots.some((root) => insideRoot(root, dbPath))) {
24
27
  throw new Error(`Refusing to open database outside storagePath: ${dbPath}`);
25
28
  }
26
29
  }
27
30
  export class TursoConnectionManager {
28
- clientFactory;
29
31
  connections = new Map();
30
32
  pending = new Map();
31
33
  closingConnections = new Map();
32
34
  closingPromise = null;
33
- constructor(clientFactory = createClient) {
35
+ clientFactory;
36
+ /** Roots whose subfolders this manager may open. Default: the two CONFIG roots. */
37
+ allowedRoots;
38
+ constructor(clientFactory = createClient, allowedRoots = defaultAllowedRoots) {
34
39
  this.clientFactory = clientFactory;
40
+ this.allowedRoots = allowedRoots;
35
41
  }
36
42
  async getConnection(dbPath) {
37
43
  if (this.closingPromise) {
@@ -41,7 +47,7 @@ export class TursoConnectionManager {
41
47
  if (closingConnection) {
42
48
  await closingConnection;
43
49
  }
44
- assertPathInsideStorage(dbPath);
50
+ assertPathInsideStorage(dbPath, this.allowedRoots());
45
51
  const existing = this.connections.get(dbPath);
46
52
  if (existing) {
47
53
  return existing;
@@ -1 +1 @@
1
- export declare function withCrossProcessWriteLock<T>(scope: "user" | "project", hash: string, fn: () => Promise<T>): Promise<T>;
1
+ export declare function withCrossProcessWriteLock<T>(scope: "user" | "project", hash: string, fn: () => Promise<T>, root?: string): Promise<T>;
@@ -49,8 +49,8 @@ function readLiveLock(path) {
49
49
  }
50
50
  return null;
51
51
  }
52
- function lockFilePath(scope, hash) {
53
- return join(CONFIG.storagePath, WRITE_LOCK_DIR, `${scope}_${hash}.lock`);
52
+ function lockFilePath(root, scope, hash) {
53
+ return join(root, WRITE_LOCK_DIR, `${scope}_${hash}.lock`);
54
54
  }
55
55
  async function publishLock(candidate, path) {
56
56
  // Match the bounded Windows hard-link retries used by the web start lock (TDR-034).
@@ -72,12 +72,12 @@ async function publishLock(candidate, path) {
72
72
  }
73
73
  }
74
74
  }
75
- export async function withCrossProcessWriteLock(scope, hash, fn) {
76
- const dir = join(CONFIG.storagePath, WRITE_LOCK_DIR);
75
+ export async function withCrossProcessWriteLock(scope, hash, fn, root = CONFIG.storagePath) {
76
+ const dir = join(root, WRITE_LOCK_DIR);
77
77
  if (!existsSync(dir)) {
78
78
  mkdirSync(dir, { recursive: true });
79
79
  }
80
- const path = lockFilePath(scope, hash);
80
+ const path = lockFilePath(root, scope, hash);
81
81
  const state = { pid: process.pid, timestamp: new Date().toISOString() };
82
82
  const deadline = Date.now() + LOCK_TIMEOUT_MS;
83
83
  const candidate = `${path}.${process.pid}.${randomUUID()}.tmp`;
@@ -1,2 +1,2 @@
1
- export declare function assertNoTursoMigrationInProgress(): void;
2
- export declare function acquireTursoOperationLock(operation: string): () => void;
1
+ export declare function assertNoTursoMigrationInProgress(root?: string): void;
2
+ export declare function acquireTursoOperationLock(operation: string, root?: string): () => void;
@@ -33,9 +33,9 @@ function readLiveLock(path) {
33
33
  }
34
34
  return null;
35
35
  }
36
- export function assertNoTursoMigrationInProgress() {
36
+ export function assertNoTursoMigrationInProgress(root = CONFIG.storagePath) {
37
37
  for (const file of [OPERATION_LOCK, LEGACY_MIGRATION_LOCK]) {
38
- const path = join(CONFIG.storagePath, file);
38
+ const path = join(root, file);
39
39
  const state = readLiveLock(path);
40
40
  if (state) {
41
41
  throw new Error(`Database migration is in progress${state.operation ? ` (${state.operation})` : ""} ` +
@@ -43,9 +43,9 @@ export function assertNoTursoMigrationInProgress() {
43
43
  }
44
44
  }
45
45
  }
46
- export function acquireTursoOperationLock(operation) {
47
- assertNoTursoMigrationInProgress();
48
- const path = join(CONFIG.storagePath, OPERATION_LOCK);
46
+ export function acquireTursoOperationLock(operation, root = CONFIG.storagePath) {
47
+ assertNoTursoMigrationInProgress(root);
48
+ const path = join(root, OPERATION_LOCK);
49
49
  const state = {
50
50
  pid: process.pid,
51
51
  timestamp: new Date().toISOString(),
@@ -1,3 +1,4 @@
1
+ import type { TursoConnectionManager } from "./connection-manager.js";
1
2
  import type { ShardInfo } from "./types.js";
2
3
  import type { TursoDb } from "./turso-db.js";
3
4
  export declare class TursoShardManager {
@@ -5,6 +6,13 @@ export declare class TursoShardManager {
5
6
  private metadataPath;
6
7
  private initPromise;
7
8
  private readonly writeLocks;
9
+ private readonly getRoot;
10
+ private readonly connections;
11
+ /** Space-bound stores pass their own root and connections (D1); defaults keep CONFIG. */
12
+ constructor(options?: {
13
+ root?: () => string;
14
+ connections?: TursoConnectionManager;
15
+ });
8
16
  reset(): void;
9
17
  withScopeWriteLock<T>(scope: "user" | "project", scopeHash: string, fn: () => Promise<T>): Promise<T>;
10
18
  private ensureInitialized;
@@ -24,6 +24,13 @@ export class TursoShardManager {
24
24
  metadataPath = "";
25
25
  initPromise = null;
26
26
  writeLocks = new Map();
27
+ getRoot;
28
+ connections;
29
+ /** Space-bound stores pass their own root and connections (D1); defaults keep CONFIG. */
30
+ constructor(options = {}) {
31
+ this.getRoot = options.root ?? (() => CONFIG.storagePath);
32
+ this.connections = options.connections ?? tursoConnectionManager;
33
+ }
27
34
  reset() {
28
35
  this.metadataDb = null;
29
36
  this.initPromise = null;
@@ -46,7 +53,7 @@ export class TursoShardManager {
46
53
  // between processes (OpenCode + Pi, or any two writers) so that shard
47
54
  // allocation, vector-count sync, and the insert itself run as one
48
55
  // critical section per scope.
49
- return await withCrossProcessWriteLock(scope, scopeHash, fn);
56
+ return await withCrossProcessWriteLock(scope, scopeHash, fn, this.getRoot());
50
57
  }
51
58
  finally {
52
59
  release();
@@ -67,8 +74,8 @@ export class TursoShardManager {
67
74
  }
68
75
  this.initPromise = (async () => {
69
76
  try {
70
- this.metadataPath = join(CONFIG.storagePath, METADATA_DB_NAME);
71
- this.metadataDb = await tursoConnectionManager.getConnection(this.metadataPath);
77
+ this.metadataPath = join(this.getRoot(), METADATA_DB_NAME);
78
+ this.metadataDb = await this.connections.getConnection(this.metadataPath);
72
79
  await this.initMetadataDb(this.metadataDb);
73
80
  }
74
81
  catch (error) {
@@ -107,7 +114,7 @@ export class TursoShardManager {
107
114
  }
108
115
  getShardPath(scope, scopeHash, shardIndex) {
109
116
  assertSafeScopeHash(scopeHash);
110
- const scopeDir = resolve(CONFIG.storagePath, `${scope}s`);
117
+ const scopeDir = resolve(this.getRoot(), `${scope}s`);
111
118
  const fullPath = resolve(join(scopeDir, `${scope}_${scopeHash}_shard_${shardIndex}.db`));
112
119
  const relativePath = relative(scopeDir, fullPath);
113
120
  if (relativePath.startsWith("..") || relativePath.includes("..")) {
@@ -117,7 +124,7 @@ export class TursoShardManager {
117
124
  }
118
125
  resolveStoredPath(storedPath, scope) {
119
126
  const fileName = basename(storedPath);
120
- return join(CONFIG.storagePath, `${scope}s`, fileName);
127
+ return join(this.getRoot(), `${scope}s`, fileName);
121
128
  }
122
129
  async getActiveShard(scope, scopeHash) {
123
130
  const metadataDb = await this.ensureInitialized();
@@ -154,7 +161,7 @@ export class TursoShardManager {
154
161
  // (disk full, permissions), the registry stays free of an orphan row that points
155
162
  // at an uninitialized file — such a row would later fail isShardValid on every
156
163
  // getWriteShard and brick all writes to this scope. initShardDb is idempotent.
157
- const shardDb = await tursoConnectionManager.getConnection(fullPath);
164
+ const shardDb = await this.connections.getConnection(fullPath);
158
165
  await this.initShardDb(shardDb);
159
166
  let result;
160
167
  try {
@@ -169,9 +176,8 @@ export class TursoShardManager {
169
176
  SELECT * FROM shards
170
177
  WHERE scope = ? AND scope_hash = ? AND shard_index = ?
171
178
  `, [scope, scopeHash, shardIndex]);
172
- if (existing) {
179
+ if (existing)
173
180
  return this.rowToShardInfo(existing);
174
- }
175
181
  }
176
182
  throw error;
177
183
  }
@@ -325,7 +331,7 @@ export class TursoShardManager {
325
331
  return true;
326
332
  }
327
333
  async syncShardVectorCount(shard) {
328
- const db = await tursoConnectionManager.getConnection(shard.dbPath);
334
+ const db = await this.connections.getConnection(shard.dbPath);
329
335
  const row = await db.get(`SELECT COUNT(*) as count FROM memories`);
330
336
  const count = Number(row?.count ?? 0);
331
337
  if (count === shard.vectorCount) {
@@ -341,13 +347,10 @@ export class TursoShardManager {
341
347
  return false;
342
348
  }
343
349
  try {
344
- const db = await tursoConnectionManager.getConnection(shard.dbPath);
350
+ const db = await this.connections.getConnection(shard.dbPath);
345
351
  const result = await db.get(`SELECT name FROM sqlite_master WHERE type='table' AND name='memories'`);
346
352
  if (!result) {
347
- log("Shard DB missing 'memories' table", {
348
- dbPath: shard.dbPath,
349
- shardId: shard.id,
350
- });
353
+ log("Shard DB missing 'memories' table", { dbPath: shard.dbPath, shardId: shard.id });
351
354
  return false;
352
355
  }
353
356
  if (!(await this.hasMatchingEmbeddingDimensions(db, shard))) {
@@ -364,12 +367,11 @@ export class TursoShardManager {
364
367
  }
365
368
  }
366
369
  async getWriteShard(scope, scopeHash) {
367
- assertNoTursoMigrationInProgress();
370
+ assertNoTursoMigrationInProgress(this.getRoot());
368
371
  for (let attempt = 0; attempt < 3; attempt++) {
369
372
  let shard = await this.getActiveShard(scope, scopeHash);
370
- if (!shard) {
373
+ if (!shard)
371
374
  return this.createShard(scope, scopeHash, 0);
372
- }
373
375
  if (!(await this.isShardValid(shard))) {
374
376
  throw new Error(`Shard ${shard.scope}/${shard.scopeHash}#${shard.shardIndex} is incompatible or corrupt. ` +
375
377
  `The original database was left untouched at ${shard.dbPath}; run the migration or restore it before writing.`);
@@ -427,11 +429,10 @@ export class TursoShardManager {
427
429
  if (!row)
428
430
  return;
429
431
  const fullPath = this.resolveStoredPath(String(row.db_path), String(row.scope));
430
- await tursoConnectionManager.closeConnection(fullPath);
432
+ await this.connections.closeConnection(fullPath);
431
433
  try {
432
- if (existsSync(fullPath)) {
434
+ if (existsSync(fullPath))
433
435
  await withSqliteFileLockRetry(() => unlinkSync(fullPath));
434
- }
435
436
  }
436
437
  catch (error) {
437
438
  log("Error deleting shard file", {
@@ -447,9 +448,9 @@ export class TursoShardManager {
447
448
  if (!row)
448
449
  return null;
449
450
  const fullPath = this.resolveStoredPath(String(row.db_path), String(row.scope));
450
- await tursoConnectionManager.closeConnection(fullPath);
451
+ await this.connections.closeConnection(fullPath);
451
452
  const archivePath = archivePathOverride ?? `${fullPath}.${reason}-${process.pid}-${Date.now()}.bak`;
452
- const scopeDir = resolve(CONFIG.storagePath, `${String(row.scope)}s`);
453
+ const scopeDir = resolve(this.getRoot(), `${String(row.scope)}s`);
453
454
  const archiveRelativePath = relative(scopeDir, resolve(archivePath));
454
455
  if (archiveRelativePath.startsWith("..") || archiveRelativePath.includes("..")) {
455
456
  throw new Error(`Archive path escapes storage directory: ${archivePath}`);
@@ -5,9 +5,13 @@ import type { Transaction } from "@libsql/client";
5
5
  export declare class TursoVectorSearch {
6
6
  insertVectorInTransaction(tx: Transaction, record: MemoryRecord): Promise<void>;
7
7
  insertVector(db: TursoDb, record: MemoryRecord): Promise<void>;
8
- searchInShard(shard: ShardInfo, queryVector: Float32Array, containerTag: string, limit: number, queryText?: string, keywordBoost?: number, keywordMode?: KeywordMode): Promise<SearchResult[]>;
8
+ searchInShard(shard: ShardInfo, queryVector: Float32Array, containerTag: string, limit: number, queryText?: string, keywordBoost?: number, keywordMode?: KeywordMode,
9
+ /** Opens the shard database; linked-space searches pass a read-only opener. */
10
+ openDb?: (dbPath: string) => Promise<TursoDb>): Promise<SearchResult[]>;
9
11
  private searchKind;
10
- searchAcrossShards(shards: ShardInfo[], queryVector: Float32Array, containerTag: string, limit: number, similarityThreshold: number, queryText?: string, keywordBoost?: number, keywordMode?: KeywordMode): Promise<{
12
+ searchAcrossShards(shards: ShardInfo[], queryVector: Float32Array, containerTag: string, limit: number, similarityThreshold: number, queryText?: string, keywordBoost?: number, keywordMode?: KeywordMode,
13
+ /** Opens each shard database; linked-space searches pass a read-only opener. */
14
+ openDb?: (dbPath: string) => Promise<TursoDb>): Promise<{
11
15
  results: SearchResult[];
12
16
  warnings: string[];
13
17
  }>;
@@ -92,8 +92,10 @@ export class TursoVectorSearch {
92
92
  ) VALUES (?, ?, vector32(?), NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
93
93
  `, [record.id, record.content, contentVector, ...commonArgs]);
94
94
  }
95
- async searchInShard(shard, queryVector, containerTag, limit, queryText, keywordBoost = 0, keywordMode = "boost") {
96
- const db = await tursoConnectionManager.getConnection(shard.dbPath);
95
+ async searchInShard(shard, queryVector, containerTag, limit, queryText, keywordBoost = 0, keywordMode = "boost",
96
+ /** Opens the shard database; linked-space searches pass a read-only opener. */
97
+ openDb = (dbPath) => tursoConnectionManager.getConnection(dbPath)) {
98
+ const db = await openDb(shard.dbPath);
97
99
  const queryJson = vectorToJson(queryVector);
98
100
  // Over-fetch aggressively when filtering by container_tag after ANN,
99
101
  // because post-filtering can discard many DiskANN neighbors.
@@ -203,11 +205,13 @@ export class TursoVectorSearch {
203
205
  return exactScanKind(db, queryJson, k, containerTag, columnName);
204
206
  }
205
207
  }
206
- async searchAcrossShards(shards, queryVector, containerTag, limit, similarityThreshold, queryText, keywordBoost = 0, keywordMode = "boost") {
208
+ async searchAcrossShards(shards, queryVector, containerTag, limit, similarityThreshold, queryText, keywordBoost = 0, keywordMode = "boost",
209
+ /** Opens each shard database; linked-space searches pass a read-only opener. */
210
+ openDb) {
207
211
  const shardErrors = [];
208
212
  const shardPromises = shards.map(async (shard) => {
209
213
  try {
210
- return await this.searchInShard(shard, queryVector, containerTag, limit, queryText, keywordBoost, keywordMode);
214
+ return await this.searchInShard(shard, queryVector, containerTag, limit, queryText, keywordBoost, keywordMode, openDb);
211
215
  }
212
216
  catch (error) {
213
217
  const message = error instanceof Error ? error.message : String(error);
@@ -1,14 +1,18 @@
1
+ import type { TursoConnectionManager } from "../turso/connection-manager.js";
1
2
  import type { UserProfile, UserProfileChangelog, UserProfileData } from "./types.js";
2
3
  import { EmbeddingService } from "../embedding.js";
3
4
  export declare const USER_PROFILES_DB_NAME = "user-profiles.db";
4
5
  export declare class UserProfileManager {
6
+ private readonly root?;
7
+ private readonly connections;
5
8
  private db;
6
9
  private dbPath;
7
10
  private initPromise;
8
11
  private coldBuffers;
9
12
  private coldBufferPath;
10
13
  private dedupCheckedCache;
11
- constructor();
14
+ /** A rooted manager (profile import, design D1) reads and writes only `root`. */
15
+ constructor(root?: string | undefined, connections?: TursoConnectionManager);
12
16
  reset(): void;
13
17
  private initialize;
14
18
  private ready;