om-memory-system 5.5.0 → 5.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/pi/extension.js +5 -0
- package/dist/cli/space-command.d.ts +6 -5
- package/dist/cli/space-command.js +154 -13
- package/dist/cli/space-transfer-command.d.ts +25 -0
- package/dist/cli/space-transfer-command.js +79 -0
- package/dist/core/memory-operations.d.ts +12 -2
- package/dist/core/memory-operations.js +14 -4
- package/dist/core/recall-summary.js +7 -5
- package/dist/core/retrieval.js +5 -2
- package/dist/core/space-tool.d.ts +9 -4
- package/dist/core/space-tool.js +35 -3
- package/dist/services/active-space-access.d.ts +6 -0
- package/dist/services/active-space-access.js +8 -0
- package/dist/services/chat/chat-runtime.js +1 -1
- package/dist/services/client.d.ts +6 -16
- package/dist/services/client.js +54 -62
- package/dist/services/context.d.ts +2 -0
- package/dist/services/context.js +12 -1
- package/dist/services/linked-search.d.ts +108 -0
- package/dist/services/linked-search.js +219 -0
- package/dist/services/space-routes.d.ts +2 -0
- package/dist/services/space-routes.js +4 -3
- package/dist/services/space-transfer-routes.d.ts +10 -0
- package/dist/services/space-transfer-routes.js +344 -0
- package/dist/services/spaces/active-space.js +14 -7
- package/dist/services/spaces/delete.d.ts +2 -0
- package/dist/services/spaces/delete.js +4 -0
- package/dist/services/spaces/export-space.d.ts +60 -0
- package/dist/services/spaces/export-space.js +367 -0
- package/dist/services/spaces/identifier-scrub.d.ts +26 -0
- package/dist/services/spaces/identifier-scrub.js +40 -0
- package/dist/services/spaces/import-space.d.ts +85 -0
- package/dist/services/spaces/import-space.js +349 -0
- package/dist/services/spaces/operations.d.ts +57 -5
- package/dist/services/spaces/operations.js +161 -5
- package/dist/services/spaces/read-store-cache.d.ts +13 -0
- package/dist/services/spaces/read-store-cache.js +34 -0
- package/dist/services/spaces/registry-store.d.ts +4 -0
- package/dist/services/spaces/registry-store.js +15 -1
- package/dist/services/spaces/registry.d.ts +32 -0
- package/dist/services/spaces/registry.js +81 -7
- package/dist/services/spaces/space-store.d.ts +50 -0
- package/dist/services/spaces/space-store.js +72 -0
- package/dist/services/spaces/transfer-service.d.ts +41 -0
- package/dist/services/spaces/transfer-service.js +408 -0
- package/dist/services/spaces/transfer-types.d.ts +163 -0
- package/dist/services/spaces/transfer-types.js +10 -0
- package/dist/services/spaces/transfer.d.ts +31 -0
- package/dist/services/spaces/transfer.js +276 -0
- package/dist/services/turso/connection-manager.d.ts +4 -2
- package/dist/services/turso/connection-manager.js +11 -5
- package/dist/services/turso/cross-process-write-lock.d.ts +1 -1
- package/dist/services/turso/cross-process-write-lock.js +5 -5
- package/dist/services/turso/operation-lock.d.ts +2 -2
- package/dist/services/turso/operation-lock.js +5 -5
- package/dist/services/turso/shard-manager.d.ts +8 -0
- package/dist/services/turso/shard-manager.js +23 -22
- package/dist/services/turso/vector-search.d.ts +6 -2
- package/dist/services/turso/vector-search.js +8 -4
- package/dist/services/user-profile/user-profile-manager.d.ts +5 -1
- package/dist/services/user-profile/user-profile-manager.js +13 -10
- package/dist/services/web-route-modules.d.ts +5 -1
- package/dist/services/web-route-modules.js +4 -0
- package/dist/services/web-server.js +1 -1
- package/dist/shared/api/space-export-schemas.d.ts +159 -0
- package/dist/shared/api/space-export-schemas.js +72 -0
- package/dist/v2/adapter.d.ts +15 -0
- package/dist/v2/adapter.js +5 -0
- package/dist/web/assets/index-2BNRLGqe.css +2 -0
- package/dist/web/assets/index-C6WnDE6h.js +170 -0
- package/dist/web/index.html +2 -2
- package/package.json +1 -1
- package/skills/omms-memory/SKILL.md +16 -14
- package/dist/web/assets/index-Dcvz3EXt.js +0 -170
- package/dist/web/assets/index-b8k5Tuei.css +0 -2
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The space transfer engine (design D3). Pure over a `SourceReader` and a
|
|
3
|
+
* `TargetWriter`: no CONFIG import, no files, no host adapters. Steps:
|
|
4
|
+
* select, skip exact duplicates, preview, write per shard, check, remove.
|
|
5
|
+
*/
|
|
6
|
+
import { tryExtractScopeFromContainerTag } from "../memory-scope.js";
|
|
7
|
+
import { formatTagsForEmbedding } from "../turso/vector-utils.js";
|
|
8
|
+
import { tagTextKey } from "./transfer-types.js";
|
|
9
|
+
export * from "./transfer-types.js";
|
|
10
|
+
/** Preview examples are for the screen only and never logged (spec). */
|
|
11
|
+
const EXAMPLE_LIMIT = 5;
|
|
12
|
+
const EXAMPLE_TEXT_LIMIT = 160;
|
|
13
|
+
const PERSONAL_GROUP_NAME = "Personal";
|
|
14
|
+
function describeError(error) {
|
|
15
|
+
return error instanceof Error ? error.message : String(error);
|
|
16
|
+
}
|
|
17
|
+
/** Both bounds are inclusive; the project filter matches the container tag exactly. */
|
|
18
|
+
function matchesFilter(row, filter) {
|
|
19
|
+
if (row.createdAt < filter.since || row.createdAt > filter.until)
|
|
20
|
+
return false;
|
|
21
|
+
if (filter.projectTag !== undefined && row.containerTag !== filter.projectTag)
|
|
22
|
+
return false;
|
|
23
|
+
return true;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Apply the filter to whatever the reader returned. Rows whose container tag
|
|
27
|
+
* cannot be parsed carry no scope, so they can be neither written to a scoped
|
|
28
|
+
* shard nor removed: the engine leaves them alone in both spaces.
|
|
29
|
+
*/
|
|
30
|
+
function selectRows(rows, filter) {
|
|
31
|
+
return rows.filter((row) => matchesFilter(row, filter) && tryExtractScopeFromContainerTag(row.containerTag) !== null);
|
|
32
|
+
}
|
|
33
|
+
/** Exact duplicates: the same id, or the same container tag and text under another id. */
|
|
34
|
+
function skipDuplicates(rows, targetIds, targetTagTexts) {
|
|
35
|
+
const kept = [];
|
|
36
|
+
let duplicates = 0;
|
|
37
|
+
for (const row of rows) {
|
|
38
|
+
if (targetIds.has(row.id) || targetTagTexts.has(tagTextKey(row.containerTag, row.text))) {
|
|
39
|
+
duplicates += 1;
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
kept.push(row);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return { kept, duplicates };
|
|
46
|
+
}
|
|
47
|
+
/** Vectors move only when the source provably used the target's model and dimensions. */
|
|
48
|
+
function canReuseVectors(source, target) {
|
|
49
|
+
return (source !== undefined &&
|
|
50
|
+
source.model !== undefined &&
|
|
51
|
+
source.model === target.model &&
|
|
52
|
+
source.dimensions === target.dimensions);
|
|
53
|
+
}
|
|
54
|
+
function groupByScope(rows) {
|
|
55
|
+
const groups = new Map();
|
|
56
|
+
for (const row of rows) {
|
|
57
|
+
const scopeRef = tryExtractScopeFromContainerTag(row.containerTag);
|
|
58
|
+
const key = `${scopeRef.scope}:${scopeRef.hash}`;
|
|
59
|
+
const group = groups.get(key) ?? { scope: scopeRef.scope, scopeHash: scopeRef.hash, rows: [] };
|
|
60
|
+
group.rows.push(row);
|
|
61
|
+
groups.set(key, group);
|
|
62
|
+
}
|
|
63
|
+
return [...groups.values()];
|
|
64
|
+
}
|
|
65
|
+
function splitTags(tags) {
|
|
66
|
+
if (!tags)
|
|
67
|
+
return [];
|
|
68
|
+
return tags
|
|
69
|
+
.split(",")
|
|
70
|
+
.map((tag) => tag.trim())
|
|
71
|
+
.filter(Boolean);
|
|
72
|
+
}
|
|
73
|
+
function requireEmbed(embed) {
|
|
74
|
+
if (!embed) {
|
|
75
|
+
throw new Error("the stored vectors cannot be reused and no embed callback was provided");
|
|
76
|
+
}
|
|
77
|
+
return embed;
|
|
78
|
+
}
|
|
79
|
+
/** Give one row a usable vector: reuse the stored one when allowed, embed otherwise. */
|
|
80
|
+
async function prepareWriteRow(row, reuseStoredVectors, embed) {
|
|
81
|
+
let vector = reuseStoredVectors ? row.vector : undefined;
|
|
82
|
+
let tagsVector = reuseStoredVectors ? row.tagsVector : undefined;
|
|
83
|
+
let embedded = false;
|
|
84
|
+
if (vector === undefined) {
|
|
85
|
+
vector = await requireEmbed(embed)(row.text);
|
|
86
|
+
embedded = true;
|
|
87
|
+
}
|
|
88
|
+
const tags = splitTags(row.tags);
|
|
89
|
+
if (tagsVector === undefined && tags.length > 0) {
|
|
90
|
+
tagsVector = await requireEmbed(embed)(formatTagsForEmbedding(tags));
|
|
91
|
+
embedded = true;
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
row: {
|
|
95
|
+
id: row.id,
|
|
96
|
+
text: row.text,
|
|
97
|
+
containerTag: row.containerTag,
|
|
98
|
+
tags: row.tags,
|
|
99
|
+
type: row.type,
|
|
100
|
+
createdAt: row.createdAt,
|
|
101
|
+
updatedAt: row.updatedAt,
|
|
102
|
+
metadata: row.metadata,
|
|
103
|
+
isPinned: row.isPinned,
|
|
104
|
+
vector,
|
|
105
|
+
tagsVector,
|
|
106
|
+
},
|
|
107
|
+
embedded,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Write one scope group in batches that fit the shard's free room, one
|
|
112
|
+
* transaction per batch, and set the shard's vector count after each batch.
|
|
113
|
+
* Progress lands in `result` and `copiedRows` per batch, so a later failure
|
|
114
|
+
* still reports how far the transfer got; earlier batches stay because a
|
|
115
|
+
* re-run skips them as duplicates.
|
|
116
|
+
*/
|
|
117
|
+
async function writeScopeGroup(target, group, maxVectorsPerShard, reuseStoredVectors, embed, result, copiedRows) {
|
|
118
|
+
let pending = group.rows;
|
|
119
|
+
while (pending.length > 0) {
|
|
120
|
+
const shard = await target.getWriteShard(group.scope, group.scopeHash);
|
|
121
|
+
const room = maxVectorsPerShard - shard.vectorCount;
|
|
122
|
+
if (room <= 0) {
|
|
123
|
+
throw new Error(`the target offered shard ${shard.key} with no room left`);
|
|
124
|
+
}
|
|
125
|
+
const batch = pending.slice(0, room);
|
|
126
|
+
const prepared = [];
|
|
127
|
+
for (const row of batch) {
|
|
128
|
+
const writeRow = await prepareWriteRow(row, reuseStoredVectors, embed);
|
|
129
|
+
prepared.push(writeRow.row);
|
|
130
|
+
result.embeddedAgain = result.embeddedAgain || writeRow.embedded;
|
|
131
|
+
}
|
|
132
|
+
await target.insertBatch(shard, prepared);
|
|
133
|
+
await target.setShardCount(shard.key, shard.vectorCount + prepared.length);
|
|
134
|
+
copiedRows.push(...batch);
|
|
135
|
+
result.copied += batch.length;
|
|
136
|
+
pending = pending.slice(batch.length);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Preview a transfer: count, project groups, at most five examples (text cut
|
|
141
|
+
* to 160 characters) and the duplicate count. Reads only; never writes.
|
|
142
|
+
*/
|
|
143
|
+
export async function previewTransfer(args) {
|
|
144
|
+
const rows = selectRows(await args.source.readRows(args.filter), args.filter);
|
|
145
|
+
const [targetIds, targetTagTexts] = await Promise.all([
|
|
146
|
+
args.target.loadExistingIds(),
|
|
147
|
+
args.target.loadExistingTagTexts(),
|
|
148
|
+
]);
|
|
149
|
+
const { kept, duplicates } = skipDuplicates(rows, targetIds, targetTagTexts);
|
|
150
|
+
const groups = new Map();
|
|
151
|
+
for (const row of kept) {
|
|
152
|
+
const scope = tryExtractScopeFromContainerTag(row.containerTag).scope;
|
|
153
|
+
const name = scope === "user" ? PERSONAL_GROUP_NAME : (row.projectName ?? row.containerTag);
|
|
154
|
+
const group = groups.get(row.containerTag) ?? { name, tag: row.containerTag, count: 0 };
|
|
155
|
+
group.count += 1;
|
|
156
|
+
groups.set(row.containerTag, group);
|
|
157
|
+
}
|
|
158
|
+
return {
|
|
159
|
+
matched: rows.length,
|
|
160
|
+
toCopy: kept.length,
|
|
161
|
+
duplicates,
|
|
162
|
+
groups: [...groups.values()],
|
|
163
|
+
examples: kept.slice(0, EXAMPLE_LIMIT).map((row) => ({
|
|
164
|
+
id: row.id,
|
|
165
|
+
text: row.text.slice(0, EXAMPLE_TEXT_LIMIT),
|
|
166
|
+
})),
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Run a transfer: select and skip duplicates at apply time, write to the
|
|
171
|
+
* target, and for a move check every copied id before removing it from the
|
|
172
|
+
* source. Failures are reported in the result, never thrown: the counts say
|
|
173
|
+
* how far the transfer got.
|
|
174
|
+
*/
|
|
175
|
+
export async function applyTransfer(args) {
|
|
176
|
+
const { source, target, filter, mode } = args;
|
|
177
|
+
const result = {
|
|
178
|
+
ok: false,
|
|
179
|
+
mode,
|
|
180
|
+
copied: 0,
|
|
181
|
+
skipped: 0,
|
|
182
|
+
removed: 0,
|
|
183
|
+
changedFromPreview: 0,
|
|
184
|
+
embeddedAgain: false,
|
|
185
|
+
missingIds: [],
|
|
186
|
+
};
|
|
187
|
+
// Steps 1 and 2: select at apply time, then drop exact duplicates.
|
|
188
|
+
let kept;
|
|
189
|
+
try {
|
|
190
|
+
const rows = selectRows(await source.readRows(filter), filter);
|
|
191
|
+
result.changedFromPreview =
|
|
192
|
+
args.previewMatched === undefined ? 0 : rows.length - args.previewMatched;
|
|
193
|
+
const [targetIds, targetTagTexts] = await Promise.all([
|
|
194
|
+
target.loadExistingIds(),
|
|
195
|
+
target.loadExistingTagTexts(),
|
|
196
|
+
]);
|
|
197
|
+
const selection = skipDuplicates(rows, targetIds, targetTagTexts);
|
|
198
|
+
kept = selection.kept;
|
|
199
|
+
result.skipped = selection.duplicates;
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
result.error = {
|
|
203
|
+
step: "select",
|
|
204
|
+
message: `reading the source or the target failed: ${describeError(error)}`,
|
|
205
|
+
};
|
|
206
|
+
return result;
|
|
207
|
+
}
|
|
208
|
+
// Step 4: write per scope group, in batches that fit the shard's free room.
|
|
209
|
+
const copiedRows = [];
|
|
210
|
+
try {
|
|
211
|
+
const [sourceEmbedding, targetEmbedding] = await Promise.all([
|
|
212
|
+
source.getEmbeddingInfo(),
|
|
213
|
+
target.getEmbeddingSettings(),
|
|
214
|
+
]);
|
|
215
|
+
const reuseStoredVectors = canReuseVectors(sourceEmbedding, targetEmbedding);
|
|
216
|
+
for (const group of groupByScope(kept)) {
|
|
217
|
+
await writeScopeGroup(target, group, args.maxVectorsPerShard, reuseStoredVectors, args.embed, result, copiedRows);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
catch (error) {
|
|
221
|
+
result.error = {
|
|
222
|
+
step: "write",
|
|
223
|
+
message: `writing to the target failed after ${result.copied} memories: ${describeError(error)}`,
|
|
224
|
+
};
|
|
225
|
+
return result;
|
|
226
|
+
}
|
|
227
|
+
if (mode === "copy") {
|
|
228
|
+
result.ok = true;
|
|
229
|
+
return result;
|
|
230
|
+
}
|
|
231
|
+
// Step 5: check that every copied id is in the target.
|
|
232
|
+
const copiedIds = copiedRows.map((row) => row.id);
|
|
233
|
+
try {
|
|
234
|
+
const found = await target.findExistingIds(copiedIds);
|
|
235
|
+
result.missingIds = copiedIds.filter((id) => !found.has(id));
|
|
236
|
+
}
|
|
237
|
+
catch (error) {
|
|
238
|
+
result.error = {
|
|
239
|
+
step: "check",
|
|
240
|
+
message: `reading the copied memories back failed: ${describeError(error)}`,
|
|
241
|
+
};
|
|
242
|
+
return result;
|
|
243
|
+
}
|
|
244
|
+
if (result.missingIds.length > 0) {
|
|
245
|
+
result.error = {
|
|
246
|
+
step: "check",
|
|
247
|
+
message: `${result.missingIds.length} copied memory id(s) missing from the target: ${result.missingIds.join(", ")}`,
|
|
248
|
+
};
|
|
249
|
+
return result;
|
|
250
|
+
}
|
|
251
|
+
// Step 6: remove only the checked ids, one transaction per source shard.
|
|
252
|
+
try {
|
|
253
|
+
const byShard = new Map();
|
|
254
|
+
for (const row of copiedRows) {
|
|
255
|
+
const shardRows = byShard.get(row.shardKey) ?? [];
|
|
256
|
+
shardRows.push(row);
|
|
257
|
+
byShard.set(row.shardKey, shardRows);
|
|
258
|
+
}
|
|
259
|
+
const before = await source.getShardCounts([...byShard.keys()]);
|
|
260
|
+
for (const [shardKey, shardRows] of byShard) {
|
|
261
|
+
await source.deleteFromShard(shardKey, shardRows.map((row) => row.id));
|
|
262
|
+
const beforeCount = before.get(shardKey) ?? shardRows.length;
|
|
263
|
+
await source.setShardCount(shardKey, Math.max(0, beforeCount - shardRows.length));
|
|
264
|
+
result.removed += shardRows.length;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
catch (error) {
|
|
268
|
+
result.error = {
|
|
269
|
+
step: "remove",
|
|
270
|
+
message: `removing from the source failed after ${result.removed} memories: ${describeError(error)}`,
|
|
271
|
+
};
|
|
272
|
+
return result;
|
|
273
|
+
}
|
|
274
|
+
result.ok = true;
|
|
275
|
+
return result;
|
|
276
|
+
}
|
|
@@ -3,12 +3,14 @@ import { TursoDb } from "./turso-db.js";
|
|
|
3
3
|
import { resettable } from "../spaces/store-handles.js";
|
|
4
4
|
export { resettable };
|
|
5
5
|
export declare class TursoConnectionManager {
|
|
6
|
-
private readonly clientFactory;
|
|
7
6
|
private readonly connections;
|
|
8
7
|
private readonly pending;
|
|
9
8
|
private readonly closingConnections;
|
|
10
9
|
private closingPromise;
|
|
11
|
-
|
|
10
|
+
private readonly clientFactory;
|
|
11
|
+
/** Roots whose subfolders this manager may open. Default: the two CONFIG roots. */
|
|
12
|
+
private readonly allowedRoots;
|
|
13
|
+
constructor(clientFactory?: typeof createClient, allowedRoots?: () => string[]);
|
|
12
14
|
getConnection(dbPath: string): Promise<TursoDb>;
|
|
13
15
|
closeConnection(dbPath: string): Promise<void>;
|
|
14
16
|
closeAll(): Promise<void>;
|
|
@@ -17,21 +17,27 @@ function insideRoot(root, dbPath) {
|
|
|
17
17
|
relativePath.startsWith(`..${sep}`) ||
|
|
18
18
|
isAbsolute(relativePath));
|
|
19
19
|
}
|
|
20
|
-
|
|
20
|
+
/** The default allowed roots: the active storage path plus the global skills path. */
|
|
21
|
+
function defaultAllowedRoots() {
|
|
21
22
|
// skills.db stays in the global storage path when a project moves storagePath.
|
|
22
|
-
|
|
23
|
+
return [CONFIG.storagePath, CONFIG.skillStoragePath].filter((root) => root !== undefined);
|
|
24
|
+
}
|
|
25
|
+
function assertPathInsideStorage(dbPath, roots) {
|
|
23
26
|
if (!roots.some((root) => insideRoot(root, dbPath))) {
|
|
24
27
|
throw new Error(`Refusing to open database outside storagePath: ${dbPath}`);
|
|
25
28
|
}
|
|
26
29
|
}
|
|
27
30
|
export class TursoConnectionManager {
|
|
28
|
-
clientFactory;
|
|
29
31
|
connections = new Map();
|
|
30
32
|
pending = new Map();
|
|
31
33
|
closingConnections = new Map();
|
|
32
34
|
closingPromise = null;
|
|
33
|
-
|
|
35
|
+
clientFactory;
|
|
36
|
+
/** Roots whose subfolders this manager may open. Default: the two CONFIG roots. */
|
|
37
|
+
allowedRoots;
|
|
38
|
+
constructor(clientFactory = createClient, allowedRoots = defaultAllowedRoots) {
|
|
34
39
|
this.clientFactory = clientFactory;
|
|
40
|
+
this.allowedRoots = allowedRoots;
|
|
35
41
|
}
|
|
36
42
|
async getConnection(dbPath) {
|
|
37
43
|
if (this.closingPromise) {
|
|
@@ -41,7 +47,7 @@ export class TursoConnectionManager {
|
|
|
41
47
|
if (closingConnection) {
|
|
42
48
|
await closingConnection;
|
|
43
49
|
}
|
|
44
|
-
assertPathInsideStorage(dbPath);
|
|
50
|
+
assertPathInsideStorage(dbPath, this.allowedRoots());
|
|
45
51
|
const existing = this.connections.get(dbPath);
|
|
46
52
|
if (existing) {
|
|
47
53
|
return existing;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare function withCrossProcessWriteLock<T>(scope: "user" | "project", hash: string, fn: () => Promise<T
|
|
1
|
+
export declare function withCrossProcessWriteLock<T>(scope: "user" | "project", hash: string, fn: () => Promise<T>, root?: string): Promise<T>;
|
|
@@ -49,8 +49,8 @@ function readLiveLock(path) {
|
|
|
49
49
|
}
|
|
50
50
|
return null;
|
|
51
51
|
}
|
|
52
|
-
function lockFilePath(scope, hash) {
|
|
53
|
-
return join(
|
|
52
|
+
function lockFilePath(root, scope, hash) {
|
|
53
|
+
return join(root, WRITE_LOCK_DIR, `${scope}_${hash}.lock`);
|
|
54
54
|
}
|
|
55
55
|
async function publishLock(candidate, path) {
|
|
56
56
|
// Match the bounded Windows hard-link retries used by the web start lock (TDR-034).
|
|
@@ -72,12 +72,12 @@ async function publishLock(candidate, path) {
|
|
|
72
72
|
}
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
|
-
export async function withCrossProcessWriteLock(scope, hash, fn) {
|
|
76
|
-
const dir = join(
|
|
75
|
+
export async function withCrossProcessWriteLock(scope, hash, fn, root = CONFIG.storagePath) {
|
|
76
|
+
const dir = join(root, WRITE_LOCK_DIR);
|
|
77
77
|
if (!existsSync(dir)) {
|
|
78
78
|
mkdirSync(dir, { recursive: true });
|
|
79
79
|
}
|
|
80
|
-
const path = lockFilePath(scope, hash);
|
|
80
|
+
const path = lockFilePath(root, scope, hash);
|
|
81
81
|
const state = { pid: process.pid, timestamp: new Date().toISOString() };
|
|
82
82
|
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
|
83
83
|
const candidate = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare function assertNoTursoMigrationInProgress(): void;
|
|
2
|
-
export declare function acquireTursoOperationLock(operation: string): () => void;
|
|
1
|
+
export declare function assertNoTursoMigrationInProgress(root?: string): void;
|
|
2
|
+
export declare function acquireTursoOperationLock(operation: string, root?: string): () => void;
|
|
@@ -33,9 +33,9 @@ function readLiveLock(path) {
|
|
|
33
33
|
}
|
|
34
34
|
return null;
|
|
35
35
|
}
|
|
36
|
-
export function assertNoTursoMigrationInProgress() {
|
|
36
|
+
export function assertNoTursoMigrationInProgress(root = CONFIG.storagePath) {
|
|
37
37
|
for (const file of [OPERATION_LOCK, LEGACY_MIGRATION_LOCK]) {
|
|
38
|
-
const path = join(
|
|
38
|
+
const path = join(root, file);
|
|
39
39
|
const state = readLiveLock(path);
|
|
40
40
|
if (state) {
|
|
41
41
|
throw new Error(`Database migration is in progress${state.operation ? ` (${state.operation})` : ""} ` +
|
|
@@ -43,9 +43,9 @@ export function assertNoTursoMigrationInProgress() {
|
|
|
43
43
|
}
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
|
-
export function acquireTursoOperationLock(operation) {
|
|
47
|
-
assertNoTursoMigrationInProgress();
|
|
48
|
-
const path = join(
|
|
46
|
+
export function acquireTursoOperationLock(operation, root = CONFIG.storagePath) {
|
|
47
|
+
assertNoTursoMigrationInProgress(root);
|
|
48
|
+
const path = join(root, OPERATION_LOCK);
|
|
49
49
|
const state = {
|
|
50
50
|
pid: process.pid,
|
|
51
51
|
timestamp: new Date().toISOString(),
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { TursoConnectionManager } from "./connection-manager.js";
|
|
1
2
|
import type { ShardInfo } from "./types.js";
|
|
2
3
|
import type { TursoDb } from "./turso-db.js";
|
|
3
4
|
export declare class TursoShardManager {
|
|
@@ -5,6 +6,13 @@ export declare class TursoShardManager {
|
|
|
5
6
|
private metadataPath;
|
|
6
7
|
private initPromise;
|
|
7
8
|
private readonly writeLocks;
|
|
9
|
+
private readonly getRoot;
|
|
10
|
+
private readonly connections;
|
|
11
|
+
/** Space-bound stores pass their own root and connections (D1); defaults keep CONFIG. */
|
|
12
|
+
constructor(options?: {
|
|
13
|
+
root?: () => string;
|
|
14
|
+
connections?: TursoConnectionManager;
|
|
15
|
+
});
|
|
8
16
|
reset(): void;
|
|
9
17
|
withScopeWriteLock<T>(scope: "user" | "project", scopeHash: string, fn: () => Promise<T>): Promise<T>;
|
|
10
18
|
private ensureInitialized;
|
|
@@ -24,6 +24,13 @@ export class TursoShardManager {
|
|
|
24
24
|
metadataPath = "";
|
|
25
25
|
initPromise = null;
|
|
26
26
|
writeLocks = new Map();
|
|
27
|
+
getRoot;
|
|
28
|
+
connections;
|
|
29
|
+
/** Space-bound stores pass their own root and connections (D1); defaults keep CONFIG. */
|
|
30
|
+
constructor(options = {}) {
|
|
31
|
+
this.getRoot = options.root ?? (() => CONFIG.storagePath);
|
|
32
|
+
this.connections = options.connections ?? tursoConnectionManager;
|
|
33
|
+
}
|
|
27
34
|
reset() {
|
|
28
35
|
this.metadataDb = null;
|
|
29
36
|
this.initPromise = null;
|
|
@@ -46,7 +53,7 @@ export class TursoShardManager {
|
|
|
46
53
|
// between processes (OpenCode + Pi, or any two writers) so that shard
|
|
47
54
|
// allocation, vector-count sync, and the insert itself run as one
|
|
48
55
|
// critical section per scope.
|
|
49
|
-
return await withCrossProcessWriteLock(scope, scopeHash, fn);
|
|
56
|
+
return await withCrossProcessWriteLock(scope, scopeHash, fn, this.getRoot());
|
|
50
57
|
}
|
|
51
58
|
finally {
|
|
52
59
|
release();
|
|
@@ -67,8 +74,8 @@ export class TursoShardManager {
|
|
|
67
74
|
}
|
|
68
75
|
this.initPromise = (async () => {
|
|
69
76
|
try {
|
|
70
|
-
this.metadataPath = join(
|
|
71
|
-
this.metadataDb = await
|
|
77
|
+
this.metadataPath = join(this.getRoot(), METADATA_DB_NAME);
|
|
78
|
+
this.metadataDb = await this.connections.getConnection(this.metadataPath);
|
|
72
79
|
await this.initMetadataDb(this.metadataDb);
|
|
73
80
|
}
|
|
74
81
|
catch (error) {
|
|
@@ -107,7 +114,7 @@ export class TursoShardManager {
|
|
|
107
114
|
}
|
|
108
115
|
getShardPath(scope, scopeHash, shardIndex) {
|
|
109
116
|
assertSafeScopeHash(scopeHash);
|
|
110
|
-
const scopeDir = resolve(
|
|
117
|
+
const scopeDir = resolve(this.getRoot(), `${scope}s`);
|
|
111
118
|
const fullPath = resolve(join(scopeDir, `${scope}_${scopeHash}_shard_${shardIndex}.db`));
|
|
112
119
|
const relativePath = relative(scopeDir, fullPath);
|
|
113
120
|
if (relativePath.startsWith("..") || relativePath.includes("..")) {
|
|
@@ -117,7 +124,7 @@ export class TursoShardManager {
|
|
|
117
124
|
}
|
|
118
125
|
resolveStoredPath(storedPath, scope) {
|
|
119
126
|
const fileName = basename(storedPath);
|
|
120
|
-
return join(
|
|
127
|
+
return join(this.getRoot(), `${scope}s`, fileName);
|
|
121
128
|
}
|
|
122
129
|
async getActiveShard(scope, scopeHash) {
|
|
123
130
|
const metadataDb = await this.ensureInitialized();
|
|
@@ -154,7 +161,7 @@ export class TursoShardManager {
|
|
|
154
161
|
// (disk full, permissions), the registry stays free of an orphan row that points
|
|
155
162
|
// at an uninitialized file — such a row would later fail isShardValid on every
|
|
156
163
|
// getWriteShard and brick all writes to this scope. initShardDb is idempotent.
|
|
157
|
-
const shardDb = await
|
|
164
|
+
const shardDb = await this.connections.getConnection(fullPath);
|
|
158
165
|
await this.initShardDb(shardDb);
|
|
159
166
|
let result;
|
|
160
167
|
try {
|
|
@@ -169,9 +176,8 @@ export class TursoShardManager {
|
|
|
169
176
|
SELECT * FROM shards
|
|
170
177
|
WHERE scope = ? AND scope_hash = ? AND shard_index = ?
|
|
171
178
|
`, [scope, scopeHash, shardIndex]);
|
|
172
|
-
if (existing)
|
|
179
|
+
if (existing)
|
|
173
180
|
return this.rowToShardInfo(existing);
|
|
174
|
-
}
|
|
175
181
|
}
|
|
176
182
|
throw error;
|
|
177
183
|
}
|
|
@@ -325,7 +331,7 @@ export class TursoShardManager {
|
|
|
325
331
|
return true;
|
|
326
332
|
}
|
|
327
333
|
async syncShardVectorCount(shard) {
|
|
328
|
-
const db = await
|
|
334
|
+
const db = await this.connections.getConnection(shard.dbPath);
|
|
329
335
|
const row = await db.get(`SELECT COUNT(*) as count FROM memories`);
|
|
330
336
|
const count = Number(row?.count ?? 0);
|
|
331
337
|
if (count === shard.vectorCount) {
|
|
@@ -341,13 +347,10 @@ export class TursoShardManager {
|
|
|
341
347
|
return false;
|
|
342
348
|
}
|
|
343
349
|
try {
|
|
344
|
-
const db = await
|
|
350
|
+
const db = await this.connections.getConnection(shard.dbPath);
|
|
345
351
|
const result = await db.get(`SELECT name FROM sqlite_master WHERE type='table' AND name='memories'`);
|
|
346
352
|
if (!result) {
|
|
347
|
-
log("Shard DB missing 'memories' table", {
|
|
348
|
-
dbPath: shard.dbPath,
|
|
349
|
-
shardId: shard.id,
|
|
350
|
-
});
|
|
353
|
+
log("Shard DB missing 'memories' table", { dbPath: shard.dbPath, shardId: shard.id });
|
|
351
354
|
return false;
|
|
352
355
|
}
|
|
353
356
|
if (!(await this.hasMatchingEmbeddingDimensions(db, shard))) {
|
|
@@ -364,12 +367,11 @@ export class TursoShardManager {
|
|
|
364
367
|
}
|
|
365
368
|
}
|
|
366
369
|
async getWriteShard(scope, scopeHash) {
|
|
367
|
-
assertNoTursoMigrationInProgress();
|
|
370
|
+
assertNoTursoMigrationInProgress(this.getRoot());
|
|
368
371
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
369
372
|
let shard = await this.getActiveShard(scope, scopeHash);
|
|
370
|
-
if (!shard)
|
|
373
|
+
if (!shard)
|
|
371
374
|
return this.createShard(scope, scopeHash, 0);
|
|
372
|
-
}
|
|
373
375
|
if (!(await this.isShardValid(shard))) {
|
|
374
376
|
throw new Error(`Shard ${shard.scope}/${shard.scopeHash}#${shard.shardIndex} is incompatible or corrupt. ` +
|
|
375
377
|
`The original database was left untouched at ${shard.dbPath}; run the migration or restore it before writing.`);
|
|
@@ -427,11 +429,10 @@ export class TursoShardManager {
|
|
|
427
429
|
if (!row)
|
|
428
430
|
return;
|
|
429
431
|
const fullPath = this.resolveStoredPath(String(row.db_path), String(row.scope));
|
|
430
|
-
await
|
|
432
|
+
await this.connections.closeConnection(fullPath);
|
|
431
433
|
try {
|
|
432
|
-
if (existsSync(fullPath))
|
|
434
|
+
if (existsSync(fullPath))
|
|
433
435
|
await withSqliteFileLockRetry(() => unlinkSync(fullPath));
|
|
434
|
-
}
|
|
435
436
|
}
|
|
436
437
|
catch (error) {
|
|
437
438
|
log("Error deleting shard file", {
|
|
@@ -447,9 +448,9 @@ export class TursoShardManager {
|
|
|
447
448
|
if (!row)
|
|
448
449
|
return null;
|
|
449
450
|
const fullPath = this.resolveStoredPath(String(row.db_path), String(row.scope));
|
|
450
|
-
await
|
|
451
|
+
await this.connections.closeConnection(fullPath);
|
|
451
452
|
const archivePath = archivePathOverride ?? `${fullPath}.${reason}-${process.pid}-${Date.now()}.bak`;
|
|
452
|
-
const scopeDir = resolve(
|
|
453
|
+
const scopeDir = resolve(this.getRoot(), `${String(row.scope)}s`);
|
|
453
454
|
const archiveRelativePath = relative(scopeDir, resolve(archivePath));
|
|
454
455
|
if (archiveRelativePath.startsWith("..") || archiveRelativePath.includes("..")) {
|
|
455
456
|
throw new Error(`Archive path escapes storage directory: ${archivePath}`);
|
|
@@ -5,9 +5,13 @@ import type { Transaction } from "@libsql/client";
|
|
|
5
5
|
export declare class TursoVectorSearch {
|
|
6
6
|
insertVectorInTransaction(tx: Transaction, record: MemoryRecord): Promise<void>;
|
|
7
7
|
insertVector(db: TursoDb, record: MemoryRecord): Promise<void>;
|
|
8
|
-
searchInShard(shard: ShardInfo, queryVector: Float32Array, containerTag: string, limit: number, queryText?: string, keywordBoost?: number, keywordMode?: KeywordMode
|
|
8
|
+
searchInShard(shard: ShardInfo, queryVector: Float32Array, containerTag: string, limit: number, queryText?: string, keywordBoost?: number, keywordMode?: KeywordMode,
|
|
9
|
+
/** Opens the shard database; linked-space searches pass a read-only opener. */
|
|
10
|
+
openDb?: (dbPath: string) => Promise<TursoDb>): Promise<SearchResult[]>;
|
|
9
11
|
private searchKind;
|
|
10
|
-
searchAcrossShards(shards: ShardInfo[], queryVector: Float32Array, containerTag: string, limit: number, similarityThreshold: number, queryText?: string, keywordBoost?: number, keywordMode?: KeywordMode
|
|
12
|
+
searchAcrossShards(shards: ShardInfo[], queryVector: Float32Array, containerTag: string, limit: number, similarityThreshold: number, queryText?: string, keywordBoost?: number, keywordMode?: KeywordMode,
|
|
13
|
+
/** Opens each shard database; linked-space searches pass a read-only opener. */
|
|
14
|
+
openDb?: (dbPath: string) => Promise<TursoDb>): Promise<{
|
|
11
15
|
results: SearchResult[];
|
|
12
16
|
warnings: string[];
|
|
13
17
|
}>;
|
|
@@ -92,8 +92,10 @@ export class TursoVectorSearch {
|
|
|
92
92
|
) VALUES (?, ?, vector32(?), NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
93
93
|
`, [record.id, record.content, contentVector, ...commonArgs]);
|
|
94
94
|
}
|
|
95
|
-
async searchInShard(shard, queryVector, containerTag, limit, queryText, keywordBoost = 0, keywordMode = "boost"
|
|
96
|
-
|
|
95
|
+
async searchInShard(shard, queryVector, containerTag, limit, queryText, keywordBoost = 0, keywordMode = "boost",
|
|
96
|
+
/** Opens the shard database; linked-space searches pass a read-only opener. */
|
|
97
|
+
openDb = (dbPath) => tursoConnectionManager.getConnection(dbPath)) {
|
|
98
|
+
const db = await openDb(shard.dbPath);
|
|
97
99
|
const queryJson = vectorToJson(queryVector);
|
|
98
100
|
// Over-fetch aggressively when filtering by container_tag after ANN,
|
|
99
101
|
// because post-filtering can discard many DiskANN neighbors.
|
|
@@ -203,11 +205,13 @@ export class TursoVectorSearch {
|
|
|
203
205
|
return exactScanKind(db, queryJson, k, containerTag, columnName);
|
|
204
206
|
}
|
|
205
207
|
}
|
|
206
|
-
async searchAcrossShards(shards, queryVector, containerTag, limit, similarityThreshold, queryText, keywordBoost = 0, keywordMode = "boost"
|
|
208
|
+
async searchAcrossShards(shards, queryVector, containerTag, limit, similarityThreshold, queryText, keywordBoost = 0, keywordMode = "boost",
|
|
209
|
+
/** Opens each shard database; linked-space searches pass a read-only opener. */
|
|
210
|
+
openDb) {
|
|
207
211
|
const shardErrors = [];
|
|
208
212
|
const shardPromises = shards.map(async (shard) => {
|
|
209
213
|
try {
|
|
210
|
-
return await this.searchInShard(shard, queryVector, containerTag, limit, queryText, keywordBoost, keywordMode);
|
|
214
|
+
return await this.searchInShard(shard, queryVector, containerTag, limit, queryText, keywordBoost, keywordMode, openDb);
|
|
211
215
|
}
|
|
212
216
|
catch (error) {
|
|
213
217
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -1,14 +1,18 @@
|
|
|
1
|
+
import type { TursoConnectionManager } from "../turso/connection-manager.js";
|
|
1
2
|
import type { UserProfile, UserProfileChangelog, UserProfileData } from "./types.js";
|
|
2
3
|
import { EmbeddingService } from "../embedding.js";
|
|
3
4
|
export declare const USER_PROFILES_DB_NAME = "user-profiles.db";
|
|
4
5
|
export declare class UserProfileManager {
|
|
6
|
+
private readonly root?;
|
|
7
|
+
private readonly connections;
|
|
5
8
|
private db;
|
|
6
9
|
private dbPath;
|
|
7
10
|
private initPromise;
|
|
8
11
|
private coldBuffers;
|
|
9
12
|
private coldBufferPath;
|
|
10
13
|
private dedupCheckedCache;
|
|
11
|
-
|
|
14
|
+
/** A rooted manager (profile import, design D1) reads and writes only `root`. */
|
|
15
|
+
constructor(root?: string | undefined, connections?: TursoConnectionManager);
|
|
12
16
|
reset(): void;
|
|
13
17
|
private initialize;
|
|
14
18
|
private ready;
|