@remnic/core 9.3.515 → 9.3.516
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +11 -11
- package/dist/access-http.d.ts +2 -1
- package/dist/access-http.js +5 -5
- package/dist/access-mcp.d.ts +1 -1
- package/dist/access-mcp.js +4 -4
- package/dist/access-schema.js +2 -2
- package/dist/{access-service-qrrIrC-0.d.ts → access-service-CZfksQuS.d.ts} +6 -2
- package/dist/access-service.d.ts +1 -1
- package/dist/access-service.js +2 -2
- package/dist/{chunk-NJ3MJQZX.js → chunk-2I5JGH3M.js} +2 -2
- package/dist/{chunk-NJ3MJQZX.js.map → chunk-2I5JGH3M.js.map} +1 -1
- package/dist/{chunk-O27WNHTT.js → chunk-5UHVGNZD.js} +2 -2
- package/dist/{chunk-3Q4H3OBR.js → chunk-5V456VRV.js} +6 -6
- package/dist/{chunk-3Q4H3OBR.js.map → chunk-5V456VRV.js.map} +1 -1
- package/dist/{chunk-EDBEWFJO.js → chunk-6BR7L222.js} +2 -2
- package/dist/{chunk-D6WE5MTW.js → chunk-FCOQXV3T.js} +6 -6
- package/dist/{chunk-RCTS5CKK.js → chunk-FK556DDH.js} +2 -2
- package/dist/{chunk-RCTS5CKK.js.map → chunk-FK556DDH.js.map} +1 -1
- package/dist/{chunk-A52AKD7C.js → chunk-FUC4LZMD.js} +2 -2
- package/dist/chunk-FUC4LZMD.js.map +1 -0
- package/dist/{chunk-FER4WARO.js → chunk-HC6EKOID.js} +20 -7
- package/dist/chunk-HC6EKOID.js.map +1 -0
- package/dist/{chunk-PIRJPV5T.js → chunk-JNANKJLN.js} +2 -2
- package/dist/chunk-JNANKJLN.js.map +1 -0
- package/dist/{chunk-7MV5CWTE.js → chunk-KXULCVOC.js} +6 -6
- package/dist/chunk-KXULCVOC.js.map +1 -0
- package/dist/{chunk-TVRN5QKH.js → chunk-PCI747N2.js} +3 -3
- package/dist/{chunk-TVRN5QKH.js.map → chunk-PCI747N2.js.map} +1 -1
- package/dist/{chunk-BLZAVUD2.js → chunk-QVJ4NWL2.js} +2 -2
- package/dist/chunk-QVJ4NWL2.js.map +1 -0
- package/dist/{chunk-EIPUHVKE.js → chunk-SML26KED.js} +7 -7
- package/dist/{chunk-JYIKKAK3.js → chunk-TTGZV5R3.js} +3 -3
- package/dist/{chunk-R26QUUQN.js → chunk-YDMVYYD2.js} +52 -11
- package/dist/chunk-YDMVYYD2.js.map +1 -0
- package/dist/{chunk-L7S47WZT.js → chunk-YNXOKMJP.js} +2 -2
- package/dist/chunk-YNXOKMJP.js.map +1 -0
- package/dist/{chunk-4Q73JBSM.js → chunk-ZEY4KYRQ.js} +38 -11
- package/dist/chunk-ZEY4KYRQ.js.map +1 -0
- package/dist/{cli-X4NJoqSe.d.ts → cli-CPe_2KB1.d.ts} +1 -1
- package/dist/cli.d.ts +2 -2
- package/dist/cli.js +16 -16
- package/dist/index.d.ts +2 -2
- package/dist/index.js +17 -17
- package/dist/mcp-memory-inspector-app.d.ts +1 -1
- package/dist/namespaces/migrate.js +9 -9
- package/dist/namespaces/search.d.ts +1 -1
- package/dist/namespaces/search.js +8 -8
- package/dist/offline-sync.d.ts +4 -0
- package/dist/offline-sync.js +1 -1
- package/dist/operator-toolkit.d.ts +3 -1
- package/dist/operator-toolkit.js +10 -10
- package/dist/orchestrator.js +9 -9
- package/dist/qmd.d.ts +1 -1
- package/dist/qmd.js +1 -1
- package/dist/search/factory.js +7 -7
- package/dist/search/index.js +7 -7
- package/dist/search/lancedb-backend.d.ts +1 -1
- package/dist/search/lancedb-backend.js +1 -1
- package/dist/search/meilisearch-backend.d.ts +1 -1
- package/dist/search/meilisearch-backend.js +1 -1
- package/dist/search/noop-backend.d.ts +1 -1
- package/dist/search/noop-backend.js +1 -1
- package/dist/search/orama-backend.d.ts +1 -1
- package/dist/search/orama-backend.js +1 -1
- package/dist/search/port.d.ts +1 -1
- package/dist/search/remote-backend.d.ts +1 -1
- package/dist/search/remote-backend.js +1 -1
- package/package.json +1 -1
- package/src/access-http.ts +14 -0
- package/src/access-service-namespace.test.ts +9 -9
- package/src/access-service.ts +4 -4
- package/src/namespaces/search.test.ts +20 -1
- package/src/namespaces/search.ts +10 -4
- package/src/offline-sync.test.ts +128 -18
- package/src/offline-sync.ts +41 -7
- package/src/operator-toolkit.ts +4 -1
- package/src/orchestrator.ts +68 -10
- package/src/qmd.ts +5 -2
- package/src/search/lancedb-backend.ts +4 -1
- package/src/search/meilisearch-backend.ts +4 -1
- package/src/search/noop-backend.ts +1 -1
- package/src/search/orama-backend.ts +4 -1
- package/src/search/port.ts +4 -1
- package/src/search/remote-backend.ts +1 -1
- package/dist/chunk-4Q73JBSM.js.map +0 -1
- package/dist/chunk-7MV5CWTE.js.map +0 -1
- package/dist/chunk-A52AKD7C.js.map +0 -1
- package/dist/chunk-BLZAVUD2.js.map +0 -1
- package/dist/chunk-FER4WARO.js.map +0 -1
- package/dist/chunk-L7S47WZT.js.map +0 -1
- package/dist/chunk-PIRJPV5T.js.map +0 -1
- package/dist/chunk-R26QUUQN.js.map +0 -1
- /package/dist/{chunk-O27WNHTT.js.map → chunk-5UHVGNZD.js.map} +0 -0
- /package/dist/{chunk-EDBEWFJO.js.map → chunk-6BR7L222.js.map} +0 -0
- /package/dist/{chunk-D6WE5MTW.js.map → chunk-FCOQXV3T.js.map} +0 -0
- /package/dist/{chunk-EIPUHVKE.js.map → chunk-SML26KED.js.map} +0 -0
- /package/dist/{chunk-JYIKKAK3.js.map → chunk-TTGZV5R3.js.map} +0 -0
package/src/offline-sync.test.ts
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
buildOfflineSyncSnapshotForPaths,
|
|
17
17
|
OFFLINE_SYNC_MAX_MTIME_MS,
|
|
18
18
|
readOfflineSyncFileContentChunk,
|
|
19
|
+
shouldPreferIncomingOfflineRuntimeFile,
|
|
19
20
|
summarizeOfflineSyncPendingChanges,
|
|
20
21
|
summarizeOfflineSyncPendingFiles,
|
|
21
22
|
} from "./offline-sync.js";
|
|
@@ -110,6 +111,32 @@ test("offline snapshot captures source-of-truth files and excludes private/inter
|
|
|
110
111
|
}
|
|
111
112
|
});
|
|
112
113
|
|
|
114
|
+
test("offline snapshot abort signal stops filesystem snapshot work", async () => {
|
|
115
|
+
const root = await tempDir("remnic-offline-snapshot-abort");
|
|
116
|
+
try {
|
|
117
|
+
await write(root, "facts/a.md", "alpha");
|
|
118
|
+
const controller = new AbortController();
|
|
119
|
+
|
|
120
|
+
await assert.rejects(
|
|
121
|
+
buildOfflineSyncSnapshot({
|
|
122
|
+
root,
|
|
123
|
+
sourceId: "remote",
|
|
124
|
+
signal: controller.signal,
|
|
125
|
+
readFileDigest: async () => {
|
|
126
|
+
controller.abort();
|
|
127
|
+
return {
|
|
128
|
+
sha256: createHash("sha256").update("alpha").digest("hex"),
|
|
129
|
+
bytes: Buffer.byteLength("alpha"),
|
|
130
|
+
};
|
|
131
|
+
},
|
|
132
|
+
}),
|
|
133
|
+
/offline sync request aborted/,
|
|
134
|
+
);
|
|
135
|
+
} finally {
|
|
136
|
+
await rm(root, { recursive: true, force: true });
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
|
|
113
140
|
test("offline sync includes retrieval debug snapshots for full-fidelity offline recall", async () => {
|
|
114
141
|
const root = await tempDir("remnic-offline-debug-snapshots");
|
|
115
142
|
try {
|
|
@@ -313,7 +340,7 @@ test("offline snapshot from base avoids rehashing unchanged files", async () =>
|
|
|
313
340
|
sourceId: "remote",
|
|
314
341
|
includeContent: false,
|
|
315
342
|
});
|
|
316
|
-
const baseCapturedAt = new Date(
|
|
343
|
+
const baseCapturedAt = new Date();
|
|
317
344
|
let readCount = 0;
|
|
318
345
|
|
|
319
346
|
const unchanged = await buildOfflineSyncSnapshotFromBase({
|
|
@@ -438,7 +465,7 @@ test("offline snapshot from base trusts mtime precision drift without rehashing"
|
|
|
438
465
|
root,
|
|
439
466
|
sourceId: "remote",
|
|
440
467
|
baseFiles: [driftedBase],
|
|
441
|
-
baseCapturedAt: new Date(
|
|
468
|
+
baseCapturedAt: new Date(),
|
|
442
469
|
includeContent: false,
|
|
443
470
|
readFileDigest: async () => {
|
|
444
471
|
throw new Error("unchanged file should reuse trusted mtime metadata");
|
|
@@ -451,6 +478,47 @@ test("offline snapshot from base trusts mtime precision drift without rehashing"
|
|
|
451
478
|
}
|
|
452
479
|
});
|
|
453
480
|
|
|
481
|
+
test("offline snapshot from base rehashes metadata-only ctime churn", async () => {
|
|
482
|
+
const root = await tempDir("remnic-offline-fast-base-ctime-only");
|
|
483
|
+
try {
|
|
484
|
+
await write(root, "facts/a.md", "alpha");
|
|
485
|
+
const filePath = path.join(root, "facts/a.md");
|
|
486
|
+
const baseSnapshot = await buildOfflineSyncSnapshot({
|
|
487
|
+
root,
|
|
488
|
+
sourceId: "remote",
|
|
489
|
+
includeContent: false,
|
|
490
|
+
});
|
|
491
|
+
const baseFile = baseSnapshot.files.find((file) => file.path === "facts/a.md");
|
|
492
|
+
assert.ok(baseFile);
|
|
493
|
+
const baseCapturedAt = new Date();
|
|
494
|
+
await new Promise<void>((resolve) => setTimeout(resolve, 1_100));
|
|
495
|
+
await chmod(filePath, 0o600);
|
|
496
|
+
await chmod(filePath, 0o644);
|
|
497
|
+
|
|
498
|
+
let digestReadCount = 0;
|
|
499
|
+
const rehashed = await buildOfflineSyncSnapshotFromBase({
|
|
500
|
+
root,
|
|
501
|
+
sourceId: "remote",
|
|
502
|
+
baseFiles: baseSnapshot.files,
|
|
503
|
+
baseCapturedAt,
|
|
504
|
+
includeContent: false,
|
|
505
|
+
readFileDigest: async ({ filePath }) => {
|
|
506
|
+
digestReadCount += 1;
|
|
507
|
+
const bytes = await readFile(filePath);
|
|
508
|
+
return {
|
|
509
|
+
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
510
|
+
bytes: bytes.byteLength,
|
|
511
|
+
};
|
|
512
|
+
},
|
|
513
|
+
});
|
|
514
|
+
|
|
515
|
+
assert.equal(digestReadCount, 1);
|
|
516
|
+
assert.equal(rehashed.files[0]?.sha256, baseFile.sha256);
|
|
517
|
+
} finally {
|
|
518
|
+
await rm(root, { recursive: true, force: true });
|
|
519
|
+
}
|
|
520
|
+
});
|
|
521
|
+
|
|
454
522
|
test("offline snapshot from base rehashes preserved-mtime rewrites when ctime changed after capture", async () => {
|
|
455
523
|
const root = await tempDir("remnic-offline-fast-base-preserved-mtime");
|
|
456
524
|
try {
|
|
@@ -463,6 +531,8 @@ test("offline snapshot from base rehashes preserved-mtime rewrites when ctime ch
|
|
|
463
531
|
});
|
|
464
532
|
const baseFile = baseSnapshot.files[0];
|
|
465
533
|
assert.ok(baseFile);
|
|
534
|
+
const baseCapturedAt = new Date();
|
|
535
|
+
await new Promise<void>((resolve) => setTimeout(resolve, 20));
|
|
466
536
|
await write(root, "facts/a.md", "bravo");
|
|
467
537
|
await utimes(filePath, baseFile.mtimeMs / 1000, baseFile.mtimeMs / 1000);
|
|
468
538
|
|
|
@@ -471,7 +541,7 @@ test("offline snapshot from base rehashes preserved-mtime rewrites when ctime ch
|
|
|
471
541
|
root,
|
|
472
542
|
sourceId: "remote",
|
|
473
543
|
baseFiles: baseSnapshot.files,
|
|
474
|
-
baseCapturedAt
|
|
544
|
+
baseCapturedAt,
|
|
475
545
|
includeContent: false,
|
|
476
546
|
readFileDigest: async ({ filePath }) => {
|
|
477
547
|
digestReadCount += 1;
|
|
@@ -525,7 +595,7 @@ test("offline snapshot apply preserves incoming mtime for future fast-base reuse
|
|
|
525
595
|
root,
|
|
526
596
|
sourceId: "local",
|
|
527
597
|
baseFiles: applied.nextBaseFiles,
|
|
528
|
-
baseCapturedAt: new Date(
|
|
598
|
+
baseCapturedAt: new Date(),
|
|
529
599
|
includeContent: false,
|
|
530
600
|
readFileDigest: async () => {
|
|
531
601
|
throw new Error("applied file should reuse preserved mtime metadata");
|
|
@@ -1526,6 +1596,50 @@ test("offline pull skips local runtime drift when remote still matches base", as
|
|
|
1526
1596
|
}
|
|
1527
1597
|
});
|
|
1528
1598
|
|
|
1599
|
+
test("offline pull treats last intent as remote-authoritative runtime state", async () => {
|
|
1600
|
+
const root = await tempDir("remnic-offline-runtime-last-intent");
|
|
1601
|
+
try {
|
|
1602
|
+
const relPath = "state/last_intent.json";
|
|
1603
|
+
const baseContent = Buffer.from("{\"intent\":\"base\"}");
|
|
1604
|
+
const localContent = Buffer.from("{\"intent\":\"local\"}");
|
|
1605
|
+
const remoteContent = Buffer.from("{\"intent\":\"remote\"}");
|
|
1606
|
+
await write(root, relPath, localContent);
|
|
1607
|
+
const baseFile = {
|
|
1608
|
+
path: relPath,
|
|
1609
|
+
sha256: createHash("sha256").update(baseContent).digest("hex"),
|
|
1610
|
+
bytes: baseContent.byteLength,
|
|
1611
|
+
mtimeMs: 1,
|
|
1612
|
+
};
|
|
1613
|
+
const incomingFile = {
|
|
1614
|
+
path: relPath,
|
|
1615
|
+
sha256: createHash("sha256").update(remoteContent).digest("hex"),
|
|
1616
|
+
bytes: remoteContent.byteLength,
|
|
1617
|
+
mtimeMs: 2,
|
|
1618
|
+
contentBase64: remoteContent.toString("base64"),
|
|
1619
|
+
};
|
|
1620
|
+
|
|
1621
|
+
assert.equal(shouldPreferIncomingOfflineRuntimeFile(relPath), true);
|
|
1622
|
+
const pull = await applyOfflineSyncSnapshot({
|
|
1623
|
+
root,
|
|
1624
|
+
snapshot: {
|
|
1625
|
+
format: "remnic.offline-sync.snapshot.v1",
|
|
1626
|
+
schemaVersion: 1,
|
|
1627
|
+
createdAt: new Date().toISOString(),
|
|
1628
|
+
sourceId: "remote",
|
|
1629
|
+
includeTranscripts: true,
|
|
1630
|
+
files: [incomingFile],
|
|
1631
|
+
},
|
|
1632
|
+
baseFiles: [baseFile],
|
|
1633
|
+
});
|
|
1634
|
+
|
|
1635
|
+
assert.equal(pull.upserted, 1);
|
|
1636
|
+
assert.equal(pull.conflicts.length, 0);
|
|
1637
|
+
assert.equal(await readUtf8(root, relPath), remoteContent.toString("utf-8"));
|
|
1638
|
+
} finally {
|
|
1639
|
+
await rm(root, { recursive: true, force: true });
|
|
1640
|
+
}
|
|
1641
|
+
});
|
|
1642
|
+
|
|
1529
1643
|
test("offline pull applies snapshots with content only for remote-changed files", async () => {
|
|
1530
1644
|
const remote = await tempDir("remnic-offline-partial-remote");
|
|
1531
1645
|
const local = await tempDir("remnic-offline-partial-local");
|
|
@@ -2108,14 +2222,13 @@ test("offline sync applies and snapshots through secure storage hooks", async ()
|
|
|
2108
2222
|
}
|
|
2109
2223
|
});
|
|
2110
2224
|
|
|
2111
|
-
test("offline snapshot fast-base
|
|
2112
|
-
const root = await tempDir("remnic-offline-fast-base-probe
|
|
2225
|
+
test("offline snapshot fast-base does not probe file headers for trusted metadata", async () => {
|
|
2226
|
+
const root = await tempDir("remnic-offline-fast-base-no-header-probe");
|
|
2113
2227
|
const relPath = "facts/same-size.md";
|
|
2114
|
-
const
|
|
2115
|
-
const newContent = Buffer.from("new!");
|
|
2228
|
+
const content = Buffer.from("fact");
|
|
2116
2229
|
const filePath = path.join(root, relPath);
|
|
2117
2230
|
try {
|
|
2118
|
-
await write(root, relPath,
|
|
2231
|
+
await write(root, relPath, content);
|
|
2119
2232
|
await chmod(filePath, 0o000);
|
|
2120
2233
|
const st = await stat(filePath);
|
|
2121
2234
|
let digestReads = 0;
|
|
@@ -2125,24 +2238,21 @@ test("offline snapshot fast-base rehashes when encrypted header probe fails", as
|
|
|
2125
2238
|
sourceId: "remote",
|
|
2126
2239
|
baseFiles: [{
|
|
2127
2240
|
path: relPath,
|
|
2128
|
-
sha256: createHash("sha256").update(
|
|
2129
|
-
bytes:
|
|
2241
|
+
sha256: createHash("sha256").update(content).digest("hex"),
|
|
2242
|
+
bytes: content.byteLength,
|
|
2130
2243
|
mtimeMs: st.mtimeMs,
|
|
2131
2244
|
}],
|
|
2132
|
-
baseCapturedAt: new Date(),
|
|
2245
|
+
baseCapturedAt: new Date(Date.now() + 60_000),
|
|
2133
2246
|
readFileDigest: async ({ path: targetPath, filePath: targetFilePath }) => {
|
|
2134
2247
|
assert.equal(targetPath, relPath);
|
|
2135
2248
|
assert.equal(targetFilePath, filePath);
|
|
2136
2249
|
digestReads += 1;
|
|
2137
|
-
|
|
2138
|
-
sha256: createHash("sha256").update(newContent).digest("hex"),
|
|
2139
|
-
bytes: newContent.byteLength,
|
|
2140
|
-
};
|
|
2250
|
+
throw new Error("trusted metadata should not read the file header or digest");
|
|
2141
2251
|
},
|
|
2142
2252
|
});
|
|
2143
2253
|
|
|
2144
|
-
assert.equal(digestReads,
|
|
2145
|
-
assert.equal(snapshot.files[0]?.sha256, createHash("sha256").update(
|
|
2254
|
+
assert.equal(digestReads, 0);
|
|
2255
|
+
assert.equal(snapshot.files[0]?.sha256, createHash("sha256").update(content).digest("hex"));
|
|
2146
2256
|
} finally {
|
|
2147
2257
|
await chmod(filePath, 0o600).catch(() => {});
|
|
2148
2258
|
await rm(root, { recursive: true, force: true });
|
package/src/offline-sync.ts
CHANGED
|
@@ -183,11 +183,13 @@ interface OfflineSyncFileRecordOptions {
|
|
|
183
183
|
includeContent: boolean;
|
|
184
184
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
185
185
|
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
186
|
+
signal?: AbortSignal;
|
|
186
187
|
}
|
|
187
188
|
|
|
188
189
|
const SYNC_INTERNAL_DIR = ".offline-sync";
|
|
189
190
|
const OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
190
191
|
const OFFLINE_SYNC_FAST_BASE_MTIME_TOLERANCE_MS = 1_000;
|
|
192
|
+
const OFFLINE_SYNC_FAST_BASE_CTIME_TOLERANCE_MS = 1;
|
|
191
193
|
const EXCLUDED_FILE_NAMES = new Set([
|
|
192
194
|
".sync-state.json",
|
|
193
195
|
]);
|
|
@@ -205,6 +207,11 @@ function sha256Buffer(buffer: Buffer): { sha256: string; bytes: number } {
|
|
|
205
207
|
return sha256Bytes(buffer);
|
|
206
208
|
}
|
|
207
209
|
|
|
210
|
+
function throwIfOfflineSyncAborted(signal: AbortSignal | undefined): void {
|
|
211
|
+
if (!signal?.aborted) return;
|
|
212
|
+
throw new Error("offline sync request aborted");
|
|
213
|
+
}
|
|
214
|
+
|
|
208
215
|
function compareByPath<T extends { path: string }>(left: T, right: T): number {
|
|
209
216
|
return left.path.localeCompare(right.path);
|
|
210
217
|
}
|
|
@@ -480,6 +487,7 @@ const REMOTE_AUTHORITATIVE_RUNTIME_STATE_FILES = new Set([
|
|
|
480
487
|
"buffer.json",
|
|
481
488
|
"embeddings.json",
|
|
482
489
|
"index_time.json",
|
|
490
|
+
"last_intent.json",
|
|
483
491
|
"last_recall.json",
|
|
484
492
|
"memory-lifecycle-ledger.jsonl",
|
|
485
493
|
"recall_impressions.jsonl",
|
|
@@ -508,22 +516,23 @@ function canReuseFastBaseFileState(
|
|
|
508
516
|
return false;
|
|
509
517
|
}
|
|
510
518
|
if (baseCapturedAtMs === null) return false;
|
|
511
|
-
|
|
519
|
+
// Node reports stat times as fractional milliseconds while Date snapshots are
|
|
520
|
+
// whole milliseconds, so allow only a tiny precision window around capture.
|
|
521
|
+
return st.ctimeMs - baseCapturedAtMs <= OFFLINE_SYNC_FAST_BASE_CTIME_TOLERANCE_MS;
|
|
512
522
|
}
|
|
513
523
|
|
|
514
524
|
async function canReuseFastBaseFileStateFromDisk(
|
|
515
525
|
baseEntry: OfflineSyncFileState,
|
|
516
|
-
filePath: string,
|
|
517
526
|
st: { size: number; mtimeMs: number; ctimeMs: number },
|
|
518
527
|
baseCapturedAtMs: number | null,
|
|
519
528
|
): Promise<boolean> {
|
|
520
|
-
|
|
521
|
-
return !(await fileIsSecureStoreEncrypted(filePath).catch(() => true));
|
|
529
|
+
return canReuseFastBaseFileState(baseEntry, st, baseCapturedAtMs);
|
|
522
530
|
}
|
|
523
531
|
|
|
524
532
|
async function readOfflineSyncFileRecord(
|
|
525
533
|
options: OfflineSyncFileRecordOptions,
|
|
526
534
|
): Promise<OfflineSyncFileRecord> {
|
|
535
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
527
536
|
const relPath = validateArchiveRelativePath(options.relPath, "offlineSyncFile.path");
|
|
528
537
|
let content: Buffer | null = null;
|
|
529
538
|
let digest: OfflineSyncFileDigest;
|
|
@@ -531,16 +540,20 @@ async function readOfflineSyncFileRecord(
|
|
|
531
540
|
content = options.readFile
|
|
532
541
|
? await options.readFile({ root: options.root.abs, path: relPath, filePath: options.filePath })
|
|
533
542
|
: await readFile(options.filePath);
|
|
543
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
534
544
|
digest = sha256Buffer(content);
|
|
535
545
|
} else if (options.readFileDigest) {
|
|
536
546
|
digest = await options.readFileDigest({ root: options.root.abs, path: relPath, filePath: options.filePath });
|
|
547
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
537
548
|
} else if (options.readFile) {
|
|
538
549
|
content = await options.readFile({ root: options.root.abs, path: relPath, filePath: options.filePath });
|
|
550
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
539
551
|
digest = sha256Buffer(content);
|
|
540
552
|
content = null;
|
|
541
553
|
} else {
|
|
542
|
-
digest = await sha256File(options.filePath);
|
|
554
|
+
digest = await sha256File(options.filePath, options.signal);
|
|
543
555
|
}
|
|
556
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
544
557
|
const st = await stat(options.filePath);
|
|
545
558
|
return {
|
|
546
559
|
path: relPath,
|
|
@@ -551,14 +564,16 @@ async function readOfflineSyncFileRecord(
|
|
|
551
564
|
};
|
|
552
565
|
}
|
|
553
566
|
|
|
554
|
-
async function sha256File(filePath: string): Promise<OfflineSyncFileDigest> {
|
|
567
|
+
async function sha256File(filePath: string, signal?: AbortSignal): Promise<OfflineSyncFileDigest> {
|
|
555
568
|
const hash = createHash("sha256");
|
|
556
569
|
let bytes = 0;
|
|
557
570
|
for await (const chunk of createReadStream(filePath)) {
|
|
571
|
+
throwIfOfflineSyncAborted(signal);
|
|
558
572
|
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
559
573
|
hash.update(buffer);
|
|
560
574
|
bytes += buffer.length;
|
|
561
575
|
}
|
|
576
|
+
throwIfOfflineSyncAborted(signal);
|
|
562
577
|
return {
|
|
563
578
|
sha256: hash.digest("hex"),
|
|
564
579
|
bytes,
|
|
@@ -600,15 +615,19 @@ export async function* iterateOfflineSyncSnapshotFileRecords(options: {
|
|
|
600
615
|
includeTranscripts?: boolean;
|
|
601
616
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
602
617
|
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
618
|
+
signal?: AbortSignal;
|
|
603
619
|
}): AsyncIterable<OfflineSyncFileRecord> {
|
|
620
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
604
621
|
const rootAbs = path.resolve(options.root);
|
|
605
622
|
const root = await prepareSafeArchiveRoot(rootAbs, "iterateOfflineSyncSnapshotFileRecords", "root");
|
|
606
623
|
const includeTranscripts = options.includeTranscripts !== false;
|
|
607
624
|
|
|
608
625
|
async function* walk(dirAbs: string): AsyncIterable<OfflineSyncFileRecord> {
|
|
626
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
609
627
|
let entries = await readdir(dirAbs, { withFileTypes: true });
|
|
610
628
|
entries = entries.sort((left, right) => left.name.localeCompare(right.name));
|
|
611
629
|
for (const entry of entries) {
|
|
630
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
612
631
|
const abs = path.join(dirAbs, entry.name);
|
|
613
632
|
const relPosix = path.relative(root.abs, abs).split(path.sep).join("/");
|
|
614
633
|
if (shouldExcludeRelPath(relPosix, includeTranscripts)) continue;
|
|
@@ -625,6 +644,7 @@ export async function* iterateOfflineSyncSnapshotFileRecords(options: {
|
|
|
625
644
|
includeContent: options.includeContent === true,
|
|
626
645
|
readFile: options.readFile,
|
|
627
646
|
readFileDigest: options.readFileDigest,
|
|
647
|
+
signal: options.signal,
|
|
628
648
|
});
|
|
629
649
|
}
|
|
630
650
|
}
|
|
@@ -640,10 +660,13 @@ export async function buildOfflineSyncSnapshot(options: {
|
|
|
640
660
|
now?: Date;
|
|
641
661
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
642
662
|
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
663
|
+
signal?: AbortSignal;
|
|
643
664
|
}): Promise<OfflineSyncSnapshot> {
|
|
665
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
644
666
|
const includeTranscripts = options.includeTranscripts !== false;
|
|
645
667
|
const files: OfflineSyncFileRecord[] = [];
|
|
646
668
|
for await (const file of iterateOfflineSyncSnapshotFileRecords(options)) files.push(file);
|
|
669
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
647
670
|
|
|
648
671
|
return {
|
|
649
672
|
format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
|
|
@@ -665,7 +688,9 @@ export async function buildOfflineSyncSnapshotFromBase(options: {
|
|
|
665
688
|
now?: Date;
|
|
666
689
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
667
690
|
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
691
|
+
signal?: AbortSignal;
|
|
668
692
|
}): Promise<OfflineSyncSnapshot> {
|
|
693
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
669
694
|
const rootAbs = path.resolve(options.root);
|
|
670
695
|
const root = await prepareSafeArchiveRoot(rootAbs, "buildOfflineSyncSnapshotFromBase", "root");
|
|
671
696
|
const includeTranscripts = options.includeTranscripts !== false;
|
|
@@ -680,9 +705,11 @@ export async function buildOfflineSyncSnapshotFromBase(options: {
|
|
|
680
705
|
const files: OfflineSyncFileRecord[] = [];
|
|
681
706
|
|
|
682
707
|
async function walk(dirAbs: string): Promise<void> {
|
|
708
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
683
709
|
let entries = await readdir(dirAbs, { withFileTypes: true });
|
|
684
710
|
entries = entries.sort((left, right) => left.name.localeCompare(right.name));
|
|
685
711
|
for (const entry of entries) {
|
|
712
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
686
713
|
const abs = path.join(dirAbs, entry.name);
|
|
687
714
|
const relPosix = path.relative(root.abs, abs).split(path.sep).join("/");
|
|
688
715
|
if (shouldExcludeRelPath(relPosix, includeTranscripts)) continue;
|
|
@@ -698,7 +725,7 @@ export async function buildOfflineSyncSnapshotFromBase(options: {
|
|
|
698
725
|
options.includeContent !== true &&
|
|
699
726
|
baseEntry &&
|
|
700
727
|
baseCapturedAtMs !== null &&
|
|
701
|
-
await canReuseFastBaseFileStateFromDisk(baseEntry,
|
|
728
|
+
await canReuseFastBaseFileStateFromDisk(baseEntry, st, baseCapturedAtMs)
|
|
702
729
|
) {
|
|
703
730
|
files.push(baseEntry);
|
|
704
731
|
continue;
|
|
@@ -710,11 +737,13 @@ export async function buildOfflineSyncSnapshotFromBase(options: {
|
|
|
710
737
|
includeContent: options.includeContent === true,
|
|
711
738
|
readFile: options.readFile,
|
|
712
739
|
readFileDigest: options.readFileDigest,
|
|
740
|
+
signal: options.signal,
|
|
713
741
|
}));
|
|
714
742
|
}
|
|
715
743
|
}
|
|
716
744
|
|
|
717
745
|
await walk(root.abs);
|
|
746
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
718
747
|
|
|
719
748
|
return {
|
|
720
749
|
format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
|
|
@@ -735,7 +764,9 @@ export async function buildOfflineSyncSnapshotForPaths(options: {
|
|
|
735
764
|
now?: Date;
|
|
736
765
|
readFile?: (target: OfflineSyncFileTarget) => Promise<Buffer>;
|
|
737
766
|
readFileDigest?: (target: OfflineSyncFileTarget) => Promise<OfflineSyncFileDigest>;
|
|
767
|
+
signal?: AbortSignal;
|
|
738
768
|
}): Promise<OfflineSyncSnapshot> {
|
|
769
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
739
770
|
const rootAbs = path.resolve(options.root);
|
|
740
771
|
const root = await prepareSafeArchiveRoot(rootAbs, "buildOfflineSyncSnapshotForPaths", "root");
|
|
741
772
|
const includeTranscripts = options.includeTranscripts !== false;
|
|
@@ -743,6 +774,7 @@ export async function buildOfflineSyncSnapshotForPaths(options: {
|
|
|
743
774
|
const seen = new Set<string>();
|
|
744
775
|
|
|
745
776
|
for (const rawPath of options.paths) {
|
|
777
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
746
778
|
const relPath = normalizeRelativePath(rawPath, "paths[]");
|
|
747
779
|
if (seen.has(relPath)) continue;
|
|
748
780
|
seen.add(relPath);
|
|
@@ -762,8 +794,10 @@ export async function buildOfflineSyncSnapshotForPaths(options: {
|
|
|
762
794
|
includeContent: options.includeContent === true,
|
|
763
795
|
readFile: options.readFile,
|
|
764
796
|
readFileDigest: options.readFileDigest,
|
|
797
|
+
signal: options.signal,
|
|
765
798
|
}));
|
|
766
799
|
}
|
|
800
|
+
throwIfOfflineSyncAborted(options.signal);
|
|
767
801
|
|
|
768
802
|
return {
|
|
769
803
|
format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
|
package/src/operator-toolkit.ts
CHANGED
|
@@ -87,7 +87,10 @@ function resolveOpenClawRemnicPluginEntry(raw: unknown): Record<string, unknown>
|
|
|
87
87
|
interface QmdRuntimeLike {
|
|
88
88
|
probe(): Promise<boolean>;
|
|
89
89
|
isAvailable(): boolean;
|
|
90
|
-
ensureCollection(
|
|
90
|
+
ensureCollection(
|
|
91
|
+
memoryDir: string,
|
|
92
|
+
execution?: { signal?: AbortSignal },
|
|
93
|
+
): Promise<"present" | "missing" | "unknown" | "skipped">;
|
|
91
94
|
debugStatus(): string;
|
|
92
95
|
}
|
|
93
96
|
|
package/src/orchestrator.ts
CHANGED
|
@@ -680,6 +680,56 @@ async function raceRecallAbort<T>(
|
|
|
680
680
|
|
|
681
681
|
/** Maximum age (ms) before a compaction-reset signal file is considered stale and removed. */
|
|
682
682
|
const COMPACTION_SIGNAL_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
683
|
+
const DEFAULT_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS = 10_000;
|
|
684
|
+
|
|
685
|
+
type SearchCollectionState = "present" | "missing" | "unknown" | "skipped";
|
|
686
|
+
|
|
687
|
+
function qmdStartupCollectionCheckTimeoutMs(): number {
|
|
688
|
+
const raw =
|
|
689
|
+
process.env.REMNIC_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS ??
|
|
690
|
+
process.env.ENGRAM_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS;
|
|
691
|
+
if (raw === undefined) return DEFAULT_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS;
|
|
692
|
+
const parsed = Number(raw);
|
|
693
|
+
return Number.isFinite(parsed) && parsed >= 1_000
|
|
694
|
+
? Math.floor(parsed)
|
|
695
|
+
: DEFAULT_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
async function qmdStartupCollectionCheckWithTimeout(
|
|
699
|
+
promise: Promise<SearchCollectionState>,
|
|
700
|
+
controller: AbortController,
|
|
701
|
+
label: string,
|
|
702
|
+
): Promise<SearchCollectionState> {
|
|
703
|
+
const timeoutMs = qmdStartupCollectionCheckTimeoutMs();
|
|
704
|
+
let timer: NodeJS.Timeout | undefined;
|
|
705
|
+
let settled = false;
|
|
706
|
+
|
|
707
|
+
const timeoutPromise = new Promise<SearchCollectionState>((resolve) => {
|
|
708
|
+
timer = setTimeout(() => {
|
|
709
|
+
if (settled) return;
|
|
710
|
+
controller.abort();
|
|
711
|
+
log.warn(
|
|
712
|
+
`QMD startup collection check for ${label} timed out after ${timeoutMs}ms; keeping search enabled fail-open`,
|
|
713
|
+
);
|
|
714
|
+
resolve("unknown");
|
|
715
|
+
}, timeoutMs);
|
|
716
|
+
timer.unref?.();
|
|
717
|
+
});
|
|
718
|
+
|
|
719
|
+
const checkedPromise = promise
|
|
720
|
+
.catch((err): SearchCollectionState => {
|
|
721
|
+
log.warn(
|
|
722
|
+
`QMD startup collection check for ${label} failed; keeping search enabled fail-open: ${err}`,
|
|
723
|
+
);
|
|
724
|
+
return "unknown";
|
|
725
|
+
})
|
|
726
|
+
.finally(() => {
|
|
727
|
+
settled = true;
|
|
728
|
+
if (timer) clearTimeout(timer);
|
|
729
|
+
});
|
|
730
|
+
|
|
731
|
+
return await Promise.race([checkedPromise, timeoutPromise]);
|
|
732
|
+
}
|
|
683
733
|
|
|
684
734
|
/** Default workspace directory when no per-agent or config workspace is available. */
|
|
685
735
|
export function defaultWorkspaceDir(): string {
|
|
@@ -2453,14 +2503,22 @@ export class Orchestrator {
|
|
|
2453
2503
|
? this.configuredNamespaces()
|
|
2454
2504
|
: [this.config.defaultNamespace];
|
|
2455
2505
|
const states = await Promise.all(
|
|
2456
|
-
namespaces.map(async (namespace) =>
|
|
2457
|
-
|
|
2458
|
-
state
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2506
|
+
namespaces.map(async (namespace) => {
|
|
2507
|
+
const collectionCheckAbort = new AbortController();
|
|
2508
|
+
const state = await qmdStartupCollectionCheckWithTimeout(
|
|
2509
|
+
this.config.namespacesEnabled
|
|
2510
|
+
? this.namespaceSearchRouter.ensureNamespaceCollection(
|
|
2511
|
+
namespace,
|
|
2512
|
+
{ signal: collectionCheckAbort.signal },
|
|
2513
|
+
)
|
|
2514
|
+
: this.qmd.ensureCollection(this.config.memoryDir, {
|
|
2515
|
+
signal: collectionCheckAbort.signal,
|
|
2516
|
+
}),
|
|
2517
|
+
collectionCheckAbort,
|
|
2518
|
+
namespace,
|
|
2519
|
+
);
|
|
2520
|
+
return { namespace, state };
|
|
2521
|
+
}),
|
|
2464
2522
|
);
|
|
2465
2523
|
const defaultState =
|
|
2466
2524
|
states.find(
|
|
@@ -2800,8 +2858,8 @@ export class Orchestrator {
|
|
|
2800
2858
|
namespaces.map(async (namespace) => ({
|
|
2801
2859
|
namespace,
|
|
2802
2860
|
state: this.config.namespacesEnabled
|
|
2803
|
-
? await this.namespaceSearchRouter.ensureNamespaceCollection(namespace)
|
|
2804
|
-
: await this.qmd.ensureCollection(this.config.memoryDir),
|
|
2861
|
+
? await this.namespaceSearchRouter.ensureNamespaceCollection(namespace, { signal })
|
|
2862
|
+
: await this.qmd.ensureCollection(this.config.memoryDir, { signal }),
|
|
2805
2863
|
})),
|
|
2806
2864
|
);
|
|
2807
2865
|
|
package/src/qmd.ts
CHANGED
|
@@ -2595,12 +2595,15 @@ export class QmdClient implements SearchBackend {
|
|
|
2595
2595
|
}
|
|
2596
2596
|
}
|
|
2597
2597
|
|
|
2598
|
-
async ensureCollection(
|
|
2598
|
+
async ensureCollection(
|
|
2599
|
+
memoryDir: string,
|
|
2600
|
+
execution?: SearchExecutionOptions,
|
|
2601
|
+
): Promise<"present" | "missing" | "unknown" | "skipped"> {
|
|
2599
2602
|
if (this.available === false && !this.daemonAvailable) return "unknown";
|
|
2600
2603
|
// If only daemon is available (no CLI), skip collection check
|
|
2601
2604
|
if (this.available === false) return "skipped";
|
|
2602
2605
|
try {
|
|
2603
|
-
const { stdout } = await this.runQmdCommand(["collection", "list"], QMD_TIMEOUT_MS);
|
|
2606
|
+
const { stdout } = await this.runQmdCommand(["collection", "list"], QMD_TIMEOUT_MS, execution?.signal);
|
|
2604
2607
|
// Parse text output: "openclaw-engram (qmd://openclaw-engram/)"
|
|
2605
2608
|
const collectionRegex = new RegExp(
|
|
2606
2609
|
`^${this.collection}\\s+\\(qmd://`,
|
|
@@ -218,7 +218,10 @@ export class LanceDbBackend implements SearchBackend {
|
|
|
218
218
|
}
|
|
219
219
|
}
|
|
220
220
|
|
|
221
|
-
async ensureCollection(
|
|
221
|
+
async ensureCollection(
|
|
222
|
+
_memoryDir: string,
|
|
223
|
+
_execution?: SearchExecutionOptions,
|
|
224
|
+
): Promise<"present" | "missing" | "unknown" | "skipped"> {
|
|
222
225
|
try {
|
|
223
226
|
await this.ensureTable();
|
|
224
227
|
return "present";
|
|
@@ -189,7 +189,10 @@ export class MeilisearchBackend implements SearchBackend {
|
|
|
189
189
|
// manages embeddings server-side per index (collection).
|
|
190
190
|
}
|
|
191
191
|
|
|
192
|
-
async ensureCollection(
|
|
192
|
+
async ensureCollection(
|
|
193
|
+
_memoryDir: string,
|
|
194
|
+
_execution?: SearchExecutionOptions,
|
|
195
|
+
): Promise<"present" | "missing" | "unknown" | "skipped"> {
|
|
193
196
|
if (!this.available) return "skipped";
|
|
194
197
|
try {
|
|
195
198
|
const client = await this.ensureClient();
|
|
@@ -51,7 +51,7 @@ export class NoopSearchBackend implements SearchBackend {
|
|
|
51
51
|
async embed(): Promise<void> {}
|
|
52
52
|
async embedCollection(_collection: string): Promise<void> {}
|
|
53
53
|
|
|
54
|
-
async ensureCollection(_memoryDir: string): Promise<"skipped"> {
|
|
54
|
+
async ensureCollection(_memoryDir: string, _execution?: SearchExecutionOptions): Promise<"skipped"> {
|
|
55
55
|
return "skipped";
|
|
56
56
|
}
|
|
57
57
|
}
|
|
@@ -251,7 +251,10 @@ export class OramaBackend implements SearchBackend {
|
|
|
251
251
|
await this.persistDbForCollection(db, collection);
|
|
252
252
|
}
|
|
253
253
|
|
|
254
|
-
async ensureCollection(
|
|
254
|
+
async ensureCollection(
|
|
255
|
+
_memoryDir: string,
|
|
256
|
+
_execution?: SearchExecutionOptions,
|
|
257
|
+
): Promise<"present" | "missing" | "unknown" | "skipped"> {
|
|
255
258
|
try {
|
|
256
259
|
await this.ensureModules();
|
|
257
260
|
await this.ensureDb();
|
package/src/search/port.ts
CHANGED
|
@@ -82,5 +82,8 @@ export interface SearchBackend {
|
|
|
82
82
|
embedCollection(collection: string): Promise<void>;
|
|
83
83
|
|
|
84
84
|
// ── Collection management ──
|
|
85
|
-
ensureCollection(
|
|
85
|
+
ensureCollection(
|
|
86
|
+
memoryDir: string,
|
|
87
|
+
execution?: SearchExecutionOptions,
|
|
88
|
+
): Promise<"present" | "missing" | "unknown" | "skipped">;
|
|
86
89
|
}
|
|
@@ -82,7 +82,7 @@ export class RemoteSearchBackend implements SearchBackend {
|
|
|
82
82
|
async embed(): Promise<void> {}
|
|
83
83
|
async embedCollection(_collection: string): Promise<void> {}
|
|
84
84
|
|
|
85
|
-
async ensureCollection(_memoryDir: string): Promise<"skipped"> {
|
|
85
|
+
async ensureCollection(_memoryDir: string, _execution?: SearchExecutionOptions): Promise<"skipped"> {
|
|
86
86
|
return "skipped";
|
|
87
87
|
}
|
|
88
88
|
|