@xdarkicex/openclaw-memory-libravdb 1.5.4 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +19 -21
- package/dist/context-engine.d.ts +3 -3
- package/dist/context-engine.js +19 -123
- package/dist/dream-promotion.d.ts +4 -6
- package/dist/dream-promotion.js +22 -13
- package/dist/index.js +25344 -29086
- package/dist/ingest-queue.d.ts +26 -3
- package/dist/ingest-queue.js +44 -23
- package/dist/libravdb-client.d.ts +59 -0
- package/dist/libravdb-client.js +296 -0
- package/dist/markdown-ingest.d.ts +10 -5
- package/dist/markdown-ingest.js +264 -32
- package/dist/memory-provider.d.ts +2 -2
- package/dist/memory-provider.js +1 -1
- package/dist/memory-runtime.d.ts +4 -33
- package/dist/memory-runtime.js +40 -51
- package/dist/plugin-runtime.d.ts +4 -6
- package/dist/plugin-runtime.js +33 -72
- package/dist/types.d.ts +3 -21
- package/openclaw.plugin.json +15 -1
- package/package.json +7 -4
- package/dist/grpc-client.d.ts +0 -44
- package/dist/grpc-client.js +0 -188
- package/dist/rpc-protobuf-codecs.d.ts +0 -71
- package/dist/rpc-protobuf-codecs.js +0 -91
- package/dist/rpc.d.ts +0 -15
- package/dist/rpc.js +0 -203
- package/dist/sidecar.d.ts +0 -40
- package/dist/sidecar.js +0 -588
package/dist/markdown-ingest.js
CHANGED
|
@@ -2,14 +2,14 @@ import fs from "node:fs";
|
|
|
2
2
|
import fsp from "node:fs/promises";
|
|
3
3
|
import os from "node:os";
|
|
4
4
|
import path from "node:path";
|
|
5
|
-
import { hashBytes } from "./markdown-hash.js";
|
|
6
5
|
import { formatError } from "./format-error.js";
|
|
7
6
|
import { IngestQueue } from "./ingest-queue.js";
|
|
8
7
|
const DEFAULT_DEBOUNCE_MS = 150;
|
|
9
8
|
const DEFAULT_TOKENIZER_ID = "markdown-ingest:v1";
|
|
10
9
|
const MARKDOWN_INGEST_VERSION = 3;
|
|
11
10
|
const HASH_BACKEND = "wasm-fnv1a64";
|
|
12
|
-
|
|
11
|
+
const STREAM_CHUNK_BYTES = 64 * 1024;
|
|
12
|
+
export function createMarkdownIngestionHandle(cfg, getClient, logger = console, fsApi = createRealFsApi()) {
|
|
13
13
|
const adapters = [];
|
|
14
14
|
const genericRoots = normalizeMarkdownRoots(cfg.markdownIngestionRoots);
|
|
15
15
|
if (isMarkdownIngestionEnabled(cfg, genericRoots)) {
|
|
@@ -19,7 +19,9 @@ export function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsA
|
|
|
19
19
|
exclude: cfg.markdownIngestionExclude,
|
|
20
20
|
debounceMs: cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS,
|
|
21
21
|
snapshotPath: resolveMarkdownSnapshotPath("generic", cfg.markdownIngestionSnapshotPath),
|
|
22
|
-
|
|
22
|
+
priorityMode: cfg.markdownIngestionPriorityMode,
|
|
23
|
+
maxTokensPerFile: cfg.markdownIngestionMaxTokensPerFile,
|
|
24
|
+
}, getClient, logger, fsApi));
|
|
23
25
|
}
|
|
24
26
|
const obsidianRoots = normalizeMarkdownRoots(cfg.markdownIngestionObsidianRoots);
|
|
25
27
|
if (cfg.markdownIngestionObsidianEnabled === true && obsidianRoots.length > 0) {
|
|
@@ -29,7 +31,9 @@ export function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsA
|
|
|
29
31
|
exclude: cfg.markdownIngestionObsidianExclude,
|
|
30
32
|
debounceMs: cfg.markdownIngestionObsidianDebounceMs ?? cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS,
|
|
31
33
|
snapshotPath: resolveMarkdownSnapshotPath("obsidian", cfg.markdownIngestionObsidianSnapshotPath),
|
|
32
|
-
|
|
34
|
+
priorityMode: cfg.markdownIngestionPriorityMode,
|
|
35
|
+
maxTokensPerFile: cfg.markdownIngestionMaxTokensPerFile,
|
|
36
|
+
}, getClient, logger, fsApi));
|
|
33
37
|
}
|
|
34
38
|
if (adapters.length === 0) {
|
|
35
39
|
return {
|
|
@@ -74,9 +78,11 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
74
78
|
excludePatterns;
|
|
75
79
|
debounceMs;
|
|
76
80
|
fsApi;
|
|
77
|
-
|
|
81
|
+
getClient;
|
|
78
82
|
logger;
|
|
79
83
|
snapshotPath;
|
|
84
|
+
priorityMode;
|
|
85
|
+
maxTokensPerFile;
|
|
80
86
|
states = new Map();
|
|
81
87
|
fileStates = new Map();
|
|
82
88
|
activeScans = new Set();
|
|
@@ -85,18 +91,31 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
85
91
|
started = false;
|
|
86
92
|
ingestQueue = null;
|
|
87
93
|
stopping = false;
|
|
94
|
+
lastAcceptMore = true;
|
|
95
|
+
lastRetryAfterMs = 0;
|
|
96
|
+
lastQueueDepth = 0;
|
|
97
|
+
lastQueueCapacity = 0;
|
|
98
|
+
lastProcessingTimeUs = 0;
|
|
99
|
+
lastNodesAccepted = 0;
|
|
100
|
+
lastNodesRejected = 0;
|
|
101
|
+
lastTokensIngested = 0;
|
|
102
|
+
lastTokenBurstLimit = 512;
|
|
103
|
+
lastWalDepth = 0;
|
|
104
|
+
lastWalCapacity = 0;
|
|
88
105
|
snapshotLoaded = false;
|
|
89
106
|
snapshotDirty = false;
|
|
90
|
-
constructor(kind, config,
|
|
107
|
+
constructor(kind, config, getClient, logger, fsApi) {
|
|
91
108
|
this.kind = kind;
|
|
92
109
|
this.roots = config.roots;
|
|
93
110
|
this.includePatterns = config.include?.length ? config.include : [];
|
|
94
111
|
this.excludePatterns = config.exclude?.length ? config.exclude : [];
|
|
95
112
|
this.debounceMs = config.debounceMs ?? DEFAULT_DEBOUNCE_MS;
|
|
96
113
|
this.fsApi = fsApi;
|
|
97
|
-
this.
|
|
114
|
+
this.getClient = getClient;
|
|
98
115
|
this.logger = logger;
|
|
99
116
|
this.snapshotPath = config.snapshotPath ?? resolveMarkdownSnapshotPath(kind);
|
|
117
|
+
this.priorityMode = config.priorityMode ?? "mtime";
|
|
118
|
+
this.maxTokensPerFile = Math.max(1, Math.trunc(config.maxTokensPerFile ?? 128_000));
|
|
100
119
|
this.tokenizerId = DEFAULT_TOKENIZER_ID;
|
|
101
120
|
this.coreDoc = true;
|
|
102
121
|
}
|
|
@@ -150,6 +169,7 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
150
169
|
scanning: false,
|
|
151
170
|
dirty: false,
|
|
152
171
|
timer: null,
|
|
172
|
+
resumeFromPath: null,
|
|
153
173
|
},
|
|
154
174
|
knownFiles: this.snapshotFilesForRoot(resolved),
|
|
155
175
|
directoryWatchers: new Map(),
|
|
@@ -167,12 +187,16 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
167
187
|
return;
|
|
168
188
|
}
|
|
169
189
|
rootState.scanState.scanning = true;
|
|
190
|
+
this.lastAcceptMore = true;
|
|
191
|
+
this.lastRetryAfterMs = 0;
|
|
170
192
|
const scan = (async () => {
|
|
171
193
|
const stats = createScanStats();
|
|
172
194
|
const startedAt = Date.now();
|
|
173
195
|
try {
|
|
174
196
|
const currentFiles = new Set();
|
|
175
|
-
|
|
197
|
+
const candidates = [];
|
|
198
|
+
await this.walkDirectory(rootState, rootState.root, currentFiles, stats, candidates);
|
|
199
|
+
await this.syncCandidates(rootState, candidates, stats);
|
|
176
200
|
if (!this.stopping) {
|
|
177
201
|
await this.pruneDeletedFiles(rootState, currentFiles, stats);
|
|
178
202
|
rootState.knownFiles = currentFiles;
|
|
@@ -198,7 +222,7 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
198
222
|
this.activeScans.delete(scan);
|
|
199
223
|
}
|
|
200
224
|
}
|
|
201
|
-
scheduleRootScan(rootState) {
|
|
225
|
+
scheduleRootScan(rootState, delayMs) {
|
|
202
226
|
if (!this.started || this.stopping) {
|
|
203
227
|
return;
|
|
204
228
|
}
|
|
@@ -214,9 +238,9 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
214
238
|
void this.scanRoot(rootState.root).catch((error) => {
|
|
215
239
|
this.logger.warn?.(`[markdown-ingest] root scan failed for ${rootState.root}: ${formatError(error)}`);
|
|
216
240
|
});
|
|
217
|
-
}, this.debounceMs);
|
|
241
|
+
}, Math.max(this.debounceMs, delayMs ?? 0));
|
|
218
242
|
}
|
|
219
|
-
async walkDirectory(rootState, dir, currentFiles, stats) {
|
|
243
|
+
async walkDirectory(rootState, dir, currentFiles, stats, candidates) {
|
|
220
244
|
if (this.shouldPruneDirectory(rootState.root, dir)) {
|
|
221
245
|
stats.directoriesPruned++;
|
|
222
246
|
return;
|
|
@@ -240,7 +264,7 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
240
264
|
}
|
|
241
265
|
const child = path.join(dir, entry.name);
|
|
242
266
|
if (entry.isDirectory()) {
|
|
243
|
-
await this.walkDirectory(rootState, child, currentFiles, stats);
|
|
267
|
+
await this.walkDirectory(rootState, child, currentFiles, stats, candidates);
|
|
244
268
|
continue;
|
|
245
269
|
}
|
|
246
270
|
if (!entry.isFile() || !isMarkdownFile(entry.name)) {
|
|
@@ -253,17 +277,74 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
253
277
|
}
|
|
254
278
|
stats.filesIncluded++;
|
|
255
279
|
currentFiles.add(child);
|
|
280
|
+
const stat = await this.safeStatWithCtime(child);
|
|
281
|
+
if (!stat) {
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
candidates.push({ path: child, size: stat.size, mtimeMs: stat.mtimeMs, ctimeMs: stat.ctimeMs, ordinal: candidates.length });
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
async syncCandidates(rootState, candidates, stats) {
|
|
288
|
+
const sorted = sortCandidates(candidates, this.priorityMode);
|
|
289
|
+
let skipping = false;
|
|
290
|
+
if (rootState.scanState.resumeFromPath) {
|
|
291
|
+
const targetExists = sorted.some((c) => c.path === rootState.scanState.resumeFromPath);
|
|
292
|
+
if (targetExists) {
|
|
293
|
+
skipping = true;
|
|
294
|
+
this.lastAcceptMore = true;
|
|
295
|
+
this.lastRetryAfterMs = 0;
|
|
296
|
+
}
|
|
297
|
+
else {
|
|
298
|
+
rootState.scanState.resumeFromPath = null;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
for (const candidate of sorted) {
|
|
302
|
+
if (skipping) {
|
|
303
|
+
if (candidate.path === rootState.scanState.resumeFromPath) {
|
|
304
|
+
skipping = false;
|
|
305
|
+
}
|
|
306
|
+
else {
|
|
307
|
+
continue;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
if (this.stopping) {
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
if (!this.lastAcceptMore) {
|
|
314
|
+
if (!this.stopping) {
|
|
315
|
+
rootState.scanState.resumeFromPath = candidate.path;
|
|
316
|
+
this.scheduleRootScan(rootState, this.lastRetryAfterMs);
|
|
317
|
+
}
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
if (this.lastWalCapacity > 0 && this.lastWalDepth > this.lastWalCapacity * 0.8) {
|
|
321
|
+
rootState.scanState.resumeFromPath = candidate.path;
|
|
322
|
+
if (!this.stopping) {
|
|
323
|
+
this.scheduleRootScan(rootState, 2000);
|
|
324
|
+
}
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
const estimatedTokens = estimateTokens(candidate.size);
|
|
328
|
+
if (estimatedTokens > this.maxTokensPerFile) {
|
|
329
|
+
stats.filesDeferred++;
|
|
330
|
+
continue;
|
|
331
|
+
}
|
|
256
332
|
try {
|
|
257
|
-
const result = await this.syncMarkdownFile(rootState,
|
|
333
|
+
const result = await this.syncMarkdownFile(rootState, candidate.path, {
|
|
334
|
+
size: candidate.size,
|
|
335
|
+
mtimeMs: candidate.mtimeMs,
|
|
336
|
+
ctimeMs: candidate.ctimeMs,
|
|
337
|
+
});
|
|
258
338
|
recordSyncResult(stats, result);
|
|
259
339
|
}
|
|
260
340
|
catch (error) {
|
|
261
341
|
stats.syncErrors++;
|
|
262
342
|
if (!this.stopping) {
|
|
263
|
-
this.logger.warn?.(`[markdown-ingest] sync failed for ${
|
|
343
|
+
this.logger.warn?.(`[markdown-ingest] sync failed for ${candidate.path}: ${formatError(error)}`);
|
|
264
344
|
}
|
|
265
345
|
}
|
|
266
346
|
}
|
|
347
|
+
rootState.scanState.resumeFromPath = null;
|
|
267
348
|
}
|
|
268
349
|
shouldPruneDirectory(root, dir) {
|
|
269
350
|
const relative = toPosixPath(path.relative(root, dir));
|
|
@@ -284,6 +365,11 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
284
365
|
try {
|
|
285
366
|
const watcher = this.fsApi.watch(dir, () => {
|
|
286
367
|
if (!this.stopping) {
|
|
368
|
+
rootState.scanState.resumeFromPath = null;
|
|
369
|
+
if (rootState.scanState.timer) {
|
|
370
|
+
clearTimeout(rootState.scanState.timer);
|
|
371
|
+
rootState.scanState.timer = null;
|
|
372
|
+
}
|
|
287
373
|
this.scheduleRootScan(rootState);
|
|
288
374
|
}
|
|
289
375
|
});
|
|
@@ -335,10 +421,10 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
335
421
|
stats.filesDeleted++;
|
|
336
422
|
}
|
|
337
423
|
}
|
|
338
|
-
async syncMarkdownFile(rootState, filePath) {
|
|
424
|
+
async syncMarkdownFile(rootState, filePath, initialStat) {
|
|
339
425
|
const sourceDoc = filePath;
|
|
340
426
|
const relativePath = toPosixPath(path.relative(rootState.root, filePath));
|
|
341
|
-
const stat = await this.
|
|
427
|
+
const stat = initialStat ?? (await this.safeStatWithCtime(filePath));
|
|
342
428
|
if (!stat) {
|
|
343
429
|
await this.deleteSourceDocument(sourceDoc);
|
|
344
430
|
this.fileStates.delete(sourceDoc);
|
|
@@ -349,14 +435,18 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
349
435
|
if (cached && cached.size === stat.size && cached.mtimeMs === stat.mtimeMs) {
|
|
350
436
|
return "unchanged";
|
|
351
437
|
}
|
|
352
|
-
const
|
|
353
|
-
|
|
438
|
+
const maxBytes = this.maxTokensPerFile * 4 + 3;
|
|
439
|
+
const streamed = await this.safeReadFileStreamed(filePath, maxBytes);
|
|
440
|
+
if (streamed === "too_large") {
|
|
441
|
+
return "skipped";
|
|
442
|
+
}
|
|
443
|
+
if (!streamed) {
|
|
354
444
|
await this.deleteSourceDocument(sourceDoc);
|
|
355
445
|
this.fileStates.delete(sourceDoc);
|
|
356
446
|
this.snapshotDirty = true;
|
|
357
447
|
return "deleted";
|
|
358
448
|
}
|
|
359
|
-
const fileHash =
|
|
449
|
+
const { text, fileHash } = streamed;
|
|
360
450
|
if (cached && cached.fileHash === fileHash) {
|
|
361
451
|
this.setFileState(sourceDoc, {
|
|
362
452
|
root: rootState.root,
|
|
@@ -368,14 +458,13 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
368
458
|
});
|
|
369
459
|
return "unchanged";
|
|
370
460
|
}
|
|
371
|
-
const text = textDecoder.decode(bytes);
|
|
372
461
|
if (this.kind === "obsidian" && this.includePatterns.length === 0 && !looksLikeObsidianNote(filePath, text)) {
|
|
373
462
|
await this.deleteSourceDocument(sourceDoc);
|
|
374
463
|
this.fileStates.delete(sourceDoc);
|
|
375
464
|
this.snapshotDirty = true;
|
|
376
465
|
return "skipped";
|
|
377
466
|
}
|
|
378
|
-
await this.ingestMarkdownDocument(sourceDoc, text, rootState.root, relativePath, fileHash, stat.size, stat.mtimeMs);
|
|
467
|
+
await this.ingestMarkdownDocument(sourceDoc, text, rootState.root, relativePath, fileHash, stat.size, stat.mtimeMs, stat.ctimeMs);
|
|
379
468
|
this.setFileState(sourceDoc, {
|
|
380
469
|
root: rootState.root,
|
|
381
470
|
sourceDoc,
|
|
@@ -390,9 +479,9 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
390
479
|
this.fileStates.set(sourceDoc, state);
|
|
391
480
|
this.snapshotDirty = true;
|
|
392
481
|
}
|
|
393
|
-
async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs) {
|
|
482
|
+
async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs, sourceCtimeMs) {
|
|
394
483
|
const queue = await this.getIngestQueue();
|
|
395
|
-
await queue.enqueueIngest(sourceDoc, text, {
|
|
484
|
+
const feedback = await queue.enqueueIngest(sourceDoc, text, {
|
|
396
485
|
tokenizerId: this.tokenizerId,
|
|
397
486
|
coreDoc: this.coreDoc,
|
|
398
487
|
sourceMeta: {
|
|
@@ -402,10 +491,44 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
402
491
|
fileHash,
|
|
403
492
|
sourceSize,
|
|
404
493
|
sourceMtimeMs: Math.trunc(sourceMtimeMs),
|
|
494
|
+
sourceCtimeMs: Math.trunc(sourceCtimeMs),
|
|
405
495
|
ingestVersion: MARKDOWN_INGEST_VERSION,
|
|
406
496
|
hashBackend: HASH_BACKEND,
|
|
407
497
|
},
|
|
408
|
-
});
|
|
498
|
+
}, this.lastTokenBurstLimit);
|
|
499
|
+
this.applyIngestFeedback(feedback);
|
|
500
|
+
}
|
|
501
|
+
applyIngestFeedback(feedback) {
|
|
502
|
+
if (feedback && typeof feedback.acceptMore === "boolean") {
|
|
503
|
+
this.lastAcceptMore = feedback.acceptMore;
|
|
504
|
+
this.lastQueueDepth = feedback.queueDepth ?? 0;
|
|
505
|
+
this.lastQueueCapacity = feedback.queueCapacity ?? 0;
|
|
506
|
+
this.lastProcessingTimeUs = feedback.processingTimeUs ?? 0;
|
|
507
|
+
this.lastNodesAccepted = feedback.nodesAccepted ?? 0;
|
|
508
|
+
this.lastNodesRejected = feedback.nodesRejected ?? 0;
|
|
509
|
+
this.lastTokensIngested = feedback.tokensIngested ?? 0;
|
|
510
|
+
if (feedback.tokenBurstLimit && feedback.tokenBurstLimit > 0) {
|
|
511
|
+
this.lastTokenBurstLimit = feedback.tokenBurstLimit;
|
|
512
|
+
}
|
|
513
|
+
this.lastWalDepth = feedback.walDepth ?? 0;
|
|
514
|
+
this.lastWalCapacity = feedback.walCapacity ?? 0;
|
|
515
|
+
if (feedback.acceptMore) {
|
|
516
|
+
this.lastRetryAfterMs = 0;
|
|
517
|
+
}
|
|
518
|
+
else {
|
|
519
|
+
this.lastRetryAfterMs = feedback.retryAfterMs || 1000;
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
else {
|
|
523
|
+
this.lastAcceptMore = true;
|
|
524
|
+
this.lastRetryAfterMs = 0;
|
|
525
|
+
this.lastQueueDepth = 0;
|
|
526
|
+
this.lastQueueCapacity = 0;
|
|
527
|
+
this.lastProcessingTimeUs = 0;
|
|
528
|
+
this.lastNodesAccepted = 0;
|
|
529
|
+
this.lastNodesRejected = 0;
|
|
530
|
+
this.lastTokensIngested = 0;
|
|
531
|
+
}
|
|
409
532
|
}
|
|
410
533
|
async deleteSourceDocument(sourceDoc) {
|
|
411
534
|
const queue = await this.getIngestQueue();
|
|
@@ -413,8 +536,42 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
413
536
|
}
|
|
414
537
|
async getIngestQueue() {
|
|
415
538
|
if (!this.ingestQueue) {
|
|
416
|
-
const
|
|
417
|
-
this.ingestQueue = new IngestQueue(
|
|
539
|
+
const client = await this.getClient();
|
|
540
|
+
this.ingestQueue = new IngestQueue((params) => client.ingestMarkdownDocument({
|
|
541
|
+
sourceDoc: params.sourceDoc,
|
|
542
|
+
text: params.text,
|
|
543
|
+
tokenizerId: params.tokenizerId,
|
|
544
|
+
coreDoc: params.coreDoc,
|
|
545
|
+
mode: params.mode,
|
|
546
|
+
sourceMeta: params.sourceMeta ? {
|
|
547
|
+
sourceRoot: params.sourceMeta.sourceRoot,
|
|
548
|
+
sourcePath: params.sourceMeta.sourcePath,
|
|
549
|
+
sourceKind: params.sourceMeta.sourceKind,
|
|
550
|
+
fileHash: params.sourceMeta.fileHash,
|
|
551
|
+
sourceSize: BigInt(params.sourceMeta.sourceSize),
|
|
552
|
+
sourceMtimeMs: BigInt(Math.trunc(params.sourceMeta.sourceMtimeMs)),
|
|
553
|
+
sourceCtimeMs: BigInt(Math.trunc(params.sourceMeta.sourceCtimeMs)),
|
|
554
|
+
ingestVersion: params.sourceMeta.ingestVersion,
|
|
555
|
+
hashBackend: params.sourceMeta.hashBackend,
|
|
556
|
+
} : undefined,
|
|
557
|
+
}).then((r) => ({
|
|
558
|
+
ok: r.ok,
|
|
559
|
+
feedback: r.feedback ? {
|
|
560
|
+
queueDepth: r.feedback.queueDepth,
|
|
561
|
+
queueCapacity: r.feedback.queueCapacity,
|
|
562
|
+
acceptMore: r.feedback.acceptMore,
|
|
563
|
+
retryAfterMs: r.feedback.retryAfterMs,
|
|
564
|
+
processingTimeUs: Number(r.feedback.processingTimeUs),
|
|
565
|
+
nodesAccepted: r.feedback.nodesAccepted,
|
|
566
|
+
nodesRejected: r.feedback.nodesRejected,
|
|
567
|
+
tokensIngested: r.feedback.tokensIngested,
|
|
568
|
+
tokenBurstLimit: r.feedback.tokenBurstLimit,
|
|
569
|
+
walDepth: r.feedback.walDepth,
|
|
570
|
+
walCapacity: r.feedback.walCapacity,
|
|
571
|
+
} : undefined,
|
|
572
|
+
})), (params) => client.deleteAuthoredDocument(params).then(() => undefined), this.logger, {
|
|
573
|
+
onChunkFeedback: (feedback) => this.applyIngestFeedback(feedback),
|
|
574
|
+
});
|
|
418
575
|
}
|
|
419
576
|
return this.ingestQueue;
|
|
420
577
|
}
|
|
@@ -426,14 +583,51 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
426
583
|
return null;
|
|
427
584
|
}
|
|
428
585
|
}
|
|
429
|
-
async
|
|
586
|
+
async safeStatWithCtime(filePath) {
|
|
430
587
|
try {
|
|
431
|
-
return await this.fsApi.
|
|
588
|
+
return await this.fsApi.stat(filePath);
|
|
432
589
|
}
|
|
433
590
|
catch {
|
|
434
591
|
return null;
|
|
435
592
|
}
|
|
436
593
|
}
|
|
594
|
+
async safeReadFileStreamed(filePath, maxBytes) {
|
|
595
|
+
let stream = null;
|
|
596
|
+
try {
|
|
597
|
+
stream = await this.fsApi.openReadStream(filePath);
|
|
598
|
+
const decoder = new TextDecoder();
|
|
599
|
+
const chunks = [];
|
|
600
|
+
let hash = 0xcbf29ce484222325n;
|
|
601
|
+
let total = 0;
|
|
602
|
+
const buffer = Buffer.allocUnsafe(STREAM_CHUNK_BYTES);
|
|
603
|
+
while (true) {
|
|
604
|
+
const { bytesRead } = await stream.read(buffer);
|
|
605
|
+
if (bytesRead === 0) {
|
|
606
|
+
break;
|
|
607
|
+
}
|
|
608
|
+
total += bytesRead;
|
|
609
|
+
if (total > maxBytes) {
|
|
610
|
+
return "too_large";
|
|
611
|
+
}
|
|
612
|
+
const chunk = buffer.subarray(0, bytesRead);
|
|
613
|
+
hash = updateFnv1a64(hash, chunk);
|
|
614
|
+
chunks.push(decoder.decode(chunk, { stream: true }));
|
|
615
|
+
}
|
|
616
|
+
chunks.push(decoder.decode());
|
|
617
|
+
return {
|
|
618
|
+
text: chunks.join(""),
|
|
619
|
+
fileHash: hash.toString(16).padStart(16, "0"),
|
|
620
|
+
};
|
|
621
|
+
}
|
|
622
|
+
catch {
|
|
623
|
+
return null;
|
|
624
|
+
}
|
|
625
|
+
finally {
|
|
626
|
+
if (stream) {
|
|
627
|
+
await stream.close().catch(() => { });
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
}
|
|
437
631
|
snapshotFilesForRoot(root) {
|
|
438
632
|
const files = new Set();
|
|
439
633
|
for (const state of this.fileStates.values()) {
|
|
@@ -497,7 +691,7 @@ class DirectoryMarkdownSourceAdapter {
|
|
|
497
691
|
}
|
|
498
692
|
}
|
|
499
693
|
logScanStats(root, stats, durationMs) {
|
|
500
|
-
this.logger.info?.(`[markdown-ingest] ${this.kind} scan complete root=${root} dirs=${stats.directoriesScanned} prunedDirs=${stats.directoriesPruned} markdown=${stats.markdownFilesSeen} included=${stats.filesIncluded} skipped=${stats.filesSkipped} unchanged=${stats.filesUnchanged} ingested=${stats.filesIngested} deleted=${stats.filesDeleted} errors=${stats.syncErrors} durationMs=${durationMs}`);
|
|
694
|
+
this.logger.info?.(`[markdown-ingest] ${this.kind} scan complete root=${root} dirs=${stats.directoriesScanned} prunedDirs=${stats.directoriesPruned} markdown=${stats.markdownFilesSeen} included=${stats.filesIncluded} skipped=${stats.filesSkipped} unchanged=${stats.filesUnchanged} ingested=${stats.filesIngested} deleted=${stats.filesDeleted} deferred=${stats.filesDeferred} errors=${stats.syncErrors} durationMs=${durationMs}`);
|
|
501
695
|
}
|
|
502
696
|
}
|
|
503
697
|
function createScanStats() {
|
|
@@ -511,8 +705,26 @@ function createScanStats() {
|
|
|
511
705
|
filesIngested: 0,
|
|
512
706
|
filesDeleted: 0,
|
|
513
707
|
syncErrors: 0,
|
|
708
|
+
filesDeferred: 0,
|
|
514
709
|
};
|
|
515
710
|
}
|
|
711
|
+
function estimateTokens(size) {
|
|
712
|
+
return Math.max(1, Math.floor(size / 4));
|
|
713
|
+
}
|
|
714
|
+
function sortCandidates(candidates, mode) {
|
|
715
|
+
return [...candidates].sort((left, right) => {
|
|
716
|
+
if (mode === "size") {
|
|
717
|
+
return right.size - left.size || left.ordinal - right.ordinal;
|
|
718
|
+
}
|
|
719
|
+
if (mode === "ctime") {
|
|
720
|
+
return right.ctimeMs - left.ctimeMs || left.ordinal - right.ordinal;
|
|
721
|
+
}
|
|
722
|
+
if (mode === "fifo") {
|
|
723
|
+
return left.ordinal - right.ordinal;
|
|
724
|
+
}
|
|
725
|
+
return right.mtimeMs - left.mtimeMs || left.ordinal - right.ordinal;
|
|
726
|
+
});
|
|
727
|
+
}
|
|
516
728
|
function recordSyncResult(stats, result) {
|
|
517
729
|
if (result === "ingested") {
|
|
518
730
|
stats.filesIngested++;
|
|
@@ -530,7 +742,6 @@ function recordSyncResult(stats, result) {
|
|
|
530
742
|
function toPosixPath(value) {
|
|
531
743
|
return value.split(path.sep).join("/");
|
|
532
744
|
}
|
|
533
|
-
const textDecoder = new TextDecoder();
|
|
534
745
|
function normalizeMarkdownRoots(roots) {
|
|
535
746
|
if (!roots?.length) {
|
|
536
747
|
return [];
|
|
@@ -545,6 +756,15 @@ function normalizeMarkdownRoots(roots) {
|
|
|
545
756
|
}
|
|
546
757
|
return [...resolved];
|
|
547
758
|
}
|
|
759
|
+
function updateFnv1a64(seed, bytes) {
|
|
760
|
+
let hash = seed;
|
|
761
|
+
const prime = 0x100000001b3n;
|
|
762
|
+
for (let i = 0; i < bytes.length; i++) {
|
|
763
|
+
hash ^= BigInt(bytes[i] ?? 0);
|
|
764
|
+
hash = BigInt.asUintN(64, hash * prime);
|
|
765
|
+
}
|
|
766
|
+
return hash;
|
|
767
|
+
}
|
|
548
768
|
function resolveMarkdownSnapshotPath(kind, configuredPath) {
|
|
549
769
|
const trimmed = configuredPath?.trim();
|
|
550
770
|
if (trimmed) {
|
|
@@ -561,10 +781,22 @@ function createRealFsApi() {
|
|
|
561
781
|
readdir: async (dir) => fsp.readdir(dir, { withFileTypes: true }),
|
|
562
782
|
readFile: async (file) => fsp.readFile(file),
|
|
563
783
|
stat: async (file) => {
|
|
564
|
-
const
|
|
565
|
-
return { size:
|
|
784
|
+
const s = await fsp.stat(file);
|
|
785
|
+
return { size: s.size, mtimeMs: s.mtimeMs, ctimeMs: s.ctimeMs };
|
|
566
786
|
},
|
|
567
787
|
watch: (dir, onChange) => fs.watch(dir, onChange),
|
|
788
|
+
openReadStream: async (file) => {
|
|
789
|
+
const handle = await fsp.open(file, "r");
|
|
790
|
+
return {
|
|
791
|
+
read: async (buffer) => {
|
|
792
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
|
|
793
|
+
return { bytesRead };
|
|
794
|
+
},
|
|
795
|
+
close: async () => {
|
|
796
|
+
await handle.close();
|
|
797
|
+
},
|
|
798
|
+
};
|
|
799
|
+
},
|
|
568
800
|
};
|
|
569
801
|
}
|
|
570
802
|
function isMarkdownFile(fileName) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import type { MemoryPromptSectionBuilder } from "openclaw/plugin-sdk/plugin-entry";
|
|
2
2
|
import type { PluginConfig } from "./types.js";
|
|
3
|
-
import type {
|
|
4
|
-
export declare function buildMemoryPromptSection(
|
|
3
|
+
import type { ClientGetter } from "./plugin-runtime.js";
|
|
4
|
+
export declare function buildMemoryPromptSection(_getClient: ClientGetter, _cfg: PluginConfig): MemoryPromptSectionBuilder;
|
package/dist/memory-provider.js
CHANGED
|
@@ -4,7 +4,7 @@ const MEMORY_PROMPT_HEADER = [
|
|
|
4
4
|
"in context via the context-engine assembler when available and relevant.",
|
|
5
5
|
"",
|
|
6
6
|
];
|
|
7
|
-
export function buildMemoryPromptSection(
|
|
7
|
+
export function buildMemoryPromptSection(_getClient, _cfg) {
|
|
8
8
|
return function memoryPromptSection({ availableTools: _availableTools, citationsMode: _citationsMode, }) {
|
|
9
9
|
// OpenClaw builds the memory prompt section synchronously for embedded runs.
|
|
10
10
|
// Actual retrieval and ranking happen in the context engine during assemble().
|
package/dist/memory-runtime.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { ClientGetter } from "./plugin-runtime.js";
|
|
2
2
|
import type { PluginConfig } from "./types.js";
|
|
3
3
|
type MemorySearchParams = {
|
|
4
4
|
query?: string;
|
|
@@ -30,7 +30,7 @@ type MemoryRuntimeStatus = {
|
|
|
30
30
|
abstractiveReady?: boolean;
|
|
31
31
|
embeddingProfile?: string;
|
|
32
32
|
};
|
|
33
|
-
export declare function buildMemoryRuntimeBridge(
|
|
33
|
+
export declare function buildMemoryRuntimeBridge(getClient: ClientGetter, cfg: PluginConfig): {
|
|
34
34
|
getMemorySearchManager(params?: {
|
|
35
35
|
agentId?: string;
|
|
36
36
|
purpose?: string;
|
|
@@ -53,37 +53,8 @@ export declare function buildMemoryRuntimeBridge(getRpc: RpcGetter, cfg: PluginC
|
|
|
53
53
|
id: string;
|
|
54
54
|
score: number;
|
|
55
55
|
text: string;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
sessionId?: string;
|
|
59
|
-
userId?: string;
|
|
60
|
-
role?: string;
|
|
61
|
-
source_doc?: string;
|
|
62
|
-
node_kind?: string;
|
|
63
|
-
ordinal?: number;
|
|
64
|
-
position?: number;
|
|
65
|
-
tier?: number;
|
|
66
|
-
authored?: boolean;
|
|
67
|
-
authority?: number;
|
|
68
|
-
access_count?: number;
|
|
69
|
-
collection?: string;
|
|
70
|
-
hop_targets?: string[] | string;
|
|
71
|
-
token_estimate?: number;
|
|
72
|
-
continuity_tail?: boolean;
|
|
73
|
-
continuity_base?: boolean;
|
|
74
|
-
continuity_bundle_id?: string;
|
|
75
|
-
elevated_guidance?: boolean;
|
|
76
|
-
source_turn_id?: string;
|
|
77
|
-
source_turn_ts?: number;
|
|
78
|
-
provenance_class?: string;
|
|
79
|
-
stability_weight?: number;
|
|
80
|
-
expanded_from_summary?: boolean;
|
|
81
|
-
parent_summary_id?: string;
|
|
82
|
-
expansion_depth?: number;
|
|
83
|
-
cascade_tier?: number;
|
|
84
|
-
[key: string]: unknown;
|
|
85
|
-
};
|
|
86
|
-
finalScore?: number;
|
|
56
|
+
metadataJson: Uint8Array<ArrayBuffer>;
|
|
57
|
+
version: bigint;
|
|
87
58
|
}[];
|
|
88
59
|
error?: undefined;
|
|
89
60
|
}>;
|