@xdarkicex/openclaw-memory-libravdb 1.5.3 → 1.5.5

package/dist/context-engine.js

@@ -559,6 +559,19 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
                 : {}),
         };
     }
+    function isGrpcAuthConfigured() {
+        const secret = process.env.LIBRAVDB_AUTH_SECRET?.trim();
+        const secretFile = process.env.LIBRAVDB_AUTH_SECRET_FILE?.trim();
+        return (typeof secret === "string" && secret.length > 0) || (typeof secretFile === "string" && secretFile.length > 0);
+    }
+    function buildGrpcAuthInitializationError(error) {
+        const code = typeof error?.code === "number" ||
+            typeof error?.code === "string"
+            ? ` code=${String(error.code)}`
+            : "";
+        return new Error(`LibraVDB gRPC auth initialization failed${code}; ` +
+            `check LIBRAVDB_AUTH_SECRET and daemon auth configuration`);
+    }
     async function runCompaction(args) {
         const request = buildCompactSessionRequest(args);
         const kernel = await getKernelOrNull("compact");
@@ -644,7 +657,10 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
                     });
                 }
                 catch (error) {
-                    // Proceed even if initialize session fails or doesn't return nonce if secret optional
+                    if (isGrpcAuthConfigured()) {
+                        throw buildGrpcAuthInitializationError(error);
+                    }
+                    // Proceed when the kernel does not require auth and the init call is unavailable.
                 }
                 return await kernel.bootstrapSession({
                     sessionId,

package/dist/index.js

@@ -839,14 +839,14 @@ var require_binary_encoding = __commonJS({
     };
     exports2.BinaryWriter = BinaryWriter;
     var BinaryReader = class {
-      constructor(buf, textDecoder3) {
+      constructor(buf, textDecoder2) {
         this.varint64 = varint_js_1.varint64read;
         this.uint32 = varint_js_1.varint32read;
         this.buf = buf;
         this.len = buf.length;
         this.pos = 0;
         this.view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
-        this.textDecoder = textDecoder3 !== null && textDecoder3 !== void 0 ? textDecoder3 : new TextDecoder();
+        this.textDecoder = textDecoder2 !== null && textDecoder2 !== void 0 ? textDecoder2 : new TextDecoder();
       }
       /**
        * Reads a tag - field number and wire type.
@@ -34240,6 +34240,17 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
       ...typeof cfg.continuityPriorContextTokens === "number" ? { continuityPriorContextTokens: cfg.continuityPriorContextTokens } : {}
     };
   }
+  function isGrpcAuthConfigured() {
+    const secret = process.env.LIBRAVDB_AUTH_SECRET?.trim();
+    const secretFile = process.env.LIBRAVDB_AUTH_SECRET_FILE?.trim();
+    return typeof secret === "string" && secret.length > 0 || typeof secretFile === "string" && secretFile.length > 0;
+  }
+  function buildGrpcAuthInitializationError(error) {
+    const code = typeof error?.code === "number" || typeof error?.code === "string" ? ` code=${String(error.code)}` : "";
+    return new Error(
+      `LibraVDB gRPC auth initialization failed${code}; check LIBRAVDB_AUTH_SECRET and daemon auth configuration`
+    );
+  }
   async function runCompaction(args) {
     const request = buildCompactSessionRequest(args);
     const kernel = await getKernelOrNull("compact");
@@ -34322,6 +34333,9 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
             clientCapabilities: [{ name: "grpc", version: "1.0" }]
           });
         } catch (error) {
+          if (isGrpcAuthConfigured()) {
+            throw buildGrpcAuthInitializationError(error);
+          }
         }
         return await kernel.bootstrapSession({
           sessionId,
@@ -38523,37 +38537,55 @@ var IngestQueue = class {
       this.options.chunkTokens = DEFAULT_OPTIONS.chunkTokens;
     }
   }
-  async enqueueIngest(sourceDoc, text, baseParams) {
+  async enqueueIngest(sourceDoc, text, baseParams, maxChunkTokens) {
     if (this.options.chunkTokens === Infinity) {
-      return this.ingestWithRetry({
+      const resp = await this.ingestWithRetry({
         ...baseParams,
         sourceDoc,
         text,
         mode: IngestMode.REPLACE
       });
-    }
-    const chunks = splitIntoChunks(text, this.options.chunkTokens);
-    if (chunks.length === 1) {
-      return this.ingestWithRetry({
-        ...baseParams,
-        sourceDoc,
-        text: chunks[0].text,
-        mode: IngestMode.REPLACE
-      });
-    }
-    for (let i = 0; i < chunks.length; i++) {
-      const isFirst = i === 0;
+      return resp.feedback;
+    }
+    let currentLimit = maxChunkTokens && maxChunkTokens > 0 ? maxChunkTokens : this.options.chunkTokens;
+    let offset = 0;
+    let isFirst = true;
+    let lastFeedback;
+    while (offset < text.length) {
+      const remainingText = text.slice(offset);
+      const chunks = splitIntoChunks(remainingText, currentLimit);
+      const chunkText = chunks[0].text;
       const chunkParams = {
         ...baseParams,
         sourceDoc,
-        text: chunks[i].text,
+        text: chunkText,
         mode: isFirst ? IngestMode.REPLACE : IngestMode.APPEND
       };
-      await this.ingestWithRetry(chunkParams);
+      const resp = await this.ingestWithRetry(chunkParams);
+      lastFeedback = resp.feedback;
+      if (lastFeedback && lastFeedback.nodesAccepted === 0 && lastFeedback.tokenBurstLimit && lastFeedback.tokenBurstLimit > 0 && lastFeedback.tokenBurstLimit < currentLimit) {
+        currentLimit = lastFeedback.tokenBurstLimit;
+        continue;
+      }
+      if (lastFeedback && lastFeedback.nodesAccepted === 0) {
+        this.logger.warn?.(
+          `[ingest-queue] Chunk permanently rejected for ${sourceDoc} at offset=${offset} length=${chunkText.length} tokenBurstLimit=${lastFeedback.tokenBurstLimit ?? "unset"}`
+        );
+      }
+      if (this.options.onChunkFeedback && lastFeedback) {
+        this.options.onChunkFeedback(lastFeedback);
+      }
+      offset += chunkText.length;
+      isFirst = false;
+      if (lastFeedback && !lastFeedback.acceptMore && offset < text.length) {
+        const delay = lastFeedback.retryAfterMs || 1e3;
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
     }
+    return lastFeedback;
   }
   async ingestWithRetry(params) {
-    await withRetry(
+    return withRetry(
       () => this.rpcCall("ingest_markdown_document", params),
       this.options.maxRetries,
       this.options.retryBaseDelayMs,
@@ -38644,6 +38676,7 @@ var DEFAULT_DEBOUNCE_MS2 = 150;
 var DEFAULT_TOKENIZER_ID = "markdown-ingest:v1";
 var MARKDOWN_INGEST_VERSION = 3;
 var HASH_BACKEND = "wasm-fnv1a64";
+var STREAM_CHUNK_BYTES = 64 * 1024;
 function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsApi = createRealFsApi2()) {
   const adapters = [];
   const genericRoots = normalizeMarkdownRoots(cfg.markdownIngestionRoots);
@@ -38656,7 +38689,9 @@ function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsApi = cr
           include: cfg.markdownIngestionInclude,
           exclude: cfg.markdownIngestionExclude,
           debounceMs: cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS2,
-          snapshotPath: resolveMarkdownSnapshotPath("generic", cfg.markdownIngestionSnapshotPath)
+          snapshotPath: resolveMarkdownSnapshotPath("generic", cfg.markdownIngestionSnapshotPath),
+          priorityMode: cfg.markdownIngestionPriorityMode,
+          maxTokensPerFile: cfg.markdownIngestionMaxTokensPerFile
         },
         getRpc,
         logger,
@@ -38674,7 +38709,9 @@ function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsApi = cr
           include: cfg.markdownIngestionObsidianInclude,
           exclude: cfg.markdownIngestionObsidianExclude,
           debounceMs: cfg.markdownIngestionObsidianDebounceMs ?? cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS2,
-          snapshotPath: resolveMarkdownSnapshotPath("obsidian", cfg.markdownIngestionObsidianSnapshotPath)
+          snapshotPath: resolveMarkdownSnapshotPath("obsidian", cfg.markdownIngestionObsidianSnapshotPath),
+          priorityMode: cfg.markdownIngestionPriorityMode,
+          maxTokensPerFile: cfg.markdownIngestionMaxTokensPerFile
         },
         getRpc,
         logger,
@@ -38731,6 +38768,8 @@ var DirectoryMarkdownSourceAdapter = class {
   getRpc;
   logger;
   snapshotPath;
+  priorityMode;
+  maxTokensPerFile;
   states = /* @__PURE__ */ new Map();
   fileStates = /* @__PURE__ */ new Map();
   activeScans = /* @__PURE__ */ new Set();
@@ -38739,6 +38778,17 @@ var DirectoryMarkdownSourceAdapter = class {
   started = false;
   ingestQueue = null;
   stopping = false;
+  lastAcceptMore = true;
+  lastRetryAfterMs = 0;
+  lastQueueDepth = 0;
+  lastQueueCapacity = 0;
+  lastProcessingTimeUs = 0;
+  lastNodesAccepted = 0;
+  lastNodesRejected = 0;
+  lastTokensIngested = 0;
+  lastTokenBurstLimit = 512;
+  lastWalDepth = 0;
+  lastWalCapacity = 0;
   snapshotLoaded = false;
   snapshotDirty = false;
   constructor(kind, config, getRpc, logger, fsApi) {
@@ -38751,6 +38801,8 @@ var DirectoryMarkdownSourceAdapter = class {
     this.getRpc = getRpc;
     this.logger = logger;
     this.snapshotPath = config.snapshotPath ?? resolveMarkdownSnapshotPath(kind);
+    this.priorityMode = config.priorityMode ?? "mtime";
+    this.maxTokensPerFile = Math.max(1, Math.trunc(config.maxTokensPerFile ?? 128e3));
     this.tokenizerId = DEFAULT_TOKENIZER_ID;
     this.coreDoc = true;
   }
@@ -38803,7 +38855,8 @@ var DirectoryMarkdownSourceAdapter = class {
       scanState: {
         scanning: false,
         dirty: false,
-        timer: null
+        timer: null,
+        resumeFromPath: null
       },
       knownFiles: this.snapshotFilesForRoot(resolved),
       directoryWatchers: /* @__PURE__ */ new Map()
@@ -38821,12 +38874,16 @@ var DirectoryMarkdownSourceAdapter = class {
       return;
     }
     rootState.scanState.scanning = true;
+    this.lastAcceptMore = true;
+    this.lastRetryAfterMs = 0;
     const scan = (async () => {
       const stats = createScanStats();
       const startedAt = Date.now();
       try {
         const currentFiles = /* @__PURE__ */ new Set();
-        await this.walkDirectory(rootState, rootState.root, currentFiles, stats);
+        const candidates = [];
+        await this.walkDirectory(rootState, rootState.root, currentFiles, stats, candidates);
+        await this.syncCandidates(rootState, candidates, stats);
         if (!this.stopping) {
           await this.pruneDeletedFiles(rootState, currentFiles, stats);
           rootState.knownFiles = currentFiles;
@@ -38850,7 +38907,7 @@ var DirectoryMarkdownSourceAdapter = class {
       this.activeScans.delete(scan);
     }
   }
-  scheduleRootScan(rootState) {
+  scheduleRootScan(rootState, delayMs) {
     if (!this.started || this.stopping) {
       return;
     }
@@ -38866,9 +38923,9 @@ var DirectoryMarkdownSourceAdapter = class {
       void this.scanRoot(rootState.root).catch((error) => {
         this.logger.warn?.(`[markdown-ingest] root scan failed for ${rootState.root}: ${formatError(error)}`);
       });
-    }, this.debounceMs);
+    }, Math.max(this.debounceMs, delayMs ?? 0));
   }
-  async walkDirectory(rootState, dir, currentFiles, stats) {
+  async walkDirectory(rootState, dir, currentFiles, stats, candidates) {
     if (this.shouldPruneDirectory(rootState.root, dir)) {
       stats.directoriesPruned++;
       return;
@@ -38891,7 +38948,7 @@ var DirectoryMarkdownSourceAdapter = class {
       }
       const child = path2.join(dir, entry.name);
       if (entry.isDirectory()) {
-        await this.walkDirectory(rootState, child, currentFiles, stats);
+        await this.walkDirectory(rootState, child, currentFiles, stats, candidates);
         continue;
       }
       if (!entry.isFile() || !isMarkdownFile(entry.name)) {
@@ -38904,16 +38961,71 @@ var DirectoryMarkdownSourceAdapter = class {
       }
       stats.filesIncluded++;
       currentFiles.add(child);
+      const stat = await this.safeStatWithCtime(child);
+      if (!stat) {
+        continue;
+      }
+      candidates.push({ path: child, size: stat.size, mtimeMs: stat.mtimeMs, ctimeMs: stat.ctimeMs, ordinal: candidates.length });
+    }
+  }
+  async syncCandidates(rootState, candidates, stats) {
+    const sorted = sortCandidates(candidates, this.priorityMode);
+    let skipping = false;
+    if (rootState.scanState.resumeFromPath) {
+      const targetExists = sorted.some((c) => c.path === rootState.scanState.resumeFromPath);
+      if (targetExists) {
+        skipping = true;
+        this.lastAcceptMore = true;
+        this.lastRetryAfterMs = 0;
+      } else {
+        rootState.scanState.resumeFromPath = null;
+      }
+    }
+    for (const candidate of sorted) {
+      if (skipping) {
+        if (candidate.path === rootState.scanState.resumeFromPath) {
+          skipping = false;
+        } else {
+          continue;
+        }
+      }
+      if (this.stopping) {
+        return;
+      }
+      if (!this.lastAcceptMore) {
+        if (!this.stopping) {
+          rootState.scanState.resumeFromPath = candidate.path;
+          this.scheduleRootScan(rootState, this.lastRetryAfterMs);
+        }
+        return;
+      }
+      if (this.lastWalCapacity > 0 && this.lastWalDepth > this.lastWalCapacity * 0.8) {
+        rootState.scanState.resumeFromPath = candidate.path;
+        if (!this.stopping) {
+          this.scheduleRootScan(rootState, 2e3);
+        }
+        return;
+      }
+      const estimatedTokens = estimateTokens(candidate.size);
+      if (estimatedTokens > this.maxTokensPerFile) {
+        stats.filesDeferred++;
+        continue;
+      }
       try {
-        const result = await this.syncMarkdownFile(rootState, child);
+        const result = await this.syncMarkdownFile(rootState, candidate.path, {
+          size: candidate.size,
+          mtimeMs: candidate.mtimeMs,
+          ctimeMs: candidate.ctimeMs
+        });
         recordSyncResult(stats, result);
       } catch (error) {
         stats.syncErrors++;
         if (!this.stopping) {
-          this.logger.warn?.(`[markdown-ingest] sync failed for ${child}: ${formatError(error)}`);
+          this.logger.warn?.(`[markdown-ingest] sync failed for ${candidate.path}: ${formatError(error)}`);
         }
       }
     }
+    rootState.scanState.resumeFromPath = null;
   }
   shouldPruneDirectory(root, dir) {
     const relative = toPosixPath(path2.relative(root, dir));
@@ -38934,6 +39046,11 @@ var DirectoryMarkdownSourceAdapter = class {
     try {
       const watcher = this.fsApi.watch(dir, () => {
         if (!this.stopping) {
+          rootState.scanState.resumeFromPath = null;
+          if (rootState.scanState.timer) {
+            clearTimeout(rootState.scanState.timer);
+            rootState.scanState.timer = null;
+          }
           this.scheduleRootScan(rootState);
         }
       });
@@ -38984,10 +39101,10 @@ var DirectoryMarkdownSourceAdapter = class {
       stats.filesDeleted++;
     }
   }
-  async syncMarkdownFile(rootState, filePath) {
+  async syncMarkdownFile(rootState, filePath, initialStat) {
     const sourceDoc = filePath;
     const relativePath = toPosixPath(path2.relative(rootState.root, filePath));
-    const stat = await this.safeStat(filePath);
+    const stat = initialStat ?? await this.safeStatWithCtime(filePath);
     if (!stat) {
       await this.deleteSourceDocument(sourceDoc);
       this.fileStates.delete(sourceDoc);
@@ -38998,14 +39115,18 @@ var DirectoryMarkdownSourceAdapter = class {
     if (cached && cached.size === stat.size && cached.mtimeMs === stat.mtimeMs) {
       return "unchanged";
     }
-    const bytes = await this.safeReadFile(filePath);
-    if (!bytes) {
+    const maxBytes = this.maxTokensPerFile * 4 + 3;
+    const streamed = await this.safeReadFileStreamed(filePath, maxBytes);
+    if (streamed === "too_large") {
+      return "skipped";
+    }
+    if (!streamed) {
       await this.deleteSourceDocument(sourceDoc);
       this.fileStates.delete(sourceDoc);
       this.snapshotDirty = true;
       return "deleted";
     }
-    const fileHash = hashBytes(bytes);
+    const { text, fileHash } = streamed;
     if (cached && cached.fileHash === fileHash) {
       this.setFileState(sourceDoc, {
         root: rootState.root,
@@ -39017,14 +39138,13 @@ var DirectoryMarkdownSourceAdapter = class {
       });
       return "unchanged";
     }
-    const text = textDecoder2.decode(bytes);
     if (this.kind === "obsidian" && this.includePatterns.length === 0 && !looksLikeObsidianNote(filePath, text)) {
       await this.deleteSourceDocument(sourceDoc);
       this.fileStates.delete(sourceDoc);
       this.snapshotDirty = true;
       return "skipped";
     }
-    await this.ingestMarkdownDocument(sourceDoc, text, rootState.root, relativePath, fileHash, stat.size, stat.mtimeMs);
+    await this.ingestMarkdownDocument(sourceDoc, text, rootState.root, relativePath, fileHash, stat.size, stat.mtimeMs, stat.ctimeMs);
     this.setFileState(sourceDoc, {
       root: rootState.root,
       sourceDoc,
@@ -39039,9 +39159,9 @@ var DirectoryMarkdownSourceAdapter = class {
     this.fileStates.set(sourceDoc, state);
     this.snapshotDirty = true;
   }
-  async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs) {
+  async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs, sourceCtimeMs) {
     const queue = await this.getIngestQueue();
-    await queue.enqueueIngest(
+    const feedback = await queue.enqueueIngest(
       sourceDoc,
       text,
       {
@@ -39054,11 +39174,44 @@ var DirectoryMarkdownSourceAdapter = class {
           fileHash,
           sourceSize,
           sourceMtimeMs: Math.trunc(sourceMtimeMs),
+          sourceCtimeMs: Math.trunc(sourceCtimeMs),
           ingestVersion: MARKDOWN_INGEST_VERSION,
           hashBackend: HASH_BACKEND
         }
-      }
+      },
+      this.lastTokenBurstLimit
     );
+    this.applyIngestFeedback(feedback);
+  }
+  applyIngestFeedback(feedback) {
+    if (feedback && typeof feedback.acceptMore === "boolean") {
+      this.lastAcceptMore = feedback.acceptMore;
+      this.lastQueueDepth = feedback.queueDepth ?? 0;
+      this.lastQueueCapacity = feedback.queueCapacity ?? 0;
+      this.lastProcessingTimeUs = feedback.processingTimeUs ?? 0;
+      this.lastNodesAccepted = feedback.nodesAccepted ?? 0;
+      this.lastNodesRejected = feedback.nodesRejected ?? 0;
+      this.lastTokensIngested = feedback.tokensIngested ?? 0;
+      if (feedback.tokenBurstLimit && feedback.tokenBurstLimit > 0) {
+        this.lastTokenBurstLimit = feedback.tokenBurstLimit;
+      }
+      this.lastWalDepth = feedback.walDepth ?? 0;
+      this.lastWalCapacity = feedback.walCapacity ?? 0;
+      if (feedback.acceptMore) {
+        this.lastRetryAfterMs = 0;
+      } else {
+        this.lastRetryAfterMs = feedback.retryAfterMs || 1e3;
+      }
+    } else {
+      this.lastAcceptMore = true;
+      this.lastRetryAfterMs = 0;
+      this.lastQueueDepth = 0;
+      this.lastQueueCapacity = 0;
+      this.lastProcessingTimeUs = 0;
+      this.lastNodesAccepted = 0;
+      this.lastNodesRejected = 0;
+      this.lastTokensIngested = 0;
+    }
   }
   async deleteSourceDocument(sourceDoc) {
     const queue = await this.getIngestQueue();
@@ -39067,7 +39220,9 @@ var DirectoryMarkdownSourceAdapter = class {
   async getIngestQueue() {
     if (!this.ingestQueue) {
       const rpc = await this.getRpc();
-      this.ingestQueue = new IngestQueue(rpc.call.bind(rpc), this.logger);
+      this.ingestQueue = new IngestQueue(rpc.call.bind(rpc), this.logger, {
+        onChunkFeedback: (feedback) => this.applyIngestFeedback(feedback)
+      });
     }
     return this.ingestQueue;
   }
@@ -39078,13 +39233,49 @@ var DirectoryMarkdownSourceAdapter = class {
       return null;
     }
   }
-  async safeReadFile(filePath) {
+  async safeStatWithCtime(filePath) {
     try {
-      return await this.fsApi.readFile(filePath);
+      return await this.fsApi.stat(filePath);
     } catch {
       return null;
     }
   }
+  async safeReadFileStreamed(filePath, maxBytes) {
+    let stream = null;
+    try {
+      stream = await this.fsApi.openReadStream(filePath);
+      const decoder = new TextDecoder();
+      const chunks = [];
+      let hash = 0xcbf29ce484222325n;
+      let total = 0;
+      const buffer = Buffer.allocUnsafe(STREAM_CHUNK_BYTES);
+      while (true) {
+        const { bytesRead } = await stream.read(buffer);
+        if (bytesRead === 0) {
+          break;
+        }
+        total += bytesRead;
+        if (total > maxBytes) {
+          return "too_large";
+        }
+        const chunk = buffer.subarray(0, bytesRead);
+        hash = updateFnv1a64(hash, chunk);
+        chunks.push(decoder.decode(chunk, { stream: true }));
+      }
+      chunks.push(decoder.decode());
+      return {
+        text: chunks.join(""),
+        fileHash: hash.toString(16).padStart(16, "0")
+      };
+    } catch {
+      return null;
+    } finally {
+      if (stream) {
+        await stream.close().catch(() => {
+        });
+      }
+    }
+  }
   snapshotFilesForRoot(root) {
     const files = /* @__PURE__ */ new Set();
     for (const state of this.fileStates.values()) {
@@ -39147,7 +39338,7 @@ var DirectoryMarkdownSourceAdapter = class {
   }
   logScanStats(root, stats, durationMs) {
     this.logger.info?.(
-      `[markdown-ingest] ${this.kind} scan complete root=${root} dirs=${stats.directoriesScanned} prunedDirs=${stats.directoriesPruned} markdown=${stats.markdownFilesSeen} included=${stats.filesIncluded} skipped=${stats.filesSkipped} unchanged=${stats.filesUnchanged} ingested=${stats.filesIngested} deleted=${stats.filesDeleted} errors=${stats.syncErrors} durationMs=${durationMs}`
+      `[markdown-ingest] ${this.kind} scan complete root=${root} dirs=${stats.directoriesScanned} prunedDirs=${stats.directoriesPruned} markdown=${stats.markdownFilesSeen} included=${stats.filesIncluded} skipped=${stats.filesSkipped} unchanged=${stats.filesUnchanged} ingested=${stats.filesIngested} deleted=${stats.filesDeleted} deferred=${stats.filesDeferred} errors=${stats.syncErrors} durationMs=${durationMs}`
     );
   }
 };
@@ -39161,9 +39352,27 @@ function createScanStats() {
     filesUnchanged: 0,
     filesIngested: 0,
     filesDeleted: 0,
-    syncErrors: 0
+    syncErrors: 0,
+    filesDeferred: 0
   };
 }
+function estimateTokens(size) {
+  return Math.max(1, Math.floor(size / 4));
+}
+function sortCandidates(candidates, mode) {
+  return [...candidates].sort((left, right) => {
+    if (mode === "size") {
+      return right.size - left.size || left.ordinal - right.ordinal;
+    }
+    if (mode === "ctime") {
+      return right.ctimeMs - left.ctimeMs || left.ordinal - right.ordinal;
+    }
+    if (mode === "fifo") {
+      return left.ordinal - right.ordinal;
+    }
+    return right.mtimeMs - left.mtimeMs || left.ordinal - right.ordinal;
+  });
+}
 function recordSyncResult(stats, result) {
   if (result === "ingested") {
     stats.filesIngested++;
@@ -39178,7 +39387,6 @@ function recordSyncResult(stats, result) {
 function toPosixPath(value) {
   return value.split(path2.sep).join("/");
 }
-var textDecoder2 = new TextDecoder();
 function normalizeMarkdownRoots(roots) {
   if (!roots?.length) {
     return [];
@@ -39193,6 +39401,15 @@ function normalizeMarkdownRoots(roots) {
   }
   return [...resolved];
 }
+function updateFnv1a64(seed, bytes) {
+  let hash = seed;
+  const prime = 0x100000001b3n;
+  for (let i = 0; i < bytes.length; i++) {
+    hash ^= BigInt(bytes[i] ?? 0);
+    hash = BigInt.asUintN(64, hash * prime);
+  }
+  return hash;
+}
 function resolveMarkdownSnapshotPath(kind, configuredPath) {
   const trimmed = configuredPath?.trim();
   if (trimmed) {
@@ -39209,10 +39426,22 @@ function createRealFsApi2() {
     readdir: async (dir) => fsp2.readdir(dir, { withFileTypes: true }),
     readFile: async (file) => fsp2.readFile(file),
     stat: async (file) => {
-      const stat = await fsp2.stat(file);
-      return { size: stat.size, mtimeMs: stat.mtimeMs };
+      const s = await fsp2.stat(file);
+      return { size: s.size, mtimeMs: s.mtimeMs, ctimeMs: s.ctimeMs };
     },
-    watch: (dir, onChange) => fs2.watch(dir, onChange)
+    watch: (dir, onChange) => fs2.watch(dir, onChange),
+    openReadStream: async (file) => {
+      const handle = await fsp2.open(file, "r");
+      return {
+        read: async (buffer) => {
+          const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
+          return { bytesRead };
+        },
+        close: async () => {
+          await handle.close();
+        }
+      };
+    }
   };
 }
 function isMarkdownFile(fileName) {

package/dist/ingest-queue.d.ts

@@ -7,6 +7,8 @@ export interface IngestQueueOptions {
     retryBaseDelayMs: number;
     /** Max retries per chunk. */
     maxRetries: number;
+    /** Called after each chunk is accepted so scan-level state stays current. */
+    onChunkFeedback?: (feedback: IngestFeedback) => void;
 }
 interface IngestMarkdownDocumentParams {
     sourceDoc: string;
@@ -20,11 +22,25 @@ interface IngestMarkdownDocumentParams {
         fileHash: string;
         sourceSize: number;
         sourceMtimeMs: number;
+        sourceCtimeMs: number;
         ingestVersion: number;
         hashBackend: string;
     };
     mode?: IngestMode;
 }
+interface IngestFeedback {
+    queueDepth: number;
+    queueCapacity: number;
+    acceptMore: boolean;
+    retryAfterMs: number;
+    processingTimeUs: number;
+    nodesAccepted: number;
+    nodesRejected: number;
+    tokensIngested: number;
+    tokenBurstLimit: number;
+    walDepth?: number;
+    walCapacity?: number;
+}
 export declare class IngestQueue {
     private readonly queue;
     private readonly rpcCall;
@@ -32,7 +48,7 @@ export declare class IngestQueue {
     private readonly options;
     private running;
     constructor(rpcCall: <T>(method: string, params: unknown) => Promise<T>, logger: LoggerLike, options?: Partial<IngestQueueOptions>);
-    enqueueIngest(sourceDoc: string, text: string, baseParams: Omit<IngestMarkdownDocumentParams, "sourceDoc" | "text" | "mode">): Promise<void>;
+    enqueueIngest(sourceDoc: string, text: string, baseParams: Omit<IngestMarkdownDocumentParams, "sourceDoc" | "text" | "mode">, maxChunkTokens?: number): Promise<IngestFeedback | undefined>;
     private ingestWithRetry;
     enqueueDelete(sourceDoc: string): Promise<void>;
 }

package/dist/ingest-queue.js

@@ -19,40 +19,59 @@ export class IngestQueue {
             this.options.chunkTokens = DEFAULT_OPTIONS.chunkTokens;
         }
     }
-    async enqueueIngest(sourceDoc, text, baseParams) {
+    async enqueueIngest(sourceDoc, text, baseParams, maxChunkTokens) {
         if (this.options.chunkTokens === Infinity) {
-            // Retry-only mode: send full text as single chunk
-            return this.ingestWithRetry({
+            const resp = await this.ingestWithRetry({
                 ...baseParams,
                 sourceDoc,
                 text,
                 mode: IngestMode.REPLACE,
             });
+            return resp.feedback;
         }
-        const chunks = splitIntoChunks(text, this.options.chunkTokens);
-        if (chunks.length === 1) {
-            return this.ingestWithRetry({
-                ...baseParams,
-                sourceDoc,
-                text: chunks[0].text,
-                mode: IngestMode.REPLACE,
-            });
-        }
-        // Multiple chunks: clear the source once, then append the remaining chunks.
-        // Sending REPLACE last deletes the earlier chunks from the same source_doc.
-        for (let i = 0; i < chunks.length; i++) {
-            const isFirst = i === 0;
+        let currentLimit = maxChunkTokens && maxChunkTokens > 0 ? maxChunkTokens : this.options.chunkTokens;
+        let offset = 0;
+        let isFirst = true;
+        let lastFeedback;
+        while (offset < text.length) {
+            const remainingText = text.slice(offset);
+            const chunks = splitIntoChunks(remainingText, currentLimit);
+            const chunkText = chunks[0].text;
             const chunkParams = {
                 ...baseParams,
                 sourceDoc,
-                text: chunks[i].text,
+                text: chunkText,
                 mode: isFirst ? IngestMode.REPLACE : IngestMode.APPEND,
             };
-            await this.ingestWithRetry(chunkParams);
+            const resp = await this.ingestWithRetry(chunkParams);
+            lastFeedback = resp.feedback;
+            if (lastFeedback &&
+                lastFeedback.nodesAccepted === 0 &&
+                lastFeedback.tokenBurstLimit &&
+                lastFeedback.tokenBurstLimit > 0 &&
+                lastFeedback.tokenBurstLimit < currentLimit) {
+                currentLimit = lastFeedback.tokenBurstLimit;
+                continue;
+            }
+            if (lastFeedback && lastFeedback.nodesAccepted === 0) {
+                this.logger.warn?.(`[ingest-queue] Chunk permanently rejected for ${sourceDoc} ` +
+                    `at offset=${offset} length=${chunkText.length} ` +
+                    `tokenBurstLimit=${lastFeedback.tokenBurstLimit ?? "unset"}`);
+            }
+            if (this.options.onChunkFeedback && lastFeedback) {
+                this.options.onChunkFeedback(lastFeedback);
+            }
+            offset += chunkText.length;
+            isFirst = false;
+            if (lastFeedback && !lastFeedback.acceptMore && offset < text.length) {
+                const delay = lastFeedback.retryAfterMs || 1000;
+                await new Promise((resolve) => setTimeout(resolve, delay));
+            }
         }
+        return lastFeedback;
     }
     async ingestWithRetry(params) {
-        await withRetry(() => this.rpcCall("ingest_markdown_document", params), this.options.maxRetries, this.options.retryBaseDelayMs, this.logger, `ingest_markdown_document(${params.sourceDoc})`);
+        return withRetry(() => this.rpcCall("ingest_markdown_document", params), this.options.maxRetries, this.options.retryBaseDelayMs, this.logger, `ingest_markdown_document(${params.sourceDoc})`);
     }
     async enqueueDelete(sourceDoc) {
         await withRetry(() => this.rpcCall("delete_authored_document", { sourceDoc }), this.options.maxRetries, this.options.retryBaseDelayMs, this.logger, `delete_authored_document(${sourceDoc})`);

package/dist/markdown-ingest.d.ts

@@ -14,14 +14,22 @@ interface FsDirentLike {
 interface FsWatcherLike extends Disposable {
     on(event: "error", handler: (error: Error) => void): void;
 }
+interface FsReadStream {
+    read(buffer: Uint8Array): Promise<{
+        bytesRead: number;
+    }>;
+    close(): Promise<void>;
+}
 interface FsApi {
     readdir(dir: string): Promise<FsDirentLike[]>;
     readFile(file: string): Promise<Uint8Array>;
     stat(file: string): Promise<{
         size: number;
         mtimeMs: number;
+        ctimeMs: number;
     }>;
     watch(dir: string, onChange: (event: string, filename: string | Buffer | null) => void): FsWatcherLike;
+    openReadStream(file: string): Promise<FsReadStream>;
 }
 export interface MarkdownSourceAdapter {
     kind: string;

package/dist/markdown-ingest.js

@@ -2,13 +2,13 @@ import fs from "node:fs";
 import fsp from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
-import { hashBytes } from "./markdown-hash.js";
 import { formatError } from "./format-error.js";
 import { IngestQueue } from "./ingest-queue.js";
 const DEFAULT_DEBOUNCE_MS = 150;
 const DEFAULT_TOKENIZER_ID = "markdown-ingest:v1";
 const MARKDOWN_INGEST_VERSION = 3;
 const HASH_BACKEND = "wasm-fnv1a64";
+const STREAM_CHUNK_BYTES = 64 * 1024;
 export function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsApi = createRealFsApi()) {
     const adapters = [];
     const genericRoots = normalizeMarkdownRoots(cfg.markdownIngestionRoots);
@@ -19,6 +19,8 @@ export function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsA
             exclude: cfg.markdownIngestionExclude,
             debounceMs: cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS,
             snapshotPath: resolveMarkdownSnapshotPath("generic", cfg.markdownIngestionSnapshotPath),
+            priorityMode: cfg.markdownIngestionPriorityMode,
+            maxTokensPerFile: cfg.markdownIngestionMaxTokensPerFile,
         }, getRpc, logger, fsApi));
     }
     const obsidianRoots = normalizeMarkdownRoots(cfg.markdownIngestionObsidianRoots);
@@ -29,6 +31,8 @@ export function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsA
             exclude: cfg.markdownIngestionObsidianExclude,
             debounceMs: cfg.markdownIngestionObsidianDebounceMs ?? cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS,
             snapshotPath: resolveMarkdownSnapshotPath("obsidian", cfg.markdownIngestionObsidianSnapshotPath),
+            priorityMode: cfg.markdownIngestionPriorityMode,
+            maxTokensPerFile: cfg.markdownIngestionMaxTokensPerFile,
         }, getRpc, logger, fsApi));
     }
     if (adapters.length === 0) {
@@ -77,6 +81,8 @@ class DirectoryMarkdownSourceAdapter {
     getRpc;
     logger;
     snapshotPath;
+    priorityMode;
+    maxTokensPerFile;
     states = new Map();
     fileStates = new Map();
     activeScans = new Set();
@@ -85,6 +91,17 @@ class DirectoryMarkdownSourceAdapter {
     started = false;
     ingestQueue = null;
     stopping = false;
+    lastAcceptMore = true;
+    lastRetryAfterMs = 0;
+    lastQueueDepth = 0;
+    lastQueueCapacity = 0;
+    lastProcessingTimeUs = 0;
+    lastNodesAccepted = 0;
+    lastNodesRejected = 0;
+    lastTokensIngested = 0;
+    lastTokenBurstLimit = 512;
+    lastWalDepth = 0;
+    lastWalCapacity = 0;
     snapshotLoaded = false;
     snapshotDirty = false;
     constructor(kind, config, getRpc, logger, fsApi) {
@@ -97,6 +114,8 @@ class DirectoryMarkdownSourceAdapter {
         this.getRpc = getRpc;
         this.logger = logger;
         this.snapshotPath = config.snapshotPath ?? resolveMarkdownSnapshotPath(kind);
+        this.priorityMode = config.priorityMode ?? "mtime";
+        this.maxTokensPerFile = Math.max(1, Math.trunc(config.maxTokensPerFile ?? 128_000));
         this.tokenizerId = DEFAULT_TOKENIZER_ID;
         this.coreDoc = true;
     }
@@ -150,6 +169,7 @@ class DirectoryMarkdownSourceAdapter {
                 scanning: false,
                 dirty: false,
                 timer: null,
+                resumeFromPath: null,
             },
             knownFiles: this.snapshotFilesForRoot(resolved),
             directoryWatchers: new Map(),
@@ -167,12 +187,16 @@ class DirectoryMarkdownSourceAdapter {
             return;
         }
         rootState.scanState.scanning = true;
+        this.lastAcceptMore = true;
+        this.lastRetryAfterMs = 0;
         const scan = (async () => {
             const stats = createScanStats();
             const startedAt = Date.now();
             try {
                 const currentFiles = new Set();
-                await this.walkDirectory(rootState, rootState.root, currentFiles, stats);
+                const candidates = [];
+                await this.walkDirectory(rootState, rootState.root, currentFiles, stats, candidates);
+                await this.syncCandidates(rootState, candidates, stats);
                 if (!this.stopping) {
                     await this.pruneDeletedFiles(rootState, currentFiles, stats);
                     rootState.knownFiles = currentFiles;
@@ -198,7 +222,7 @@ class DirectoryMarkdownSourceAdapter {
             this.activeScans.delete(scan);
         }
     }
-    scheduleRootScan(rootState) {
+    scheduleRootScan(rootState, delayMs) {
         if (!this.started || this.stopping) {
             return;
         }
@@ -214,9 +238,9 @@ class DirectoryMarkdownSourceAdapter {
             void this.scanRoot(rootState.root).catch((error) => {
                 this.logger.warn?.(`[markdown-ingest] root scan failed for ${rootState.root}: ${formatError(error)}`);
             });
-        }, this.debounceMs);
+        }, Math.max(this.debounceMs, delayMs ?? 0));
     }
-    async walkDirectory(rootState, dir, currentFiles, stats) {
+    async walkDirectory(rootState, dir, currentFiles, stats, candidates) {
         if (this.shouldPruneDirectory(rootState.root, dir)) {
             stats.directoriesPruned++;
             return;
@@ -240,7 +264,7 @@ class DirectoryMarkdownSourceAdapter {
             }
             const child = path.join(dir, entry.name);
             if (entry.isDirectory()) {
-                await this.walkDirectory(rootState, child, currentFiles, stats);
+                await this.walkDirectory(rootState, child, currentFiles, stats, candidates);
                 continue;
             }
             if (!entry.isFile() || !isMarkdownFile(entry.name)) {
@@ -253,17 +277,74 @@ class DirectoryMarkdownSourceAdapter {
             }
             stats.filesIncluded++;
             currentFiles.add(child);
+            const stat = await this.safeStatWithCtime(child);
+            if (!stat) {
+                continue;
+            }
+            candidates.push({ path: child, size: stat.size, mtimeMs: stat.mtimeMs, ctimeMs: stat.ctimeMs, ordinal: candidates.length });
+        }
+    }
+    async syncCandidates(rootState, candidates, stats) {
+        const sorted = sortCandidates(candidates, this.priorityMode);
+        let skipping = false;
+        if (rootState.scanState.resumeFromPath) {
+            const targetExists = sorted.some((c) => c.path === rootState.scanState.resumeFromPath);
+            if (targetExists) {
+                skipping = true;
+                this.lastAcceptMore = true;
+                this.lastRetryAfterMs = 0;
+            }
+            else {
+                rootState.scanState.resumeFromPath = null;
+            }
+        }
+        for (const candidate of sorted) {
+            if (skipping) {
+                if (candidate.path === rootState.scanState.resumeFromPath) {
+                    skipping = false;
+                }
+                else {
+                    continue;
+                }
+            }
+            if (this.stopping) {
+                return;
+            }
+            if (!this.lastAcceptMore) {
+                if (!this.stopping) {
+                    rootState.scanState.resumeFromPath = candidate.path;
+                    this.scheduleRootScan(rootState, this.lastRetryAfterMs);
+                }
+                return;
+            }
+            if (this.lastWalCapacity > 0 && this.lastWalDepth > this.lastWalCapacity * 0.8) {
+                rootState.scanState.resumeFromPath = candidate.path;
+                if (!this.stopping) {
+                    this.scheduleRootScan(rootState, 2000);
+                }
+                return;
+            }
+            const estimatedTokens = estimateTokens(candidate.size);
+            if (estimatedTokens > this.maxTokensPerFile) {
+                stats.filesDeferred++;
+                continue;
+            }
             try {
-                const result = await this.syncMarkdownFile(rootState, child);
+                const result = await this.syncMarkdownFile(rootState, candidate.path, {
+                    size: candidate.size,
+                    mtimeMs: candidate.mtimeMs,
+                    ctimeMs: candidate.ctimeMs,
+                });
                 recordSyncResult(stats, result);
             }
             catch (error) {
                 stats.syncErrors++;
                 if (!this.stopping) {
-                    this.logger.warn?.(`[markdown-ingest] sync failed for ${child}: ${formatError(error)}`);
+                    this.logger.warn?.(`[markdown-ingest] sync failed for ${candidate.path}: ${formatError(error)}`);
                 }
             }
         }
+        rootState.scanState.resumeFromPath = null;
     }
     shouldPruneDirectory(root, dir) {
         const relative = toPosixPath(path.relative(root, dir));
@@ -284,6 +365,11 @@ class DirectoryMarkdownSourceAdapter {
         try {
             const watcher = this.fsApi.watch(dir, () => {
                 if (!this.stopping) {
+                    rootState.scanState.resumeFromPath = null;
+                    if (rootState.scanState.timer) {
+                        clearTimeout(rootState.scanState.timer);
+                        rootState.scanState.timer = null;
+                    }
                     this.scheduleRootScan(rootState);
                 }
             });
@@ -335,10 +421,10 @@ class DirectoryMarkdownSourceAdapter {
             stats.filesDeleted++;
         }
     }
-    async syncMarkdownFile(rootState, filePath) {
+    async syncMarkdownFile(rootState, filePath, initialStat) {
         const sourceDoc = filePath;
         const relativePath = toPosixPath(path.relative(rootState.root, filePath));
-        const stat = await this.safeStat(filePath);
+        const stat = initialStat ?? (await this.safeStatWithCtime(filePath));
         if (!stat) {
             await this.deleteSourceDocument(sourceDoc);
             this.fileStates.delete(sourceDoc);
@@ -349,14 +435,18 @@ class DirectoryMarkdownSourceAdapter {
         if (cached && cached.size === stat.size && cached.mtimeMs === stat.mtimeMs) {
             return "unchanged";
         }
-        const bytes = await this.safeReadFile(filePath);
-        if (!bytes) {
+        const maxBytes = this.maxTokensPerFile * 4 + 3;
+        const streamed = await this.safeReadFileStreamed(filePath, maxBytes);
+        if (streamed === "too_large") {
+            return "skipped";
+        }
+        if (!streamed) {
             await this.deleteSourceDocument(sourceDoc);
             this.fileStates.delete(sourceDoc);
             this.snapshotDirty = true;
             return "deleted";
         }
-        const fileHash = hashBytes(bytes);
+        const { text, fileHash } = streamed;
         if (cached && cached.fileHash === fileHash) {
             this.setFileState(sourceDoc, {
                 root: rootState.root,
@@ -368,14 +458,13 @@ class DirectoryMarkdownSourceAdapter {
             });
             return "unchanged";
         }
-        const text = textDecoder.decode(bytes);
         if (this.kind === "obsidian" && this.includePatterns.length === 0 && !looksLikeObsidianNote(filePath, text)) {
             await this.deleteSourceDocument(sourceDoc);
             this.fileStates.delete(sourceDoc);
             this.snapshotDirty = true;
             return "skipped";
         }
-        await this.ingestMarkdownDocument(sourceDoc, text, rootState.root, relativePath, fileHash, stat.size, stat.mtimeMs);
+        await this.ingestMarkdownDocument(sourceDoc, text, rootState.root, relativePath, fileHash, stat.size, stat.mtimeMs, stat.ctimeMs);
         this.setFileState(sourceDoc, {
             root: rootState.root,
             sourceDoc,
@@ -390,9 +479,9 @@ class DirectoryMarkdownSourceAdapter {
         this.fileStates.set(sourceDoc, state);
         this.snapshotDirty = true;
     }
-    async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs) {
+    async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs, sourceCtimeMs) {
         const queue = await this.getIngestQueue();
-        await queue.enqueueIngest(sourceDoc, text, {
+        const feedback = await queue.enqueueIngest(sourceDoc, text, {
             tokenizerId: this.tokenizerId,
             coreDoc: this.coreDoc,
             sourceMeta: {
@@ -402,10 +491,44 @@ class DirectoryMarkdownSourceAdapter {
                 fileHash,
                 sourceSize,
                 sourceMtimeMs: Math.trunc(sourceMtimeMs),
+                sourceCtimeMs: Math.trunc(sourceCtimeMs),
                 ingestVersion: MARKDOWN_INGEST_VERSION,
                 hashBackend: HASH_BACKEND,
             },
-        });
+        }, this.lastTokenBurstLimit);
+        this.applyIngestFeedback(feedback);
+    }
+    applyIngestFeedback(feedback) {
+        if (feedback && typeof feedback.acceptMore === "boolean") {
+            this.lastAcceptMore = feedback.acceptMore;
+            this.lastQueueDepth = feedback.queueDepth ?? 0;
+            this.lastQueueCapacity = feedback.queueCapacity ?? 0;
+            this.lastProcessingTimeUs = feedback.processingTimeUs ?? 0;
+            this.lastNodesAccepted = feedback.nodesAccepted ?? 0;
+            this.lastNodesRejected = feedback.nodesRejected ?? 0;
+            this.lastTokensIngested = feedback.tokensIngested ?? 0;
+            if (feedback.tokenBurstLimit && feedback.tokenBurstLimit > 0) {
+                this.lastTokenBurstLimit = feedback.tokenBurstLimit;
+            }
+            this.lastWalDepth = feedback.walDepth ?? 0;
+            this.lastWalCapacity = feedback.walCapacity ?? 0;
+            if (feedback.acceptMore) {
+                this.lastRetryAfterMs = 0;
+            }
+            else {
+                this.lastRetryAfterMs = feedback.retryAfterMs || 1000;
+            }
+        }
+        else {
+            this.lastAcceptMore = true;
+            this.lastRetryAfterMs = 0;
+            this.lastQueueDepth = 0;
+            this.lastQueueCapacity = 0;
+            this.lastProcessingTimeUs = 0;
+            this.lastNodesAccepted = 0;
+            this.lastNodesRejected = 0;
+            this.lastTokensIngested = 0;
+        }
     }
     async deleteSourceDocument(sourceDoc) {
         const queue = await this.getIngestQueue();
@@ -414,7 +537,9 @@ class DirectoryMarkdownSourceAdapter {
     async getIngestQueue() {
         if (!this.ingestQueue) {
             const rpc = await this.getRpc();
-            this.ingestQueue = new IngestQueue(rpc.call.bind(rpc), this.logger);
+            this.ingestQueue = new IngestQueue(rpc.call.bind(rpc), this.logger, {
+                onChunkFeedback: (feedback) => this.applyIngestFeedback(feedback),
+            });
         }
         return this.ingestQueue;
     }
@@ -426,14 +551,51 @@ class DirectoryMarkdownSourceAdapter {
             return null;
         }
     }
-    async safeReadFile(filePath) {
+    async safeStatWithCtime(filePath) {
         try {
-            return await this.fsApi.readFile(filePath);
+            return await this.fsApi.stat(filePath);
         }
         catch {
             return null;
         }
     }
+    async safeReadFileStreamed(filePath, maxBytes) {
+        let stream = null;
+        try {
+            stream = await this.fsApi.openReadStream(filePath);
+            const decoder = new TextDecoder();
+            const chunks = [];
+            let hash = 0xcbf29ce484222325n;
+            let total = 0;
+            const buffer = Buffer.allocUnsafe(STREAM_CHUNK_BYTES);
+            while (true) {
+                const { bytesRead } = await stream.read(buffer);
+                if (bytesRead === 0) {
+                    break;
+                }
+                total += bytesRead;
+                if (total > maxBytes) {
+                    return "too_large";
+                }
+                const chunk = buffer.subarray(0, bytesRead);
+                hash = updateFnv1a64(hash, chunk);
+                chunks.push(decoder.decode(chunk, { stream: true }));
+            }
+            chunks.push(decoder.decode());
+            return {
+                text: chunks.join(""),
+                fileHash: hash.toString(16).padStart(16, "0"),
+            };
+        }
+        catch {
+            return null;
+        }
+        finally {
+            if (stream) {
+                await stream.close().catch(() => { });
+            }
+        }
+    }
     snapshotFilesForRoot(root) {
         const files = new Set();
         for (const state of this.fileStates.values()) {
@@ -497,7 +659,7 @@ class DirectoryMarkdownSourceAdapter {
         }
     }
     logScanStats(root, stats, durationMs) {
-        this.logger.info?.(`[markdown-ingest] ${this.kind} scan complete root=${root} dirs=${stats.directoriesScanned} prunedDirs=${stats.directoriesPruned} markdown=${stats.markdownFilesSeen} included=${stats.filesIncluded} skipped=${stats.filesSkipped} unchanged=${stats.filesUnchanged} ingested=${stats.filesIngested} deleted=${stats.filesDeleted} errors=${stats.syncErrors} durationMs=${durationMs}`);
+        this.logger.info?.(`[markdown-ingest] ${this.kind} scan complete root=${root} dirs=${stats.directoriesScanned} prunedDirs=${stats.directoriesPruned} markdown=${stats.markdownFilesSeen} included=${stats.filesIncluded} skipped=${stats.filesSkipped} unchanged=${stats.filesUnchanged} ingested=${stats.filesIngested} deleted=${stats.filesDeleted} deferred=${stats.filesDeferred} errors=${stats.syncErrors} durationMs=${durationMs}`);
     }
 }
 function createScanStats() {
@@ -511,8 +673,26 @@ function createScanStats() {
         filesIngested: 0,
         filesDeleted: 0,
         syncErrors: 0,
+        filesDeferred: 0,
     };
 }
+function estimateTokens(size) {
+    return Math.max(1, Math.floor(size / 4));
+}
+function sortCandidates(candidates, mode) {
+    return [...candidates].sort((left, right) => {
+        if (mode === "size") {
+            return right.size - left.size || left.ordinal - right.ordinal;
+        }
+        if (mode === "ctime") {
+            return right.ctimeMs - left.ctimeMs || left.ordinal - right.ordinal;
+        }
+        if (mode === "fifo") {
+            return left.ordinal - right.ordinal;
+        }
+        return right.mtimeMs - left.mtimeMs || left.ordinal - right.ordinal;
+    });
+}
 function recordSyncResult(stats, result) {
     if (result === "ingested") {
         stats.filesIngested++;
@@ -530,7 +710,6 @@ function recordSyncResult(stats, result) {
 function toPosixPath(value) {
     return value.split(path.sep).join("/");
 }
-const textDecoder = new TextDecoder();
 function normalizeMarkdownRoots(roots) {
     if (!roots?.length) {
         return [];
@@ -545,6 +724,15 @@ function normalizeMarkdownRoots(roots) {
     }
     return [...resolved];
 }
+function updateFnv1a64(seed, bytes) {
+    let hash = seed;
+    const prime = 0x100000001b3n;
+    for (let i = 0; i < bytes.length; i++) {
+        hash ^= BigInt(bytes[i] ?? 0);
+        hash = BigInt.asUintN(64, hash * prime);
+    }
+    return hash;
+}
 function resolveMarkdownSnapshotPath(kind, configuredPath) {
     const trimmed = configuredPath?.trim();
     if (trimmed) {
@@ -561,10 +749,22 @@ function createRealFsApi() {
         readdir: async (dir) => fsp.readdir(dir, { withFileTypes: true }),
         readFile: async (file) => fsp.readFile(file),
         stat: async (file) => {
-            const stat = await fsp.stat(file);
-            return { size: stat.size, mtimeMs: stat.mtimeMs };
+            const s = await fsp.stat(file);
+            return { size: s.size, mtimeMs: s.mtimeMs, ctimeMs: s.ctimeMs };
         },
         watch: (dir, onChange) => fs.watch(dir, onChange),
+        openReadStream: async (file) => {
+            const handle = await fsp.open(file, "r");
+            return {
+                read: async (buffer) => {
+                    const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
+                    return { bytesRead };
+                },
+                close: async () => {
+                    await handle.close();
+                },
+            };
+        },
     };
 }
 function isMarkdownFile(fileName) {

package/dist/types.d.ts

@@ -48,6 +48,8 @@ export interface PluginConfig {
     markdownIngestionInclude?: string[];
     markdownIngestionExclude?: string[];
     markdownIngestionDebounceMs?: number;
+    markdownIngestionPriorityMode?: "mtime" | "ctime" | "size" | "fifo";
+    markdownIngestionMaxTokensPerFile?: number;
     markdownIngestionSnapshotPath?: string;
     markdownIngestionObsidianSnapshotPath?: string;
     dreamPromotionEnabled?: boolean;

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "libravdb-memory",
   "name": "LibraVDB Memory",
   "description": "Persistent vector memory with three-tier hybrid scoring",
-  "version": "1.5.3",
+  "version": "1.5.5",
   "kind": [
     "memory",
     "context-engine"
@@ -263,6 +263,20 @@
         "type": "number",
         "default": 150
       },
+      "markdownIngestionPriorityMode": {
+        "type": "string",
+        "enum": [
+          "mtime",
+          "ctime",
+          "size",
+          "fifo"
+        ],
+        "default": "mtime"
+      },
+      "markdownIngestionMaxTokensPerFile": {
+        "type": "number",
+        "default": 128000
+      },
       "markdownIngestionSnapshotPath": {
         "type": "string"
       },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xdarkicex/openclaw-memory-libravdb",
-  "version": "1.5.3",
+  "version": "1.5.5",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",