@xdarkicex/openclaw-memory-libravdb 1.4.67 → 1.4.69

package/dist/context-engine.js

@@ -275,6 +275,17 @@ function resolvePredictiveCompactionTokenCount(args) {
     return (normalizeCurrentTokenCount(args.currentTokenCount) ??
         approximateMessagesTokens(args.messages) + approximateTokenCount(args.prompt ?? ""));
 }
+function resolveAfterTurnPredictiveCompactionTokenCount(args) {
+    const currentTokenCount = normalizeCurrentTokenCount(args.currentTokenCount);
+    const forwardedMessageTokens = normalizeCurrentTokenCount(approximateMessagesTokens(args.messages));
+    if (currentTokenCount == null) {
+        return forwardedMessageTokens;
+    }
+    if (forwardedMessageTokens == null) {
+        return currentTokenCount;
+    }
+    return Math.max(currentTokenCount, forwardedMessageTokens);
+}
 export function normalizeKernelMessage(message) {
     return {
         role: message.role,
@@ -564,11 +575,15 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
     }
     async function performAfterTurnPredictiveCompaction(args) {
         const dynamicCompactThreshold = getDynamicCompactThreshold(args.tokenBudget);
-        const predictiveTargetSize = resolvePredictiveCompactionTarget({
+        const currentContextTokens = resolveAfterTurnPredictiveCompactionTokenCount({
             currentTokenCount: args.currentTokenCount,
+            messages: args.messages,
+        });
+        const predictiveTargetSize = resolvePredictiveCompactionTarget({
+            currentTokenCount: currentContextTokens,
             threshold: dynamicCompactThreshold,
         });
-        if (args.currentTokenCount == null ||
+        if (currentContextTokens == null ||
             dynamicCompactThreshold == null ||
             predictiveTargetSize == null) {
             return;
@@ -577,7 +592,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
             logger,
             phase: "afterTurn",
             sessionId: args.sessionId,
-            currentTokenCount: args.currentTokenCount,
+            currentTokenCount: currentContextTokens,
             threshold: dynamicCompactThreshold,
             targetSize: predictiveTargetSize,
             tokenBudget: args.tokenBudget,
@@ -587,13 +602,13 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
             targetSize: predictiveTargetSize,
             tokenBudget: args.tokenBudget,
             force: true,
-            currentTokenCount: args.currentTokenCount,
+            currentTokenCount: currentContextTokens,
         });
         logPredictiveCompactionOutcome({
             logger,
             phase: "afterTurn",
             sessionId: args.sessionId,
-            currentTokenCount: args.currentTokenCount,
+            currentTokenCount: currentContextTokens,
             threshold: dynamicCompactThreshold,
             targetSize: predictiveTargetSize,
             tokenBudget: args.tokenBudget,
@@ -803,6 +818,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
                     });
                     await performAfterTurnPredictiveCompaction({
                         sessionId,
+                        messages,
                         tokenBudget: args.tokenBudget,
                         currentTokenCount,
                     });
@@ -818,6 +834,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
                 });
                 await performAfterTurnPredictiveCompaction({
                     sessionId,
+                    messages,
                     tokenBudget: args.tokenBudget,
                     currentTokenCount,
                 });

package/dist/index.js

@@ -9467,7 +9467,7 @@ var require_service_config = __commonJS({
     exports2.validateRetryThrottling = validateRetryThrottling;
     exports2.validateServiceConfig = validateServiceConfig;
     exports2.extractAndSelectServiceConfig = extractAndSelectServiceConfig;
-    var os3 = __require("os");
+    var os4 = __require("os");
     var constants_1 = require_constants();
     var DURATION_REGEX = /^\d+(\.\d{1,9})?s$/;
     var CLIENT_LANGUAGE_STRING = "node";
@@ -9766,7 +9766,7 @@ var require_service_config = __commonJS({
         if (Array.isArray(validatedConfig.clientHostname)) {
           let hostnameMatched = false;
           for (const hostname2 of validatedConfig.clientHostname) {
-            if (hostname2 === os3.hostname()) {
+            if (hostname2 === os4.hostname()) {
               hostnameMatched = true;
             }
           }
@@ -24066,7 +24066,7 @@ var require_subchannel_call = __commonJS({
     Object.defineProperty(exports2, "__esModule", { value: true });
     exports2.Http2SubchannelCall = void 0;
     var http2 = __require("http2");
-    var os3 = __require("os");
+    var os4 = __require("os");
     var constants_1 = require_constants();
     var metadata_1 = require_metadata();
     var stream_decoder_1 = require_stream_decoder();
@@ -24074,7 +24074,7 @@ var require_subchannel_call = __commonJS({
     var constants_2 = require_constants();
     var TRACER_NAME = "subchannel_call";
     function getSystemErrorName(errno) {
-      for (const [name, num] of Object.entries(os3.constants.errno)) {
+      for (const [name, num] of Object.entries(os4.constants.errno)) {
         if (num === errno) {
           return name;
         }
@@ -33927,6 +33927,19 @@ function buildBudgetFallbackContext(messages, tokenBudget) {
 function resolvePredictiveCompactionTokenCount(args) {
   return normalizeCurrentTokenCount(args.currentTokenCount) ?? approximateMessagesTokens(args.messages) + approximateTokenCount(args.prompt ?? "");
 }
+function resolveAfterTurnPredictiveCompactionTokenCount(args) {
+  const currentTokenCount = normalizeCurrentTokenCount(args.currentTokenCount);
+  const forwardedMessageTokens = normalizeCurrentTokenCount(
+    approximateMessagesTokens(args.messages)
+  );
+  if (currentTokenCount == null) {
+    return forwardedMessageTokens;
+  }
+  if (forwardedMessageTokens == null) {
+    return currentTokenCount;
+  }
+  return Math.max(currentTokenCount, forwardedMessageTokens);
+}
 function normalizeKernelMessage(message) {
   return {
     role: message.role,
@@ -34184,18 +34197,22 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
   }
   async function performAfterTurnPredictiveCompaction(args) {
     const dynamicCompactThreshold = getDynamicCompactThreshold(args.tokenBudget);
-    const predictiveTargetSize = resolvePredictiveCompactionTarget({
+    const currentContextTokens = resolveAfterTurnPredictiveCompactionTokenCount({
       currentTokenCount: args.currentTokenCount,
+      messages: args.messages
+    });
+    const predictiveTargetSize = resolvePredictiveCompactionTarget({
+      currentTokenCount: currentContextTokens,
       threshold: dynamicCompactThreshold
     });
-    if (args.currentTokenCount == null || dynamicCompactThreshold == null || predictiveTargetSize == null) {
+    if (currentContextTokens == null || dynamicCompactThreshold == null || predictiveTargetSize == null) {
       return;
     }
     logPredictiveCompactionAttempt({
       logger,
       phase: "afterTurn",
       sessionId: args.sessionId,
-      currentTokenCount: args.currentTokenCount,
+      currentTokenCount: currentContextTokens,
       threshold: dynamicCompactThreshold,
       targetSize: predictiveTargetSize,
       tokenBudget: args.tokenBudget
@@ -34205,13 +34222,13 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
       targetSize: predictiveTargetSize,
       tokenBudget: args.tokenBudget,
       force: true,
-      currentTokenCount: args.currentTokenCount
+      currentTokenCount: currentContextTokens
     });
     logPredictiveCompactionOutcome({
       logger,
       phase: "afterTurn",
       sessionId: args.sessionId,
-      currentTokenCount: args.currentTokenCount,
+      currentTokenCount: currentContextTokens,
       threshold: dynamicCompactThreshold,
       targetSize: predictiveTargetSize,
       tokenBudget: args.tokenBudget,
@@ -34428,6 +34445,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
           });
           await performAfterTurnPredictiveCompaction({
             sessionId,
+            messages,
             tokenBudget: args.tokenBudget,
             currentTokenCount
           });
@@ -34443,6 +34461,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
         });
         await performAfterTurnPredictiveCompaction({
           sessionId,
+          messages,
           tokenBudget: args.tokenBudget,
           currentTokenCount
         });
@@ -34523,6 +34542,7 @@ function isRecord(value) {
 // src/markdown-ingest.ts
 import fs2 from "node:fs";
 import fsp2 from "node:fs/promises";
+import os2 from "node:os";
 import path2 from "node:path";
 
 // node_modules/.pnpm/@bufbuild+protobuf@1.7.2/node_modules/@bufbuild/protobuf/dist/proxy/index.js
@@ -38484,7 +38504,8 @@ function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsApi = cr
           roots: genericRoots,
           include: cfg.markdownIngestionInclude,
           exclude: cfg.markdownIngestionExclude,
-          debounceMs: cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS2
+          debounceMs: cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS2,
+          snapshotPath: resolveMarkdownSnapshotPath("generic", cfg.markdownIngestionSnapshotPath)
         },
         getRpc,
         logger,
@@ -38493,7 +38514,7 @@ function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsApi = cr
     );
   }
   const obsidianRoots = normalizeMarkdownRoots(cfg.markdownIngestionObsidianRoots);
-  if (cfg.markdownIngestionObsidianEnabled !== false && obsidianRoots.length > 0) {
+  if (cfg.markdownIngestionObsidianEnabled === true && obsidianRoots.length > 0) {
     adapters.push(
       new DirectoryMarkdownSourceAdapter(
         "obsidian",
@@ -38501,7 +38522,8 @@ function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsApi = cr
           roots: obsidianRoots,
           include: cfg.markdownIngestionObsidianInclude,
           exclude: cfg.markdownIngestionObsidianExclude,
-          debounceMs: cfg.markdownIngestionObsidianDebounceMs ?? cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS2
+          debounceMs: cfg.markdownIngestionObsidianDebounceMs ?? cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS2,
+          snapshotPath: resolveMarkdownSnapshotPath("obsidian", cfg.markdownIngestionObsidianSnapshotPath)
         },
         getRpc,
         logger,
@@ -38557,6 +38579,7 @@ var DirectoryMarkdownSourceAdapter = class {
   fsApi;
   getRpc;
   logger;
+  snapshotPath;
   states = /* @__PURE__ */ new Map();
   fileStates = /* @__PURE__ */ new Map();
   activeScans = /* @__PURE__ */ new Set();
@@ -38565,6 +38588,8 @@ var DirectoryMarkdownSourceAdapter = class {
   started = false;
   ingestQueue = null;
   stopping = false;
+  snapshotLoaded = false;
+  snapshotDirty = false;
   constructor(kind, config, getRpc, logger, fsApi) {
     this.kind = kind;
     this.roots = config.roots;
@@ -38574,6 +38599,7 @@ var DirectoryMarkdownSourceAdapter = class {
     this.fsApi = fsApi;
     this.getRpc = getRpc;
     this.logger = logger;
+    this.snapshotPath = config.snapshotPath ?? resolveMarkdownSnapshotPath(kind);
     this.tokenizerId = DEFAULT_TOKENIZER_ID;
     this.coreDoc = true;
   }
@@ -38581,6 +38607,7 @@ var DirectoryMarkdownSourceAdapter = class {
     if (this.started) {
       return;
     }
+    await this.loadSnapshot();
     this.started = true;
     this.stopping = false;
     await this.refresh();
@@ -38608,8 +38635,10 @@ var DirectoryMarkdownSourceAdapter = class {
     if (this.activeScans.size > 0) {
       await Promise.allSettled([...this.activeScans]);
     }
+    await this.saveSnapshotIfDirty();
     this.states.clear();
     this.fileStates.clear();
+    this.snapshotLoaded = false;
     this.started = false;
   }
   getRootState(root) {
@@ -38625,7 +38654,7 @@ var DirectoryMarkdownSourceAdapter = class {
         dirty: false,
         timer: null
       },
-      knownFiles: /* @__PURE__ */ new Set(),
+      knownFiles: this.snapshotFilesForRoot(resolved),
       directoryWatchers: /* @__PURE__ */ new Map()
     };
     this.states.set(resolved, created);
@@ -38642,12 +38671,16 @@ var DirectoryMarkdownSourceAdapter = class {
     }
     rootState.scanState.scanning = true;
     const scan = (async () => {
+      const stats = createScanStats();
+      const startedAt = Date.now();
       try {
         const currentFiles = /* @__PURE__ */ new Set();
-        await this.walkDirectory(rootState, rootState.root, currentFiles);
+        await this.walkDirectory(rootState, rootState.root, currentFiles, stats);
         if (!this.stopping) {
-          await this.pruneDeletedFiles(rootState, currentFiles);
+          await this.pruneDeletedFiles(rootState, currentFiles, stats);
           rootState.knownFiles = currentFiles;
+          await this.saveSnapshotIfDirty();
+          this.logScanStats(rootState.root, stats, Date.now() - startedAt);
         }
       } finally {
         rootState.scanState.scanning = false;
@@ -38684,7 +38717,12 @@ var DirectoryMarkdownSourceAdapter = class {
       });
     }, this.debounceMs);
   }
-  async walkDirectory(rootState, dir, currentFiles) {
+  async walkDirectory(rootState, dir, currentFiles, stats) {
+    if (this.shouldPruneDirectory(rootState.root, dir)) {
+      stats.directoriesPruned++;
+      return;
+    }
+    stats.directoriesScanned++;
     await this.ensureDirectoryWatcher(rootState, dir);
     let entries;
     try {
@@ -38702,25 +38740,42 @@ var DirectoryMarkdownSourceAdapter = class {
       }
       const child = path2.join(dir, entry.name);
       if (entry.isDirectory()) {
-        await this.walkDirectory(rootState, child, currentFiles);
+        await this.walkDirectory(rootState, child, currentFiles, stats);
         continue;
       }
       if (!entry.isFile() || !isMarkdownFile(entry.name)) {
         continue;
       }
+      stats.markdownFilesSeen++;
       if (!this.shouldIncludeFile(rootState.root, child)) {
+        stats.filesSkipped++;
         continue;
       }
+      stats.filesIncluded++;
       currentFiles.add(child);
       try {
-        await this.syncMarkdownFile(rootState, child);
+        const result = await this.syncMarkdownFile(rootState, child);
+        recordSyncResult(stats, result);
       } catch (error) {
+        stats.syncErrors++;
         if (!this.stopping) {
           this.logger.warn?.(`[markdown-ingest] sync failed for ${child}: ${formatError(error)}`);
         }
       }
     }
   }
+  shouldPruneDirectory(root, dir) {
+    const relative = toPosixPath(path2.relative(root, dir));
+    if (!relative || relative === "." || relative.startsWith("..")) {
+      return false;
+    }
+    for (const pattern of this.excludePatterns) {
+      if (matchesExcludedDirectory(relative, pattern)) {
+        return true;
+      }
+    }
+    return false;
+  }
   async ensureDirectoryWatcher(rootState, dir) {
     if (rootState.directoryWatchers.has(dir)) {
       return;
@@ -38761,7 +38816,7 @@ var DirectoryMarkdownSourceAdapter = class {
     }
     return true;
   }
-  async pruneDeletedFiles(rootState, currentFiles) {
+  async pruneDeletedFiles(rootState, currentFiles, stats) {
     const removed = [];
     for (const previous of rootState.knownFiles) {
       if (!currentFiles.has(previous)) {
@@ -38774,6 +38829,8 @@ var DirectoryMarkdownSourceAdapter = class {
     for (const filePath of removed) {
       await this.deleteSourceDocument(filePath);
       this.fileStates.delete(filePath);
+      this.snapshotDirty = true;
+      stats.filesDeleted++;
     }
   }
   async syncMarkdownFile(rootState, filePath) {
@@ -38783,21 +38840,23 @@ var DirectoryMarkdownSourceAdapter = class {
     if (!stat) {
       await this.deleteSourceDocument(sourceDoc);
       this.fileStates.delete(sourceDoc);
-      return;
+      this.snapshotDirty = true;
+      return "deleted";
     }
     const cached = this.fileStates.get(sourceDoc);
     if (cached && cached.size === stat.size && cached.mtimeMs === stat.mtimeMs) {
-      return;
+      return "unchanged";
     }
     const bytes = await this.safeReadFile(filePath);
     if (!bytes) {
       await this.deleteSourceDocument(sourceDoc);
       this.fileStates.delete(sourceDoc);
-      return;
+      this.snapshotDirty = true;
+      return "deleted";
     }
     const fileHash = hashBytes(bytes);
     if (cached && cached.fileHash === fileHash) {
-      this.fileStates.set(sourceDoc, {
+      this.setFileState(sourceDoc, {
         root: rootState.root,
         sourceDoc,
         relativePath,
@@ -38805,16 +38864,17 @@ var DirectoryMarkdownSourceAdapter = class {
         size: stat.size,
         mtimeMs: stat.mtimeMs
       });
-      return;
+      return "unchanged";
     }
     const text = textDecoder2.decode(bytes);
     if (this.kind === "obsidian" && this.includePatterns.length === 0 && !looksLikeObsidianNote(filePath, text)) {
       await this.deleteSourceDocument(sourceDoc);
       this.fileStates.delete(sourceDoc);
-      return;
+      this.snapshotDirty = true;
+      return "skipped";
     }
     await this.ingestMarkdownDocument(sourceDoc, text, rootState.root, relativePath, fileHash, stat.size, stat.mtimeMs);
-    this.fileStates.set(sourceDoc, {
+    this.setFileState(sourceDoc, {
       root: rootState.root,
       sourceDoc,
       relativePath,
@@ -38822,6 +38882,11 @@ var DirectoryMarkdownSourceAdapter = class {
       size: stat.size,
       mtimeMs: stat.mtimeMs
     });
+    return "ingested";
+  }
+  setFileState(sourceDoc, state) {
+    this.fileStates.set(sourceDoc, state);
+    this.snapshotDirty = true;
   }
   async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs) {
     const queue = await this.getIngestQueue();
@@ -38869,7 +38934,96 @@ var DirectoryMarkdownSourceAdapter = class {
       return null;
     }
   }
+  snapshotFilesForRoot(root) {
+    const files = /* @__PURE__ */ new Set();
+    for (const state of this.fileStates.values()) {
+      if (state.root === root) {
+        files.add(state.sourceDoc);
+      }
+    }
+    return files;
+  }
+  async loadSnapshot() {
+    if (this.snapshotLoaded) {
+      return;
+    }
+    this.snapshotLoaded = true;
+    let raw;
+    try {
+      raw = await fsp2.readFile(this.snapshotPath, "utf8");
+    } catch (error) {
+      if (!formatError(error).includes("ENOENT")) {
+        this.logger.warn?.(`[markdown-ingest] failed to read snapshot ${this.snapshotPath}: ${formatError(error)}`);
+      }
+      return;
+    }
+    try {
+      const parsed = JSON.parse(raw);
+      if (parsed.ingestVersion !== MARKDOWN_INGEST_VERSION || parsed.hashBackend !== HASH_BACKEND || !parsed.files) {
+        return;
+      }
+      const configuredRoots = new Set(this.roots.map((root) => path2.resolve(root)));
+      for (const [sourceDoc, state] of Object.entries(parsed.files)) {
+        if (isValidSnapshotState(sourceDoc, state) && configuredRoots.has(path2.resolve(state.root))) {
+          this.fileStates.set(sourceDoc, state);
+        }
+      }
+      this.logger.info?.(`[markdown-ingest] loaded ${this.fileStates.size} ${this.kind} file snapshots from ${this.snapshotPath}`);
+    } catch (error) {
+      this.logger.warn?.(`[markdown-ingest] failed to parse snapshot ${this.snapshotPath}: ${formatError(error)}`);
+    }
+  }
+  async saveSnapshotIfDirty() {
+    if (!this.snapshotDirty) {
+      return;
+    }
+    const payload = {
+      version: 1,
+      ingestVersion: MARKDOWN_INGEST_VERSION,
+      hashBackend: HASH_BACKEND,
+      files: Object.fromEntries([...this.fileStates.entries()].sort(([left], [right]) => left.localeCompare(right)))
+    };
+    try {
+      await fsp2.mkdir(path2.dirname(this.snapshotPath), { recursive: true });
+      const tmp = `${this.snapshotPath}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
+      await fsp2.writeFile(tmp, `${JSON.stringify(payload, null, 2)}
+`);
+      await fsp2.rename(tmp, this.snapshotPath);
+      this.snapshotDirty = false;
+    } catch (error) {
+      this.logger.warn?.(`[markdown-ingest] failed to write snapshot ${this.snapshotPath}: ${formatError(error)}`);
+    }
+  }
+  logScanStats(root, stats, durationMs) {
+    this.logger.info?.(
+      `[markdown-ingest] ${this.kind} scan complete root=${root} dirs=${stats.directoriesScanned} prunedDirs=${stats.directoriesPruned} markdown=${stats.markdownFilesSeen} included=${stats.filesIncluded} skipped=${stats.filesSkipped} unchanged=${stats.filesUnchanged} ingested=${stats.filesIngested} deleted=${stats.filesDeleted} errors=${stats.syncErrors} durationMs=${durationMs}`
+    );
+  }
 };
+function createScanStats() {
+  return {
+    directoriesScanned: 0,
+    directoriesPruned: 0,
+    markdownFilesSeen: 0,
+    filesIncluded: 0,
+    filesSkipped: 0,
+    filesUnchanged: 0,
+    filesIngested: 0,
+    filesDeleted: 0,
+    syncErrors: 0
+  };
+}
+function recordSyncResult(stats, result) {
+  if (result === "ingested") {
+    stats.filesIngested++;
+  } else if (result === "unchanged") {
+    stats.filesUnchanged++;
+  } else if (result === "deleted") {
+    stats.filesDeleted++;
+  } else {
+    stats.filesSkipped++;
+  }
+}
 function toPosixPath(value) {
   return value.split(path2.sep).join("/");
 }
@@ -38888,11 +39042,16 @@ function normalizeMarkdownRoots(roots) {
   }
   return [...resolved];
 }
-function isMarkdownIngestionEnabled(cfg, roots) {
-  if (cfg.markdownIngestionEnabled === false) {
-    return false;
+function resolveMarkdownSnapshotPath(kind, configuredPath) {
+  const trimmed = configuredPath?.trim();
+  if (trimmed) {
+    return path2.resolve(trimmed);
   }
-  return roots.length > 0;
+  const stateDir = process.env.OPENCLAW_STATE_DIR?.trim() || path2.join(os2.homedir(), ".openclaw");
+  return path2.join(stateDir, `libravdb-markdown-ingest-${kind}.json`);
+}
+function isMarkdownIngestionEnabled(cfg, roots) {
+  return cfg.markdownIngestionEnabled === true && roots.length > 0;
 }
 function createRealFsApi2() {
   return {
@@ -38913,6 +39072,17 @@ function matchesGlob(value, pattern) {
   const escaped = pattern.split("*").map((part) => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join(".*");
   return new RegExp(`^${escaped}$`).test(value);
 }
+function matchesExcludedDirectory(relativeDir, pattern) {
+  const normalized = relativeDir.replace(/\/+$/, "");
+  return matchesGlob(normalized, pattern) || matchesGlob(`${normalized}/`, pattern) || matchesGlob(`${normalized}/.probe`, pattern);
+}
+function isValidSnapshotState(sourceDoc, value) {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  const state = value;
+  return state.sourceDoc === sourceDoc && typeof state.root === "string" && typeof state.relativePath === "string" && typeof state.fileHash === "string" && typeof state.size === "number" && Number.isFinite(state.size) && typeof state.mtimeMs === "number" && Number.isFinite(state.mtimeMs);
+}
 function looksLikeObsidianNote(filePath, text) {
   const frontmatterStart = parseFrontmatterStart(text);
   if (frontmatterStart == null) {
@@ -39439,7 +39609,7 @@ var GrpcKernelClient = class {
 // src/sidecar.ts
 import fs3 from "node:fs";
 import net from "node:net";
-import os2 from "node:os";
+import os3 from "node:os";
 import path4 from "node:path";
 var STARTUP_CONNECT_MAX_RETRIES = 5;
 var STARTUP_CONNECT_BASE_DELAY_MS = 100;
@@ -39710,7 +39880,7 @@ function resolveConfiguredEndpoint(cfg) {
 function daemonProvisioningHint() {
   return "If you installed the npm package, install and start libravdbd separately; the package does not provision the daemon binary, ONNX Runtime, or model assets.";
 }
-function defaultEndpoint(platform = process.platform, homeDir = os2.homedir(), pathExists = fs3.existsSync) {
+function defaultEndpoint(platform = process.platform, homeDir = os3.homedir(), pathExists = fs3.existsSync) {
   const envEndpoint = normalizeConfiguredEndpoint(process.env.LIBRAVDB_RPC_ENDPOINT);
   if (envEndpoint) {
     return envEndpoint;

package/dist/markdown-ingest.js

@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import fsp from "node:fs/promises";
+import os from "node:os";
 import path from "node:path";
 import { hashBytes } from "./markdown-hash.js";
 import { formatError } from "./format-error.js";
@@ -17,15 +18,17 @@ export function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsA
             include: cfg.markdownIngestionInclude,
             exclude: cfg.markdownIngestionExclude,
             debounceMs: cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS,
+            snapshotPath: resolveMarkdownSnapshotPath("generic", cfg.markdownIngestionSnapshotPath),
         }, getRpc, logger, fsApi));
     }
     const obsidianRoots = normalizeMarkdownRoots(cfg.markdownIngestionObsidianRoots);
-    if (cfg.markdownIngestionObsidianEnabled !== false && obsidianRoots.length > 0) {
+    if (cfg.markdownIngestionObsidianEnabled === true && obsidianRoots.length > 0) {
         adapters.push(new DirectoryMarkdownSourceAdapter("obsidian", {
             roots: obsidianRoots,
             include: cfg.markdownIngestionObsidianInclude,
             exclude: cfg.markdownIngestionObsidianExclude,
             debounceMs: cfg.markdownIngestionObsidianDebounceMs ?? cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS,
+            snapshotPath: resolveMarkdownSnapshotPath("obsidian", cfg.markdownIngestionObsidianSnapshotPath),
         }, getRpc, logger, fsApi));
     }
     if (adapters.length === 0) {
@@ -73,6 +76,7 @@ class DirectoryMarkdownSourceAdapter {
     fsApi;
     getRpc;
     logger;
+    snapshotPath;
     states = new Map();
     fileStates = new Map();
     activeScans = new Set();
@@ -81,6 +85,8 @@ class DirectoryMarkdownSourceAdapter {
     started = false;
     ingestQueue = null;
     stopping = false;
+    snapshotLoaded = false;
+    snapshotDirty = false;
     constructor(kind, config, getRpc, logger, fsApi) {
         this.kind = kind;
         this.roots = config.roots;
@@ -90,6 +96,7 @@ class DirectoryMarkdownSourceAdapter {
         this.fsApi = fsApi;
         this.getRpc = getRpc;
         this.logger = logger;
+        this.snapshotPath = config.snapshotPath ?? resolveMarkdownSnapshotPath(kind);
         this.tokenizerId = DEFAULT_TOKENIZER_ID;
         this.coreDoc = true;
     }
@@ -97,6 +104,7 @@ class DirectoryMarkdownSourceAdapter {
         if (this.started) {
             return;
         }
+        await this.loadSnapshot();
         this.started = true;
         this.stopping = false;
         await this.refresh();
@@ -124,8 +132,10 @@ class DirectoryMarkdownSourceAdapter {
         if (this.activeScans.size > 0) {
             await Promise.allSettled([...this.activeScans]);
         }
+        await this.saveSnapshotIfDirty();
         this.states.clear();
         this.fileStates.clear();
+        this.snapshotLoaded = false;
         this.started = false;
     }
     getRootState(root) {
@@ -141,7 +151,7 @@ class DirectoryMarkdownSourceAdapter {
                 dirty: false,
                 timer: null,
             },
-            knownFiles: new Set(),
+            knownFiles: this.snapshotFilesForRoot(resolved),
             directoryWatchers: new Map(),
         };
         this.states.set(resolved, created);
@@ -158,12 +168,16 @@ class DirectoryMarkdownSourceAdapter {
         }
         rootState.scanState.scanning = true;
         const scan = (async () => {
+            const stats = createScanStats();
+            const startedAt = Date.now();
             try {
                 const currentFiles = new Set();
-                await this.walkDirectory(rootState, rootState.root, currentFiles);
+                await this.walkDirectory(rootState, rootState.root, currentFiles, stats);
                 if (!this.stopping) {
-                    await this.pruneDeletedFiles(rootState, currentFiles);
+                    await this.pruneDeletedFiles(rootState, currentFiles, stats);
                     rootState.knownFiles = currentFiles;
+                    await this.saveSnapshotIfDirty();
+                    this.logScanStats(rootState.root, stats, Date.now() - startedAt);
                 }
             }
             finally {
@@ -202,7 +216,12 @@ class DirectoryMarkdownSourceAdapter {
             });
         }, this.debounceMs);
     }
-    async walkDirectory(rootState, dir, currentFiles) {
+    async walkDirectory(rootState, dir, currentFiles, stats) {
+        if (this.shouldPruneDirectory(rootState.root, dir)) {
+            stats.directoriesPruned++;
+            return;
+        }
+        stats.directoriesScanned++;
         await this.ensureDirectoryWatcher(rootState, dir);
         let entries;
         try {
@@ -221,26 +240,43 @@ class DirectoryMarkdownSourceAdapter {
             }
             const child = path.join(dir, entry.name);
             if (entry.isDirectory()) {
-                await this.walkDirectory(rootState, child, currentFiles);
+                await this.walkDirectory(rootState, child, currentFiles, stats);
                 continue;
             }
             if (!entry.isFile() || !isMarkdownFile(entry.name)) {
                 continue;
             }
+            stats.markdownFilesSeen++;
             if (!this.shouldIncludeFile(rootState.root, child)) {
+                stats.filesSkipped++;
                 continue;
             }
+            stats.filesIncluded++;
             currentFiles.add(child);
             try {
-                await this.syncMarkdownFile(rootState, child);
+                const result = await this.syncMarkdownFile(rootState, child);
+                recordSyncResult(stats, result);
             }
             catch (error) {
+                stats.syncErrors++;
                 if (!this.stopping) {
                     this.logger.warn?.(`[markdown-ingest] sync failed for ${child}: ${formatError(error)}`);
                 }
             }
         }
     }
+    shouldPruneDirectory(root, dir) {
+        const relative = toPosixPath(path.relative(root, dir));
+        if (!relative || relative === "." || relative.startsWith("..")) {
+            return false;
+        }
+        for (const pattern of this.excludePatterns) {
+            if (matchesExcludedDirectory(relative, pattern)) {
+                return true;
+            }
+        }
+        return false;
+    }
     async ensureDirectoryWatcher(rootState, dir) {
         if (rootState.directoryWatchers.has(dir)) {
             return;
@@ -282,7 +318,7 @@ class DirectoryMarkdownSourceAdapter {
         }
         return true;
     }
-    async pruneDeletedFiles(rootState, currentFiles) {
+    async pruneDeletedFiles(rootState, currentFiles, stats) {
         const removed = [];
         for (const previous of rootState.knownFiles) {
             if (!currentFiles.has(previous)) {
@@ -295,6 +331,8 @@ class DirectoryMarkdownSourceAdapter {
         for (const filePath of removed) {
             await this.deleteSourceDocument(filePath);
             this.fileStates.delete(filePath);
+            this.snapshotDirty = true;
+            stats.filesDeleted++;
         }
     }
     async syncMarkdownFile(rootState, filePath) {
@@ -304,21 +342,23 @@ class DirectoryMarkdownSourceAdapter {
         if (!stat) {
             await this.deleteSourceDocument(sourceDoc);
             this.fileStates.delete(sourceDoc);
-            return;
+            this.snapshotDirty = true;
+            return "deleted";
         }
         const cached = this.fileStates.get(sourceDoc);
         if (cached && cached.size === stat.size && cached.mtimeMs === stat.mtimeMs) {
-            return;
+            return "unchanged";
         }
         const bytes = await this.safeReadFile(filePath);
         if (!bytes) {
             await this.deleteSourceDocument(sourceDoc);
             this.fileStates.delete(sourceDoc);
-            return;
+            this.snapshotDirty = true;
+            return "deleted";
         }
         const fileHash = hashBytes(bytes);
         if (cached && cached.fileHash === fileHash) {
-            this.fileStates.set(sourceDoc, {
+            this.setFileState(sourceDoc, {
                 root: rootState.root,
                 sourceDoc,
                 relativePath,
@@ -326,16 +366,17 @@ class DirectoryMarkdownSourceAdapter {
                 size: stat.size,
                 mtimeMs: stat.mtimeMs,
             });
-            return;
+            return "unchanged";
         }
         const text = textDecoder.decode(bytes);
         if (this.kind === "obsidian" && this.includePatterns.length === 0 && !looksLikeObsidianNote(filePath, text)) {
             await this.deleteSourceDocument(sourceDoc);
             this.fileStates.delete(sourceDoc);
-            return;
+            this.snapshotDirty = true;
+            return "skipped";
         }
         await this.ingestMarkdownDocument(sourceDoc, text, rootState.root, relativePath, fileHash, stat.size, stat.mtimeMs);
-        this.fileStates.set(sourceDoc, {
+        this.setFileState(sourceDoc, {
             root: rootState.root,
             sourceDoc,
             relativePath,
@@ -343,6 +384,11 @@ class DirectoryMarkdownSourceAdapter {
             size: stat.size,
             mtimeMs: stat.mtimeMs,
         });
+        return "ingested";
+    }
+    setFileState(sourceDoc, state) {
+        this.fileStates.set(sourceDoc, state);
+        this.snapshotDirty = true;
     }
     async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs) {
         const queue = await this.getIngestQueue();
@@ -388,6 +434,98 @@ class DirectoryMarkdownSourceAdapter {
             return null;
         }
     }
+    snapshotFilesForRoot(root) {
+        const files = new Set();
+        for (const state of this.fileStates.values()) {
+            if (state.root === root) {
+                files.add(state.sourceDoc);
+            }
+        }
+        return files;
+    }
+    async loadSnapshot() {
+        if (this.snapshotLoaded) {
+            return;
+        }
+        this.snapshotLoaded = true;
+        let raw;
+        try {
+            raw = await fsp.readFile(this.snapshotPath, "utf8");
+        }
+        catch (error) {
+            if (!formatError(error).includes("ENOENT")) {
+                this.logger.warn?.(`[markdown-ingest] failed to read snapshot ${this.snapshotPath}: ${formatError(error)}`);
+            }
+            return;
+        }
+        try {
+            const parsed = JSON.parse(raw);
+            if (parsed.ingestVersion !== MARKDOWN_INGEST_VERSION || parsed.hashBackend !== HASH_BACKEND || !parsed.files) {
+                return;
+            }
+            const configuredRoots = new Set(this.roots.map((root) => path.resolve(root)));
+            for (const [sourceDoc, state] of Object.entries(parsed.files)) {
+                if (isValidSnapshotState(sourceDoc, state) && configuredRoots.has(path.resolve(state.root))) {
+                    this.fileStates.set(sourceDoc, state);
+                }
+            }
+            this.logger.info?.(`[markdown-ingest] loaded ${this.fileStates.size} ${this.kind} file snapshots from ${this.snapshotPath}`);
+        }
+        catch (error) {
+            this.logger.warn?.(`[markdown-ingest] failed to parse snapshot ${this.snapshotPath}: ${formatError(error)}`);
+        }
+    }
+    async saveSnapshotIfDirty() {
+        if (!this.snapshotDirty) {
+            return;
+        }
+        const payload = {
+            version: 1,
+            ingestVersion: MARKDOWN_INGEST_VERSION,
+            hashBackend: HASH_BACKEND,
+            files: Object.fromEntries([...this.fileStates.entries()].sort(([left], [right]) => left.localeCompare(right))),
+        };
+        try {
+            await fsp.mkdir(path.dirname(this.snapshotPath), { recursive: true });
+            const tmp = `${this.snapshotPath}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
+            await fsp.writeFile(tmp, `${JSON.stringify(payload, null, 2)}\n`);
+            await fsp.rename(tmp, this.snapshotPath);
+            this.snapshotDirty = false;
+        }
+        catch (error) {
+            this.logger.warn?.(`[markdown-ingest] failed to write snapshot ${this.snapshotPath}: ${formatError(error)}`);
+        }
+    }
+    logScanStats(root, stats, durationMs) {
+        this.logger.info?.(`[markdown-ingest] ${this.kind} scan complete root=${root} dirs=${stats.directoriesScanned} prunedDirs=${stats.directoriesPruned} markdown=${stats.markdownFilesSeen} included=${stats.filesIncluded} skipped=${stats.filesSkipped} unchanged=${stats.filesUnchanged} ingested=${stats.filesIngested} deleted=${stats.filesDeleted} errors=${stats.syncErrors} durationMs=${durationMs}`);
+    }
+}
+function createScanStats() {
+    return {
+        directoriesScanned: 0,
+        directoriesPruned: 0,
+        markdownFilesSeen: 0,
+        filesIncluded: 0,
+        filesSkipped: 0,
+        filesUnchanged: 0,
+        filesIngested: 0,
+        filesDeleted: 0,
+        syncErrors: 0,
+    };
+}
+function recordSyncResult(stats, result) {
+    if (result === "ingested") {
+        stats.filesIngested++;
+    }
+    else if (result === "unchanged") {
+        stats.filesUnchanged++;
+    }
+    else if (result === "deleted") {
+        stats.filesDeleted++;
+    }
+    else {
+        stats.filesSkipped++;
+    }
 }
 function toPosixPath(value) {
     return value.split(path.sep).join("/");
@@ -407,11 +545,16 @@ function normalizeMarkdownRoots(roots) {
     }
     return [...resolved];
 }
-function isMarkdownIngestionEnabled(cfg, roots) {
-    if (cfg.markdownIngestionEnabled === false) {
-        return false;
+function resolveMarkdownSnapshotPath(kind, configuredPath) {
+    const trimmed = configuredPath?.trim();
+    if (trimmed) {
+        return path.resolve(trimmed);
     }
-    return roots.length > 0;
+    const stateDir = process.env.OPENCLAW_STATE_DIR?.trim() || path.join(os.homedir(), ".openclaw");
+    return path.join(stateDir, `libravdb-markdown-ingest-${kind}.json`);
+}
+function isMarkdownIngestionEnabled(cfg, roots) {
+    return cfg.markdownIngestionEnabled === true && roots.length > 0;
 }
 function createRealFsApi() {
     return {
@@ -435,6 +578,24 @@ function matchesGlob(value, pattern) {
         .join(".*");
     return new RegExp(`^${escaped}$`).test(value);
 }
+function matchesExcludedDirectory(relativeDir, pattern) {
+    const normalized = relativeDir.replace(/\/+$/, "");
+    return matchesGlob(normalized, pattern) || matchesGlob(`${normalized}/`, pattern) || matchesGlob(`${normalized}/.probe`, pattern);
+}
+function isValidSnapshotState(sourceDoc, value) {
+    if (!value || typeof value !== "object") {
+        return false;
+    }
+    const state = value;
+    return (state.sourceDoc === sourceDoc &&
+        typeof state.root === "string" &&
+        typeof state.relativePath === "string" &&
+        typeof state.fileHash === "string" &&
+        typeof state.size === "number" &&
+        Number.isFinite(state.size) &&
+        typeof state.mtimeMs === "number" &&
+        Number.isFinite(state.mtimeMs));
+}
 function looksLikeObsidianNote(filePath, text) {
     const frontmatterStart = parseFrontmatterStart(text);
     if (frontmatterStart == null) {

package/dist/types.d.ts

@@ -48,6 +48,8 @@ export interface PluginConfig {
     markdownIngestionInclude?: string[];
     markdownIngestionExclude?: string[];
     markdownIngestionDebounceMs?: number;
+    markdownIngestionSnapshotPath?: string;
+    markdownIngestionObsidianSnapshotPath?: string;
     dreamPromotionEnabled?: boolean;
     dreamPromotionDiaryPath?: string;
     dreamPromotionUserId?: string;

package/docs/configuration.md

@@ -98,6 +98,10 @@ The plugin exposes `ingestionGateThreshold` for host-side gating decisions:
 | `markdownIngestionObsidianExclude` | string[] | — | Obsidian glob exclude patterns |
 | `markdownIngestionObsidianDebounceMs` | number | `150` | Obsidian debounce window |
 
+Configured markdown roots are ignored unless the matching enable flag is set to
+`true`. Set `markdownIngestionEnabled: true` for generic roots and
+`markdownIngestionObsidianEnabled: true` for Obsidian vault roots.
+
 ## Dream promotion
 
 | Key | Type | Default | Notes |

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "libravdb-memory",
   "name": "LibraVDB Memory",
   "description": "Persistent vector memory with three-tier hybrid scoring",
-  "version": "1.4.67",
+  "version": "1.4.69",
   "kind": [
     "memory",
     "context-engine"
@@ -206,6 +206,12 @@
         "type": "number",
         "default": 150
       },
+      "markdownIngestionSnapshotPath": {
+        "type": "string"
+      },
+      "markdownIngestionObsidianSnapshotPath": {
+        "type": "string"
+      },
       "dreamPromotionEnabled": {
         "type": "boolean",
         "default": false

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xdarkicex/openclaw-memory-libravdb",
-  "version": "1.4.67",
+  "version": "1.4.69",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",