@xdarkicex/openclaw-memory-libravdb 1.4.46 → 1.4.48

package/dist/ingest-queue.d.ts

@@ -0,0 +1,39 @@
+import type { LoggerLike } from "./types.js";
+import { IngestMode } from "@xdarkicex/libravdb-contracts";
+export interface IngestQueueOptions {
+    /** Max tokens per chunk. Infinity = chunking disabled (retry-only mode). */
+    chunkTokens: number;
+    /** Base delay for exponential backoff retry in ms. */
+    retryBaseDelayMs: number;
+    /** Max retries per chunk. */
+    maxRetries: number;
+}
+interface IngestMarkdownDocumentParams {
+    sourceDoc: string;
+    text: string;
+    tokenizerId: string;
+    coreDoc: boolean;
+    sourceMeta: {
+        sourceRoot: string;
+        sourcePath: string;
+        sourceKind: string;
+        fileHash: string;
+        sourceSize: number;
+        sourceMtimeMs: number;
+        ingestVersion: number;
+        hashBackend: string;
+    };
+    mode?: IngestMode;
+}
+export declare class IngestQueue {
+    private readonly queue;
+    private readonly rpcCall;
+    private readonly logger;
+    private readonly options;
+    private running;
+    constructor(rpcCall: <T>(method: string, params: unknown) => Promise<T>, logger: LoggerLike, options?: Partial<IngestQueueOptions>);
+    enqueueIngest(sourceDoc: string, text: string, baseParams: Omit<IngestMarkdownDocumentParams, "sourceDoc" | "text" | "mode">): Promise<void>;
+    private ingestWithRetry;
+    enqueueDelete(sourceDoc: string): Promise<void>;
+}
+export {};

package/dist/ingest-queue.js

@@ -0,0 +1,124 @@
+import { IngestMode } from "@xdarkicex/libravdb-contracts";
+const DEFAULT_OPTIONS = {
+    chunkTokens: 8192,
+    retryBaseDelayMs: 500,
+    maxRetries: 4,
+};
+export class IngestQueue {
+    queue = [];
+    rpcCall;
+    logger;
+    options;
+    running = false;
+    constructor(rpcCall, logger, options = {}) {
+        this.rpcCall = rpcCall;
+        this.logger = logger;
+        this.options = { ...DEFAULT_OPTIONS, ...options };
+    }
+    async enqueueIngest(sourceDoc, text, baseParams) {
+        if (this.options.chunkTokens === Infinity) {
+            // Retry-only mode: send full text as single chunk
+            return this.ingestWithRetry({
+                ...baseParams,
+                sourceDoc,
+                text,
+                mode: IngestMode.REPLACE,
+            });
+        }
+        const chunks = splitIntoChunks(text, this.options.chunkTokens);
+        if (chunks.length === 1) {
+            return this.ingestWithRetry({
+                ...baseParams,
+                sourceDoc,
+                text: chunks[0].text,
+                mode: IngestMode.REPLACE,
+            });
+        }
+        // Multiple chunks: clear the source once, then append the remaining chunks.
+        // Sending REPLACE last deletes the earlier chunks from the same source_doc.
+        for (let i = 0; i < chunks.length; i++) {
+            const isFirst = i === 0;
+            const chunkParams = {
+                ...baseParams,
+                sourceDoc,
+                text: chunks[i].text,
+                mode: isFirst ? IngestMode.REPLACE : IngestMode.APPEND,
+            };
+            await this.ingestWithRetry(chunkParams);
+        }
+    }
+    async ingestWithRetry(params) {
+        await withRetry(() => this.rpcCall("ingest_markdown_document", params), this.options.maxRetries, this.options.retryBaseDelayMs, this.logger, `ingest_markdown_document(${params.sourceDoc})`);
+    }
+    async enqueueDelete(sourceDoc) {
+        await withRetry(() => this.rpcCall("delete_authored_document", { sourceDoc }), this.options.maxRetries, this.options.retryBaseDelayMs, this.logger, `delete_authored_document(${sourceDoc})`);
+    }
+}
+function splitIntoChunks(text, maxTokens) {
+    // Approximate: 4 chars per token for typical English text
+    const maxChars = maxTokens * 4;
+    if (text.length <= maxChars) {
+        return [{ text, ordinal: 0 }];
+    }
+    const chunks = [];
+    let offset = 0;
+    let ordinal = 0;
+    while (offset < text.length) {
+        let end = Math.min(offset + maxChars, text.length);
+        // Walk back up to 256 chars looking for a sentence boundary
+        const probeLimit = Math.min(256, end - offset);
+        let hardCut = end;
+        for (let i = 0; i < probeLimit; i++) {
+            const pos = end - i;
+            const ch = text.charAt(pos);
+            if (ch === "\n" && text.charAt(pos + 1) === "\n") {
+                hardCut = pos + 2;
+                break;
+            }
+        }
+        if (hardCut === end) {
+            for (let i = 0; i < probeLimit; i++) {
+                const pos = end - i;
+                if (text.charAt(pos) === "\n") {
+                    hardCut = pos + 1;
+                    break;
+                }
+            }
+        }
+        if (hardCut === end) {
+            for (let i = 0; i < probeLimit; i++) {
+                const pos = end - i;
+                if (text.charAt(pos) === " ") {
+                    hardCut = pos;
+                    break;
+                }
+            }
+        }
+        chunks.push({ text: text.slice(offset, hardCut), ordinal });
+        ordinal++;
+        offset = hardCut;
+    }
+    return chunks;
+}
+async function withRetry(fn, maxRetries, baseDelayMs, logger, label) {
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (err) {
+            lastError = err;
+            if (attempt < maxRetries) {
+                // Full jitter: random * cap
+                const cap = baseDelayMs * Math.pow(2, attempt);
+                const delay = Math.random() * cap;
+                logger.warn?.(`[ingest-queue] ${label} failed (attempt ${attempt + 1}/${maxRetries + 1}), retrying in ${Math.round(delay)}ms: ${err}`);
+                await sleep(delay);
+            }
+        }
+    }
+    throw lastError instanceof Error ? lastError : new Error(String(lastError));
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/markdown-ingest.js

@@ -3,6 +3,7 @@ import fsp from "node:fs/promises";
 import path from "node:path";
 import { hashBytes } from "./markdown-hash.js";
 import { formatError } from "./format-error.js";
+import { IngestQueue } from "./ingest-queue.js";
 const DEFAULT_DEBOUNCE_MS = 150;
 const DEFAULT_TOKENIZER_ID = "markdown-ingest:v1";
 const MARKDOWN_INGEST_VERSION = 3;
@@ -78,6 +79,7 @@ class DirectoryMarkdownSourceAdapter {
     tokenizerId;
     coreDoc;
     started = false;
+    ingestQueue = null;
     stopping = false;
     constructor(kind, config, getRpc, logger, fsApi) {
         this.kind = kind;
@@ -343,10 +345,8 @@ class DirectoryMarkdownSourceAdapter {
         });
     }
     async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs) {
-        const rpc = await this.getRpc();
-        const params = {
-            sourceDoc,
-            text,
+        const queue = await this.getIngestQueue();
+        await queue.enqueueIngest(sourceDoc, text, {
             tokenizerId: this.tokenizerId,
             coreDoc: this.coreDoc,
             sourceMeta: {
@@ -359,13 +359,18 @@ class DirectoryMarkdownSourceAdapter {
                 ingestVersion: MARKDOWN_INGEST_VERSION,
                 hashBackend: HASH_BACKEND,
             },
-        };
-        await rpc.call("ingest_markdown_document", params);
+        });
     }
     async deleteSourceDocument(sourceDoc) {
-        const rpc = await this.getRpc();
-        const params = { sourceDoc };
-        await rpc.call("delete_authored_document", params);
+        const queue = await this.getIngestQueue();
+        await queue.enqueueDelete(sourceDoc);
+    }
+    async getIngestQueue() {
+        if (!this.ingestQueue) {
+            const rpc = await this.getRpc();
+            this.ingestQueue = new IngestQueue(rpc.call.bind(rpc), this.logger);
+        }
+        return this.ingestQueue;
     }
     async safeStat(filePath) {
         try {

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "libravdb-memory",
   "name": "LibraVDB Memory",
   "description": "Persistent vector memory with three-tier hybrid scoring",
-  "version": "1.4.46",
+  "version": "1.4.48",
   "kind": [
     "memory",
     "context-engine"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xdarkicex/openclaw-memory-libravdb",
-  "version": "1.4.46",
+  "version": "1.4.48",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -73,7 +73,7 @@
     "@grpc/proto-loader": "^0.8.0",
     "@openclaw/plugin-inspector": "0.3.7",
     "@types/node": "^20.11.0",
-    "@xdarkicex/libravdb-contracts": "^0.0.1",
+    "@xdarkicex/libravdb-contracts": "^0.0.6",
     "esbuild": "^0.27.0",
     "typescript": "^6.0.3"
   },