@xdarkicex/openclaw-memory-libravdb 1.4.45 → 1.4.47

package/dist/ingest-queue.d.ts

@@ -0,0 +1,39 @@
+import type { LoggerLike } from "./types.js";
+import { IngestMode } from "@xdarkicex/libravdb-contracts";
+export interface IngestQueueOptions {
+    /** Max tokens per chunk. Infinity = chunking disabled (retry-only mode). */
+    chunkTokens: number;
+    /** Base delay for exponential backoff retry in ms. */
+    retryBaseDelayMs: number;
+    /** Max retries per chunk. */
+    maxRetries: number;
+}
+interface IngestMarkdownDocumentParams {
+    sourceDoc: string;
+    text: string;
+    tokenizerId: string;
+    coreDoc: boolean;
+    sourceMeta: {
+        sourceRoot: string;
+        sourcePath: string;
+        sourceKind: string;
+        fileHash: string;
+        sourceSize: number;
+        sourceMtimeMs: number;
+        ingestVersion: number;
+        hashBackend: string;
+    };
+    mode?: IngestMode;
+}
+export declare class IngestQueue {
+    private readonly queue;
+    private readonly rpcCall;
+    private readonly logger;
+    private readonly options;
+    private running;
+    constructor(rpcCall: <T>(method: string, params: unknown) => Promise<T>, logger: LoggerLike, options?: Partial<IngestQueueOptions>);
+    enqueueIngest(sourceDoc: string, text: string, baseParams: Omit<IngestMarkdownDocumentParams, "sourceDoc" | "text" | "mode">): Promise<void>;
+    private ingestWithRetry;
+    enqueueDelete(sourceDoc: string): Promise<void>;
+}
+export {};

package/dist/ingest-queue.js

@@ -0,0 +1,123 @@
+import { IngestMode } from "@xdarkicex/libravdb-contracts";
+const DEFAULT_OPTIONS = {
+    chunkTokens: 8192,
+    retryBaseDelayMs: 500,
+    maxRetries: 4,
+};
+export class IngestQueue {
+    queue = [];
+    rpcCall;
+    logger;
+    options;
+    running = false;
+    constructor(rpcCall, logger, options = {}) {
+        this.rpcCall = rpcCall;
+        this.logger = logger;
+        this.options = { ...DEFAULT_OPTIONS, ...options };
+    }
+    async enqueueIngest(sourceDoc, text, baseParams) {
+        if (this.options.chunkTokens === Infinity) {
+            // Retry-only mode: send full text as single chunk
+            return this.ingestWithRetry({
+                ...baseParams,
+                sourceDoc,
+                text,
+                mode: IngestMode.REPLACE,
+            });
+        }
+        const chunks = splitIntoChunks(text, this.options.chunkTokens);
+        if (chunks.length === 1) {
+            return this.ingestWithRetry({
+                ...baseParams,
+                sourceDoc,
+                text: chunks[0].text,
+                mode: IngestMode.REPLACE,
+            });
+        }
+        // Multiple chunks: use APPEND mode for all but last (which can be REPLACE)
+        for (let i = 0; i < chunks.length; i++) {
+            const isLast = i === chunks.length - 1;
+            const chunkParams = {
+                ...baseParams,
+                sourceDoc,
+                text: chunks[i].text,
+                mode: isLast ? IngestMode.REPLACE : IngestMode.APPEND,
+            };
+            await this.ingestWithRetry(chunkParams);
+        }
+    }
+    async ingestWithRetry(params) {
+        await withRetry(() => this.rpcCall("ingest_markdown_document", params), this.options.maxRetries, this.options.retryBaseDelayMs, this.logger, `ingest_markdown_document(${params.sourceDoc})`);
+    }
+    async enqueueDelete(sourceDoc) {
+        await withRetry(() => this.rpcCall("delete_authored_document", { sourceDoc }), this.options.maxRetries, this.options.retryBaseDelayMs, this.logger, `delete_authored_document(${sourceDoc})`);
+    }
+}
+function splitIntoChunks(text, maxTokens) {
+    // Approximate: 4 chars per token for typical English text
+    const maxChars = maxTokens * 4;
+    if (text.length <= maxChars) {
+        return [{ text, ordinal: 0 }];
+    }
+    const chunks = [];
+    let offset = 0;
+    let ordinal = 0;
+    while (offset < text.length) {
+        let end = Math.min(offset + maxChars, text.length);
+        // Walk back up to 256 chars looking for a sentence boundary
+        const probeLimit = Math.min(256, end - offset);
+        let hardCut = end;
+        for (let i = 0; i < probeLimit; i++) {
+            const pos = end - i;
+            const ch = text.charAt(pos);
+            if (ch === "\n" && text.charAt(pos + 1) === "\n") {
+                hardCut = pos + 2;
+                break;
+            }
+        }
+        if (hardCut === end) {
+            for (let i = 0; i < probeLimit; i++) {
+                const pos = end - i;
+                if (text.charAt(pos) === "\n") {
+                    hardCut = pos + 1;
+                    break;
+                }
+            }
+        }
+        if (hardCut === end) {
+            for (let i = 0; i < probeLimit; i++) {
+                const pos = end - i;
+                if (text.charAt(pos) === " ") {
+                    hardCut = pos;
+                    break;
+                }
+            }
+        }
+        chunks.push({ text: text.slice(offset, hardCut), ordinal });
+        ordinal++;
+        offset = hardCut;
+    }
+    return chunks;
+}
+async function withRetry(fn, maxRetries, baseDelayMs, logger, label) {
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (err) {
+            lastError = err;
+            if (attempt < maxRetries) {
+                // Full jitter: random * cap
+                const cap = baseDelayMs * Math.pow(2, attempt);
+                const delay = Math.random() * cap;
+                logger.warn?.(`[ingest-queue] ${label} failed (attempt ${attempt + 1}/${maxRetries + 1}), retrying in ${Math.round(delay)}ms: ${err}`);
+                await sleep(delay);
+            }
+        }
+    }
+    throw lastError instanceof Error ? lastError : new Error(String(lastError));
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/lifecycle-hooks.js

@@ -1,3 +1,4 @@
+import { formatError } from "./format-error.js";
 export function createBeforeResetHook(runtime, logger = console) {
     return async (event, ctx) => {
         const typedEvent = asBeforeResetEvent(event);
@@ -56,9 +57,3 @@ function asSessionEndEvent(value) {
 function isRecord(value) {
     return typeof value === "object" && value !== null;
 }
-function formatError(error) {
-    if (error instanceof Error && error.message.trim()) {
-        return error.message;
-    }
-    return String(error);
-}

package/dist/markdown-ingest.js

@@ -2,6 +2,8 @@ import fs from "node:fs";
 import fsp from "node:fs/promises";
 import path from "node:path";
 import { hashBytes } from "./markdown-hash.js";
+import { formatError } from "./format-error.js";
+import { IngestQueue } from "./ingest-queue.js";
 const DEFAULT_DEBOUNCE_MS = 150;
 const DEFAULT_TOKENIZER_ID = "markdown-ingest:v1";
 const MARKDOWN_INGEST_VERSION = 3;
@@ -77,6 +79,7 @@ class DirectoryMarkdownSourceAdapter {
     tokenizerId;
     coreDoc;
     started = false;
+    ingestQueue = null;
     stopping = false;
     constructor(kind, config, getRpc, logger, fsApi) {
         this.kind = kind;
@@ -342,10 +345,8 @@ class DirectoryMarkdownSourceAdapter {
         });
     }
     async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs) {
-        const rpc = await this.getRpc();
-        const params = {
-            sourceDoc,
-            text,
+        const queue = await this.getIngestQueue();
+        await queue.enqueueIngest(sourceDoc, text, {
             tokenizerId: this.tokenizerId,
             coreDoc: this.coreDoc,
             sourceMeta: {
@@ -358,13 +359,18 @@ class DirectoryMarkdownSourceAdapter {
                 ingestVersion: MARKDOWN_INGEST_VERSION,
                 hashBackend: HASH_BACKEND,
             },
-        };
-        await rpc.call("ingest_markdown_document", params);
+        });
     }
     async deleteSourceDocument(sourceDoc) {
-        const rpc = await this.getRpc();
-        const params = { sourceDoc };
-        await rpc.call("delete_authored_document", params);
+        const queue = await this.getIngestQueue();
+        await queue.enqueueDelete(sourceDoc);
+    }
+    async getIngestQueue() {
+        if (!this.ingestQueue) {
+            const rpc = await this.getRpc();
+            this.ingestQueue = new IngestQueue(rpc.call.bind(rpc), this.logger);
+        }
+        return this.ingestQueue;
     }
     async safeStat(filePath) {
         try {
@@ -429,12 +435,6 @@ function matchesGlob(value, pattern) {
         .join(".*");
     return new RegExp(`^${escaped}$`).test(value);
 }
-function formatError(error) {
-    if (error instanceof Error) {
-        return error.message;
-    }
-    return String(error);
-}
 function looksLikeObsidianNote(filePath, text) {
     if (!text.startsWith("---\n")) {
         return hasInlineObsidianTag(text);

package/dist/plugin-runtime.js

@@ -1,6 +1,7 @@
 import { RpcClient } from "./rpc.js";
 import { GrpcKernelClient } from "./grpc-client.js";
 import { daemonProvisioningHint, startSidecar } from "./sidecar.js";
+import { formatError } from "./format-error.js";
 import { readFileSync } from "node:fs";
 export const DEFAULT_RPC_TIMEOUT_MS = 30000;
 export const STARTUP_HEALTH_TIMEOUT_MS = 2000;
@@ -124,12 +125,6 @@ function loadSecretFromEnv() {
     }
     return undefined;
 }
-function formatError(error) {
-    if (error instanceof Error && error.message.trim()) {
-        return error.message;
-    }
-    return String(error);
-}
 export function enrichStartupError(error, healthMessage) {
     const rawMessage = error instanceof Error ? error.message : String(error);
     const message = rawMessage.trim() || "LibraVDB daemon startup failed";

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "libravdb-memory",
   "name": "LibraVDB Memory",
   "description": "Persistent vector memory with three-tier hybrid scoring",
-  "version": "1.4.45",
+  "version": "1.4.47",
   "kind": [
     "memory",
     "context-engine"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xdarkicex/openclaw-memory-libravdb",
-  "version": "1.4.45",
+  "version": "1.4.47",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -73,7 +73,7 @@
     "@grpc/proto-loader": "^0.8.0",
     "@openclaw/plugin-inspector": "0.3.7",
     "@types/node": "^20.11.0",
-    "@xdarkicex/libravdb-contracts": "^0.0.1",
+    "@xdarkicex/libravdb-contracts": "^0.0.6",
     "esbuild": "^0.27.0",
     "typescript": "^6.0.3"
   },