@xdarkicex/openclaw-memory-libravdb 1.5.4 → 1.6.0

package/dist/ingest-queue.d.ts

@@ -7,6 +7,8 @@ export interface IngestQueueOptions {
     retryBaseDelayMs: number;
     /** Max retries per chunk. */
     maxRetries: number;
+    /** Called after each chunk is accepted so scan-level state stays current. */
+    onChunkFeedback?: (feedback: IngestFeedback) => void;
 }
 interface IngestMarkdownDocumentParams {
     sourceDoc: string;
@@ -20,19 +22,40 @@ interface IngestMarkdownDocumentParams {
         fileHash: string;
         sourceSize: number;
         sourceMtimeMs: number;
+        sourceCtimeMs: number;
         ingestVersion: number;
         hashBackend: string;
     };
     mode?: IngestMode;
 }
+interface IngestFeedback {
+    queueDepth: number;
+    queueCapacity: number;
+    acceptMore: boolean;
+    retryAfterMs: number;
+    processingTimeUs: number;
+    nodesAccepted: number;
+    nodesRejected: number;
+    tokensIngested: number;
+    tokenBurstLimit: number;
+    walDepth?: number;
+    walCapacity?: number;
+}
+interface IngestMarkdownDocumentResponse {
+    ok: boolean;
+    feedback?: IngestFeedback;
+}
 export declare class IngestQueue {
     private readonly queue;
-    private readonly rpcCall;
+    private readonly ingestDocument;
+    private readonly deleteDocument;
     private readonly logger;
     private readonly options;
     private running;
-    constructor(rpcCall: <T>(method: string, params: unknown) => Promise<T>, logger: LoggerLike, options?: Partial<IngestQueueOptions>);
-    enqueueIngest(sourceDoc: string, text: string, baseParams: Omit<IngestMarkdownDocumentParams, "sourceDoc" | "text" | "mode">): Promise<void>;
+    constructor(ingestDocument: (params: IngestMarkdownDocumentParams) => Promise<IngestMarkdownDocumentResponse>, deleteDocument: (params: {
+        sourceDoc: string;
+    }) => Promise<unknown>, logger: LoggerLike, options?: Partial<IngestQueueOptions>);
+    enqueueIngest(sourceDoc: string, text: string, baseParams: Omit<IngestMarkdownDocumentParams, "sourceDoc" | "text" | "mode">, maxChunkTokens?: number): Promise<IngestFeedback | undefined>;
     private ingestWithRetry;
     enqueueDelete(sourceDoc: string): Promise<void>;
 }

package/dist/ingest-queue.js

@@ -6,12 +6,14 @@ const DEFAULT_OPTIONS = {
 };
 export class IngestQueue {
     queue = [];
-    rpcCall;
+    ingestDocument;
+    deleteDocument;
     logger;
     options;
     running = false;
-    constructor(rpcCall, logger, options = {}) {
-        this.rpcCall = rpcCall;
+    constructor(ingestDocument, deleteDocument, logger, options = {}) {
+        this.ingestDocument = ingestDocument;
+        this.deleteDocument = deleteDocument;
         this.logger = logger;
         this.options = { ...DEFAULT_OPTIONS, ...options };
         if (!(this.options.chunkTokens > 0)) {
@@ -19,43 +21,62 @@ export class IngestQueue {
             this.options.chunkTokens = DEFAULT_OPTIONS.chunkTokens;
         }
     }
-    async enqueueIngest(sourceDoc, text, baseParams) {
+    async enqueueIngest(sourceDoc, text, baseParams, maxChunkTokens) {
         if (this.options.chunkTokens === Infinity) {
-            // Retry-only mode: send full text as single chunk
-            return this.ingestWithRetry({
+            const resp = await this.ingestWithRetry({
                 ...baseParams,
                 sourceDoc,
                 text,
                 mode: IngestMode.REPLACE,
             });
+            return resp.feedback;
         }
-        const chunks = splitIntoChunks(text, this.options.chunkTokens);
-        if (chunks.length === 1) {
-            return this.ingestWithRetry({
-                ...baseParams,
-                sourceDoc,
-                text: chunks[0].text,
-                mode: IngestMode.REPLACE,
-            });
-        }
-        // Multiple chunks: clear the source once, then append the remaining chunks.
-        // Sending REPLACE last deletes the earlier chunks from the same source_doc.
-        for (let i = 0; i < chunks.length; i++) {
-            const isFirst = i === 0;
+        let currentLimit = maxChunkTokens && maxChunkTokens > 0 ? maxChunkTokens : this.options.chunkTokens;
+        let offset = 0;
+        let isFirst = true;
+        let lastFeedback;
+        while (offset < text.length) {
+            const remainingText = text.slice(offset);
+            const chunks = splitIntoChunks(remainingText, currentLimit);
+            const chunkText = chunks[0].text;
             const chunkParams = {
                 ...baseParams,
                 sourceDoc,
-                text: chunks[i].text,
+                text: chunkText,
                 mode: isFirst ? IngestMode.REPLACE : IngestMode.APPEND,
             };
-            await this.ingestWithRetry(chunkParams);
+            const resp = await this.ingestWithRetry(chunkParams);
+            lastFeedback = resp.feedback;
+            if (lastFeedback &&
+                lastFeedback.nodesAccepted === 0 &&
+                lastFeedback.tokenBurstLimit &&
+                lastFeedback.tokenBurstLimit > 0 &&
+                lastFeedback.tokenBurstLimit < currentLimit) {
+                currentLimit = lastFeedback.tokenBurstLimit;
+                continue;
+            }
+            if (lastFeedback && lastFeedback.nodesAccepted === 0) {
+                this.logger.warn?.(`[ingest-queue] Chunk permanently rejected for ${sourceDoc} ` +
+                    `at offset=${offset} length=${chunkText.length} ` +
+                    `tokenBurstLimit=${lastFeedback.tokenBurstLimit ?? "unset"}`);
+            }
+            if (this.options.onChunkFeedback && lastFeedback) {
+                this.options.onChunkFeedback(lastFeedback);
+            }
+            offset += chunkText.length;
+            isFirst = false;
+            if (lastFeedback && !lastFeedback.acceptMore && offset < text.length) {
+                const delay = lastFeedback.retryAfterMs || 1000;
+                await new Promise((resolve) => setTimeout(resolve, delay));
+            }
         }
+        return lastFeedback;
     }
     async ingestWithRetry(params) {
-        await withRetry(() => this.rpcCall("ingest_markdown_document", params), this.options.maxRetries, this.options.retryBaseDelayMs, this.logger, `ingest_markdown_document(${params.sourceDoc})`);
+        return withRetry(() => this.ingestDocument(params), this.options.maxRetries, this.options.retryBaseDelayMs, this.logger, `ingest_markdown_document(${params.sourceDoc})`);
     }
     async enqueueDelete(sourceDoc) {
-        await withRetry(() => this.rpcCall("delete_authored_document", { sourceDoc }), this.options.maxRetries, this.options.retryBaseDelayMs, this.logger, `delete_authored_document(${sourceDoc})`);
+        await withRetry(() => this.deleteDocument({ sourceDoc }), this.options.maxRetries, this.options.retryBaseDelayMs, this.logger, `delete_authored_document(${sourceDoc})`);
     }
 }
 function splitIntoChunks(text, maxTokens) {

package/dist/libravdb-client.d.ts

@@ -0,0 +1,59 @@
+import type { Interceptor } from "@connectrpc/connect";
+import type { PartialMessage } from "@bufbuild/protobuf";
+import type { AfterTurnKernelRequest, AfterTurnKernelResponse, AssembleContextInternalRequest, AssembleContextInternalResponse, BootstrapSessionKernelRequest, BootstrapSessionKernelResponse, CompactSessionRequest, CompactSessionResponse, DeleteAuthoredDocumentRequest, DeleteAuthoredDocumentResponse, DreamPromotionResponse, ExportMemoryRequest, ExportMemoryResponse, FlushNamespaceRequest, FlushNamespaceResponse, FlushRequest, FlushResponse, HealthRequest, HealthResponse, IngestMarkdownDocumentRequest, IngestMarkdownDocumentResponse, IngestMessageKernelRequest, IngestMessageKernelResponse, ListCollectionRequest, ListCollectionResponse, ListLifecycleJournalRequest, ListLifecycleJournalResponse, MarkMemorySupersededRequest, MarkMemorySupersededResponse, MemoryStatusRequest, MemoryStatusResponse, PromoteDreamEntriesRequest, RankCandidatesRequest, RankCandidatesResponse, RebuildIndexRequest, RebuildIndexResponse, ReindexAuthoredDocumentRequest, ReindexAuthoredDocumentResponse, SearchTextCollectionsRequest, SearchTextRequest, SearchTextResponse, SessionLifecycleHintRequest, SessionLifecycleHintResponse } from "@xdarkicex/libravdb-contracts";
+export interface LibravDBClientOptions {
+    endpoint?: string;
+    secret?: string;
+    timeoutMs?: number;
+    tlsCaPath?: string;
+    tlsMode?: "auto" | "tls" | "insecure";
+    tlsClientCertPath?: string;
+    tlsClientKeyPath?: string;
+}
+export declare function resolveClientEndpoint(configuredEndpoint?: string): string;
+interface RpcMutex {
+    current: Promise<void>;
+    lock(): Promise<() => void>;
+}
+export interface AuthInterceptorState {
+    readonly secret: string | undefined;
+    nonceHex: string | undefined;
+    bootstrap(): Promise<void>;
+    readonly rpcMutex: RpcMutex;
+}
+export declare function createAuthInterceptor(state: AuthInterceptorState): Interceptor;
+export declare class LibravDBClient {
+    private client;
+    private readonly secret;
+    private nonceHex;
+    private closed;
+    constructor(options?: LibravDBClientOptions);
+    bootstrapHandshake(): Promise<void>;
+    private guardOpen;
+    health(req?: PartialMessage<HealthRequest>): Promise<HealthResponse>;
+    status(req?: PartialMessage<MemoryStatusRequest>): Promise<MemoryStatusResponse>;
+    flush(req?: PartialMessage<FlushRequest>): Promise<FlushResponse>;
+    sessionLifecycleHint(req: PartialMessage<SessionLifecycleHintRequest>): Promise<SessionLifecycleHintResponse>;
+    listLifecycleJournal(req: PartialMessage<ListLifecycleJournalRequest>): Promise<ListLifecycleJournalResponse>;
+    ingestMarkdownDocument(req: PartialMessage<IngestMarkdownDocumentRequest>): Promise<IngestMarkdownDocumentResponse>;
+    promoteDreamEntries(req: PartialMessage<PromoteDreamEntriesRequest>): Promise<DreamPromotionResponse>;
+    reindexAuthoredDocument(req: PartialMessage<ReindexAuthoredDocumentRequest>): Promise<ReindexAuthoredDocumentResponse>;
+    deleteAuthoredDocument(req: PartialMessage<DeleteAuthoredDocumentRequest>): Promise<DeleteAuthoredDocumentResponse>;
+    markMemorySuperseded(req: PartialMessage<MarkMemorySupersededRequest>): Promise<MarkMemorySupersededResponse>;
+    searchText(req: PartialMessage<SearchTextRequest>): Promise<SearchTextResponse>;
+    searchTextCollections(req: PartialMessage<SearchTextCollectionsRequest>): Promise<SearchTextResponse>;
+    listCollection(req: PartialMessage<ListCollectionRequest>): Promise<ListCollectionResponse>;
+    exportMemory(req: PartialMessage<ExportMemoryRequest>): Promise<ExportMemoryResponse>;
+    flushNamespace(req: PartialMessage<FlushNamespaceRequest>): Promise<FlushNamespaceResponse>;
+    rebuildIndex(req: PartialMessage<RebuildIndexRequest>, opts?: {
+        timeoutMs?: number;
+    }): Promise<RebuildIndexResponse>;
+    bootstrapSessionKernel(req: PartialMessage<BootstrapSessionKernelRequest>): Promise<BootstrapSessionKernelResponse>;
+    ingestMessageKernel(req: PartialMessage<IngestMessageKernelRequest>): Promise<IngestMessageKernelResponse>;
+    afterTurnKernel(req: PartialMessage<AfterTurnKernelRequest>): Promise<AfterTurnKernelResponse>;
+    assembleContextInternal(req: PartialMessage<AssembleContextInternalRequest>): Promise<AssembleContextInternalResponse>;
+    compactSession(req: PartialMessage<CompactSessionRequest>): Promise<CompactSessionResponse>;
+    rankCandidates(req: PartialMessage<RankCandidatesRequest>): Promise<RankCandidatesResponse>;
+    close(): void;
+}
+export {};

package/dist/libravdb-client.js

@@ -0,0 +1,296 @@
+import { createPromiseClient } from "@connectrpc/connect";
+import { createGrpcTransport } from "@connectrpc/connect-node";
+import { LibravDB } from "@xdarkicex/libravdb-contracts/client";
+import { createHmac } from "node:crypto";
+import fs from "node:fs";
+import net from "node:net";
+import os from "node:os";
+import path from "node:path";
+export function resolveClientEndpoint(configuredEndpoint) {
+    if (configuredEndpoint && configuredEndpoint !== "auto")
+        return configuredEndpoint;
+    if (process.env.LIBRAVDB_GRPC_ENDPOINT)
+        return process.env.LIBRAVDB_GRPC_ENDPOINT;
+    if (process.platform === "win32")
+        return "tcp:127.0.0.1:37421";
+    const sockName = "libravdb.sock";
+    const candidateDirs = [
+        path.join(os.homedir(), ".libravdbd", "run"),
+        "/opt/homebrew/var/libravdbd/run",
+        "/usr/local/var/libravdbd/run",
+    ];
+    for (const dir of candidateDirs) {
+        const fullPath = path.join(dir, sockName);
+        if (fs.existsSync(fullPath))
+            return `unix:${fullPath}`;
+    }
+    return `unix:${path.join(os.homedir(), ".libravdbd", "run", sockName)}`;
+}
+function createRpcMutex() {
+    return {
+        current: Promise.resolve(),
+        async lock() {
+            let release;
+            const p = new Promise(r => release = r);
+            const prev = this.current;
+            this.current = prev.then(() => p);
+            await prev;
+            return release;
+        }
+    };
+}
+export function createAuthInterceptor(state) {
+    return (next) => async (req) => {
+        // Health does not participate in the nonce chain — bypass the
+        // mutex entirely so recovery can call Health without deadlocking.
+        if (req.method.name === "Health") {
+            return next(req);
+        }
+        const release = await state.rpcMutex.lock();
+        try {
+            // Lost the nonce? Recover inside the lock so queued requests
+            // wait for the chain to be restored instead of failing spuriously.
+            if (state.secret && !state.nonceHex) {
+                await state.bootstrap();
+                if (!state.nonceHex) {
+                    throw new Error("LibraVDB: bootstrap handshake did not return a nonce");
+                }
+            }
+            if (state.secret && state.nonceHex) {
+                const hmac = createHmac("sha256", state.secret);
+                hmac.update(state.nonceHex);
+                req.header.set("x-libravdb-nonce", state.nonceHex);
+                req.header.set("x-libravdb-auth", hmac.digest("hex"));
+            }
+            let res;
+            try {
+                res = await next(req);
+            }
+            catch (error) {
+                if (state.secret && state.nonceHex) {
+                    state.nonceHex = undefined;
+                }
+                throw error;
+            }
+            if (state.secret) {
+                const nextNonce = res.header.get("x-libravdb-nonce") || res.trailer.get("x-libravdb-nonce");
+                if (nextNonce) {
+                    state.nonceHex = nextNonce;
+                }
+                else {
+                    state.nonceHex = undefined;
+                }
+            }
+            return res;
+        }
+        finally {
+            release();
+        }
+    };
+}
+export class LibravDBClient {
+    client;
+    secret;
+    nonceHex;
+    closed = false;
+    constructor(options = {}) {
+        this.secret = options.secret ?? loadSecretFromEnv();
+        const rawEndpoint = resolveClientEndpoint(options.endpoint);
+        const isUnix = rawEndpoint.startsWith("unix:");
+        const socketPath = isUnix ? rawEndpoint.slice(5) : undefined;
+        const credMode = resolveCredentialMode(rawEndpoint, options.tlsMode);
+        const isInsecure = isUnix || credMode === "insecure";
+        const targetUrl = isUnix
+            ? "http://localhost"
+            : rawEndpoint.replace(/^tcp:/, isInsecure ? "http://" : "https://");
+        let rootCerts = null;
+        let clientKey = null;
+        let clientCert = null;
+        if (!isInsecure && options.tlsCaPath) {
+            rootCerts = fs.readFileSync(options.tlsCaPath);
+        }
+        if (options.tlsClientCertPath && options.tlsClientKeyPath) {
+            clientCert = fs.readFileSync(options.tlsClientCertPath);
+            clientKey = fs.readFileSync(options.tlsClientKeyPath);
+        }
+        const rpcMutex = createRpcMutex();
+        const self = this;
+        const authInterceptor = createAuthInterceptor({
+            secret: this.secret,
+            get nonceHex() { return self.nonceHex; },
+            set nonceHex(v) { self.nonceHex = v; },
+            bootstrap: () => self.bootstrapHandshake(),
+            rpcMutex,
+        });
+        const transport = createGrpcTransport({
+            baseUrl: targetUrl,
+            httpVersion: "2",
+            nodeOptions: isUnix
+                ? { createConnection: () => net.connect(socketPath) }
+                : {
+                    ...(rootCerts ? { ca: rootCerts } : {}),
+                    ...(clientKey ? { key: clientKey } : {}),
+                    ...(clientCert ? { cert: clientCert } : {}),
+                    ...(isInsecure ? { rejectUnauthorized: false } : {}),
+                },
+            defaultTimeoutMs: options.timeoutMs ?? 30000,
+            interceptors: [authInterceptor],
+        });
+        this.client = createPromiseClient(LibravDB, transport);
+    }
+    async bootstrapHandshake() {
+        this.guardOpen();
+        try {
+            await this.client.health({ service: "" }, {
+                onHeader: (headers) => {
+                    const nonce = headers.get("x-libravdb-nonce");
+                    if (nonce)
+                        this.nonceHex = nonce;
+                },
+            });
+        }
+        catch (error) {
+            throw new Error(`LibraVDB: failed to handshake with daemon: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    guardOpen() {
+        if (this.closed) {
+            throw new Error("LibravDB client is closed");
+        }
+    }
+    // ── Session lifecycle ────────────────────────────────────────────
+    async health(req = {}) {
+        this.guardOpen();
+        return this.client.health(req);
+    }
+    async status(req = {}) {
+        this.guardOpen();
+        return this.client.status(req);
+    }
+    async flush(req = {}) {
+        this.guardOpen();
+        return this.client.flush(req);
+    }
+    async sessionLifecycleHint(req) {
+        this.guardOpen();
+        return this.client.sessionLifecycleHint(req);
+    }
+    async listLifecycleJournal(req) {
+        this.guardOpen();
+        return this.client.listLifecycleJournal(req);
+    }
+    // ── Ingest ───────────────────────────────────────────────────────
+    async ingestMarkdownDocument(req) {
+        this.guardOpen();
+        return this.client.ingestMarkdownDocument(req);
+    }
+    async promoteDreamEntries(req) {
+        this.guardOpen();
+        return this.client.promoteDreamEntries(req);
+    }
+    async reindexAuthoredDocument(req) {
+        this.guardOpen();
+        return this.client.reindexAuthoredDocument(req);
+    }
+    async deleteAuthoredDocument(req) {
+        this.guardOpen();
+        return this.client.deleteAuthoredDocument(req);
+    }
+    async markMemorySuperseded(req) {
+        this.guardOpen();
+        return this.client.markMemorySuperseded(req);
+    }
+    // ── Search / query ───────────────────────────────────────────────
+    async searchText(req) {
+        this.guardOpen();
+        return this.client.searchText(req);
+    }
+    async searchTextCollections(req) {
+        this.guardOpen();
+        return this.client.searchTextCollections(req);
+    }
+    async listCollection(req) {
+        this.guardOpen();
+        return this.client.listCollection(req);
+    }
+    // ── Memory ───────────────────────────────────────────────────────
+    async exportMemory(req) {
+        this.guardOpen();
+        return this.client.exportMemory(req);
+    }
+    async flushNamespace(req) {
+        this.guardOpen();
+        return this.client.flushNamespace(req);
+    }
+    // ── Index ────────────────────────────────────────────────────────
+    async rebuildIndex(req, opts) {
+        this.guardOpen();
+        return this.client.rebuildIndex(req, opts);
+    }
+    // ── Kernel ───────────────────────────────────────────────────────
+    async bootstrapSessionKernel(req) {
+        this.guardOpen();
+        return this.client.bootstrapSessionKernel(req);
+    }
+    async ingestMessageKernel(req) {
+        this.guardOpen();
+        return this.client.ingestMessageKernel(req);
+    }
+    async afterTurnKernel(req) {
+        this.guardOpen();
+        return this.client.afterTurnKernel(req);
+    }
+    async assembleContextInternal(req) {
+        this.guardOpen();
+        return this.client.assembleContextInternal(req);
+    }
+    async compactSession(req) {
+        this.guardOpen();
+        return this.client.compactSession(req);
+    }
+    async rankCandidates(req) {
+        this.guardOpen();
+        return this.client.rankCandidates(req);
+    }
+    close() {
+        this.closed = true;
+    }
+}
+function resolveCredentialMode(endpoint, tlsMode) {
+    if (tlsMode === "tls")
+        return "tls";
+    if (tlsMode === "insecure")
+        return "insecure";
+    const target = endpoint.startsWith("tcp:") ? endpoint.slice(4) : endpoint;
+    if (target.startsWith("unix:"))
+        return "insecure";
+    const host = extractHost(target);
+    const normalized = host.toLowerCase();
+    return normalized === "localhost" || normalized === "127.0.0.1" || normalized === "::1"
+        ? "insecure"
+        : "tls";
+}
+function extractHost(target) {
+    const withoutDns = target.startsWith("dns:///") ? target.slice("dns:///".length) : target;
+    if (withoutDns.startsWith("[")) {
+        const close = withoutDns.indexOf("]");
+        return close > 0 ? withoutDns.slice(1, close) : withoutDns;
+    }
+    const sep = withoutDns.lastIndexOf(":");
+    return sep > 0 ? withoutDns.slice(0, sep) : withoutDns;
+}
+function loadSecretFromEnv() {
+    const secret = process.env.LIBRAVDB_AUTH_SECRET;
+    if (secret)
+        return secret;
+    const secretPath = process.env.LIBRAVDB_AUTH_SECRET_FILE;
+    if (secretPath) {
+        try {
+            return fs.readFileSync(secretPath, "utf8").trim();
+        }
+        catch {
+            return undefined;
+        }
+    }
+    return undefined;
+}

package/dist/markdown-ingest.d.ts

@@ -1,11 +1,8 @@
 import type { LoggerLike, PluginConfig } from "./types.js";
+import type { ClientGetter } from "./plugin-runtime.js";
 type Disposable = {
     close(): void;
 };
-interface RpcLike {
-    call<T>(method: string, params: unknown): Promise<T>;
-}
-type RpcGetterLike = () => Promise<RpcLike>;
 interface FsDirentLike {
     name: string;
     isDirectory(): boolean;
@@ -14,14 +11,22 @@ interface FsDirentLike {
 interface FsWatcherLike extends Disposable {
     on(event: "error", handler: (error: Error) => void): void;
 }
+interface FsReadStream {
+    read(buffer: Uint8Array): Promise<{
+        bytesRead: number;
+    }>;
+    close(): Promise<void>;
+}
 interface FsApi {
     readdir(dir: string): Promise<FsDirentLike[]>;
     readFile(file: string): Promise<Uint8Array>;
     stat(file: string): Promise<{
         size: number;
         mtimeMs: number;
+        ctimeMs: number;
     }>;
     watch(dir: string, onChange: (event: string, filename: string | Buffer | null) => void): FsWatcherLike;
+    openReadStream(file: string): Promise<FsReadStream>;
 }
 export interface MarkdownSourceAdapter {
     kind: string;
@@ -39,5 +44,5 @@ export interface MarkdownIngestionSnapshot {
     size: number;
     mtimeMs: number;
 }
-export declare function createMarkdownIngestionHandle(cfg: PluginConfig, getRpc: RpcGetterLike, logger?: LoggerLike, fsApi?: FsApi): MarkdownIngestionHandle;
+export declare function createMarkdownIngestionHandle(cfg: PluginConfig, getClient: ClientGetter, logger?: LoggerLike, fsApi?: FsApi): MarkdownIngestionHandle;
 export {};