@xdarkicex/openclaw-memory-libravdb 1.4.5 → 1.4.7

package/dist/markdown-hash.js

@@ -0,0 +1,82 @@
+const WASM_BYTES = new Uint8Array([
+    0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x07, 0x01, 0x60,
+    0x02, 0x7f, 0x7f, 0x01, 0x7e, 0x03, 0x02, 0x01, 0x00, 0x04, 0x05, 0x01,
+    0x70, 0x01, 0x01, 0x01, 0x05, 0x03, 0x01, 0x00, 0x10, 0x06, 0x09, 0x01,
+    0x7f, 0x01, 0x41, 0x80, 0x80, 0xc0, 0x00, 0x0b, 0x07, 0x19, 0x02, 0x06,
+    0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x02, 0x00, 0x0c, 0x68, 0x61, 0x73,
+    0x68, 0x5f, 0x66, 0x6e, 0x76, 0x31, 0x61, 0x36, 0x34, 0x00, 0x00, 0x0a,
+    0x41, 0x01, 0x3f, 0x01, 0x01, 0x7e, 0x42, 0x83, 0x87, 0xf4, 0x9c, 0x87,
+    0xf6, 0xc3, 0xb2, 0x14, 0x21, 0x02, 0x02, 0x40, 0x20, 0x01, 0x45, 0x0d,
+    0x00, 0x03, 0x40, 0x20, 0x02, 0x20, 0x00, 0x31, 0x00, 0x00, 0x85, 0x42,
+    0xb3, 0x83, 0x80, 0x80, 0x80, 0x20, 0x7e, 0x21, 0x02, 0x20, 0x00, 0x41,
+    0x01, 0x6a, 0x21, 0x00, 0x20, 0x01, 0x41, 0x7f, 0x6a, 0x22, 0x01, 0x0d,
+    0x00, 0x0b, 0x0b, 0x20, 0x02, 0x0b,
+]);
+const textEncoder = new TextEncoder();
+const FNV_OFFSET_BASIS = 0xcbf29ce484222325n;
+const FNV_PRIME = 0x100000001b3n;
+class Fnv64Fallback {
+    kind = "js-fnv1a64";
+    hash(bytes) {
+        let hash = FNV_OFFSET_BASIS;
+        for (let i = 0; i < bytes.length; i++) {
+            hash ^= BigInt(bytes[i] ?? 0);
+            hash = BigInt.asUintN(64, hash * FNV_PRIME);
+        }
+        return hash.toString(16).padStart(16, "0");
+    }
+}
+class WasmFnv64 {
+    kind = "wasm-fnv1a64";
+    memory;
+    hashFn;
+    view;
+    constructor() {
+        const module = new WebAssembly.Module(WASM_BYTES);
+        const instance = new WebAssembly.Instance(module, {});
+        const exports = instance.exports;
+        this.memory = exports.memory;
+        this.hashFn = exports.hash_fnv1a64;
+        this.view = new Uint8Array(this.memory.buffer);
+    }
+    hash(bytes) {
+        this.ensureCapacity(bytes.length);
+        this.view.set(bytes, 0);
+        const raw = this.hashFn(0, bytes.length);
+        return BigInt.asUintN(64, raw).toString(16).padStart(16, "0");
+    }
+    ensureCapacity(size) {
+        if (this.view.byteLength >= size) {
+            return;
+        }
+        const pageSize = 65536;
+        const requiredPages = Math.ceil(size / pageSize);
+        const currentPages = this.memory.buffer.byteLength / pageSize;
+        const deltaPages = requiredPages - currentPages;
+        if (deltaPages > 0) {
+            this.memory.grow(deltaPages);
+            this.view = new Uint8Array(this.memory.buffer);
+        }
+    }
+}
+let backend = null;
+function getBackend() {
+    if (!backend) {
+        try {
+            backend = new WasmFnv64();
+        }
+        catch {
+            backend = new Fnv64Fallback();
+        }
+    }
+    return backend;
+}
+export function hashBytes(bytes) {
+    return getBackend().hash(bytes);
+}
+export function hashText(text) {
+    return hashBytes(textEncoder.encode(text));
+}
+export function getHashBackendName() {
+    return getBackend().kind;
+}

package/dist/markdown-ingest.d.ts

@@ -0,0 +1,43 @@
+import type { LoggerLike, PluginConfig } from "./types.js";
+type Disposable = {
+    close(): void;
+};
+interface RpcLike {
+    call<T>(method: string, params: unknown): Promise<T>;
+}
+type RpcGetterLike = () => Promise<RpcLike>;
+interface FsDirentLike {
+    name: string;
+    isDirectory(): boolean;
+    isFile(): boolean;
+}
+interface FsWatcherLike extends Disposable {
+    on(event: "error", handler: (error: Error) => void): void;
+}
+interface FsApi {
+    readdir(dir: string): Promise<FsDirentLike[]>;
+    readFile(file: string): Promise<Uint8Array>;
+    stat(file: string): Promise<{
+        size: number;
+        mtimeMs: number;
+    }>;
+    watch(dir: string, onChange: (event: string, filename: string | Buffer | null) => void): FsWatcherLike;
+}
+export interface MarkdownSourceAdapter {
+    kind: string;
+    start(): Promise<void>;
+    refresh(): Promise<void>;
+    stop(): Promise<void>;
+}
+export interface MarkdownIngestionHandle {
+    start(): Promise<void>;
+    refresh(): Promise<void>;
+    stop(): Promise<void>;
+}
+export interface MarkdownIngestionSnapshot {
+    fileHash: string;
+    size: number;
+    mtimeMs: number;
+}
+export declare function createMarkdownIngestionHandle(cfg: PluginConfig, getRpc: RpcGetterLike, logger?: LoggerLike, fsApi?: FsApi): MarkdownIngestionHandle;
+export {};

package/dist/markdown-ingest.js

@@ -0,0 +1,464 @@
+import fs from "node:fs";
+import fsp from "node:fs/promises";
+import path from "node:path";
+import { hashBytes } from "./markdown-hash.js";
+const DEFAULT_DEBOUNCE_MS = 150;
+const DEFAULT_TOKENIZER_ID = "markdown-ingest:v1";
+const MARKDOWN_INGEST_VERSION = 3;
+const HASH_BACKEND = "wasm-fnv1a64";
+export function createMarkdownIngestionHandle(cfg, getRpc, logger = console, fsApi = createRealFsApi()) {
+    const adapters = [];
+    const genericRoots = normalizeMarkdownRoots(cfg.markdownIngestionRoots);
+    if (isMarkdownIngestionEnabled(cfg, genericRoots)) {
+        adapters.push(new DirectoryMarkdownSourceAdapter("generic", {
+            roots: genericRoots,
+            include: cfg.markdownIngestionInclude,
+            exclude: cfg.markdownIngestionExclude,
+            debounceMs: cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS,
+        }, getRpc, logger, fsApi));
+    }
+    const obsidianRoots = normalizeMarkdownRoots(cfg.markdownIngestionObsidianRoots);
+    if (cfg.markdownIngestionObsidianEnabled !== false && obsidianRoots.length > 0) {
+        adapters.push(new DirectoryMarkdownSourceAdapter("obsidian", {
+            roots: obsidianRoots,
+            include: cfg.markdownIngestionObsidianInclude,
+            exclude: cfg.markdownIngestionObsidianExclude,
+            debounceMs: cfg.markdownIngestionObsidianDebounceMs ?? cfg.markdownIngestionDebounceMs ?? DEFAULT_DEBOUNCE_MS,
+        }, getRpc, logger, fsApi));
+    }
+    if (adapters.length === 0) {
+        return {
+            async start() { },
+            async refresh() { },
+            async stop() { },
+        };
+    }
+    const adapter = new CompositeMarkdownSourceAdapter(adapters);
+    return {
+        start: () => adapter.start(),
+        refresh: () => adapter.refresh(),
+        stop: () => adapter.stop(),
+    };
+}
+class CompositeMarkdownSourceAdapter {
+    adapters;
+    kind = "composite";
+    constructor(adapters) {
+        this.adapters = adapters;
+    }
+    async start() {
+        for (const adapter of this.adapters) {
+            await adapter.start();
+        }
+    }
+    async refresh() {
+        for (const adapter of this.adapters) {
+            await adapter.refresh();
+        }
+    }
+    async stop() {
+        for (const adapter of this.adapters) {
+            await adapter.stop();
+        }
+    }
+}
+class DirectoryMarkdownSourceAdapter {
+    kind;
+    roots;
+    includePatterns;
+    excludePatterns;
+    debounceMs;
+    fsApi;
+    getRpc;
+    logger;
+    states = new Map();
+    fileStates = new Map();
+    tokenizerId;
+    coreDoc;
+    started = false;
+    constructor(kind, config, getRpc, logger, fsApi) {
+        this.kind = kind;
+        this.roots = config.roots;
+        this.includePatterns = config.include?.length ? config.include : [];
+        this.excludePatterns = config.exclude?.length ? config.exclude : [];
+        this.debounceMs = config.debounceMs ?? DEFAULT_DEBOUNCE_MS;
+        this.fsApi = fsApi;
+        this.getRpc = getRpc;
+        this.logger = logger;
+        this.tokenizerId = DEFAULT_TOKENIZER_ID;
+        this.coreDoc = true;
+    }
+    async start() {
+        if (this.started) {
+            return;
+        }
+        this.started = true;
+        await this.refresh();
+    }
+    async refresh() {
+        if (!this.started) {
+            return;
+        }
+        for (const root of this.roots) {
+            await this.scanRoot(root);
+        }
+    }
+    async stop() {
+        for (const state of this.states.values()) {
+            if (state.scanState.timer) {
+                clearTimeout(state.scanState.timer);
+                state.scanState.timer = null;
+            }
+            for (const watcher of state.directoryWatchers.values()) {
+                watcher.close();
+            }
+            state.directoryWatchers.clear();
+        }
+        this.states.clear();
+        this.fileStates.clear();
+        this.started = false;
+    }
+    getRootState(root) {
+        const resolved = path.resolve(root);
+        const existing = this.states.get(resolved);
+        if (existing) {
+            return existing;
+        }
+        const created = {
+            root: resolved,
+            scanState: {
+                scanning: false,
+                dirty: false,
+                timer: null,
+            },
+            knownFiles: new Set(),
+            directoryWatchers: new Map(),
+        };
+        this.states.set(resolved, created);
+        return created;
+    }
+    async scanRoot(root) {
+        const rootState = this.getRootState(root);
+        if (rootState.scanState.scanning) {
+            rootState.scanState.dirty = true;
+            return;
+        }
+        rootState.scanState.scanning = true;
+        try {
+            const currentFiles = new Set();
+            await this.walkDirectory(rootState, rootState.root, currentFiles);
+            await this.pruneDeletedFiles(rootState, currentFiles);
+            rootState.knownFiles = currentFiles;
+        }
+        finally {
+            rootState.scanState.scanning = false;
+            if (rootState.scanState.dirty) {
+                rootState.scanState.dirty = false;
+                this.scheduleRootScan(rootState);
+            }
+        }
+    }
+    scheduleRootScan(rootState) {
+        if (rootState.scanState.scanning) {
+            rootState.scanState.dirty = true;
+            return;
+        }
+        if (rootState.scanState.timer) {
+            return;
+        }
+        rootState.scanState.timer = setTimeout(() => {
+            rootState.scanState.timer = null;
+            void this.scanRoot(rootState.root).catch((error) => {
+                this.logger.warn?.(`[markdown-ingest] root scan failed for ${rootState.root}: ${formatError(error)}`);
+            });
+        }, this.debounceMs);
+    }
+    async walkDirectory(rootState, dir, currentFiles) {
+        await this.ensureDirectoryWatcher(rootState, dir);
+        let entries;
+        try {
+            entries = await this.fsApi.readdir(dir);
+        }
+        catch (error) {
+            const message = formatError(error);
+            if (!message.includes("ENOENT")) {
+                this.logger.warn?.(`[markdown-ingest] readdir failed for ${dir}: ${message}`);
+            }
+            return;
+        }
+        for (const entry of entries) {
+            const child = path.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                await this.walkDirectory(rootState, child, currentFiles);
+                continue;
+            }
+            if (!entry.isFile() || !isMarkdownFile(entry.name)) {
+                continue;
+            }
+            if (!this.shouldIncludeFile(rootState.root, child)) {
+                continue;
+            }
+            currentFiles.add(child);
+            try {
+                await this.syncMarkdownFile(rootState, child);
+            }
+            catch (error) {
+                this.logger.warn?.(`[markdown-ingest] sync failed for ${child}: ${formatError(error)}`);
+            }
+        }
+    }
+    async ensureDirectoryWatcher(rootState, dir) {
+        if (rootState.directoryWatchers.has(dir)) {
+            return;
+        }
+        try {
+            const watcher = this.fsApi.watch(dir, () => {
+                this.scheduleRootScan(rootState);
+            });
+            watcher.on("error", (error) => {
+                this.logger.warn?.(`[markdown-ingest] watch error for ${dir}: ${formatError(error)}`);
+            });
+            rootState.directoryWatchers.set(dir, watcher);
+        }
+        catch (error) {
+            this.logger.warn?.(`[markdown-ingest] watch unavailable for ${dir}: ${formatError(error)}`);
+        }
+    }
+    shouldIncludeFile(root, filePath) {
+        if (isOpenClawMemoryFile(filePath)) {
+            return true;
+        }
+        const relative = toPosixPath(path.relative(root, filePath));
+        if (this.excludePatterns.length > 0) {
+            for (const pattern of this.excludePatterns) {
+                if (matchesGlob(relative, pattern)) {
+                    return false;
+                }
+            }
+        }
+        if (this.includePatterns.length > 0) {
+            for (const pattern of this.includePatterns) {
+                if (matchesGlob(relative, pattern)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+        return true;
+    }
+    async pruneDeletedFiles(rootState, currentFiles) {
+        const removed = [];
+        for (const previous of rootState.knownFiles) {
+            if (!currentFiles.has(previous)) {
+                removed.push(previous);
+            }
+        }
+        if (removed.length === 0) {
+            return;
+        }
+        for (const filePath of removed) {
+            await this.deleteSourceDocument(filePath);
+            this.fileStates.delete(filePath);
+        }
+    }
+    async syncMarkdownFile(rootState, filePath) {
+        const sourceDoc = filePath;
+        const relativePath = toPosixPath(path.relative(rootState.root, filePath));
+        const stat = await this.safeStat(filePath);
+        if (!stat) {
+            await this.deleteSourceDocument(sourceDoc);
+            this.fileStates.delete(sourceDoc);
+            return;
+        }
+        const cached = this.fileStates.get(sourceDoc);
+        if (cached && cached.size === stat.size && cached.mtimeMs === stat.mtimeMs) {
+            return;
+        }
+        const bytes = await this.safeReadFile(filePath);
+        if (!bytes) {
+            await this.deleteSourceDocument(sourceDoc);
+            this.fileStates.delete(sourceDoc);
+            return;
+        }
+        const fileHash = hashBytes(bytes);
+        if (cached && cached.fileHash === fileHash) {
+            this.fileStates.set(sourceDoc, {
+                root: rootState.root,
+                sourceDoc,
+                relativePath,
+                fileHash,
+                size: stat.size,
+                mtimeMs: stat.mtimeMs,
+            });
+            return;
+        }
+        const text = textDecoder.decode(bytes);
+        if (this.kind === "obsidian" && this.includePatterns.length === 0 && !looksLikeObsidianNote(filePath, text)) {
+            await this.deleteSourceDocument(sourceDoc);
+            this.fileStates.delete(sourceDoc);
+            return;
+        }
+        await this.ingestMarkdownDocument(sourceDoc, text, rootState.root, relativePath, fileHash, stat.size, stat.mtimeMs);
+        this.fileStates.set(sourceDoc, {
+            root: rootState.root,
+            sourceDoc,
+            relativePath,
+            fileHash,
+            size: stat.size,
+            mtimeMs: stat.mtimeMs,
+        });
+    }
+    async ingestMarkdownDocument(sourceDoc, text, sourceRoot, sourcePath, fileHash, sourceSize, sourceMtimeMs) {
+        const rpc = await this.getRpc();
+        const params = {
+            sourceDoc,
+            text,
+            tokenizerId: this.tokenizerId,
+            coreDoc: this.coreDoc,
+            sourceMeta: {
+                sourceRoot,
+                sourcePath,
+                sourceKind: this.kind,
+                fileHash,
+                sourceSize,
+                sourceMtimeMs,
+                ingestVersion: MARKDOWN_INGEST_VERSION,
+                hashBackend: HASH_BACKEND,
+            },
+        };
+        await rpc.call("ingest_markdown_document", params);
+    }
+    async deleteSourceDocument(sourceDoc) {
+        const rpc = await this.getRpc();
+        const params = { sourceDoc };
+        await rpc.call("delete_authored_document", params);
+    }
+    async safeStat(filePath) {
+        try {
+            return await this.fsApi.stat(filePath);
+        }
+        catch {
+            return null;
+        }
+    }
+    async safeReadFile(filePath) {
+        try {
+            return await this.fsApi.readFile(filePath);
+        }
+        catch {
+            return null;
+        }
+    }
+}
+function toPosixPath(value) {
+    return value.split(path.sep).join("/");
+}
+const textDecoder = new TextDecoder();
+function normalizeMarkdownRoots(roots) {
+    if (!roots?.length) {
+        return [];
+    }
+    const resolved = new Set();
+    for (const root of roots) {
+        const trimmed = root.trim();
+        if (!trimmed) {
+            continue;
+        }
+        resolved.add(path.resolve(trimmed));
+    }
+    return [...resolved];
+}
+function isMarkdownIngestionEnabled(cfg, roots) {
+    if (cfg.markdownIngestionEnabled === false) {
+        return false;
+    }
+    return roots.length > 0;
+}
+function createRealFsApi() {
+    return {
+        readdir: async (dir) => fsp.readdir(dir, { withFileTypes: true }),
+        readFile: async (file) => fsp.readFile(file),
+        stat: async (file) => {
+            const stat = await fsp.stat(file);
+            return { size: stat.size, mtimeMs: stat.mtimeMs };
+        },
+        watch: (dir, onChange) => fs.watch(dir, onChange),
+    };
+}
+function isMarkdownFile(fileName) {
+    const lower = fileName.toLowerCase();
+    return lower.endsWith(".md") || lower.endsWith(".markdown");
+}
+function matchesGlob(value, pattern) {
+    const escaped = pattern
+        .split("*")
+        .map((part) => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&"))
+        .join(".*");
+    return new RegExp(`^${escaped}$`).test(value);
+}
+function formatError(error) {
+    if (error instanceof Error) {
+        return error.message;
+    }
+    return String(error);
+}
+function looksLikeObsidianNote(filePath, text) {
+    if (!text.startsWith("---\n")) {
+        return hasInlineObsidianTag(text);
+    }
+    const frontmatterEnd = findFrontmatterEnd(text, 4);
+    if (frontmatterEnd < 0) {
+        return hasInlineObsidianTag(text);
+    }
+    const frontmatter = text.slice(4, frontmatterEnd);
+    const lines = frontmatter.split("\n");
+    for (const line of lines) {
+        const trimmed = line.trimStart();
+        if (trimmed.startsWith("tags:") ||
+            trimmed.startsWith("tag:") ||
+            trimmed.startsWith("openclaw:") ||
+            trimmed.startsWith("memory:")) {
+            return true;
+        }
+    }
+    return hasInlineObsidianTag(text.slice(frontmatterEnd + 4));
+}
+function findFrontmatterEnd(text, offset) {
+    for (let i = offset; i < text.length - 3; i++) {
+        if (text.charCodeAt(i) !== 45 || text.charCodeAt(i + 1) !== 45 || text.charCodeAt(i + 2) !== 45) {
+            continue;
+        }
+        const next = text.charCodeAt(i + 3);
+        if (next === 10) {
+            return i;
+        }
+        if (next === 13 && text.charCodeAt(i + 4) === 10) {
+            return i;
+        }
+    }
+    return -1;
+}
+function hasInlineObsidianTag(text) {
+    let inFence = false;
+    const lines = text.split("\n");
+    for (const line of lines) {
+        const trimmed = line.trimStart();
+        if (trimmed.startsWith("```") || trimmed.startsWith("~~~")) {
+            inFence = !inFence;
+            continue;
+        }
+        if (inFence) {
+            continue;
+        }
+        if (trimmed.startsWith("#")) {
+            continue;
+        }
+        if (/(^|[^A-Za-z0-9_])#([A-Za-z][A-Za-z0-9/_-]*)\b/.test(line)) {
+            return true;
+        }
+    }
+    return false;
+}
+function isOpenClawMemoryFile(filePath) {
+    return path.basename(filePath).toLowerCase() === "memory.md";
+}

package/dist/memory-provider.d.ts

@@ -0,0 +1,4 @@
+import type { MemoryPromptSectionBuilder } from "openclaw/plugin-sdk/plugin-entry";
+import type { PluginConfig, RecallCache, SearchResult } from "./types.js";
+import type { RpcGetter } from "./plugin-runtime.js";
+export declare function buildMemoryPromptSection(_getRpc: RpcGetter, _cfg: PluginConfig, _recallCache: RecallCache<SearchResult>): MemoryPromptSectionBuilder;

package/dist/memory-provider.js

@@ -0,0 +1,13 @@
+const MEMORY_PROMPT_HEADER = [
+    "## Memory",
+    "LibraVDB persistent memory is configured. Recalled memories may appear",
+    "in context via the context-engine assembler when available and relevant.",
+    "",
+];
+export function buildMemoryPromptSection(_getRpc, _cfg, _recallCache) {
+    return function memoryPromptSection({ availableTools: _availableTools, citationsMode: _citationsMode, }) {
+        // OpenClaw builds the memory prompt section synchronously for embedded runs.
+        // Actual retrieval and ranking happen in the context engine during assemble().
+        return [...MEMORY_PROMPT_HEADER];
+    };
+}