@byted-las/contextlake-openclaw 1.0.0

package/README.md

@@ -0,0 +1,64 @@
+# ContextLake OpenClaw Plugin
+
+A powerful, serverless-first Knowledge Base / Knowledge Lake (知识库/知识湖) plugin for the OpenClaw Agent framework.
+
+This plugin allows OpenClaw agents to natively understand, index, and retrieve local and remote documents (PDF, Word, Markdown, Text) using vector similarity search. It features a pluggable architecture supporting both completely offline operations (via LanceDB and local GGUF models) and cloud-based operations.
+
+## Features
+
+- **Multi-modal Document Ingestion**: Automatically extracts text from PDF, DOCX, TXT, and MD files.
+- **Agentic Tools**: Exposes `contextlake-ingest`, `contextlake-retrieve`, `contextlake-list`, and `contextlake-delete` skills directly to the LLM.
+- **Slash Commands**: Quick chat actions (`/contextlake-search`, `/contextlake-ingest`) for fast human-in-the-loop operations.
+- **CLI Management**: Full command-line interface for bulk operations and CI/CD integration.
+- **Local-First & Pluggable**: 
+  - Uses `LanceDB` for local embedded vector storage.
+  - Uses `node-llama-cpp` for offline, private embedding generation (`bge-small-zh` by default).
+  - Can be easily configured to use remote embedding APIs (like OpenAI or Doubao ARK) or remote storage (TOS).
+
+## Installation
+
+Assuming you are in an OpenClaw environment:
+
+```bash
+npm install
+npm run build
+```
+
+Then register the plugin in your OpenClaw configuration:
+
+```json
+{
+  "plugins": [
+    {
+      "package": "contextlake-openclaw",
+      "config": {
+        "metadata_storage": {
+          "type": "local",
+          "lancedb_uri": "./data/contextlake"
+        },
+        "file_storage": {
+          "type": "local",
+          "local_base_dir": "./data/files"
+        }
+      }
+    }
+  ]
+}
+```
+
+## Agent Tools (Skills)
+
+The LLM agent automatically has access to the following tools:
+
+1. **`contextlake-ingest`**: Uploads and indexes files.
+2. **`contextlake-retrieve`**: Performs vector search on the indexed documents based on a query.
+3. **`contextlake-list`**: Lists all currently indexed documents.
+4. **`contextlake-delete`**: Removes documents by ID or filter.
+
+You can simply talk to the agent:
+> "帮我把这个 `manual.pdf` 注入到知识库中。"
+> "在知识湖中检索关于架构设计的文档。"
+
+## Development
+
+For detailed architecture and internal workflows, please refer to [AGENTS.md](./AGENTS.md).

package/bin/contextlake-openclaw.js

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+
+// 临时执行脚本，如果是生产环境可能需要直接执行 dist 下的 js 文件
+require('ts-node').register();
+require('../scripts/cli.ts');

package/dist/index.d.ts

@@ -0,0 +1,113 @@
+declare const plugin: {
+    id: string;
+    name: string;
+    version: string;
+    description: string;
+    configSchema: {
+        type: string;
+        properties: {
+            metadata_storage: {
+                type: string;
+                properties: {
+                    type: {
+                        type: string;
+                        enum: string[];
+                        default: string;
+                    };
+                    lancedb_uri: {
+                        type: string;
+                        default: string;
+                    };
+                    remote_api_endpoint: {
+                        type: string;
+                    };
+                    remote_api_key: {
+                        type: string;
+                    };
+                    embedding: {
+                        type: string;
+                        properties: {
+                            provider: {
+                                type: string;
+                                enum: string[];
+                                default: string;
+                            };
+                            model_name: {
+                                type: string;
+                                default: string;
+                            };
+                            api_key: {
+                                type: string;
+                            };
+                            api_base: {
+                                type: string;
+                            };
+                        };
+                        default: {
+                            provider: string;
+                            model_name: string;
+                        };
+                    };
+                };
+                default: {
+                    type: string;
+                    lancedb_uri: string;
+                };
+            };
+            file_storage: {
+                type: string;
+                properties: {
+                    type: {
+                        type: string;
+                        enum: string[];
+                        default: string;
+                    };
+                    local_base_dir: {
+                        type: string;
+                        default: string;
+                    };
+                    tos: {
+                        type: string;
+                        properties: {
+                            access_key: {
+                                type: string;
+                            };
+                            secret_key: {
+                                type: string;
+                            };
+                            region: {
+                                type: string;
+                            };
+                            path: {
+                                type: string;
+                                description: string;
+                            };
+                            endpoint: {
+                                type: string;
+                            };
+                            sts_token: {
+                                type: string;
+                            };
+                        };
+                    };
+                };
+                default: {
+                    type: string;
+                    local_base_dir: string;
+                };
+            };
+            storage_policy: {
+                type: string;
+                properties: {
+                    max_inline_size_kb: {
+                        type: string;
+                        default: number;
+                        description: string;
+                    };
+                };
+            };
+        };
+    };
+    register(ctx: any): void;
+};
+export default plugin;

package/dist/index.js

@@ -0,0 +1,73 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const commands_1 = require("./src/commands");
+const plugin = {
+    id: 'contextlake-openclaw',
+    name: 'ContextLake',
+    version: '1.1.0',
+    description: 'A lightweight knowledge base plugin for OpenClaw using LanceDB and TOS, with data profiling support',
+    configSchema: {
+        type: 'object',
+        properties: {
+            metadata_storage: {
+                type: 'object',
+                properties: {
+                    type: { type: 'string', enum: ['local', 'remote'], default: 'local' },
+                    lancedb_uri: { type: 'string', default: './data/contextlake' },
+                    remote_api_endpoint: { type: 'string' },
+                    remote_api_key: { type: 'string' },
+                    embedding: {
+                        type: 'object',
+                        properties: {
+                            provider: { type: 'string', enum: ['local', 'remote', 'openai'], default: 'local' },
+                            model_name: { type: 'string', default: 'hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf' },
+                            api_key: { type: 'string' },
+                            api_base: { type: 'string' }
+                        },
+                        default: { provider: 'local', model_name: 'hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf' }
+                    }
+                },
+                default: { type: 'local', lancedb_uri: './data/contextlake' }
+            },
+            file_storage: {
+                type: 'object',
+                properties: {
+                    type: { type: 'string', enum: ['local', 'tos'], default: 'local' },
+                    local_base_dir: { type: 'string', default: './data/files' },
+                    tos: {
+                        type: 'object',
+                        properties: {
+                            access_key: { type: 'string' },
+                            secret_key: { type: 'string' },
+                            region: { type: 'string' },
+                            path: { type: 'string', description: 'TOS path in format tos://bucket/base_path/' },
+                            endpoint: { type: 'string' },
+                            sts_token: { type: 'string' }
+                        }
+                    }
+                },
+                default: { type: 'local', local_base_dir: './data/files' }
+            },
+            storage_policy: {
+                type: 'object',
+                properties: {
+                    max_inline_size_kb: { type: 'number', default: 1024, description: 'Files smaller than this size (in KB) will be stored directly in LanceDB' }
+                }
+            }
+        }
+    },
+    register(ctx) {
+        const logger = ctx.logger || {
+            info: (msg, ...args) => console.log(msg, ...args),
+            warn: (msg, ...args) => console.warn(msg, ...args),
+            error: (msg, ...args) => console.error(msg, ...args),
+            debug: (msg, ...args) => console.debug(msg, ...args),
+        };
+        // Add logging
+        logger.info(`[${new Date().toISOString()}] [ContextLake] Plugin register started`);
+        // Delegate all registrations to commands/index.ts
+        (0, commands_1.registerAll)(ctx, logger);
+        logger.info(`[${new Date().toISOString()}] [ContextLake] Plugin register completed`);
+    }
+};
+exports.default = plugin;

package/dist/src/client/lancedb.d.ts

@@ -0,0 +1,30 @@
+import * as lancedb from '@lancedb/lancedb';
+import { EmbeddingProvider } from '../service/embedding/interface';
+export interface LanceDBConfig {
+    uri: string;
+}
+export interface DocumentSchema {
+    id: string;
+    vector: number[];
+    text: string;
+    source: string;
+    file_type: string;
+    storage_type: string;
+    url: string;
+    metadata: string;
+    created_at: number;
+    binary_data?: Buffer;
+}
+export declare class ContextLakeLanceDBClient {
+    private db;
+    private table;
+    private config;
+    private embeddingProvider;
+    constructor(config: LanceDBConfig, embeddingProvider: EmbeddingProvider);
+    connect(): Promise<void>;
+    getTable(tableName?: string, dim?: number): Promise<lancedb.Table>;
+    addAssets(docs: DocumentSchema[]): Promise<void>;
+    search(query: string, limit?: number, filter?: string): Promise<any[]>;
+    delete(filter: string): Promise<void>;
+    list(limit?: number, filter?: string): Promise<any[]>;
+}

package/dist/src/client/lancedb.js

@@ -0,0 +1,113 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ContextLakeLanceDBClient = void 0;
+const lancedb = __importStar(require("@lancedb/lancedb"));
+class ContextLakeLanceDBClient {
+    db = null;
+    table = null;
+    config;
+    embeddingProvider;
+    constructor(config, embeddingProvider) {
+        this.config = config;
+        this.embeddingProvider = embeddingProvider;
+    }
+    async connect() {
+        if (!this.db) {
+            this.db = await lancedb.connect(this.config.uri);
+        }
+    }
+    async getTable(tableName = 'documents', dim = 0) {
+        if (this.table)
+            return this.table;
+        await this.connect();
+        const tableNames = await this.db.tableNames();
+        if (tableNames.includes(tableName)) {
+            this.table = await this.db.openTable(tableName);
+        }
+        else {
+            if (dim <= 0) {
+                // Fallback: use embedding provider to infer dimension only if needed
+                const dummyVec = await this.embeddingProvider.generateEmbedding("init");
+                dim = dummyVec.length;
+            }
+            // @ts-ignore
+            this.table = await this.db.createTable(tableName, [
+                {
+                    id: 'schema_init',
+                    vector: Array(dim).fill(0),
+                    text: '',
+                    source: '',
+                    file_type: '',
+                    storage_type: '',
+                    url: '',
+                    metadata: '{}',
+                    created_at: 0,
+                    binary_data: Buffer.from('')
+                }
+            ]);
+            await this.table.delete('id = "schema_init"');
+        }
+        return this.table;
+    }
+    async addAssets(docs) {
+        const table = await this.getTable();
+        // @ts-ignore
+        await table.add(docs);
+    }
+    async search(query, limit = 5, filter) {
+        const vector = await this.embeddingProvider.generateEmbedding(query);
+        const table = await this.getTable();
+        // @ts-ignore
+        let search = table.vectorSearch(vector).limit(limit);
+        if (filter) {
+            search = search.where(filter);
+        }
+        return await search.toArray();
+    }
+    async delete(filter) {
+        const table = await this.getTable();
+        await table.delete(filter);
+    }
+    async list(limit = 100, filter) {
+        const table = await this.getTable();
+        let query = table.query().limit(limit);
+        if (filter) {
+            query = query.where(filter);
+        }
+        return await query.toArray();
+    }
+}
+exports.ContextLakeLanceDBClient = ContextLakeLanceDBClient;

package/dist/src/client/tos.d.ts

@@ -0,0 +1,19 @@
+import { StorageProvider } from '../service/storage/interface';
+export interface TosConfig {
+    access_key?: string;
+    secret_key?: string;
+    region: string;
+    endpoint?: string;
+    path: string;
+    sts_token?: string;
+}
+export declare class ContextLakeTosClient implements StorageProvider {
+    private client;
+    private bucket;
+    private basePath;
+    constructor(config: TosConfig);
+    private parseTosUrl;
+    uploadFile(fileName: string, buffer: Buffer): Promise<string>;
+    downloadFile(key: string): Promise<Buffer>;
+    deleteFile(key: string): Promise<void>;
+}

package/dist/src/client/tos.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ContextLakeTosClient = void 0;
+const tos_sdk_1 = require("@volcengine/tos-sdk");
+class ContextLakeTosClient {
+    client;
+    bucket;
+    basePath;
+    constructor(config) {
+        const tosConfig = {
+            accessKeyId: config.access_key,
+            accessKeySecret: config.secret_key,
+            region: config.region,
+            endpoint: config.endpoint,
+            securityToken: config.sts_token,
+        };
+        this.client = new tos_sdk_1.TosClient(tosConfig);
+        if (!config.path) {
+            throw new Error('TOS configuration requires "path" (e.g. tos://bucket/path/)');
+        }
+        if (!config.path.startsWith('tos://')) {
+            throw new Error('TOS path must start with tos://');
+        }
+        const parts = config.path.substring(6).split('/');
+        this.bucket = parts[0];
+        this.basePath = parts.slice(1).join('/');
+        if (this.basePath && !this.basePath.endsWith('/')) {
+            this.basePath += '/';
+        }
+    }
+    parseTosUrl(key) {
+        if (key.startsWith('tos://')) {
+            const pathParts = key.split('/').slice(3);
+            return pathParts.join('/');
+        }
+        return key;
+    }
+    async uploadFile(fileName, buffer) {
+        const key = `${this.basePath}${fileName}`;
+        await this.client.putObject({
+            bucket: this.bucket,
+            key,
+            body: buffer,
+        });
+        return `tos://${this.bucket}/${key}`;
+    }
+    async downloadFile(key) {
+        const actualKey = this.parseTosUrl(key);
+        const result = await this.client.getObject({
+            bucket: this.bucket,
+            key: actualKey,
+        });
+        // Check if result.data is a stream or buffer
+        if (Buffer.isBuffer(result.data)) {
+            return result.data;
+        }
+        // Read stream to buffer
+        // @ts-ignore
+        const stream = result.data.content || result.data;
+        if (stream && stream.toArray) {
+            // Optimized path for some stream implementations (like node-fetch/minipass)
+            return Buffer.concat(await stream.toArray());
+        }
+        if (stream && stream[Symbol.asyncIterator]) {
+            const chunks = [];
+            for await (const chunk of stream) {
+                chunks.push(Buffer.from(chunk));
+            }
+            return Buffer.concat(chunks);
+        }
+        return Buffer.from(result.data);
+    }
+    async deleteFile(key) {
+        const actualKey = this.parseTosUrl(key);
+        await this.client.deleteObject({
+            bucket: this.bucket,
+            key: actualKey,
+        });
+    }
+}
+exports.ContextLakeTosClient = ContextLakeTosClient;

package/dist/src/commands/cli.d.ts

@@ -0,0 +1,6 @@
+export declare function getCliCommands(pluginConfig: any, logger: any): {
+    ingestAction: (files: any, options: any) => Promise<void>;
+    searchAction: (query: any, options: any) => Promise<void>;
+    listAction: (options: any) => Promise<void>;
+    deleteAction: (options: any) => Promise<void>;
+};

package/dist/src/commands/cli.js

@@ -0,0 +1,78 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getCliCommands = getCliCommands;
+const ingest_1 = require("../lib/actions/ingest");
+const retrieve_1 = require("../lib/actions/retrieve");
+const manage_1 = require("../lib/actions/manage");
+function getCliCommands(pluginConfig, logger) {
+    return {
+        ingestAction: async (files, options) => {
+            logger.info(`[${new Date().toISOString()}] [ContextLake] CLI ingest started`, { files, options });
+            try {
+                const metadata = options.metadata ? JSON.parse(options.metadata) : {};
+                const result = await (0, ingest_1.ingestAssets)({
+                    files,
+                    metadata,
+                    chunkSize: parseInt(options.chunkSize),
+                    overlap: parseInt(options.overlap)
+                }, pluginConfig, logger);
+                // eslint-disable-next-line no-console
+                console.log(JSON.stringify(result, null, 2));
+                logger.info(`[${new Date().toISOString()}] [ContextLake] CLI ingest success`);
+            }
+            catch (e) {
+                console.error('Error:', e.message);
+                logger.error(`[${new Date().toISOString()}] [ContextLake] CLI ingest failed`, { error: e.message, stack: e.stack });
+            }
+        },
+        searchAction: async (query, options) => {
+            logger.info(`[${new Date().toISOString()}] [ContextLake] CLI search started`, { query, options });
+            try {
+                const result = await (0, retrieve_1.retrieveAssets)({
+                    query,
+                    top_k: parseInt(options.topK),
+                    filter: options.filter,
+                    include_binary: options.binary
+                }, pluginConfig, logger);
+                // eslint-disable-next-line no-console
+                console.log(JSON.stringify(result, null, 2));
+                logger.info(`[${new Date().toISOString()}] [ContextLake] CLI search success`);
+            }
+            catch (e) {
+                console.error('Error:', e.message);
+                logger.error(`[${new Date().toISOString()}] [ContextLake] CLI search failed`, { error: e.message, stack: e.stack });
+            }
+        },
+        listAction: async (options) => {
+            logger.info(`[${new Date().toISOString()}] [ContextLake] CLI list started`, { options });
+            try {
+                const result = await (0, manage_1.listAssets)({
+                    limit: parseInt(options.limit)
+                }, pluginConfig, logger);
+                // eslint-disable-next-line no-console
+                console.log(JSON.stringify(result, null, 2));
+                logger.info(`[${new Date().toISOString()}] [ContextLake] CLI list success`);
+            }
+            catch (e) {
+                console.error('Error:', e.message);
+                logger.error(`[${new Date().toISOString()}] [ContextLake] CLI list failed`, { error: e.message, stack: e.stack });
+            }
+        },
+        deleteAction: async (options) => {
+            logger.info(`[${new Date().toISOString()}] [ContextLake] CLI delete started`, { options });
+            try {
+                const result = await (0, manage_1.deleteAssets)({
+                    file_ids: options.ids,
+                    filter: options.filter
+                }, pluginConfig, logger);
+                // eslint-disable-next-line no-console
+                console.log(JSON.stringify(result, null, 2));
+                logger.info(`[${new Date().toISOString()}] [ContextLake] CLI delete success`);
+            }
+            catch (e) {
+                console.error('Error:', e.message);
+                logger.error(`[${new Date().toISOString()}] [ContextLake] CLI delete failed`, { error: e.message, stack: e.stack });
+            }
+        }
+    };
+}

package/dist/src/commands/index.d.ts

@@ -0,0 +1 @@
+export declare function registerAll(ctx: any, logger: any): void;