@byted-las/contextlake-openclaw 1.0.0

package/dist/src/commands/tools.js

@@ -0,0 +1,286 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getAgentTools = getAgentTools;
+const ingest_1 = require("../lib/actions/ingest");
+const retrieve_1 = require("../lib/actions/retrieve");
+const manage_1 = require("../lib/actions/manage");
+function getAgentTools(pluginConfig, logger) {
+    return {
+        ingestTool: {
+            name: 'contextlake-ingest',
+            label: 'ContextLake Ingest',
+            description: `Upload, ingest, and index documents into the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
+Use this tool when the user wants to "将知识注入", "上传文件", "入库", "添加文档", "ingest files", or "add knowledge".
+Supports processing of various file types including PDF, Word, Markdown, and Text.
+Automatically handles text extraction, cleaning, chunking, embedding generation, and storage.
+
+Example User Queries:
+- "帮我把这个文档注入到知识湖中"
+- "上传这份 PDF 到知识库"
+- "Please ingest these documents into ContextLake"
+- "将 /path/to/doc.txt 添加到知识库"`,
+            parameters: {
+                type: 'object',
+                properties: {
+                    files: {
+                        type: 'array',
+                        items: { type: 'string' },
+                        description: 'List of file paths to ingest'
+                    },
+                    metadata: {
+                        type: 'object',
+                        description: 'Optional JSON metadata to attach to documents',
+                        additionalProperties: true
+                    },
+                    chunkSize: {
+                        type: 'integer',
+                        description: 'Chunk size for text splitting'
+                    },
+                    overlap: {
+                        type: 'integer',
+                        description: 'Overlap size for text splitting'
+                    }
+                },
+                required: ['files'],
+                additionalProperties: false
+            },
+            schema: {
+                type: 'object',
+                properties: {
+                    files: {
+                        type: 'array',
+                        items: { type: 'string' },
+                        description: 'List of file paths to ingest'
+                    },
+                    metadata: {
+                        type: 'object',
+                        description: 'Optional JSON metadata to attach to documents',
+                        additionalProperties: true
+                    },
+                    chunkSize: {
+                        type: 'integer',
+                        description: 'Chunk size for text splitting'
+                    },
+                    overlap: {
+                        type: 'integer',
+                        description: 'Overlap size for text splitting'
+                    }
+                },
+                required: ['files'],
+                additionalProperties: false
+            },
+            async execute(toolCallId, params) {
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Executing ingest skill, toolCallId: ${toolCallId}`, { params: JSON.stringify(params) });
+                try {
+                    let actualParams = params;
+                    if (typeof params === 'string') {
+                        try {
+                            actualParams = JSON.parse(params);
+                        }
+                        catch (e) {
+                            logger.warn(`[ContextLake] Received string params, possibly toolCallId?`, { params });
+                            return {
+                                success: false,
+                                error: `Invalid params format: received string "${params}", expected object with 'files' array.`
+                            };
+                        }
+                    }
+                    if (!actualParams.files && actualParams.params && actualParams.params.files) {
+                        actualParams = actualParams.params;
+                    }
+                    if (!actualParams.files || !Array.isArray(actualParams.files)) {
+                        return {
+                            success: false,
+                            error: `Invalid params: 'files' must be an array. Received keys: ${Object.keys(actualParams)}`
+                        };
+                    }
+                    const result = await (0, ingest_1.ingestAssets)(actualParams, pluginConfig, logger);
+                    logger.info(`[${new Date().toISOString()}] [ContextLake] Ingest skill completed successfully`, { resultSummary: Array.isArray(result) ? `Processed ${result.length} items` : 'Success' });
+                    return {
+                        success: true,
+                        result
+                    };
+                }
+                catch (error) {
+                    logger.error(`[${new Date().toISOString()}] [ContextLake] Ingest skill failed`, { error: error.message, stack: error.stack });
+                    return {
+                        success: false,
+                        error: error.message
+                    };
+                }
+            }
+        },
+        retrieveTool: {
+            name: 'contextlake-retrieve',
+            label: 'ContextLake Retrieve',
+            description: `Search, query, and retrieve relevant information from the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
+Use this tool when the user wants to "搜索知识", "获取信息", "召回文档", "查询知识库", "search knowledge base", or "retrieve documents".
+Uses vector similarity search to find semantically related document chunks.
+
+Example User Queries:
+- "知识库里有关于产品安装的文档吗？"
+- "帮我从知识湖中召回关于财务报表的资料"
+- "Search the knowledge base for deployment guides"
+- "根据知识库内容，回答如何配置网关"`,
+            parameters: {
+                type: 'object',
+                properties: {
+                    query: { type: 'string', description: 'Search query' },
+                    top_k: { type: 'integer', description: 'Number of results to return' },
+                    filter: { type: 'string', description: 'Filter string' },
+                    include_binary: { type: 'boolean', description: 'Whether to include binary content' }
+                },
+                required: ['query'],
+                additionalProperties: false
+            },
+            schema: {
+                type: 'object',
+                properties: {
+                    query: { type: 'string', description: 'Search query' },
+                    top_k: { type: 'integer', description: 'Number of results to return' },
+                    filter: { type: 'string', description: 'Filter string' },
+                    include_binary: { type: 'boolean', description: 'Whether to include binary content' }
+                },
+                required: ['query'],
+                additionalProperties: false
+            },
+            async execute(toolCallId, params) {
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Executing retrieve skill, toolCallId: ${toolCallId}`, { params: JSON.stringify(params) });
+                try {
+                    let actualParams = params;
+                    if (typeof params === 'string') {
+                        try {
+                            actualParams = JSON.parse(params);
+                        }
+                        catch (e) {
+                            logger.warn(`[ContextLake] Received string params for retrieve, attempting fallback parsing`, { params });
+                            // For retrieve, if it's a string, maybe they just passed the query directly
+                            actualParams = { query: params };
+                        }
+                    }
+                    else if (params && params.params) {
+                        actualParams = params.params;
+                    }
+                    if (!actualParams || typeof actualParams.query !== 'string') {
+                        return {
+                            success: false,
+                            error: `Invalid params: 'query' is required and must be a string. Received: ${JSON.stringify(actualParams)}`
+                        };
+                    }
+                    const result = await (0, retrieve_1.retrieveAssets)(actualParams, pluginConfig, logger);
+                    logger.info(`[${new Date().toISOString()}] [ContextLake] Retrieve skill completed`, { resultCount: Array.isArray(result) ? result.length : 0 });
+                    return {
+                        success: true,
+                        result
+                    };
+                }
+                catch (error) {
+                    logger.error(`[${new Date().toISOString()}] [ContextLake] Retrieve skill failed`, { error: error.message, stack: error.stack });
+                    return {
+                        success: false,
+                        error: error.message
+                    };
+                }
+            }
+        },
+        listTool: {
+            name: 'contextlake-list',
+            label: 'ContextLake List',
+            description: `List documents and assets currently in the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
+Use this tool when the user wants to "列出所有知识", "查看知识库文件", "显示文档列表", "list knowledge", or "show documents".
+
+Example User Queries:
+- "知识湖里目前有哪些文件？"
+- "列出前10个知识库文档"
+- "Show me all documents in the knowledge base"`,
+            parameters: {
+                type: 'object',
+                properties: {
+                    limit: { type: 'integer', description: 'Limit for list action' }
+                },
+                required: [],
+                additionalProperties: false
+            },
+            schema: {
+                type: 'object',
+                properties: {
+                    limit: { type: 'integer', description: 'Limit for list action' }
+                },
+                required: []
+            },
+            async execute(toolCallId, params) {
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Executing list skill, toolCallId: ${toolCallId}`, { params: JSON.stringify(params) });
+                try {
+                    let actualParams = params;
+                    if (params && params.params) {
+                        actualParams = params.params;
+                    }
+                    const result = await (0, manage_1.listAssets)(actualParams, pluginConfig, logger);
+                    logger.info(`[${new Date().toISOString()}] [ContextLake] List skill completed`, { count: Array.isArray(result) ? result.length : 0 });
+                    return {
+                        success: true,
+                        result
+                    };
+                }
+                catch (error) {
+                    logger.error(`[${new Date().toISOString()}] [ContextLake] List skill failed`, { error: error.message, stack: error.stack });
+                    return {
+                        success: false,
+                        error: error.message
+                    };
+                }
+            }
+        },
+        deleteTool: {
+            name: 'contextlake-delete',
+            label: 'ContextLake Delete',
+            description: `Delete documents and assets from the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
+Use this tool when the user wants to "删除某个文档", "清理知识库", "移除文件", "delete knowledge", or "remove documents".
+Supports deleting documents by their specific IDs or by applying a SQL-like filter.
+
+Example User Queries:
+- "删除 ID 为 12345 的文档"
+- "Please delete the old architecture document from the knowledge base"
+- "从知识湖中移除 category 是 finance 的所有记录"`,
+            parameters: {
+                type: 'object',
+                properties: {
+                    file_ids: { type: 'array', items: { type: 'string' }, description: 'File IDs to delete' },
+                    filter: { type: 'string', description: 'Filter string for deletion' }
+                },
+                required: [],
+                additionalProperties: false
+            },
+            schema: {
+                type: 'object',
+                properties: {
+                    file_ids: { type: 'array', items: { type: 'string' }, description: 'File IDs to delete' },
+                    filter: { type: 'string', description: 'Filter string for deletion' }
+                },
+                required: []
+            },
+            async execute(toolCallId, params) {
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Executing delete skill, toolCallId: ${toolCallId}`, { params: JSON.stringify(params) });
+                try {
+                    let actualParams = params;
+                    if (params && params.params) {
+                        actualParams = params.params;
+                    }
+                    const result = await (0, manage_1.deleteAssets)(actualParams, pluginConfig, logger);
+                    logger.info(`[${new Date().toISOString()}] [ContextLake] Delete skill completed`, { result });
+                    return {
+                        success: true,
+                        result
+                    };
+                }
+                catch (error) {
+                    logger.error(`[${new Date().toISOString()}] [ContextLake] Delete skill failed`, { error: error.message, stack: error.stack });
+                    return {
+                        success: false,
+                        error: error.message
+                    };
+                }
+            }
+        }
+    };
+}

package/dist/src/lib/actions/ingest.d.ts

@@ -0,0 +1,8 @@
+interface IngestParams {
+    files: string[];
+    metadata?: Record<string, any>;
+    chunkSize?: number;
+    overlap?: number;
+}
+export declare function ingestAssets(params: IngestParams, config: any, logger?: any): Promise<any>;
+export {};

package/dist/src/lib/actions/ingest.js

@@ -0,0 +1,123 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ingestAssets = ingestAssets;
+const factory_1 = require("../../service/storage/factory");
+const factory_2 = require("../../service/metadata/factory");
+const loader_1 = require("../../processor/loader");
+const path = __importStar(require("path"));
+// @ts-ignore
+const uuid_1 = require("uuid");
+async function ingestAssets(params, config, logger) {
+    if (logger) {
+        logger.info(`[ContextLake-Action] Calling ingestAssets with params: ${JSON.stringify(params)}`);
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.log(`[ContextLake-Action] Calling ingestAssets with params: ${JSON.stringify(params)}`);
+    }
+    const storageConfig = config.file_storage || { type: 'local', local_base_dir: './data/files' };
+    const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
+    const storageProvider = (0, factory_1.createStorageProvider)(storageConfig);
+    const metadataProvider = (0, factory_2.createMetadataProvider)(metaConfig);
+    await metadataProvider.connect();
+    const maxInlineSize = (config.storage_policy?.max_inline_size_kb || 1024) * 1024;
+    const results = [];
+    for (const filePath of params.files) {
+        try {
+            const fileName = path.basename(filePath);
+            const { buffer, text, type } = await (0, loader_1.processFile)(filePath);
+            const createdAt = Date.now(); // Current timestamp
+            let fileUrl = '';
+            let storageType = '';
+            let binaryData = Buffer.alloc(0);
+            // Decide storage strategy
+            if (buffer.length <= maxInlineSize) {
+                binaryData = buffer;
+                fileUrl = `inline://${fileName}`; // Virtual URL for inline
+                storageType = 'inline';
+            }
+            else {
+                fileUrl = await storageProvider.uploadFile(fileName, buffer);
+                storageType = storageConfig.type;
+            }
+            const chunks = (0, loader_1.splitText)(text, params.chunkSize || 500, params.overlap || 50);
+            const docs = [];
+            // If no text extracted (e.g. image), store one entry with empty text but with metadata/binary
+            if (chunks.length === 0) {
+                const vector = await metadataProvider.generateEmbedding(fileName); // Embed filename as fallback
+                docs.push({
+                    id: (0, uuid_1.v4)(),
+                    vector,
+                    text: '',
+                    source: fileName,
+                    file_type: type,
+                    storage_type: storageType,
+                    url: fileUrl,
+                    metadata: JSON.stringify(params.metadata || {}),
+                    created_at: createdAt,
+                    binary_data: binaryData
+                });
+            }
+            else {
+                for (const chunk of chunks) {
+                    const vector = await metadataProvider.generateEmbedding(chunk);
+                    docs.push({
+                        id: (0, uuid_1.v4)(),
+                        vector,
+                        text: chunk,
+                        source: fileName,
+                        file_type: type,
+                        storage_type: storageType,
+                        url: fileUrl,
+                        metadata: JSON.stringify(params.metadata || {}),
+                        created_at: createdAt,
+                        binary_data: binaryData // Only attach to first chunk
+                    });
+                    // Clear binary data for subsequent chunks of the same file to avoid duplication
+                    binaryData = Buffer.alloc(0);
+                }
+            }
+            await metadataProvider.addAssets(docs);
+            results.push({ file: fileName, status: 'success', chunks: docs.length });
+        }
+        catch (error) {
+            // @ts-ignore
+            results.push({ file: filePath, status: 'error', message: error.message });
+        }
+    }
+    // Ensure plain JSON serialization
+    return JSON.parse(JSON.stringify(results));
+}

package/dist/src/lib/actions/manage.d.ts

@@ -0,0 +1,15 @@
+interface ListParams {
+    limit?: number;
+}
+interface DeleteParams {
+    file_ids?: string[];
+    filter?: string;
+}
+export declare function listAssets(params: ListParams, config: any, logger?: any): Promise<any>;
+export declare function deleteAssets(params: DeleteParams, config: any, logger?: any): Promise<{
+    status: string;
+    message: string;
+    deleted_count: number;
+    storage_cleaned: number;
+}>;
+export {};

package/dist/src/lib/actions/manage.js

@@ -0,0 +1,91 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.listAssets = listAssets;
+exports.deleteAssets = deleteAssets;
+const factory_1 = require("../../service/metadata/factory");
+const factory_2 = require("../../service/storage/factory");
+async function listAssets(params, config, logger) {
+    if (logger) {
+        logger.info(`[ContextLake-Action] Calling listAssets with params: ${JSON.stringify(params)}`);
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.log(`[ContextLake-Action] Calling listAssets with params: ${JSON.stringify(params)}`);
+    }
+    // Ensure config has default if not provided
+    const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
+    const metadataProvider = (0, factory_1.createMetadataProvider)(metaConfig);
+    await metadataProvider.connect();
+    const docs = await metadataProvider.list(params.limit || 100);
+    // Group by file (since we store chunks)
+    const fileMap = new Map();
+    docs.forEach((doc) => {
+        // Basic grouping by source filename. 
+        const key = doc.source;
+        if (!fileMap.has(key)) {
+            fileMap.set(key, {
+                source: doc.source,
+                file_type: doc.file_type,
+                storage_type: doc.storage_type,
+                url: doc.url,
+                chunks_count: 0,
+                metadata: doc.metadata
+            });
+        }
+        fileMap.get(key).chunks_count++;
+    });
+    // Cleanly serialize before sending back
+    return JSON.parse(JSON.stringify(Array.from(fileMap.values())));
+}
+async function deleteAssets(params, config, logger) {
+    if (logger) {
+        logger.info(`[ContextLake-Action] Calling deleteAssets with params: ${JSON.stringify(params)}`);
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.log(`[ContextLake-Action] Calling deleteAssets with params: ${JSON.stringify(params)}`);
+    }
+    const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
+    const metadataProvider = (0, factory_1.createMetadataProvider)(metaConfig);
+    // file_storage config is optional for deletion (we might not need to delete from storage if inline)
+    // Check if file_storage config exists before creating provider
+    let storageProvider;
+    if (config.file_storage && config.file_storage.type) {
+        storageProvider = (0, factory_2.createStorageProvider)(config.file_storage);
+    }
+    await metadataProvider.connect();
+    let deleteFilter = params.filter || '';
+    if (params.file_ids && params.file_ids.length > 0) {
+        // Construct filter
+        deleteFilter = params.file_ids.map(id => `id = '${id}'`).join(' OR ');
+    }
+    if (!deleteFilter)
+        throw new Error("Delete requires file_ids or filter");
+    // Retrieve documents to be deleted to get their URLs for storage cleanup
+    // Note: This might be expensive if deleting many files. 
+    // Optimization: If filter is complex, we might skip storage cleanup or do it asynchronously.
+    const docsToDelete = await metadataProvider.search('', 1000, deleteFilter);
+    // 1. Delete from Metadata
+    await metadataProvider.delete(deleteFilter);
+    // 2. Delete from Storage (Best effort)
+    const deletedUrls = new Set();
+    if (storageProvider) {
+        for (const doc of docsToDelete) {
+            if (doc.url && !deletedUrls.has(doc.url) && !doc.url.startsWith('inline://')) {
+                try {
+                    await storageProvider.deleteFile(doc.url);
+                    deletedUrls.add(doc.url);
+                }
+                catch (e) {
+                    console.error(`Failed to delete file ${doc.url}:`, e);
+                }
+            }
+        }
+    }
+    return {
+        status: 'success',
+        message: 'Documents deleted',
+        deleted_count: docsToDelete.length,
+        storage_cleaned: deletedUrls.size
+    };
+}

package/dist/src/lib/actions/retrieve.d.ts

@@ -0,0 +1,8 @@
+interface RetrieveParams {
+    query: string;
+    top_k?: number;
+    filter?: string;
+    include_binary?: boolean;
+}
+export declare function retrieveAssets(params: RetrieveParams, config: any, logger?: any): Promise<any[]>;
+export {};

package/dist/src/lib/actions/retrieve.js

@@ -0,0 +1,73 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.retrieveAssets = retrieveAssets;
+const factory_1 = require("../../service/storage/factory");
+const factory_2 = require("../../service/metadata/factory");
+async function retrieveAssets(params, config, logger) {
+    if (logger) {
+        logger.info(`[ContextLake-Action] Calling retrieveAssets with params: ${JSON.stringify(params)}`);
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.log(`[ContextLake-Action] Calling retrieveAssets with params: ${JSON.stringify(params)}`);
+    }
+    const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
+    const metadataProvider = (0, factory_2.createMetadataProvider)(metaConfig);
+    let storageProvider;
+    if (config.file_storage && config.file_storage.type) {
+        storageProvider = (0, factory_1.createStorageProvider)(config.file_storage);
+    }
+    await metadataProvider.connect();
+    const results = await metadataProvider.search(params.query, params.top_k || 5, params.filter);
+    // Post-process to handle binary data retrieval if requested
+    if (params.include_binary) {
+        const enrichedResults = await Promise.all(results.map(async (doc) => {
+            // Remove raw binary_data buffer to avoid clone errors in postMessage
+            const { binary_data, ...rest } = doc;
+            if (rest.vector) {
+                rest.vector = Array.from(rest.vector);
+            }
+            try {
+                if (rest.storage_type === 'inline') {
+                    // Return inline binary data (Base64)
+                    return {
+                        ...rest,
+                        binary_content: binary_data ? binary_data.toString('base64') : null
+                    };
+                }
+                else if (rest.storage_type === 'local' || rest.storage_type === 'tos') {
+                    if (storageProvider) {
+                        // Download from storage provider
+                        const buffer = await storageProvider.downloadFile(rest.url);
+                        return {
+                            ...rest,
+                            binary_content: buffer.toString('base64')
+                        };
+                    }
+                    else {
+                        return { ...rest, binary_error: 'Storage provider not configured' };
+                    }
+                }
+                return rest;
+            }
+            catch (e) {
+                return { ...rest, binary_error: 'Failed to fetch content' };
+            }
+        }));
+        return enrichedResults;
+    }
+    // Remove binary_data from default response to keep it light and serializable
+    return results.map((doc) => {
+        const { binary_data, ...rest } = doc;
+        // Ensure all fields are standard serializable types (remove complex objects/buffers)
+        if (rest.vector) {
+            // Keep vector as standard array of numbers if it exists, but some clients might choke on large arrays.
+            // Usually vector is fine, but LanceDB might return custom objects like Float32Array which fail structuredClone in postMessage
+            rest.vector = Array.from(rest.vector);
+        }
+        // LanceDB records might be Proxy objects or contain non-clonable getters.
+        // We stringify and parse to get a clean, plain Javascript Object.
+        const safeDoc = JSON.parse(JSON.stringify(rest));
+        return safeDoc;
+    });
+}

package/dist/src/processor/loader.d.ts

@@ -0,0 +1,7 @@
+export interface FileData {
+    buffer: Buffer;
+    text: string;
+    type: string;
+}
+export declare function processFile(filePath: string): Promise<FileData>;
+export declare function splitText(text: string, chunkSize?: number, overlap?: number): string[];