@byted-las/contextlake-openclaw 1.0.0 → 1.0.3

package/src/commands/slashcmd.ts

@@ -1,10 +1,13 @@
-// @ts-ignore
-import { PluginContext } from 'openclaw/plugin-sdk';
-import { ingestAssets } from '../lib/actions/ingest';
+import { ingestSource } from '../lib/actions/ingest';
 import { retrieveAssets } from '../lib/actions/retrieve';
 import { listAssets, deleteAssets } from '../lib/actions/manage';
+import { connectDataSource, listDataSources, ConnectParams } from '../lib/actions/profiler';
+import { ContextLakeConfig } from '../utils/config';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
 
-export function getSlashCommands(pluginConfig: any, logger: any) {
+export function getSlashCommands(pluginConfig: ContextLakeConfig, logger: any) {
     return {
         ingestHandler: async (commandCtx: any) => {
             const rawArgs = commandCtx.args || "";
@@ -13,12 +16,20 @@ export function getSlashCommands(pluginConfig: any, logger: any) {
             logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command ingest started`, { args });
             try {
                 if (args.length === 0) {
-                    return { text: `**Error:** Missing files. Usage: /contextlake-ingest /path/to/file1 /path/to/file2` };
+                    return { text: `**Error:** Missing datasource_name. Usage: /contextlake-ingest <datasource_name>` };
                 }
-                const result = await ingestAssets({
-                    files: args,
-                    metadata: {}
-                }, pluginConfig, logger);
+                
+                const datasource_name = args[0];
+
+                const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
+                const dsDir = path.join(BASE_DIR, datasource_name);
+                const dbPath = path.join(dsDir, 'catalog_db');
+
+                if (!fs.existsSync(dbPath)) {
+                    return { text: `**Error:** Data source "${datasource_name}" has not been profiled yet.\n\nPlease run the profiler first using:\n\`/contextlake-profiler <datasource_name> <vendor> <bucket> <prefix> [endpoint] [ak] [sk] [region]\`` };
+                }
+
+                const result = await ingestSource({ datasource_name }, pluginConfig, logger);
                 
                 logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command ingest completed`, { resultCount: result.length });
                 return { text: `**Ingest Results (${result.length} files processed):**\n\`\`\`json\n${JSON.stringify(result, null, 2)}\n\`\`\`` };
@@ -90,6 +101,54 @@ export function getSlashCommands(pluginConfig: any, logger: any) {
                 logger.error(`[ContextLake] Slash delete failed`, { error: e.message });
                 return { text: `**Error executing delete:** ${e.message}` };
             }
-        }
+        },
+        profilerHandler: async (commandCtx: any) => {
+            const rawArgs = commandCtx.args || "";
+            const args = rawArgs.split(' ').filter((arg: string) => arg.trim() !== '');
+
+            logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command profiler started`, { args });
+            try {
+                if (args.length < 4) {
+                    return { text: `**Error:** Missing arguments. Usage: /contextlake-profiler <datasource_name> <vendor> <bucket> <prefix> [endpoint] [ak] [sk] [region]` };
+                }
+                
+                const [datasource_name, vendor, bucket, prefix, endpoint, access_key, secret_key, region] = args;
+                
+                if (!['volcengine', 'alibaba', 'tencent', 'aws', 'local'].includes(vendor)) {
+                     return { text: `**Error:** Invalid vendor. Must be one of: volcengine, alibaba, tencent, aws, local` };
+                }
+                
+                const params: ConnectParams = {
+                    datasource_name,
+                    vendor: vendor as ConnectParams['vendor'],
+                    bucket,
+                    prefix,
+                    endpoint,
+                    access_key,
+                    secret_key,
+                    region,
+                };
+                
+                const result = await connectDataSource(params);
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command profiler completed`, { result });
+                return { text: `**Profiler Results:**\n\`\`\`json\n${JSON.stringify(result, null, 2)}\n\`\`\`` };
+            } catch (e: any) {
+                logger.error(`[ContextLake] Slash profiler failed`, { error: e.message });
+                return { text: `**Error executing profiler:** ${e.message}` };
+            }
+        },
+
+        listDatasourceHandler: async (commandCtx: any) => {
+            logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command list-datasource started`);
+            try {
+                const result = await listDataSources();
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command list-datasource completed`, { result });
+                return { text: `**Data Sources:**\n\`\`\`json\n${JSON.stringify(result, null, 2)}\n\`\`\`` };
+            } catch (e: any) {
+                logger.error(`[ContextLake] Slash list-datasource failed`, { error: e.message });
+                return { text: `**Error executing list-datasource:** ${e.message}` };
+            }
+        },
+
     };
 }

package/src/commands/tools.ts

@@ -1,72 +1,68 @@
-import { ingestAssets } from '../lib/actions/ingest';
+import { ingestSource } from '../lib/actions/ingest';
 import { retrieveAssets } from '../lib/actions/retrieve';
 import { listAssets, deleteAssets } from '../lib/actions/manage';
+import { connectDataSource, listDataSources } from '../lib/actions/profiler';
+import { getLasTools } from '../lib/actions/las-tools';
+import { ContextLakeConfig } from '../utils/config';
+// @ts-ignore
+import type { AnyAgentTool } from 'openclaw/plugin-sdk';
+
+export function getAgentTools(pluginConfig: ContextLakeConfig, logger: any): {
+    ingestTool: AnyAgentTool;
+    retrieveTool: AnyAgentTool;
+    listTool: AnyAgentTool;
+    deleteTool: AnyAgentTool;
+    lasDataProfilerTool: AnyAgentTool;
+    listDatasourceTool: AnyAgentTool;
+    lasTools: AnyAgentTool[];
+} {
+    const lasTools = getLasTools(pluginConfig, logger);
 
-export function getAgentTools(pluginConfig: any, logger: any) {
     return {
+        lasTools,
+        listDatasourceTool: {
+            name: 'contextlake-list-datasource',
+            label: 'ContextLake List Datasources',
+            description: `List all connected and profiled data sources.`,
+            parameters: {
+                type: 'object',
+                properties: {},
+                required: [],
+                additionalProperties: false
+            },
+            async execute(toolCallId: string, params: any) {
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Executing list-datasource skill, toolCallId: ${toolCallId}`);
+                try {
+                    const result = await listDataSources();
+                    return {
+                        content: [{ type: "text", text: JSON.stringify(result) }],
+                        details: result
+                    } as any;
+                } catch (error: any) {
+                    logger.error(`[${new Date().toISOString()}] [ContextLake] list-datasource skill failed`, { error: error.message });
+                    return {
+                        content: [{ type: "text", text: String(error.message) }],
+                        details: { error: error.message }
+                    } as any;
+                }
+            }
+        },
         ingestTool: {
             name: 'contextlake-ingest',
             label: 'ContextLake Ingest',
-            description: `Upload, ingest, and index documents into the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
+            description: `Process and ingest all files from a connected data source into the knowledge base.
 Use this tool when the user wants to "将知识注入", "上传文件", "入库", "添加文档", "ingest files", or "add knowledge".
-Supports processing of various file types including PDF, Word, Markdown, and Text.
-Automatically handles text extraction, cleaning, chunking, embedding generation, and storage.
-
-Example User Queries:
-- "帮我把这个文档注入到知识湖中"
-- "上传这份 PDF 到知识库"
-- "Please ingest these documents into ContextLake"
-- "将 /path/to/doc.txt 添加到知识库"`,
+Supports multimodal files (text, images, audio, video, pdf) by using LAS models to understand and embed them.
+Must be called after a data source has been successfully profiled via \`las-data-profiler\`.`,
             parameters: {
                 type: 'object',
                 properties: {
-                    files: { 
-                        type: 'array', 
-                        items: { type: 'string' }, 
-                        description: 'List of file paths to ingest' 
-                    },
-                    metadata: { 
-                        type: 'object', 
-                        description: 'Optional JSON metadata to attach to documents',
-                        additionalProperties: true
-                    },
-                    chunkSize: { 
-                        type: 'integer', 
-                        description: 'Chunk size for text splitting' 
-                    },
-                    overlap: { 
-                        type: 'integer', 
-                        description: 'Overlap size for text splitting' 
-                    }
-                },
-                required: ['files'],
-                additionalProperties: false
-            },
-            schema: {
-                type: 'object',
-                properties: {
-                    files: { 
-                        type: 'array', 
-                        items: { type: 'string' }, 
-                        description: 'List of file paths to ingest' 
-                    },
-                    metadata: { 
-                        type: 'object', 
-                        description: 'Optional JSON metadata to attach to documents',
-                        additionalProperties: true
-                    },
-                    chunkSize: { 
-                        type: 'integer', 
-                        description: 'Chunk size for text splitting' 
-                    },
-                    overlap: { 
-                        type: 'integer', 
-                        description: 'Overlap size for text splitting' 
-                    }
+                    datasource_name: { type: 'string', description: 'Name of the data source previously profiled' }
                 },
-                required: ['files'],
+                required: ['datasource_name'],
                 additionalProperties: false
             },
+            
             async execute(toolCallId: string, params: any) {
                 logger.info(`[${new Date().toISOString()}] [ContextLake] Executing ingest skill, toolCallId: ${toolCallId}`, { params: JSON.stringify(params) });
 
@@ -78,35 +74,35 @@ Example User Queries:
                         } catch (e) {
                             logger.warn(`[ContextLake] Received string params, possibly toolCallId?`, { params });
                             return {
-                                success: false,
-                                error: `Invalid params format: received string "${params}", expected object with 'files' array.`
-                            };
+                                content: [{ type: "text", text: `Invalid params format: received string "${params}", expected object with 'datasource_name'.` }],
+                                details: { error: `Invalid params format: received string "${params}", expected object with 'datasource_name'.` }
+                            } as any;
                         }
                     }
                     
-                    if (!actualParams.files && actualParams.params && actualParams.params.files) {
+                    if (!actualParams.datasource_name && actualParams.params && actualParams.params.datasource_name) {
                         actualParams = actualParams.params;
                     }
 
-                    if (!actualParams.files || !Array.isArray(actualParams.files)) {
+                    if (!actualParams.datasource_name) {
                         return {
-                            success: false,
-                            error: `Invalid params: 'files' must be an array. Received keys: ${Object.keys(actualParams)}`
-                        };
+                                content: [{ type: "text", text: `Invalid params: 'datasource_name' is required. Received keys: ${Object.keys(actualParams)}` }],
+                                details: { error: `Invalid params: 'datasource_name' is required. Received keys: ${Object.keys(actualParams)}` }
+                            } as any;
                     }
 
-                    const result = await ingestAssets(actualParams, pluginConfig, logger);
+                    const result = await ingestSource(actualParams, pluginConfig, logger);
                     logger.info(`[${new Date().toISOString()}] [ContextLake] Ingest skill completed successfully`, { resultSummary: Array.isArray(result) ? `Processed ${result.length} items` : 'Success' });
                     return {
-                        success: true,
-                        result
-                    };
+                        content: [{ type: "text", text: JSON.stringify(result) }],
+                        details: result
+                    } as any;
                 } catch (error: any) {
                     logger.error(`[${new Date().toISOString()}] [ContextLake] Ingest skill failed`, { error: error.message, stack: error.stack });
                     return {
-                        success: false,
-                        error: error.message
-                    };
+                        content: [{ type: "text", text: String(error.message) }],
+                        details: { error: error.message }
+                    } as any;
                 }
             }
         },
@@ -133,17 +129,7 @@ Example User Queries:
                 required: ['query'],
                 additionalProperties: false
             },
-            schema: {
-                type: 'object',
-                properties: {
-                    query: { type: 'string', description: 'Search query' },
-                    top_k: { type: 'integer', description: 'Number of results to return' },
-                    filter: { type: 'string', description: 'Filter string' },
-                    include_binary: { type: 'boolean', description: 'Whether to include binary content' }
-                },
-                required: ['query'],
-                additionalProperties: false
-            },
+            
             async execute(toolCallId: string, params: any) {
                 logger.info(`[${new Date().toISOString()}] [ContextLake] Executing retrieve skill, toolCallId: ${toolCallId}`, { params: JSON.stringify(params) });
                 
@@ -163,23 +149,25 @@ Example User Queries:
                     
                     if (!actualParams || typeof actualParams.query !== 'string') {
                          return {
-                             success: false,
-                             error: `Invalid params: 'query' is required and must be a string. Received: ${JSON.stringify(actualParams)}`
-                         };
+                                content: [{ type: "text", text: `Invalid params: 'query' is required and must be a string. Received: ${JSON.stringify(actualParams)}` }],
+                                details: { error: `Invalid params: 'query' is required and must be a string. Received: ${JSON.stringify(actualParams)}` }
+                            } as any;
                     }
 
                     const result = await retrieveAssets(actualParams, pluginConfig, logger);
                     logger.info(`[${new Date().toISOString()}] [ContextLake] Retrieve skill completed`, { resultCount: Array.isArray(result) ? result.length : 0 });
                     return {
-                        success: true,
-                        result
-                    };
+                        content: [{ type: "text", text: JSON.stringify(result) }],
+                        details: result
+                    } as any;
                 } catch (error: any) {
                     logger.error(`[${new Date().toISOString()}] [ContextLake] Retrieve skill failed`, { error: error.message, stack: error.stack });
                     return {
-                        success: false,
-                        error: error.message
-                    };
+                        content: [{ type: "text", text: String(error.message
+                    ) }],
+                        details: { error: error.message
+                     }
+                    } as any;
                 }
             }
         },
@@ -201,13 +189,7 @@ Example User Queries:
                 required: [],
                 additionalProperties: false
             },
-            schema: {
-                type: 'object',
-                properties: {
-                    limit: { type: 'integer', description: 'Limit for list action' }
-                },
-                required: []
-            },
+            
             async execute(toolCallId: string, params: any) {
                 logger.info(`[${new Date().toISOString()}] [ContextLake] Executing list skill, toolCallId: ${toolCallId}`, { params: JSON.stringify(params) });
 
@@ -219,15 +201,17 @@ Example User Queries:
                     const result = await listAssets(actualParams, pluginConfig, logger);
                     logger.info(`[${new Date().toISOString()}] [ContextLake] List skill completed`, { count: Array.isArray(result) ? result.length : 0 });
                     return {
-                        success: true,
-                        result
-                    };
+                        content: [{ type: "text", text: JSON.stringify(result) }],
+                        details: result
+                    } as any;
                 } catch (error: any) {
                     logger.error(`[${new Date().toISOString()}] [ContextLake] List skill failed`, { error: error.message, stack: error.stack });
                     return {
-                        success: false,
-                        error: error.message
-                    };
+                        content: [{ type: "text", text: String(error.message
+                    ) }],
+                        details: { error: error.message
+                     }
+                    } as any;
                 }
             }
         },
@@ -251,14 +235,7 @@ Example User Queries:
                 required: [],
                 additionalProperties: false
             },
-            schema: {
-                type: 'object',
-                properties: {
-                    file_ids: { type: 'array', items: { type: 'string' }, description: 'File IDs to delete' },
-                    filter: { type: 'string', description: 'Filter string for deletion' }
-                },
-                required: []
-            },
+            
             async execute(toolCallId: string, params: any) {
                 logger.info(`[${new Date().toISOString()}] [ContextLake] Executing delete skill, toolCallId: ${toolCallId}`, { params: JSON.stringify(params) });
 
@@ -270,15 +247,63 @@ Example User Queries:
                     const result = await deleteAssets(actualParams, pluginConfig, logger);
                     logger.info(`[${new Date().toISOString()}] [ContextLake] Delete skill completed`, { result });
                     return {
-                        success: true,
-                        result
-                    };
+                        content: [{ type: "text", text: JSON.stringify(result) }],
+                        details: result
+                    } as any;
                 } catch (error: any) {
                     logger.error(`[${new Date().toISOString()}] [ContextLake] Delete skill failed`, { error: error.message, stack: error.stack });
                     return {
-                        success: false,
-                        error: error.message
-                    };
+                        content: [{ type: "text", text: String(error.message
+                    ) }],
+                        details: { error: error.message
+                     }
+                    } as any;
+                }
+            }
+        },
+        lasDataProfilerTool: {
+            name: 'las-data-profiler',
+            label: 'LAS Data Profiler',
+            description: 'Connect to a data source (TOS/OSS/COS/S3/Local) and profile its structure, schemas, and media metadata into LanceDB',
+            parameters: {
+                type: 'object',
+                properties: {
+                    datasource_name: { type: 'string', description: 'Name of the data source' },
+                    vendor: { type: 'string', enum: ['volcengine', 'alibaba', 'tencent', 'aws', 'local'], description: 'Data source type' },
+                    endpoint: { type: 'string', description: 'S3 Endpoint URL (not needed for local)' },
+                    access_key: { type: 'string', description: 'Credential ID for the data source' },
+                    secret_key: { type: 'string', description: 'Credential value for the data source' },
+                    region: { type: 'string', description: 'Region identifier (e.g. cn-beijing)' },
+                    bucket: { type: 'string', description: 'Bucket name (or local root directory for local vendor)' },
+                    prefix: { type: 'string', description: 'Path prefix to limit scan scope' },
+                    sample_rows: { type: 'integer', description: 'Number of rows to sample per structured file' }
+                },
+                required: ['datasource_name', 'vendor', 'bucket', 'prefix'],
+                additionalProperties: false
+            },
+            
+            async execute(toolCallId: string, params: any) {
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Executing las-data-profiler skill, toolCallId: ${toolCallId}`, { params: JSON.stringify(params) });
+
+                try {
+                    let actualParams = params;
+                    if (params && params.params) {
+                        actualParams = params.params;
+                    }
+                    const result = await connectDataSource(actualParams);
+                    logger.info(`[${new Date().toISOString()}] [ContextLake] las-data-profiler skill completed`, { result });
+                    return {
+                        content: [{ type: "text", text: JSON.stringify(result) }],
+                        details: result
+                    } as any;
+                } catch (error: any) {
+                    logger.error(`[${new Date().toISOString()}] [ContextLake] las-data-profiler skill failed`, { error: error.message, stack: error.stack });
+                    return {
+                        content: [{ type: "text", text: String(error.message
+                    ) }],
+                        details: { error: error.message
+                     }
+                    } as any;
                 }
             }
         }