@byted-las/contextlake-openclaw 1.0.0 → 1.0.3

package/dist/index.d.ts

@@ -1,3 +1,4 @@
+import type { OpenClawPluginApi } from 'openclaw/plugin-sdk';
 declare const plugin: {
     id: string;
     name: string;
@@ -108,6 +109,6 @@ declare const plugin: {
             };
         };
     };
-    register(ctx: any): void;
+    register(ctx: OpenClawPluginApi): void;
 };
 export default plugin;

package/dist/index.js

@@ -4,7 +4,7 @@ const commands_1 = require("./src/commands");
 const plugin = {
     id: 'contextlake-openclaw',
     name: 'ContextLake',
-    version: '1.1.0',
+    version: '1.0.2',
     description: 'A lightweight knowledge base plugin for OpenClaw using LanceDB and TOS, with data profiling support',
     configSchema: {
         type: 'object',
@@ -58,10 +58,10 @@ const plugin = {
     },
     register(ctx) {
         const logger = ctx.logger || {
-            info: (msg, ...args) => console.log(msg, ...args),
-            warn: (msg, ...args) => console.warn(msg, ...args),
-            error: (msg, ...args) => console.error(msg, ...args),
-            debug: (msg, ...args) => console.debug(msg, ...args),
+            info: (msg) => console.log(msg),
+            warn: (msg) => console.warn(msg),
+            error: (msg) => console.error(msg),
+            debug: (msg) => console.debug(msg),
         };
         // Add logging
         logger.info(`[${new Date().toISOString()}] [ContextLake] Plugin register started`);

package/dist/src/client/lancedb.js

@@ -60,7 +60,7 @@ class ContextLakeLanceDBClient {
         else {
             if (dim <= 0) {
                 // Fallback: use embedding provider to infer dimension only if needed
-                const dummyVec = await this.embeddingProvider.generateEmbedding("init");
+                const dummyVec = await this.embeddingProvider.generateEmbedding('init');
                 dim = dummyVec.length;
             }
             // @ts-ignore
@@ -88,10 +88,18 @@ class ContextLakeLanceDBClient {
         await table.add(docs);
     }
     async search(query, limit = 5, filter) {
-        const vector = await this.embeddingProvider.generateEmbedding(query);
+        const normalizedLimit = Number.isFinite(limit) ? Math.max(1, Math.floor(limit)) : 5;
         const table = await this.getTable();
+        if (!query || !query.trim()) {
+            let fallbackQuery = table.query().limit(normalizedLimit);
+            if (filter) {
+                fallbackQuery = fallbackQuery.where(filter);
+            }
+            return await fallbackQuery.toArray();
+        }
+        const vector = await this.embeddingProvider.generateMultimodalEmbedding([{ type: 'text', text: query }]);
         // @ts-ignore
-        let search = table.vectorSearch(vector).limit(limit);
+        let search = table.vectorSearch(vector).limit(normalizedLimit);
         if (filter) {
             search = search.where(filter);
         }
@@ -102,8 +110,9 @@ class ContextLakeLanceDBClient {
         await table.delete(filter);
     }
     async list(limit = 100, filter) {
+        const normalizedLimit = Number.isFinite(limit) ? Math.max(1, Math.floor(limit)) : 100;
         const table = await this.getTable();
-        let query = table.query().limit(limit);
+        let query = table.query().limit(normalizedLimit);
         if (filter) {
             query = query.where(filter);
         }

package/dist/src/commands/cli.d.ts

@@ -1,6 +1,9 @@
-export declare function getCliCommands(pluginConfig: any, logger: any): {
-    ingestAction: (files: any, options: any) => Promise<void>;
+import { ContextLakeConfig } from '../utils/config';
+export declare function getCliCommands(pluginConfig: ContextLakeConfig, logger: any): {
+    connectAction: (datasource_name: string, options: any) => Promise<void>;
+    ingestAction: (datasource_name: string) => Promise<void>;
     searchAction: (query: any, options: any) => Promise<void>;
     listAction: (options: any) => Promise<void>;
     deleteAction: (options: any) => Promise<void>;
+    onboardAction: () => Promise<void>;
 };

package/dist/src/commands/cli.js

@@ -1,20 +1,78 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getCliCommands = getCliCommands;
+// @ts-ignore
 const ingest_1 = require("../lib/actions/ingest");
 const retrieve_1 = require("../lib/actions/retrieve");
 const manage_1 = require("../lib/actions/manage");
+const profiler_1 = require("../lib/actions/profiler");
+const credentials_1 = require("../utils/credentials");
+function parseOptionalInt(value, fallback) {
+    const parsed = Number.parseInt(String(value), 10);
+    return Number.isFinite(parsed) ? parsed : fallback;
+}
+function parseMetadata(metadata) {
+    if (!metadata) {
+        return {};
+    }
+    if (typeof metadata === 'object') {
+        return metadata;
+    }
+    if (typeof metadata !== 'string') {
+        throw new Error('metadata must be a JSON object or JSON string');
+    }
+    try {
+        const parsed = JSON.parse(metadata);
+        if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+            throw new Error('metadata must be a JSON object');
+        }
+        return parsed;
+    }
+    catch (error) {
+        throw new Error(`Invalid metadata JSON: ${error.message}`);
+    }
+}
 function getCliCommands(pluginConfig, logger) {
     return {
-        ingestAction: async (files, options) => {
-            logger.info(`[${new Date().toISOString()}] [ContextLake] CLI ingest started`, { files, options });
+        connectAction: async (datasource_name, options) => {
+            logger.info(`[${new Date().toISOString()}] [ContextLake] CLI connect started`, { datasource_name, options });
+            try {
+                const params = {
+                    datasource_name,
+                    vendor: options.vendor,
+                    endpoint: options.endpoint,
+                    access_key: options.ak,
+                    secret_key: options.sk,
+                    region: options.region,
+                    bucket: options.bucket,
+                    prefix: options.prefix,
+                    sample_rows: parseInt(options.sampleRows),
+                };
+                // eslint-disable-next-line no-console
+                console.log(`[contextlake connect] Connecting to datasource "${datasource_name}"...`);
+                // eslint-disable-next-line no-console
+                console.log(`  vendor: ${params.vendor}`);
+                // eslint-disable-next-line no-console
+                console.log(`  bucket: ${params.bucket}`);
+                // eslint-disable-next-line no-console
+                console.log(`  prefix: ${params.prefix}`);
+                const result = await (0, profiler_1.connectDataSource)(params);
+                // eslint-disable-next-line no-console
+                console.log(JSON.stringify(result, null, 2));
+                logger.info(`[${new Date().toISOString()}] [ContextLake] CLI connect success`);
+            }
+            catch (e) {
+                // eslint-disable-next-line no-console
+                console.error('Error:', e.message);
+                logger.error(`[${new Date().toISOString()}] [ContextLake] CLI connect failed`, { error: e.message, stack: e.stack });
+                process.exitCode = 1;
+            }
+        },
+        ingestAction: async (datasource_name) => {
+            logger.info(`[${new Date().toISOString()}] [ContextLake] CLI ingest started`, { datasource_name });
             try {
-                const metadata = options.metadata ? JSON.parse(options.metadata) : {};
-                const result = await (0, ingest_1.ingestAssets)({
-                    files,
-                    metadata,
-                    chunkSize: parseInt(options.chunkSize),
-                    overlap: parseInt(options.overlap)
+                const result = await (0, ingest_1.ingestSource)({
+                    datasource_name
                 }, pluginConfig, logger);
                 // eslint-disable-next-line no-console
                 console.log(JSON.stringify(result, null, 2));
@@ -30,7 +88,7 @@ function getCliCommands(pluginConfig, logger) {
             try {
                 const result = await (0, retrieve_1.retrieveAssets)({
                     query,
-                    top_k: parseInt(options.topK),
+                    top_k: parseOptionalInt(options.topK, 5),
                     filter: options.filter,
                     include_binary: options.binary
                 }, pluginConfig, logger);
@@ -47,7 +105,7 @@ function getCliCommands(pluginConfig, logger) {
             logger.info(`[${new Date().toISOString()}] [ContextLake] CLI list started`, { options });
             try {
                 const result = await (0, manage_1.listAssets)({
-                    limit: parseInt(options.limit)
+                    limit: parseOptionalInt(options.limit, 100)
                 }, pluginConfig, logger);
                 // eslint-disable-next-line no-console
                 console.log(JSON.stringify(result, null, 2));
@@ -73,6 +131,32 @@ function getCliCommands(pluginConfig, logger) {
                 console.error('Error:', e.message);
                 logger.error(`[${new Date().toISOString()}] [ContextLake] CLI delete failed`, { error: e.message, stack: e.stack });
             }
+        },
+        onboardAction: async () => {
+            logger.info(`[${new Date().toISOString()}] [ContextLake] CLI onboard started`);
+            try {
+                const currentCreds = (0, credentials_1.loadCredentials)();
+                // eslint-disable-next-line no-console
+                console.log('Welcome to ContextLake Onboarding!');
+                // eslint-disable-next-line no-console
+                console.log('Please provide your credentials below. Press enter to keep the current value.');
+                const lasApiKey = await (0, credentials_1.promptForInput)('LAS_API_KEY', currentCreds.LAS_API_KEY);
+                const volcengineAccessKey = await (0, credentials_1.promptForInput)('VOLCENGINE_ACCESS_KEY', currentCreds.VOLCENGINE_ACCESS_KEY);
+                const volcengineSecretKey = await (0, credentials_1.promptForInput)('VOLCENGINE_SECRET_KEY', currentCreds.VOLCENGINE_SECRET_KEY);
+                const newCreds = {
+                    LAS_API_KEY: lasApiKey,
+                    VOLCENGINE_ACCESS_KEY: volcengineAccessKey,
+                    VOLCENGINE_SECRET_KEY: volcengineSecretKey
+                };
+                (0, credentials_1.saveCredentials)(newCreds);
+                // eslint-disable-next-line no-console
+                console.log('Credentials saved successfully!');
+                logger.info(`[${new Date().toISOString()}] [ContextLake] CLI onboard success`);
+            }
+            catch (e) {
+                console.error('Error during onboarding:', e.message);
+                logger.error(`[${new Date().toISOString()}] [ContextLake] CLI onboard failed`, { error: e.message, stack: e.stack });
+            }
         }
     };
 }

package/dist/src/commands/index.d.ts

@@ -1 +1,2 @@
-export declare function registerAll(ctx: any, logger: any): void;
+import type { OpenClawPluginApi, PluginLogger } from 'openclaw/plugin-sdk';
+export declare function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger): void;

package/dist/src/commands/index.js

@@ -5,7 +5,6 @@ const tools_1 = require("./tools");
 const cli_1 = require("./cli");
 const slashcmd_1 = require("./slashcmd");
 const config_1 = require("../utils/config");
-const las_data_profiler_1 = require("../skills/las-data-profiler");
 function registerAll(ctx, logger) {
     const pluginConfig = (0, config_1.getPluginConfig)(ctx);
     // Register Agent Tools
@@ -19,9 +18,17 @@ function registerAll(ctx, logger) {
         logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.listTool.name}`);
         ctx.registerTool(tools.deleteTool);
         logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.deleteTool.name}`);
+        ctx.registerTool(tools.lasDataProfilerTool);
+        logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.lasDataProfilerTool.name}`);
+        ctx.registerTool(tools.listDatasourceTool);
+        logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.listDatasourceTool.name}`);
+        for (const lasTool of tools.lasTools) {
+            ctx.registerTool(lasTool);
+            logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${lasTool.name}`);
+        }
     }
     catch (error) {
-        logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering agent tools: ${error.message}`, { stack: error.stack });
+        logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering agent tools: ${error.message}${error.stack ? '\\n' + error.stack : ''}`);
         throw error;
     }
     // Register CLI
@@ -43,37 +50,10 @@ function registerAll(ctx, logger) {
                 .requiredOption('--bucket <bucket>', 'Bucket name (or local root directory for local vendor)')
                 .requiredOption('--prefix <prefix>', 'Path prefix to limit scan scope')
                 .option('--sample-rows <number>', 'Number of rows to sample per structured file', '100')
-                .action(async (datasource_name, options) => {
-                try {
-                    const params = {
-                        datasource_name,
-                        vendor: options.vendor,
-                        endpoint: options.endpoint,
-                        access_key: options.ak,
-                        secret_key: options.sk,
-                        region: options.region,
-                        bucket: options.bucket,
-                        prefix: options.prefix,
-                        sample_rows: parseInt(options.sampleRows),
-                    };
-                    console.log(`[contextlake connect] Connecting to datasource "${datasource_name}"...`);
-                    console.log(`  vendor: ${params.vendor}`);
-                    console.log(`  bucket: ${params.bucket}`);
-                    console.log(`  prefix: ${params.prefix}`);
-                    const result = await (0, las_data_profiler_1.connectDataSource)(params);
-                    console.log(JSON.stringify(result, null, 2));
-                }
-                catch (e) {
-                    console.error('Error:', e.message);
-                    process.exitCode = 1;
-                }
-            });
+                .action(commands.connectAction);
             // Ingest
-            contextlake.command('ingest <files...>')
-                .description('Ingest one or more files into the knowledge base')
-                .option('-c, --chunk-size <number>', 'Chunk size for text splitting', '500')
-                .option('-o, --overlap <number>', 'Chunk overlap size', '50')
-                .option('-m, --metadata <json>', 'JSON metadata to attach to the documents')
+            contextlake.command('ingest <datasource_name>')
+                .description('Process and ingest all files from a connected data source into the knowledge base')
                 .action(commands.ingestAction);
             // Search
             contextlake.command('search <query>')
@@ -93,11 +73,15 @@ function registerAll(ctx, logger) {
                 .option('--ids <ids...>', 'List of specific file IDs to delete')
                 .option('-f, --filter <string>', 'Filter string to match documents for deletion')
                 .action(commands.deleteAction);
+            // Onboard
+            contextlake.command('onboard')
+                .description('Configure credentials for ContextLake')
+                .action(commands.onboardAction);
         }, { commands: ['contextlake'] });
         logger.info(`[${new Date().toISOString()}] [ContextLake] CLI commands registered`);
     }
     catch (error) {
-        logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering CLI commands: ${error.message}`, { stack: error.stack });
+        logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering CLI commands: ${error.message}${error.stack ? '\\n' + error.stack : ''}`);
         throw error;
     }
     // Register Slash Commands
@@ -109,7 +93,7 @@ function registerAll(ctx, logger) {
         const slashCommands = (0, slashcmd_1.getSlashCommands)(pluginConfig, logger);
         ctx.registerCommand({
             name: 'contextlake-ingest',
-            description: 'Ingest files into the knowledge base (usage: /contextlake-ingest file1 file2)',
+            description: 'Process and ingest all files from a connected data source (usage: /contextlake-ingest <datasource_name>)',
             acceptsArgs: true,
             handler: slashCommands.ingestHandler
         });
@@ -131,9 +115,21 @@ function registerAll(ctx, logger) {
             acceptsArgs: true,
             handler: slashCommands.deleteHandler
         });
+        ctx.registerCommand({
+            name: 'contextlake-profiler',
+            description: 'Connect to a data source and profile its structure (usage: /contextlake-profiler <datasource_name> <vendor> <bucket> <prefix>)',
+            acceptsArgs: true,
+            handler: slashCommands.profilerHandler
+        });
+        ctx.registerCommand({
+            name: 'contextlake-list-datasource',
+            description: 'List all connected and profiled data sources (usage: /contextlake-list-datasource)',
+            acceptsArgs: false,
+            handler: slashCommands.listDatasourceHandler
+        });
         logger.info(`[${new Date().toISOString()}] [ContextLake] Slash commands registered`);
     }
     catch (error) {
-        logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering Slash commands: ${error.message}`, { stack: error.stack });
+        logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering Slash commands: ${error.message}${error.stack ? '\\n' + error.stack : ''}`);
     }
 }

package/dist/src/commands/slashcmd.d.ts

@@ -1,4 +1,5 @@
-export declare function getSlashCommands(pluginConfig: any, logger: any): {
+import { ContextLakeConfig } from '../utils/config';
+export declare function getSlashCommands(pluginConfig: ContextLakeConfig, logger: any): {
     ingestHandler: (commandCtx: any) => Promise<{
         text: string;
     }>;
@@ -11,4 +12,10 @@ export declare function getSlashCommands(pluginConfig: any, logger: any): {
     deleteHandler: (commandCtx: any) => Promise<{
         text: string;
     }>;
+    profilerHandler: (commandCtx: any) => Promise<{
+        text: string;
+    }>;
+    listDatasourceHandler: (commandCtx: any) => Promise<{
+        text: string;
+    }>;
 };

package/dist/src/commands/slashcmd.js

@@ -1,9 +1,46 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getSlashCommands = getSlashCommands;
 const ingest_1 = require("../lib/actions/ingest");
 const retrieve_1 = require("../lib/actions/retrieve");
 const manage_1 = require("../lib/actions/manage");
+const profiler_1 = require("../lib/actions/profiler");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const os = __importStar(require("os"));
 function getSlashCommands(pluginConfig, logger) {
     return {
         ingestHandler: async (commandCtx) => {
@@ -12,12 +49,16 @@ function getSlashCommands(pluginConfig, logger) {
             logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command ingest started`, { args });
             try {
                 if (args.length === 0) {
-                    return { text: `**Error:** Missing files. Usage: /contextlake-ingest /path/to/file1 /path/to/file2` };
+                    return { text: `**Error:** Missing datasource_name. Usage: /contextlake-ingest <datasource_name>` };
                 }
-                const result = await (0, ingest_1.ingestAssets)({
-                    files: args,
-                    metadata: {}
-                }, pluginConfig, logger);
+                const datasource_name = args[0];
+                const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
+                const dsDir = path.join(BASE_DIR, datasource_name);
+                const dbPath = path.join(dsDir, 'catalog_db');
+                if (!fs.existsSync(dbPath)) {
+                    return { text: `**Error:** Data source "${datasource_name}" has not been profiled yet.\n\nPlease run the profiler first using:\n\`/contextlake-profiler <datasource_name> <vendor> <bucket> <prefix> [endpoint] [ak] [sk] [region]\`` };
+                }
+                const result = await (0, ingest_1.ingestSource)({ datasource_name }, pluginConfig, logger);
                 logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command ingest completed`, { resultCount: result.length });
                 return { text: `**Ingest Results (${result.length} files processed):**\n\`\`\`json\n${JSON.stringify(result, null, 2)}\n\`\`\`` };
             }
@@ -86,6 +127,49 @@ function getSlashCommands(pluginConfig, logger) {
                 logger.error(`[ContextLake] Slash delete failed`, { error: e.message });
                 return { text: `**Error executing delete:** ${e.message}` };
             }
-        }
+        },
+        profilerHandler: async (commandCtx) => {
+            const rawArgs = commandCtx.args || "";
+            const args = rawArgs.split(' ').filter((arg) => arg.trim() !== '');
+            logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command profiler started`, { args });
+            try {
+                if (args.length < 4) {
+                    return { text: `**Error:** Missing arguments. Usage: /contextlake-profiler <datasource_name> <vendor> <bucket> <prefix> [endpoint] [ak] [sk] [region]` };
+                }
+                const [datasource_name, vendor, bucket, prefix, endpoint, access_key, secret_key, region] = args;
+                if (!['volcengine', 'alibaba', 'tencent', 'aws', 'local'].includes(vendor)) {
+                    return { text: `**Error:** Invalid vendor. Must be one of: volcengine, alibaba, tencent, aws, local` };
+                }
+                const params = {
+                    datasource_name,
+                    vendor: vendor,
+                    bucket,
+                    prefix,
+                    endpoint,
+                    access_key,
+                    secret_key,
+                    region,
+                };
+                const result = await (0, profiler_1.connectDataSource)(params);
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command profiler completed`, { result });
+                return { text: `**Profiler Results:**\n\`\`\`json\n${JSON.stringify(result, null, 2)}\n\`\`\`` };
+            }
+            catch (e) {
+                logger.error(`[ContextLake] Slash profiler failed`, { error: e.message });
+                return { text: `**Error executing profiler:** ${e.message}` };
+            }
+        },
+        listDatasourceHandler: async (commandCtx) => {
+            logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command list-datasource started`);
+            try {
+                const result = await (0, profiler_1.listDataSources)();
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command list-datasource completed`, { result });
+                return { text: `**Data Sources:**\n\`\`\`json\n${JSON.stringify(result, null, 2)}\n\`\`\`` };
+            }
+            catch (e) {
+                logger.error(`[ContextLake] Slash list-datasource failed`, { error: e.message });
+                return { text: `**Error executing list-datasource:** ${e.message}` };
+            }
+        },
     };
 }