@byted-las/contextlake-openclaw 1.0.2 → 1.0.3

package/dist/src/client/lancedb.js

@@ -97,7 +97,7 @@ class ContextLakeLanceDBClient {
             }
             return await fallbackQuery.toArray();
         }
-        const vector = await this.embeddingProvider.generateEmbedding(query);
+        const vector = await this.embeddingProvider.generateMultimodalEmbedding([{ type: 'text', text: query }]);
         // @ts-ignore
         let search = table.vectorSearch(vector).limit(normalizedLimit);
         if (filter) {

package/dist/src/commands/cli.d.ts

@@ -1,8 +1,9 @@
 import { ContextLakeConfig } from '../utils/config';
 export declare function getCliCommands(pluginConfig: ContextLakeConfig, logger: any): {
     connectAction: (datasource_name: string, options: any) => Promise<void>;
-    ingestAction: (files: any, options: any) => Promise<void>;
+    ingestAction: (datasource_name: string) => Promise<void>;
     searchAction: (query: any, options: any) => Promise<void>;
     listAction: (options: any) => Promise<void>;
     deleteAction: (options: any) => Promise<void>;
+    onboardAction: () => Promise<void>;
 };

package/dist/src/commands/cli.js

@@ -6,6 +6,7 @@ const ingest_1 = require("../lib/actions/ingest");
 const retrieve_1 = require("../lib/actions/retrieve");
 const manage_1 = require("../lib/actions/manage");
 const profiler_1 = require("../lib/actions/profiler");
+const credentials_1 = require("../utils/credentials");
 function parseOptionalInt(value, fallback) {
     const parsed = Number.parseInt(String(value), 10);
     return Number.isFinite(parsed) ? parsed : fallback;
@@ -67,15 +68,11 @@ function getCliCommands(pluginConfig, logger) {
                 process.exitCode = 1;
             }
         },
-        ingestAction: async (files, options) => {
-            logger.info(`[${new Date().toISOString()}] [ContextLake] CLI ingest started`, { files, options });
+        ingestAction: async (datasource_name) => {
+            logger.info(`[${new Date().toISOString()}] [ContextLake] CLI ingest started`, { datasource_name });
             try {
-                const metadata = parseMetadata(options.metadata);
-                const result = await (0, ingest_1.ingestAssets)({
-                    files,
-                    metadata,
-                    chunkSize: parseOptionalInt(options.chunkSize, 500),
-                    overlap: parseOptionalInt(options.overlap, 50)
+                const result = await (0, ingest_1.ingestSource)({
+                    datasource_name
                 }, pluginConfig, logger);
                 // eslint-disable-next-line no-console
                 console.log(JSON.stringify(result, null, 2));
@@ -134,6 +131,32 @@ function getCliCommands(pluginConfig, logger) {
                 console.error('Error:', e.message);
                 logger.error(`[${new Date().toISOString()}] [ContextLake] CLI delete failed`, { error: e.message, stack: e.stack });
             }
+        },
+        onboardAction: async () => {
+            logger.info(`[${new Date().toISOString()}] [ContextLake] CLI onboard started`);
+            try {
+                const currentCreds = (0, credentials_1.loadCredentials)();
+                // eslint-disable-next-line no-console
+                console.log('Welcome to ContextLake Onboarding!');
+                // eslint-disable-next-line no-console
+                console.log('Please provide your credentials below. Press enter to keep the current value.');
+                const lasApiKey = await (0, credentials_1.promptForInput)('LAS_API_KEY', currentCreds.LAS_API_KEY);
+                const volcengineAccessKey = await (0, credentials_1.promptForInput)('VOLCENGINE_ACCESS_KEY', currentCreds.VOLCENGINE_ACCESS_KEY);
+                const volcengineSecretKey = await (0, credentials_1.promptForInput)('VOLCENGINE_SECRET_KEY', currentCreds.VOLCENGINE_SECRET_KEY);
+                const newCreds = {
+                    LAS_API_KEY: lasApiKey,
+                    VOLCENGINE_ACCESS_KEY: volcengineAccessKey,
+                    VOLCENGINE_SECRET_KEY: volcengineSecretKey
+                };
+                (0, credentials_1.saveCredentials)(newCreds);
+                // eslint-disable-next-line no-console
+                console.log('Credentials saved successfully!');
+                logger.info(`[${new Date().toISOString()}] [ContextLake] CLI onboard success`);
+            }
+            catch (e) {
+                console.error('Error during onboarding:', e.message);
+                logger.error(`[${new Date().toISOString()}] [ContextLake] CLI onboard failed`, { error: e.message, stack: e.stack });
+            }
         }
     };
 }

package/dist/src/commands/index.js

@@ -20,6 +20,12 @@ function registerAll(ctx, logger) {
         logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.deleteTool.name}`);
         ctx.registerTool(tools.lasDataProfilerTool);
         logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.lasDataProfilerTool.name}`);
+        ctx.registerTool(tools.listDatasourceTool);
+        logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.listDatasourceTool.name}`);
+        for (const lasTool of tools.lasTools) {
+            ctx.registerTool(lasTool);
+            logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${lasTool.name}`);
+        }
     }
     catch (error) {
         logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering agent tools: ${error.message}${error.stack ? '\\n' + error.stack : ''}`);
@@ -46,11 +52,8 @@ function registerAll(ctx, logger) {
                 .option('--sample-rows <number>', 'Number of rows to sample per structured file', '100')
                 .action(commands.connectAction);
             // Ingest
-            contextlake.command('ingest <files...>')
-                .description('Ingest one or more files into the knowledge base')
-                .option('-c, --chunk-size <number>', 'Chunk size for text splitting', '500')
-                .option('-o, --overlap <number>', 'Chunk overlap size', '50')
-                .option('-m, --metadata <json>', 'JSON metadata to attach to the documents')
+            contextlake.command('ingest <datasource_name>')
+                .description('Process and ingest all files from a connected data source into the knowledge base')
                 .action(commands.ingestAction);
             // Search
             contextlake.command('search <query>')
@@ -70,6 +73,10 @@ function registerAll(ctx, logger) {
                 .option('--ids <ids...>', 'List of specific file IDs to delete')
                 .option('-f, --filter <string>', 'Filter string to match documents for deletion')
                 .action(commands.deleteAction);
+            // Onboard
+            contextlake.command('onboard')
+                .description('Configure credentials for ContextLake')
+                .action(commands.onboardAction);
         }, { commands: ['contextlake'] });
         logger.info(`[${new Date().toISOString()}] [ContextLake] CLI commands registered`);
     }
@@ -86,7 +93,7 @@ function registerAll(ctx, logger) {
         const slashCommands = (0, slashcmd_1.getSlashCommands)(pluginConfig, logger);
         ctx.registerCommand({
             name: 'contextlake-ingest',
-            description: 'Ingest files into the knowledge base (usage: /contextlake-ingest file1 file2)',
+            description: 'Process and ingest all files from a connected data source (usage: /contextlake-ingest <datasource_name>)',
             acceptsArgs: true,
             handler: slashCommands.ingestHandler
         });
@@ -108,6 +115,18 @@ function registerAll(ctx, logger) {
             acceptsArgs: true,
             handler: slashCommands.deleteHandler
         });
+        ctx.registerCommand({
+            name: 'contextlake-profiler',
+            description: 'Connect to a data source and profile its structure (usage: /contextlake-profiler <datasource_name> <vendor> <bucket> <prefix>)',
+            acceptsArgs: true,
+            handler: slashCommands.profilerHandler
+        });
+        ctx.registerCommand({
+            name: 'contextlake-list-datasource',
+            description: 'List all connected and profiled data sources (usage: /contextlake-list-datasource)',
+            acceptsArgs: false,
+            handler: slashCommands.listDatasourceHandler
+        });
         logger.info(`[${new Date().toISOString()}] [ContextLake] Slash commands registered`);
     }
     catch (error) {

package/dist/src/commands/slashcmd.d.ts

@@ -12,4 +12,10 @@ export declare function getSlashCommands(pluginConfig: ContextLakeConfig, logger
     deleteHandler: (commandCtx: any) => Promise<{
         text: string;
     }>;
+    profilerHandler: (commandCtx: any) => Promise<{
+        text: string;
+    }>;
+    listDatasourceHandler: (commandCtx: any) => Promise<{
+        text: string;
+    }>;
 };

package/dist/src/commands/slashcmd.js

@@ -1,9 +1,46 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getSlashCommands = getSlashCommands;
 const ingest_1 = require("../lib/actions/ingest");
 const retrieve_1 = require("../lib/actions/retrieve");
 const manage_1 = require("../lib/actions/manage");
+const profiler_1 = require("../lib/actions/profiler");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const os = __importStar(require("os"));
 function getSlashCommands(pluginConfig, logger) {
     return {
         ingestHandler: async (commandCtx) => {
@@ -12,12 +49,16 @@ function getSlashCommands(pluginConfig, logger) {
             logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command ingest started`, { args });
             try {
                 if (args.length === 0) {
-                    return { text: `**Error:** Missing files. Usage: /contextlake-ingest /path/to/file1 /path/to/file2` };
+                    return { text: `**Error:** Missing datasource_name. Usage: /contextlake-ingest <datasource_name>` };
                 }
-                const result = await (0, ingest_1.ingestAssets)({
-                    files: args,
-                    metadata: {}
-                }, pluginConfig, logger);
+                const datasource_name = args[0];
+                const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
+                const dsDir = path.join(BASE_DIR, datasource_name);
+                const dbPath = path.join(dsDir, 'catalog_db');
+                if (!fs.existsSync(dbPath)) {
+                    return { text: `**Error:** Data source "${datasource_name}" has not been profiled yet.\n\nPlease run the profiler first using:\n\`/contextlake-profiler <datasource_name> <vendor> <bucket> <prefix> [endpoint] [ak] [sk] [region]\`` };
+                }
+                const result = await (0, ingest_1.ingestSource)({ datasource_name }, pluginConfig, logger);
                 logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command ingest completed`, { resultCount: result.length });
                 return { text: `**Ingest Results (${result.length} files processed):**\n\`\`\`json\n${JSON.stringify(result, null, 2)}\n\`\`\`` };
             }
@@ -86,6 +127,49 @@ function getSlashCommands(pluginConfig, logger) {
                 logger.error(`[ContextLake] Slash delete failed`, { error: e.message });
                 return { text: `**Error executing delete:** ${e.message}` };
             }
-        }
+        },
+        profilerHandler: async (commandCtx) => {
+            const rawArgs = commandCtx.args || "";
+            const args = rawArgs.split(' ').filter((arg) => arg.trim() !== '');
+            logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command profiler started`, { args });
+            try {
+                if (args.length < 4) {
+                    return { text: `**Error:** Missing arguments. Usage: /contextlake-profiler <datasource_name> <vendor> <bucket> <prefix> [endpoint] [ak] [sk] [region]` };
+                }
+                const [datasource_name, vendor, bucket, prefix, endpoint, access_key, secret_key, region] = args;
+                if (!['volcengine', 'alibaba', 'tencent', 'aws', 'local'].includes(vendor)) {
+                    return { text: `**Error:** Invalid vendor. Must be one of: volcengine, alibaba, tencent, aws, local` };
+                }
+                const params = {
+                    datasource_name,
+                    vendor: vendor,
+                    bucket,
+                    prefix,
+                    endpoint,
+                    access_key,
+                    secret_key,
+                    region,
+                };
+                const result = await (0, profiler_1.connectDataSource)(params);
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command profiler completed`, { result });
+                return { text: `**Profiler Results:**\n\`\`\`json\n${JSON.stringify(result, null, 2)}\n\`\`\`` };
+            }
+            catch (e) {
+                logger.error(`[ContextLake] Slash profiler failed`, { error: e.message });
+                return { text: `**Error executing profiler:** ${e.message}` };
+            }
+        },
+        listDatasourceHandler: async (commandCtx) => {
+            logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command list-datasource started`);
+            try {
+                const result = await (0, profiler_1.listDataSources)();
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command list-datasource completed`, { result });
+                return { text: `**Data Sources:**\n\`\`\`json\n${JSON.stringify(result, null, 2)}\n\`\`\`` };
+            }
+            catch (e) {
+                logger.error(`[ContextLake] Slash list-datasource failed`, { error: e.message });
+                return { text: `**Error executing list-datasource:** ${e.message}` };
+            }
+        },
     };
 }

package/dist/src/commands/tools.d.ts

@@ -6,4 +6,6 @@ export declare function getAgentTools(pluginConfig: ContextLakeConfig, logger: a
     listTool: AnyAgentTool;
     deleteTool: AnyAgentTool;
     lasDataProfilerTool: AnyAgentTool;
+    listDatasourceTool: AnyAgentTool;
+    lasTools: AnyAgentTool[];
 };

package/dist/src/commands/tools.js

@@ -5,44 +5,52 @@ const ingest_1 = require("../lib/actions/ingest");
 const retrieve_1 = require("../lib/actions/retrieve");
 const manage_1 = require("../lib/actions/manage");
 const profiler_1 = require("../lib/actions/profiler");
+const las_tools_1 = require("../lib/actions/las-tools");
 function getAgentTools(pluginConfig, logger) {
+    const lasTools = (0, las_tools_1.getLasTools)(pluginConfig, logger);
     return {
+        lasTools,
+        listDatasourceTool: {
+            name: 'contextlake-list-datasource',
+            label: 'ContextLake List Datasources',
+            description: `List all connected and profiled data sources.`,
+            parameters: {
+                type: 'object',
+                properties: {},
+                required: [],
+                additionalProperties: false
+            },
+            async execute(toolCallId, params) {
+                logger.info(`[${new Date().toISOString()}] [ContextLake] Executing list-datasource skill, toolCallId: ${toolCallId}`);
+                try {
+                    const result = await (0, profiler_1.listDataSources)();
+                    return {
+                        content: [{ type: "text", text: JSON.stringify(result) }],
+                        details: result
+                    };
+                }
+                catch (error) {
+                    logger.error(`[${new Date().toISOString()}] [ContextLake] list-datasource skill failed`, { error: error.message });
+                    return {
+                        content: [{ type: "text", text: String(error.message) }],
+                        details: { error: error.message }
+                    };
+                }
+            }
+        },
         ingestTool: {
             name: 'contextlake-ingest',
             label: 'ContextLake Ingest',
-            description: `Upload, ingest, and index documents into the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
+            description: `Process and ingest all files from a connected data source into the knowledge base.
 Use this tool when the user wants to "将知识注入", "上传文件", "入库", "添加文档", "ingest files", or "add knowledge".
-Supports processing of various file types including PDF, Word, Markdown, and Text.
-Automatically handles text extraction, cleaning, chunking, embedding generation, and storage.
-
-Example User Queries:
-- "帮我把这个文档注入到知识湖中"
-- "上传这份 PDF 到知识库"
-- "Please ingest these documents into ContextLake"
-- "将 /path/to/doc.txt 添加到知识库"`,
+Supports multimodal files (text, images, audio, video, pdf) by using LAS models to understand and embed them.
+Must be called after a data source has been successfully profiled via \`las-data-profiler\`.`,
             parameters: {
                 type: 'object',
                 properties: {
-                    files: {
-                        type: 'array',
-                        items: { type: 'string' },
-                        description: 'List of file paths to ingest'
-                    },
-                    metadata: {
-                        type: 'object',
-                        description: 'Optional JSON metadata to attach to documents',
-                        additionalProperties: true
-                    },
-                    chunkSize: {
-                        type: 'integer',
-                        description: 'Chunk size for text splitting'
-                    },
-                    overlap: {
-                        type: 'integer',
-                        description: 'Overlap size for text splitting'
-                    }
+                    datasource_name: { type: 'string', description: 'Name of the data source previously profiled' }
                 },
-                required: ['files'],
+                required: ['datasource_name'],
                 additionalProperties: false
             },
             async execute(toolCallId, params) {
@@ -56,21 +64,21 @@ Example User Queries:
                         catch (e) {
                             logger.warn(`[ContextLake] Received string params, possibly toolCallId?`, { params });
                             return {
-                                content: [{ type: "text", text: `Invalid params format: received string "${params}", expected object with 'files' array.` }],
-                                details: { error: `Invalid params format: received string "${params}", expected object with 'files' array.` }
+                                content: [{ type: "text", text: `Invalid params format: received string "${params}", expected object with 'datasource_name'.` }],
+                                details: { error: `Invalid params format: received string "${params}", expected object with 'datasource_name'.` }
                             };
                         }
                     }
-                    if (!actualParams.files && actualParams.params && actualParams.params.files) {
+                    if (!actualParams.datasource_name && actualParams.params && actualParams.params.datasource_name) {
                         actualParams = actualParams.params;
                     }
-                    if (!actualParams.files || !Array.isArray(actualParams.files)) {
+                    if (!actualParams.datasource_name) {
                         return {
-                            content: [{ type: "text", text: `Invalid params: 'files' must be an array. Received keys: ${Object.keys(actualParams)}` }],
-                            details: { error: `Invalid params: 'files' must be an array. Received keys: ${Object.keys(actualParams)}` }
+                            content: [{ type: "text", text: `Invalid params: 'datasource_name' is required. Received keys: ${Object.keys(actualParams)}` }],
+                            details: { error: `Invalid params: 'datasource_name' is required. Received keys: ${Object.keys(actualParams)}` }
                         };
                     }
-                    const result = await (0, ingest_1.ingestAssets)(actualParams, pluginConfig, logger);
+                    const result = await (0, ingest_1.ingestSource)(actualParams, pluginConfig, logger);
                     logger.info(`[${new Date().toISOString()}] [ContextLake] Ingest skill completed successfully`, { resultSummary: Array.isArray(result) ? `Processed ${result.length} items` : 'Success' });
                     return {
                         content: [{ type: "text", text: JSON.stringify(result) }],
@@ -81,8 +89,7 @@ Example User Queries:
                     logger.error(`[${new Date().toISOString()}] [ContextLake] Ingest skill failed`, { error: error.message, stack: error.stack });
                     return {
                         content: [{ type: "text", text: String(error.message) }],
-                        details: { error: error.message
-                        }
+                        details: { error: error.message }
                     };
                 }
             }

package/dist/src/lib/actions/ingest-source.d.ts

@@ -0,0 +1,15 @@
+import { ContextLakeConfig } from '../../utils/config';
+export interface IngestSourceParams {
+    datasource_name: string;
+}
+export declare function ingestSource(params: IngestSourceParams, config: ContextLakeConfig, logger?: any): Promise<({
+    file: any;
+    status: string;
+    chunks: number;
+    message?: undefined;
+} | {
+    file: any;
+    status: string;
+    message: any;
+    chunks?: undefined;
+})[]>;

package/dist/src/lib/actions/ingest-source.js

@@ -0,0 +1,193 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ingestSource = ingestSource;
+const factory_1 = require("../../service/metadata/factory");
+const las_api_1 = require("./las-api");
+const lancedb = __importStar(require("@lancedb/lancedb"));
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+// @ts-ignore
+const uuid_1 = require("uuid");
+const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
+async function ingestSource(params, config, logger) {
+    if (logger) {
+        logger.info(`[ContextLake-Action] Calling ingestSource with params: ${JSON.stringify(params)}`);
+    }
+    else {
+        // eslint-disable-next-line no-console
+        console.log(`[ContextLake-Action] Calling ingestSource with params: ${JSON.stringify(params)}`);
+    }
+    const dsDir = path.join(BASE_DIR, params.datasource_name);
+    const dbPath = path.join(dsDir, 'catalog_db');
+    if (!fs.existsSync(dbPath)) {
+        throw new Error(`Data source database not found at ${dbPath}. Please run profiler connect first.`);
+    }
+    const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
+    const metadataProvider = (0, factory_1.createMetadataProvider)(metaConfig);
+    await metadataProvider.connect();
+    const lasClient = new las_api_1.LasApiClient(config, logger);
+    const results = [];
+    // Connect to the profiler LanceDB to read the file catalog
+    const profilerDb = await lancedb.connect(dbPath);
+    const tableNames = await profilerDb.tableNames();
+    if (!tableNames.includes('file_catalog')) {
+        throw new Error(`table 'file_catalog' not found in ${dbPath}`);
+    }
+    const catalogTable = await profilerDb.openTable('file_catalog');
+    const files = await catalogTable.query().toArray();
+    logger?.info(`[ContextLake-Action] Found ${files.length} files in catalog`);
+    // Simple chunking for text
+    const splitText = (text, chunkSize = 500, overlap = 50) => {
+        const chunks = [];
+        if (!text)
+            return chunks;
+        let i = 0;
+        while (i < text.length) {
+            chunks.push(text.slice(i, i + chunkSize));
+            i += chunkSize - overlap;
+        }
+        return chunks;
+    };
+    const processText = async (text, fileInfo) => {
+        const chunks = splitText(text);
+        const docs = [];
+        for (const chunk of chunks) {
+            const vector = await metadataProvider.generateMultimodalEmbedding([{ type: 'text', text: chunk }]);
+            docs.push({
+                id: (0, uuid_1.v4)(),
+                vector,
+                text: chunk,
+                source: fileInfo.key,
+                file_type: fileInfo.category,
+                storage_type: 'source',
+                url: fileInfo.url || `tos://${fileInfo.bucket}/${fileInfo.key}`,
+                metadata: JSON.stringify({ datasource: params.datasource_name }),
+                created_at: Date.now(),
+                binary_data: Buffer.from('')
+            });
+        }
+        return docs;
+    };
+    for (const file of files) {
+        try {
+            logger?.info(`[ContextLake-Action] Processing file: ${file.key}, type: ${file.media_type}`);
+            let docs = [];
+            const fileUrl = file.url || `tos://${file.bucket}/${file.key}`;
+            if (file.media_type === 'pdf') {
+                // PDF Parse
+                const result = await lasClient.submitAndPoll('las_pdf_parse_doubao', {
+                    url: fileUrl
+                });
+                const markdown = result.data?.markdown || '';
+                docs = await processText(markdown, file);
+            }
+            else if (file.media_type === 'image') {
+                // Multimodal Embedding directly
+                const vector = await metadataProvider.generateMultimodalEmbedding([
+                    { type: 'image_url', image_url: { url: fileUrl } },
+                    { type: 'text', text: 'This is an image from the dataset.' }
+                ]);
+                docs.push({
+                    id: (0, uuid_1.v4)(),
+                    vector,
+                    text: 'Image from dataset',
+                    source: file.key,
+                    file_type: 'image',
+                    storage_type: 'source',
+                    url: fileUrl,
+                    metadata: JSON.stringify({ datasource: params.datasource_name }),
+                    created_at: Date.now(),
+                    binary_data: Buffer.from('')
+                });
+            }
+            else if (file.media_type === 'audio') {
+                // ASR
+                const result = await lasClient.submitAndPoll('las_asr_pro', {
+                    audio: { url: fileUrl, format: file.key.split('.').pop() || 'wav' },
+                    request: { model_name: 'bigmodel' }
+                });
+                const text = result.data?.result?.text || '';
+                docs = await processText(text, file);
+            }
+            else if (file.media_type === 'video') {
+                // Video understanding -> text -> embedding
+                const result = await lasClient.submitAndPoll('las_long_video_understand', {
+                    video_url: fileUrl,
+                    query: "详细描述这个视频的内容",
+                    model_name: "doubao-seed-2-0-lite-260215"
+                });
+                // Assuming video output is a text description somewhere in the response. 
+                // Note: the exact structure depends on the API return, adjusting to generic text.
+                const text = JSON.stringify(result.data || '');
+                // Also need audio extract and ASR for video
+                // 1. Extract audio
+                // The output_path_template needs a unique path per video
+                const audioOutputPath = `tos://${file.bucket}/.tmp/audio/${(0, uuid_1.v4)()}.wav`;
+                await lasClient.process('las_audio_extract_and_split', {
+                    input_path: fileUrl,
+                    output_path_template: audioOutputPath,
+                    output_format: 'wav'
+                });
+                // 2. ASR on the extracted audio
+                // Wait briefly for object to be available if needed (often synchronous but tos takes a ms)
+                const asrResult = await lasClient.submitAndPoll('las_asr_pro', {
+                    audio: { url: audioOutputPath.replace('{index}.{output_file_ext}', '0.wav'), format: 'wav' },
+                    request: { model_name: 'bigmodel' }
+                });
+                const audioText = asrResult.data?.result?.text || '';
+                // Combine video text and audio text
+                const combinedText = `Video Description: ${text}\n\nAudio Transcription: ${audioText}`;
+                docs = await processText(combinedText, file);
+            }
+            else if (file.category === 'structured' || file.category === 'non-structured') {
+                // If we had a direct text content, we could process it here. 
+                // Assuming basic local download or similar is available, but for now we skip raw file reading from TOS in this demo script unless implemented.
+                // Fallback just logs
+                logger?.warn(`[ContextLake-Action] Skipping raw text/structured download for ${file.key} - implement TOS download if needed`);
+            }
+            if (docs.length > 0) {
+                await metadataProvider.addAssets(docs);
+                results.push({ file: file.key, status: 'success', chunks: docs.length });
+            }
+        }
+        catch (error) {
+            logger?.error(`[ContextLake-Action] Error processing ${file.key}: ${error.message}`);
+            results.push({ file: file.key, status: 'error', message: error.message });
+        }
+    }
+    return results;
+}

package/dist/src/lib/actions/ingest.d.ts

@@ -1,9 +1,15 @@
 import { ContextLakeConfig } from '../../utils/config';
-interface IngestParams {
-    files: string[];
-    metadata?: Record<string, any>;
-    chunkSize?: number;
-    overlap?: number;
+export interface IngestSourceParams {
+    datasource_name: string;
 }
-export declare function ingestAssets(params: IngestParams, config: ContextLakeConfig, logger?: any): Promise<any>;
-export {};
+export declare function ingestSource(params: IngestSourceParams, config: ContextLakeConfig, logger?: any): Promise<({
+    file: any;
+    status: string;
+    chunks: number;
+    message?: undefined;
+} | {
+    file: any;
+    status: string;
+    message: any;
+    chunks?: undefined;
+})[]>;