@byted-las/contextlake-openclaw 1.0.6 → 1.0.8

package/src/commands/index.ts

@@ -12,9 +12,6 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
     try {
         const tools = getAgentTools(pluginConfig, logger);
         
-        ctx.registerTool(tools.ingestTool );
-        logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.ingestTool.name}`);
-        
         ctx.registerTool(tools.retrieveTool );
         logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.retrieveTool.name}`);
         
@@ -23,9 +20,6 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
         
         ctx.registerTool(tools.deleteTool );
         logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.deleteTool.name}`);
-        
-        ctx.registerTool(tools.lasDataProfilerTool );
-        logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.lasDataProfilerTool.name}`);
 
         ctx.registerTool(tools.listS3ObjectsTool );
         logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.listS3ObjectsTool.name}`);
@@ -65,21 +59,6 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
             
             const commands = getCliCommands(pluginConfig, logger);
             
-            // connect -- data source profiling (las-data-profiler)
-            contextlake.command('connect <datasource_name> <url>')
-                .description('Connect to a data source and profile its structure, schemas, and media metadata into LanceDB')
-                .option('--endpoint <url>', 'S3 Endpoint URL (not needed for local)')
-                .option('--ak <credential_id>', 'Credential ID for the data source')
-                .option('--sk <credential_value>', 'Credential value for the data source')
-                .option('--region <region>', 'Region identifier (e.g. cn-beijing)')
-                .option('--sample-rows <number>', 'Number of rows to sample per structured file', '100')
-                .action(commands.connectAction);
-
-            // Ingest
-            contextlake.command('ingest <datasource_name>')
-                .description('Process and ingest all files from a connected data source into the knowledge base')
-                .action(commands.ingestAction);
-
             // Search
             contextlake.command('search <query>')
                 .description('Search the knowledge base for relevant documents')
@@ -122,13 +101,6 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
 
         const slashCommands = getSlashCommands(pluginConfig, logger);
         
-        ctx.registerCommand({
-            name: 'contextlake-ingest',
-            description: 'Process and ingest all files from a connected data source (usage: /contextlake-ingest <datasource_name>)',
-            acceptsArgs: true,
-            handler: slashCommands.ingestHandler
-        });
-
         ctx.registerCommand({
             name: 'contextlake-list',
             description: 'List documents currently in the knowledge base',
@@ -150,13 +122,6 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
             handler: slashCommands.deleteHandler
         });
 
-        ctx.registerCommand({
-            name: 'contextlake-profiler',
-            description: 'Connect to a data source and profile its structure (usage: /contextlake-profiler <datasource_name> <vendor> <bucket> <prefix>)',
-            acceptsArgs: true,
-            handler: slashCommands.profilerHandler
-        });
-
         ctx.registerCommand({
             name: 'contextlake-list-datasource',
             description: 'List all connected and profiled data sources (usage: /contextlake-list-datasource)',

package/src/commands/slashcmd.ts

@@ -9,36 +9,6 @@ import * as os from 'os';
 
 export function getSlashCommands(pluginConfig: ContextLakeConfig, logger: any) {
     return {
-        ingestHandler: async (commandCtx: any) => {
-            const rawArgs = commandCtx.args || "";
-            const args = rawArgs.split(' ').filter((arg: string) => arg.trim() !== '');
-
-            logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command ingest started`, { args });
-            try {
-                if (args.length === 0) {
-                    return { text: `**Error:** Missing datasource_name. Usage: /contextlake-ingest <datasource_name>` };
-                }
-                
-                const datasource_name = args[0];
-
-                const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
-                const dsDir = path.join(BASE_DIR, datasource_name);
-                const dbPath = path.join(dsDir, 'catalog_db');
-
-                if (!fs.existsSync(dbPath)) {
-                    return { text: `**Error:** Data source "${datasource_name}" has not been profiled yet.\n\nPlease run the profiler first using:\n\`/contextlake-profiler <datasource_name> <vendor> <bucket> <prefix> [endpoint] [ak] [sk] [region]\`` };
-                }
-
-                const result = await ingestSource({ datasource_name }, pluginConfig, logger);
-                
-                logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command ingest completed`, { resultCount: result.length });
-                return { text: `**Ingest Results (${result.length} files processed):**\n\`\`\`json\n${JSON.stringify(result, null, 2)}\n\`\`\`` };
-            } catch (e: any) {
-                logger.error(`[ContextLake] Slash ingest failed`, { error: e.message });
-                return { text: `**Error executing ingest:** ${e.message}` };
-            }
-        },
-
         listHandler: async (commandCtx: any) => {
             const rawArgs = commandCtx.args || "";
             const args = rawArgs.split(' ').filter((arg: string) => arg.trim() !== '');
@@ -102,35 +72,6 @@ export function getSlashCommands(pluginConfig: ContextLakeConfig, logger: any) {
                 return { text: `**Error executing delete:** ${e.message}` };
             }
         },
-        profilerHandler: async (commandCtx: any) => {
-            const rawArgs = commandCtx.args || "";
-            const args = rawArgs.split(' ').filter((arg: string) => arg.trim() !== '');
-
-            logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command profiler started`, { args });
-            try {
-                if (args.length < 2) {
-                    return { text: `**Error:** Missing arguments. Usage: /contextlake-profiler <datasource_name> <url> [endpoint] [ak] [sk] [region]` };
-                }
-                
-                const [datasource_name, url, endpoint, access_key, secret_key, region] = args;
-                
-                const params: ConnectParams = {
-                    datasource_name,
-                    url,
-                    endpoint,
-                    access_key,
-                    secret_key,
-                    region,
-                };
-                
-                const result = await connectDataSource(params);
-                logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command profiler completed`, { result });
-                return { text: `**Profiler Results:**\n\`\`\`json\n${JSON.stringify(result, null, 2)}\n\`\`\`` };
-            } catch (e: any) {
-                logger.error(`[ContextLake] Slash profiler failed`, { error: e.message });
-                return { text: `**Error executing profiler:** ${e.message}` };
-            }
-        },
 
         listDatasourceHandler: async (commandCtx: any) => {
             logger.info(`[${new Date().toISOString()}] [ContextLake] Slash command list-datasource started`);

package/src/commands/tools.ts

@@ -10,11 +10,9 @@ import { ContextLakeConfig } from '../utils/config';
 import type { AnyAgentTool } from 'openclaw/plugin-sdk';
 
 export function getAgentTools(pluginConfig: ContextLakeConfig, logger: any): {
-    ingestTool: AnyAgentTool;
     retrieveTool: AnyAgentTool;
     listTool: AnyAgentTool;
     deleteTool: AnyAgentTool;
-    lasDataProfilerTool: AnyAgentTool;
     listDatasourceTool: AnyAgentTool;
     listS3ObjectsTool: AnyAgentTool;
     readS3ObjectTool: AnyAgentTool;
@@ -54,65 +52,6 @@ export function getAgentTools(pluginConfig: ContextLakeConfig, logger: any): {
                 }
             }
         },
-        ingestTool: {
-            name: 'contextlake-ingest',
-            label: 'ContextLake Ingest',
-            description: `Process and ingest all files from a connected data source into the knowledge base.
-Use this tool when the user wants to "将知识注入", "上传文件", "入库", "添加文档", "ingest files", or "add knowledge".
-Supports multimodal files (text, images, audio, video, pdf) by using LAS models to understand and embed them.
-Must be called after a data source has been successfully profiled via \`las-data-profiler\`.`,
-            parameters: {
-                type: 'object',
-                properties: {
-                    datasource_name: { type: 'string', description: 'Name of the data source previously profiled' }
-                },
-                required: ['datasource_name'],
-                additionalProperties: false
-            },
-            
-            async execute(toolCallId: string, params: any) {
-                logger.info(`[${new Date().toISOString()}] [ContextLake] Executing ingest skill, toolCallId: ${toolCallId}`, { params: JSON.stringify(params) });
-
-                try {
-                    let actualParams = params;
-                    if (typeof params === 'string') {
-                        try {
-                            actualParams = JSON.parse(params);
-                        } catch (e) {
-                            logger.warn(`[ContextLake] Received string params, possibly toolCallId?`, { params });
-                            return {
-                                content: [{ type: "text", text: `Invalid params format: received string "${params}", expected object with 'datasource_name'.` }],
-                                details: { error: `Invalid params format: received string "${params}", expected object with 'datasource_name'.` }
-                            } as any;
-                        }
-                    }
-                    
-                    if (!actualParams.datasource_name && actualParams.params && actualParams.params.datasource_name) {
-                        actualParams = actualParams.params;
-                    }
-
-                    if (!actualParams.datasource_name) {
-                        return {
-                                content: [{ type: "text", text: `Invalid params: 'datasource_name' is required. Received keys: ${Object.keys(actualParams)}` }],
-                                details: { error: `Invalid params: 'datasource_name' is required. Received keys: ${Object.keys(actualParams)}` }
-                            } as any;
-                    }
-
-                    const result = await ingestSource(actualParams, pluginConfig, logger);
-                    logger.info(`[${new Date().toISOString()}] [ContextLake] Ingest skill completed successfully`, { resultSummary: Array.isArray(result) ? `Processed ${result.length} items` : 'Success' });
-                    return {
-                        content: [{ type: "text", text: JSON.stringify(result) }],
-                        details: result
-                    } as any;
-                } catch (error: any) {
-                    logger.error(`[${new Date().toISOString()}] [ContextLake] Ingest skill failed`, { error: error.message, stack: error.stack });
-                    return {
-                        content: [{ type: "text", text: String(error.message) }],
-                        details: { error: error.message }
-                    } as any;
-                }
-            }
-        },
         retrieveTool: {
             name: 'contextlake-retrieve',
             label: 'ContextLake Retrieve',
@@ -268,44 +207,6 @@ Example User Queries:
                 }
             }
         },
-        lasDataProfilerTool: {
-            name: 'las-data-profiler',
-            label: 'LAS Data Profiler',
-            description: 'Connect to a data source (TOS/OSS/COS/S3/Local) and profile its structure, schemas, and media metadata into LanceDB',
-            parameters: {
-                type: 'object',
-                properties: {
-                    datasource_name: { type: 'string', description: 'Name of the data source' },
-                    url: { type: 'string', description: 'Data source URL (e.g. tos://bucket/prefix, oss://..., s3://..., file:///path)' },
-                    sample_rows: { type: 'integer', description: 'Number of rows to sample per structured file' }
-                },
-                required: ['datasource_name', 'url'],
-                additionalProperties: false
-            },
-            
-            async execute(toolCallId: string, params: any) {
-                logger.info(`[${new Date().toISOString()}] [ContextLake] Executing las-data-profiler skill, toolCallId: ${toolCallId}`, { params: JSON.stringify(params) });
-
-                try {
-                    let actualParams = params;
-                    if (params && params.params) {
-                        actualParams = params.params;
-                    }
-                    const result = await connectDataSource(actualParams);
-                    logger.info(`[${new Date().toISOString()}] [ContextLake] las-data-profiler skill completed`, { result });
-                    return {
-                        content: [{ type: "text", text: JSON.stringify(result) }],
-                        details: result
-                    } as any;
-                } catch (error: any) {
-                    logger.error(`[${new Date().toISOString()}] [ContextLake] las-data-profiler skill failed`, { error: error.message, stack: error.stack });
-                    return {
-                        content: [{ type: "text", text: String(error.message) }],
-                        details: { error: error.message }
-                    } as any;
-                }
-            }
-        },
         listS3ObjectsTool: {
             name: 'list-s3-objects',
             label: 'List S3 Objects',

package/src/skills/contextlake-ingest/SKILL.md

@@ -1,42 +1,59 @@
 ---
 name: contextlake-ingest
 description: |
-  Upload, ingest, and index documents into the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
-  Use this tool when the user wants to "将知识注入", "上传文件", "入库", "添加文档", "ingest files", or "add knowledge".
-  Supports processing of various file types including PDF, Word, Markdown, and Text.
-  Automatically handles text extraction, cleaning, chunking, embedding generation, and storage.
-
-  Example User Queries:
-  - "帮我把这个文档注入到知识湖中"
-  - "上传这份 PDF 到知识库"
-  - "Please ingest these documents into ContextLake"
-  - "将 /path/to/doc.txt 添加到知识库"
-
-  Example Tool Call:
-  ```json
-  {
-    "files": ["/absolute/path/to/document.pdf"],
-    "metadata": { "category": "tech" }
-  }
-  ```
-
-parameters:
-  files:
-    type: array
-    items:
-      type: string
-    description: List of file paths to ingest
-    required: true
-  metadata:
-    type: object
-    description: Optional JSON metadata to attach to documents
-    required: false
-  chunkSize:
-    type: integer
-    description: Chunk size for text splitting
-    required: false
-  overlap:
-    type: integer
-    description: Overlap size for text splitting
-    required: false
+  A guide for OpenClaw agents on how to perform data ingestion into ContextLake.
+  Use this skill to understand the correct workflow for profiling a dataset and then ingesting it.
+  
+  When a user asks to "ingest data", "导入数据", "接入数据源", or "处理某个桶里的数据", 
+  you MUST follow the two-step workflow outlined below using the specific tools provided.
+
+  IMPORTANT RULE: You are STRICTLY FORBIDDEN from using any tools outside of this project for ingestion.
+  Do not use custom Python scripts, external CLI tools, or unrelated plugins.
+---
+
+# ContextLake Data Ingestion Workflow Guide
+
+To successfully ingest data into the ContextLake Knowledge Base, you **MUST** follow a strict two-step process.
+
+## Step 1: Data Profiling
+Before data can be ingested, the source MUST first be profiled to extract its schema, structure, and media metadata into a local LanceDB catalog.
+
+**Tool to use:** OpenClaw Agent using `byted-las-data-profiler` skill to guide the profiling process.
+
+### How it works:
+- **Purpose:** Connects to an S3-compatible source (TOS, OSS, AWS S3, etc.) or local file system and catalogs the files.
+- **Underlying tools used:** The agent will use `list-s3-objects`, `read-s3-object`, and `write-lance-catalog` to perform the profiling step by step.
+
+*Wait for Step 1 to complete successfully before proceeding to Step 2.*
+
 ---
+
+## Step 2: Data Ingestion
+Once the data source is successfully profiled and the catalog is created, you can proceed to ingest the data into ContextLake.
+
+**Tool to use:** OpenClaw Agent using basic tools guided by this skill.
+
+### How it works:
+- **Purpose:** Reads the LanceDB catalog created in Step 1, processes the multimodal files (text, images, audio, video, PDF) using LAS models, chunks the data, generates embeddings, and indexes them into the ContextLake Knowledge Base.
+- **Underlying tools used:** 
+  1. Use `read-lance-catalog` to read the catalog of files from `~/.openclaw/contextlake/profiler/{datasource_name}/catalog_db`.
+  2. For each file, use appropriate LAS tools (like `las_pdf_parse_doubao`, `las_image_resample`, `las_long_video_understand`) to extract text and features.
+  3. Chunk and process the text.
+  4. Use the embedding tool or model to generate vectors.
+  5. Save the final chunks and vectors into the main ContextLake knowledge base.
+
+*Note: You are acting as the ingestion pipeline. You must coordinate the reading of the catalog and the processing of each file type using the available LAS tools.*
+
+---
+
+## Auxiliary Tools (Use only when necessary)
+If you need to verify the catalog contents between Step 1 and Step 2, or if ingestion fails and you need to debug:
+
+- **`read-lance-catalog`**: Use this tool to read the records from the catalog database created in Step 1.
+  - Requires `db_path` (e.g., `~/.openclaw/contextlake/profiler/{datasource_name}/catalog_db`) and `table_name` (usually "files").
+- **`list-s3-objects` / `read-s3-object`**: Use these tools to manually inspect the raw source files if profiling fails.
+- **`contextlake-list-datasource`**: Use this tool to see all data sources that have been connected/profiled.
+
+## Error Handling
+- If `contextlake-ingest` fails saying the datasource is not found, ensure that the data profiling step completed successfully for that exact `datasource_name`.
+- Always report the results of both steps to the user clearly.

package/src/skills/las-data-profiler/SKILL.md

@@ -1,11 +1,13 @@
 ---
 name: byted-las-data-profiler
 description: |
-  Volcengine TOS Dataset Profiling Tool. Based on the S3-compatible protocol, it scans the file structure in TOS buckets and catalogs them.
-  It writes the catalog index to a local LanceDB. It is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
+  Volcengine TOS Dataset Profiling Tool. Based on the S3-compatible protocol, it scans the file structure in TOS buckets and catalogs them,
+  performs schema inference and column semantic analysis on structured data (JSONL/CSV/Parquet/JSON),
+  extracts key meta-information for media files (Image/Audio/Video/PDF) by reading only header bytes,
+  and writes all results to a local LanceDB. It is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
 
   IMPORTANT RULE: You are STRICTLY FORBIDDEN from writing or executing Python scripts to access S3/TOS or LanceDB.
-  You MUST exclusively use the provided tools (`list-s3-objects`, `read-s3-object`, `write-lance-catalog`, `las-data-profiler`) to accomplish the profiling tasks.
+  You MUST exclusively use the provided tools (`list-s3-objects`, `read-s3-object`, `write-lance-catalog`) via formal tool calls to accomplish the profiling tasks. DO NOT execute them as bash/shell commands.
 ---
 
 ## Trigger Scenarios
@@ -16,24 +18,28 @@ Be sure to use this Skill when the user mentions the following scenarios:
 - Need to understand what a batch of data files contains and what their schema looks like
 - Need to extract meta-information such as image resolution, audio/video duration, PDF page count, etc.
 - Need to write the meta-information of object storage or local files into LanceDB
+- Mentions TOS, boto3, or object storage data profiling
 - Mentions keywords like "dataset scanning", "file cataloging", "data catalog", "data profiling", etc.
+- Need to batch identify the type and size of remote/local files and build an index
+- Need to quickly understand the structure of an unfamiliar dataset (what files are there, how the schema is, field meanings)
+- Need to connect/dock a data source for profiling
+- Mentions "connect" data source, docking data source
 
-## Overall Workflow
-When instructed to profile a dataset, you should prefer using the `las-data-profiler` tool directly, which automatically handles the S3 listing and LanceDB writing using internal TypeScript logic.
-If you need to perform custom exploration, you can use `list-s3-objects` to traverse the bucket and `read-s3-object` to read file headers, and `write-lance-catalog` to save results.
+## Overview
+This Skill acts as a Dataset Profiling Guide. You should use the `list-s3-objects` tool to traverse the S3 bucket or local directory, use `read-s3-object` to read file contents or headers, parse the schema or media metadata, and finally use `write-lance-catalog` to save the catalog into a local LanceDB.
 
-## Parameter Description (for `las-data-profiler` tool)
-| Parameter | Description | Example |
-|-----------|-------------|---------|
-| datasource_name | The name of the data source | my_tos_data |
-| vendor | volcengine / alibaba / tencent / aws / local | volcengine |
-| endpoint | S3 Endpoint URL (not required for local) | https://tos-s3-cn-beijing.volces.com |
-| access_key | AK | - |
-| secret_key | SK | - |
-| region | Region identifier | cn-beijing |
-| bucket | Bucket name (root directory path when local) | my-data-bucket |
-| prefix | Path prefix to limit the scan scope | datasets/2024/ |
+1. **Cataloging**: Use `list-s3-objects` to record the meta-information (path, size, etc.) of files.
+2. **Understanding Structured Data**: Use `read-s3-object` to sample JSONL / CSV / TSV / Parquet / JSON.
+3. **Extracting Media Meta-information**: Use `read-s3-object` with `maxBytes` to read only the file header (without downloading the full file) for images, audio, video, and PDFs to extract key attributes.
+4. **Writing to LanceDB**: Use `write-lance-catalog` to save the results.
 
 ## Output Location
 - LanceDB table storage path: `~/.openclaw/contextlake/profiler/{datasource_name}/catalog_db`
-- Configuration file: `~/.openclaw/contextlake/profiler/{datasource_name}/env.sh`
+- Table names: `files`, `structured_schemas`, `media_metadata`
+
+## Available Tools for this Skill
+- `list-s3-objects`: To traverse and list files in the bucket/directory. (Call this as an Agent Tool, NOT a bash command).
+- `read-s3-object`: To read specific bytes of a file for schema inference or metadata extraction. (Call this as an Agent Tool, NOT a bash command).
+- `write-lance-catalog`: To write the profiling results to the LanceDB catalog. (Call this as an Agent Tool, NOT a bash command).
+
+Always report the final profiling summary back to the user once the `write-lance-catalog` completes successfully.

package/dist/src/lib/actions/ingest-source.d.ts

@@ -1,15 +0,0 @@
-import { ContextLakeConfig } from '../../utils/config';
-export interface IngestSourceParams {
-    datasource_name: string;
-}
-export declare function ingestSource(params: IngestSourceParams, config: ContextLakeConfig, logger?: any): Promise<({
-    file: any;
-    status: string;
-    chunks: number;
-    message?: undefined;
-} | {
-    file: any;
-    status: string;
-    message: any;
-    chunks?: undefined;
-})[]>;

package/dist/src/lib/actions/ingest-source.js

@@ -1,193 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ingestSource = ingestSource;
-const factory_1 = require("../../service/metadata/factory");
-const las_api_1 = require("./las-api");
-const lancedb = __importStar(require("@lancedb/lancedb"));
-const path = __importStar(require("path"));
-const fs = __importStar(require("fs"));
-const os = __importStar(require("os"));
-// @ts-ignore
-const uuid_1 = require("uuid");
-const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
-async function ingestSource(params, config, logger) {
-    if (logger) {
-        logger.info(`[ContextLake-Action] Calling ingestSource with params: ${JSON.stringify(params)}`);
-    }
-    else {
-        // eslint-disable-next-line no-console
-        console.log(`[ContextLake-Action] Calling ingestSource with params: ${JSON.stringify(params)}`);
-    }
-    const dsDir = path.join(BASE_DIR, params.datasource_name);
-    const dbPath = path.join(dsDir, 'catalog_db');
-    if (!fs.existsSync(dbPath)) {
-        throw new Error(`Data source database not found at ${dbPath}. Please run profiler connect first.`);
-    }
-    const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
-    const metadataProvider = (0, factory_1.createMetadataProvider)(metaConfig);
-    await metadataProvider.connect();
-    const lasClient = new las_api_1.LasApiClient(config, logger);
-    const results = [];
-    // Connect to the profiler LanceDB to read the file catalog
-    const profilerDb = await lancedb.connect(dbPath);
-    const tableNames = await profilerDb.tableNames();
-    if (!tableNames.includes('file_catalog')) {
-        throw new Error(`table 'file_catalog' not found in ${dbPath}`);
-    }
-    const catalogTable = await profilerDb.openTable('file_catalog');
-    const files = await catalogTable.query().toArray();
-    logger?.info(`[ContextLake-Action] Found ${files.length} files in catalog`);
-    // Simple chunking for text
-    const splitText = (text, chunkSize = 500, overlap = 50) => {
-        const chunks = [];
-        if (!text)
-            return chunks;
-        let i = 0;
-        while (i < text.length) {
-            chunks.push(text.slice(i, i + chunkSize));
-            i += chunkSize - overlap;
-        }
-        return chunks;
-    };
-    const processText = async (text, fileInfo) => {
-        const chunks = splitText(text);
-        const docs = [];
-        for (const chunk of chunks) {
-            const vector = await metadataProvider.generateMultimodalEmbedding([{ type: 'text', text: chunk }]);
-            docs.push({
-                id: (0, uuid_1.v4)(),
-                vector,
-                text: chunk,
-                source: fileInfo.key,
-                file_type: fileInfo.category,
-                storage_type: 'source',
-                url: fileInfo.url || `tos://${fileInfo.bucket}/${fileInfo.key}`,
-                metadata: JSON.stringify({ datasource: params.datasource_name }),
-                created_at: Date.now(),
-                binary_data: Buffer.from('')
-            });
-        }
-        return docs;
-    };
-    for (const file of files) {
-        try {
-            logger?.info(`[ContextLake-Action] Processing file: ${file.key}, type: ${file.media_type}`);
-            let docs = [];
-            const fileUrl = file.url || `tos://${file.bucket}/${file.key}`;
-            if (file.media_type === 'pdf') {
-                // PDF Parse
-                const result = await lasClient.submitAndPoll('las_pdf_parse_doubao', {
-                    url: fileUrl
-                });
-                const markdown = result.data?.markdown || '';
-                docs = await processText(markdown, file);
-            }
-            else if (file.media_type === 'image') {
-                // Multimodal Embedding directly
-                const vector = await metadataProvider.generateMultimodalEmbedding([
-                    { type: 'image_url', image_url: { url: fileUrl } },
-                    { type: 'text', text: 'This is an image from the dataset.' }
-                ]);
-                docs.push({
-                    id: (0, uuid_1.v4)(),
-                    vector,
-                    text: 'Image from dataset',
-                    source: file.key,
-                    file_type: 'image',
-                    storage_type: 'source',
-                    url: fileUrl,
-                    metadata: JSON.stringify({ datasource: params.datasource_name }),
-                    created_at: Date.now(),
-                    binary_data: Buffer.from('')
-                });
-            }
-            else if (file.media_type === 'audio') {
-                // ASR
-                const result = await lasClient.submitAndPoll('las_asr_pro', {
-                    audio: { url: fileUrl, format: file.key.split('.').pop() || 'wav' },
-                    request: { model_name: 'bigmodel' }
-                });
-                const text = result.data?.result?.text || '';
-                docs = await processText(text, file);
-            }
-            else if (file.media_type === 'video') {
-                // Video understanding -> text -> embedding
-                const result = await lasClient.submitAndPoll('las_long_video_understand', {
-                    video_url: fileUrl,
-                    query: "详细描述这个视频的内容",
-                    model_name: "doubao-seed-2-0-lite-260215"
-                });
-                // Assuming video output is a text description somewhere in the response. 
-                // Note: the exact structure depends on the API return, adjusting to generic text.
-                const text = JSON.stringify(result.data || '');
-                // Also need audio extract and ASR for video
-                // 1. Extract audio
-                // The output_path_template needs a unique path per video
-                const audioOutputPath = `tos://${file.bucket}/.tmp/audio/${(0, uuid_1.v4)()}.wav`;
-                await lasClient.process('las_audio_extract_and_split', {
-                    input_path: fileUrl,
-                    output_path_template: audioOutputPath,
-                    output_format: 'wav'
-                });
-                // 2. ASR on the extracted audio
-                // Wait briefly for object to be available if needed (often synchronous but tos takes a ms)
-                const asrResult = await lasClient.submitAndPoll('las_asr_pro', {
-                    audio: { url: audioOutputPath.replace('{index}.{output_file_ext}', '0.wav'), format: 'wav' },
-                    request: { model_name: 'bigmodel' }
-                });
-                const audioText = asrResult.data?.result?.text || '';
-                // Combine video text and audio text
-                const combinedText = `Video Description: ${text}\n\nAudio Transcription: ${audioText}`;
-                docs = await processText(combinedText, file);
-            }
-            else if (file.category === 'structured' || file.category === 'non-structured') {
-                // If we had a direct text content, we could process it here. 
-                // Assuming basic local download or similar is available, but for now we skip raw file reading from TOS in this demo script unless implemented.
-                // Fallback just logs
-                logger?.warn(`[ContextLake-Action] Skipping raw text/structured download for ${file.key} - implement TOS download if needed`);
-            }
-            if (docs.length > 0) {
-                await metadataProvider.addAssets(docs);
-                results.push({ file: file.key, status: 'success', chunks: docs.length });
-            }
-        }
-        catch (error) {
-            logger?.error(`[ContextLake-Action] Error processing ${file.key}: ${error.message}`);
-            results.push({ file: file.key, status: 'error', message: error.message });
-        }
-    }
-    return results;
-}