@byted-las/contextlake-openclaw 1.0.0 → 1.0.3

package/src/lib/actions/ingest.ts

@@ -1,103 +1,179 @@
 import { createStorageProvider } from '../../service/storage/factory';
 import { createMetadataProvider } from '../../service/metadata/factory';
-import { processFile, splitText } from '../../processor/loader';
+import { LasApiClient } from './las-api';
+import { ContextLakeConfig } from '../../utils/config';
+import { getLasTools } from './las-tools';
+import * as lancedb from '@lancedb/lancedb';
 import * as path from 'path';
+import * as fs from 'fs';
+import * as os from 'os';
 // @ts-ignore
 import { v4 as uuidv4 } from 'uuid';
 
-interface IngestParams {
-  files: string[];
-  metadata?: Record<string, any>;
-  chunkSize?: number;
-  overlap?: number;
+export interface IngestSourceParams {
+  datasource_name: string;
 }
 
-export async function ingestAssets(params: IngestParams, config: any, logger?: any) {
+const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
+
+export async function ingestSource(params: IngestSourceParams, config: ContextLakeConfig, logger?: any) {
   if (logger) {
-      logger.info(`[ContextLake-Action] Calling ingestAssets with params: ${JSON.stringify(params)}`);
+    logger.info(`[ContextLake-Action] Calling ingestSource with params: ${JSON.stringify(params)}`);
   } else {
-      // eslint-disable-next-line no-console
-      console.log(`[ContextLake-Action] Calling ingestAssets with params: ${JSON.stringify(params)}`);
+    // eslint-disable-next-line no-console
+    console.log(`[ContextLake-Action] Calling ingestSource with params: ${JSON.stringify(params)}`);
+  }
+
+  const dsDir = path.join(BASE_DIR, params.datasource_name);
+  const dbPath = path.join(dsDir, 'catalog_db');
+
+  if (!fs.existsSync(dbPath)) {
+    throw new Error(`Data source database not found at ${dbPath}. Please run profiler connect first.`);
   }
 
-  const storageConfig = config.file_storage || { type: 'local', local_base_dir: './data/files' };
   const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
-  
-  const storageProvider = createStorageProvider(storageConfig);
-  const metadataProvider = createMetadataProvider(metaConfig);
-  
+  const metadataProvider = createMetadataProvider(metaConfig as any);
   await metadataProvider.connect();
-  const maxInlineSize = (config.storage_policy?.max_inline_size_kb || 1024) * 1024;
 
+  const lasClient = new LasApiClient(config, logger);
   const results = [];
 
-  for (const filePath of params.files) {
+  // Connect to the profiler LanceDB to read the file catalog
+  const profilerDb = await lancedb.connect(dbPath);
+  const tableNames = await profilerDb.tableNames();
+  
+  if (!tableNames.includes('file_catalog')) {
+    throw new Error(`table 'file_catalog' not found in ${dbPath}`);
+  }
+  
+  const catalogTable = await profilerDb.openTable('file_catalog');
+  const files = await catalogTable.query().toArray();
+  
+  logger?.info(`[ContextLake-Action] Found ${files.length} files in catalog`);
+
+  // Simple chunking for text
+  const splitText = (text: string, chunkSize: number = 500, overlap: number = 50) => {
+    const chunks: string[] = [];
+    if (!text) return chunks;
+    let i = 0;
+    while (i < text.length) {
+      chunks.push(text.slice(i, i + chunkSize));
+      i += chunkSize - overlap;
+    }
+    return chunks;
+  };
+
+  const processText = async (text: string, fileInfo: any) => {
+    const chunks = splitText(text);
+    const docs = [];
+    for (const chunk of chunks) {
+      const vector = await metadataProvider.generateMultimodalEmbedding!([{ type: 'text', text: chunk }]);
+      docs.push({
+        id: uuidv4(),
+        vector,
+        text: chunk,
+        source: fileInfo.key,
+        file_type: fileInfo.category,
+        storage_type: 'source',
+        url: fileInfo.url || `tos://${fileInfo.bucket}/${fileInfo.key}`,
+        metadata: JSON.stringify({ datasource: params.datasource_name }),
+        created_at: Date.now(),
+        binary_data: Buffer.from('')
+      });
+    }
+    return docs;
+  };
+
+  for (const file of files) {
     try {
-      const fileName = path.basename(filePath);
-      const { buffer, text, type } = await processFile(filePath);
-      const createdAt = Date.now(); // Current timestamp
-      
-      let fileUrl = '';
-      let storageType = '';
-      let binaryData: Buffer = Buffer.alloc(0);
+      logger?.info(`[ContextLake-Action] Processing file: ${file.key}, type: ${file.media_type}`);
+      let docs: any[] = [];
+      const fileUrl = file.url || `tos://${file.bucket}/${file.key}`;
 
-      // Decide storage strategy
-      if (buffer.length <= maxInlineSize) {
-        binaryData = buffer;
-        fileUrl = `inline://${fileName}`; // Virtual URL for inline
-        storageType = 'inline';
-      } else {
-        fileUrl = await storageProvider.uploadFile(fileName, buffer);
-        storageType = storageConfig.type;
+      if (file.media_type === 'pdf') {
+        // PDF Parse
+        const result = await lasClient.submitAndPoll('las_pdf_parse_doubao', {
+          url: fileUrl
+        });
+        const markdown = result.data?.markdown || '';
+        docs = await processText(markdown, file);
+      } else if (file.media_type === 'image') {
+        // Multimodal Embedding directly
+        const vector = await metadataProvider.generateMultimodalEmbedding!([
+            { type: 'image_url', image_url: { url: fileUrl } },
+            { type: 'text', text: 'This is an image from the dataset.' }
+        ]);
+        docs.push({
+          id: uuidv4(),
+          vector,
+          text: 'Image from dataset',
+          source: file.key,
+          file_type: 'image',
+          storage_type: 'source',
+          url: fileUrl,
+          metadata: JSON.stringify({ datasource: params.datasource_name }),
+          created_at: Date.now(),
+          binary_data: Buffer.from('')
+        });
+      } else if (file.media_type === 'audio') {
+        // ASR
+        const result = await lasClient.submitAndPoll('las_asr_pro', {
+          audio: { url: fileUrl, format: (file.key as string).split('.').pop() || 'wav' },
+          request: { model_name: 'bigmodel' }
+        });
+        const text = result.data?.result?.text || '';
+        docs = await processText(text, file);
+      } else if (file.media_type === 'video') {
+        // Video understanding -> text -> embedding
+        const result = await lasClient.submitAndPoll('las_long_video_understand', {
+          video_url: fileUrl,
+          query: "详细描述这个视频的内容",
+          model_name: "doubao-seed-2-0-lite-260215"
+        });
+        // Assuming video output is a text description somewhere in the response. 
+        // Note: the exact structure depends on the API return, adjusting to generic text.
+        const text = JSON.stringify(result.data || ''); 
+        
+        // Also need audio extract and ASR for video
+        // 1. Extract audio
+        // The output_path_template needs a unique path per video
+        const audioOutputPath = `tos://${file.bucket}/.tmp/audio/${uuidv4()}.wav`;
+        await lasClient.process('las_audio_extract_and_split', {
+            input_path: fileUrl,
+            output_path_template: audioOutputPath,
+            output_format: 'wav'
+        });
+        
+        // 2. ASR on the extracted audio
+        // Wait briefly for object to be available if needed (often synchronous but tos takes a ms)
+        const asrResult = await lasClient.submitAndPoll('las_asr_pro', {
+            audio: { url: audioOutputPath.replace('{index}.{output_file_ext}', '0.wav'), format: 'wav' },
+            request: { model_name: 'bigmodel' }
+        });
+        
+        const audioText = asrResult.data?.result?.text || '';
+        
+        // Combine video text and audio text
+        const combinedText = `Video Description: ${text}\n\nAudio Transcription: ${audioText}`;
+        docs = await processText(combinedText, file);
+        
+      } else if (file.category === 'structured' || file.category === 'non-structured') {
+          // If we had a direct text content, we could process it here. 
+          // Assuming basic local download or similar is available, but for now we skip raw file reading from TOS in this demo script unless implemented.
+          // Fallback just logs
+          logger?.warn(`[ContextLake-Action] Skipping raw text/structured download for ${file.key} - implement TOS download if needed`);
       }
 
-      const chunks = splitText(text, params.chunkSize || 500, params.overlap || 50);
-      const docs = [];
-      
-      // If no text extracted (e.g. image), store one entry with empty text but with metadata/binary
-      if (chunks.length === 0) {
-          const vector = await metadataProvider.generateEmbedding(fileName); // Embed filename as fallback
-           docs.push({
-            id: uuidv4(),
-            vector,
-            text: '',
-            source: fileName,
-            file_type: type,
-            storage_type: storageType,
-            url: fileUrl,
-            metadata: JSON.stringify(params.metadata || {}),
-            created_at: createdAt,
-            binary_data: binaryData
-          });
-      } else {
-           for (const chunk of chunks) {
-            const vector = await metadataProvider.generateEmbedding(chunk);
-            docs.push({
-                id: uuidv4(),
-                vector,
-                text: chunk,
-                source: fileName,
-                file_type: type,
-                storage_type: storageType,
-                url: fileUrl,
-                metadata: JSON.stringify(params.metadata || {}),
-                created_at: createdAt,
-                binary_data: binaryData // Only attach to first chunk
-            });
-            // Clear binary data for subsequent chunks of the same file to avoid duplication
-            binaryData = Buffer.alloc(0); 
-          }
+      if (docs.length > 0) {
+        await metadataProvider.addAssets(docs);
+        results.push({ file: file.key, status: 'success', chunks: docs.length });
       }
 
-      await metadataProvider.addAssets(docs);
-      results.push({ file: fileName, status: 'success', chunks: docs.length });
-
     } catch (error: any) {
-      // @ts-ignore
-      results.push({ file: filePath, status: 'error', message: error.message });
+      logger?.error(`[ContextLake-Action] Error processing ${file.key}: ${error.message}`);
+      results.push({ file: file.key, status: 'error', message: error.message });
     }
   }
 
-  // Ensure plain JSON serialization
-  return JSON.parse(JSON.stringify(results));
+  return results;
 }

package/src/lib/actions/las-api.ts

@@ -0,0 +1,119 @@
+import { ContextLakeConfig } from '../../utils/config';
+
+export class LasApiClient {
+    private endpoint: string;
+    private apiKey: string;
+    private logger: any;
+
+    constructor(config: ContextLakeConfig, logger: any) {
+        this.logger = logger;
+        this.apiKey = config.metadata_storage?.embedding?.api_key || process.env.LAS_API_KEY || '';
+        this.endpoint = config.metadata_storage?.embedding?.api_base || process.env.LAS_BASE_URL || 'https://operator.las.cn-beijing.volces.com';
+        
+        // Remove trailing slash
+        if (this.endpoint.endsWith('/')) {
+            this.endpoint = this.endpoint.slice(0, -1);
+        }
+    }
+
+    private async request(path: string, body: any) {
+        if (!this.apiKey) {
+            throw new Error('LAS_API_KEY is not configured. Please set it in config or environment variables.');
+        }
+
+        const url = `${this.endpoint}${path}`;
+        this.logger.debug(`[LasApiClient] Requesting ${url}`, { body: JSON.stringify(body) });
+
+        const response = await fetch(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`
+            },
+            body: JSON.stringify(body)
+        });
+
+        if (!response.ok) {
+            let errorText = '';
+            try {
+                errorText = await response.text();
+            } catch (e) { }
+            throw new Error(`LAS API Error: ${response.status} ${response.statusText} - ${errorText}`);
+        }
+
+        const result = await response.json();
+        return result;
+    }
+
+    async process(operatorId: string, data: any, version: string = 'v1') {
+        const result = await this.request('/api/v1/process', {
+            operator_id: operatorId,
+            operator_version: version,
+            data
+        });
+        return result;
+    }
+
+    async submit(operatorId: string, data: any, version: string = 'v1') {
+        const result = await this.request('/api/v1/submit', {
+            operator_id: operatorId,
+            operator_version: version,
+            data
+        });
+        return result;
+    }
+
+    async poll(operatorId: string, taskId: string, version: string = 'v1') {
+        const result = await this.request('/api/v1/poll', {
+            operator_id: operatorId,
+            operator_version: version,
+            task_id: taskId
+        });
+        return result;
+    }
+
+    async submitAndPoll(operatorId: string, data: any, version: string = 'v1', pollIntervalMs: number = 3000, maxRetries: number = 200) {
+        const submitResult = await this.submit(operatorId, data, version);
+        
+        if (!submitResult?.metadata?.task_id) {
+            throw new Error(`Failed to submit task for ${operatorId}. Response: ${JSON.stringify(submitResult)}`);
+        }
+        
+        const taskId = submitResult.metadata.task_id;
+        this.logger.info(`[LasApiClient] Task submitted: ${taskId} for ${operatorId}`);
+
+        let retries = 0;
+        while (retries < maxRetries) {
+            await new Promise(resolve => setTimeout(resolve, pollIntervalMs));
+            
+            const pollResult = await this.poll(operatorId, taskId, version);
+            const status = pollResult?.metadata?.task_status;
+            
+            this.logger.debug(`[LasApiClient] Task ${taskId} status: ${status}`);
+
+            if (status === 'COMPLETED') {
+                return pollResult;
+            } else if (status === 'FAILED' || status === 'TIMEOUT') {
+                const errorMsg = pollResult?.metadata?.error_msg || 'Unknown error';
+                throw new Error(`Task ${taskId} failed with status: ${status}. Message: ${errorMsg}`);
+            }
+            
+            retries++;
+        }
+        
+        throw new Error(`Task ${taskId} timed out after ${maxRetries} polling attempts.`);
+    }
+
+    async multimodalEmbedding(model: string, input: any[], encodingFormat: string = 'float', dimensions?: number, instructions?: string, sparseEmbedding?: any) {
+        const body: any = {
+            model,
+            input,
+            encoding_format: encodingFormat
+        };
+        if (dimensions) body.dimensions = dimensions;
+        if (instructions) body.instructions = instructions;
+        if (sparseEmbedding) body.sparse_embedding = sparseEmbedding;
+
+        return await this.request('/api/v1/embeddings/multimodal', body);
+    }
+}

package/src/lib/actions/las-tools.ts

@@ -0,0 +1,196 @@
+import { LasApiClient } from './las-api';
+import { ContextLakeConfig } from '../../utils/config';
+// @ts-ignore
+import type { AnyAgentTool } from 'openclaw/plugin-sdk';
+
+export function getLasTools(pluginConfig: ContextLakeConfig, logger: any): AnyAgentTool[] {
+    const apiClient = new LasApiClient(pluginConfig, logger);
+
+    const callApi = async (method: string, args: any[]) => {
+        try {
+            // @ts-ignore
+            return await apiClient[method](...args);
+        } catch (error: any) {
+            logger.error(`[LasTools] API ${method} failed`, { error: error.message });
+            return { error: error.message };
+        }
+    };
+
+    return [
+        {
+            name: 'las_image_resample',
+            label: 'LAS Image Resample',
+            description: `Resample/Resize an image and save it to TOS. 
+Parameters in data:
+- image_src_type (string, default: "image_url"): "image_url" or "image_tos"
+- image (string, required): URL or tos:// path
+- tos_dir (string, required): tos:// output directory
+- image_suffix (string): ".jpg" or ".png"
+- target_size (array of integers): e.g. [1024, 1024]
+- target_dpi (array of integers): e.g. [72, 72]
+- method (string): "nearest", "bilinear", "bicubic", "lanczos"`,
+            parameters: {
+                type: 'object',
+                properties: { data: { type: 'object', additionalProperties: true } },
+                required: ['data']
+            },
+            async execute(toolCallId: string, params: any) {
+                return await callApi('process', ['las_image_resample', params.data]);
+            }
+        },
+        {
+            name: 'las_audio_extract_and_split',
+            label: 'LAS Audio Extract and Split',
+            description: `Extract audio from video and split it into chunks.
+Parameters in data:
+- input_path (string, required): tos:// video path
+- output_path_template (string, required): e.g. tos://bucket/{index}.{output_file_ext}
+- split_duration (number): duration in seconds, default 30.0
+- output_format (string): "wav", "mp3", "flac"
+- timeout (integer)
+- extra_params (array of string): ffmpeg params`,
+            parameters: {
+                type: 'object',
+                properties: { data: { type: 'object', additionalProperties: true } },
+                required: ['data']
+            },
+            async execute(toolCallId: string, params: any) {
+                return await callApi('process', ['las_audio_extract_and_split', params.data]);
+            }
+        },
+        {
+            name: 'las_audio_convert',
+            label: 'LAS Audio Convert',
+            description: `Convert audio format.
+Parameters in data:
+- input_path (string, required): tos:// audio path
+- output_path (string, required): tos:// output path
+- output_format (string): "wav", "mp3", "flac"
+- extra_params (array of string): ffmpeg params`,
+            parameters: {
+                type: 'object',
+                properties: { data: { type: 'object', additionalProperties: true } },
+                required: ['data']
+            },
+            async execute(toolCallId: string, params: any) {
+                return await callApi('process', ['las_audio_convert', params.data]);
+            }
+        },
+        {
+            name: 'las_asr_pro',
+            label: 'LAS ASR Pro (Speech Recognition)',
+            description: `Perform automatic speech recognition (ASR).
+Parameters in data:
+- resource: "bigasr" or "seedasr"
+- audio (object, required): { url: string, language: string, format: string }
+- request (object, required): { model_name: "bigmodel", ... }
+- user (object): { uid: string }`,
+            parameters: {
+                type: 'object',
+                properties: { data: { type: 'object', additionalProperties: true } },
+                required: ['data']
+            },
+            async execute(toolCallId: string, params: any) {
+                return await callApi('submitAndPoll', ['las_asr_pro', params.data]);
+            }
+        },
+        {
+            name: 'las_seed_2_0',
+            label: 'LAS Seed 2.0 (Audio)',
+            description: `ASR with Seed 2.0.
+Parameters in data:
+- audio (object, required): { url, format, language }
+- request (object, required): { model_name: "seedasr", ... }`,
+            parameters: {
+                type: 'object',
+                properties: { data: { type: 'object', additionalProperties: true } },
+                required: ['data']
+            },
+            async execute(toolCallId: string, params: any) {
+                return await callApi('submitAndPoll', ['las_seed_2_0', params.data]);
+            }
+        },
+        {
+            name: 'las_bare_image_text_embedding',
+            label: 'LAS Multimodal Embedding',
+            description: `Multimodal Embedding (image and text).
+Parameters:
+- model (string, required): "doubao-embedding-vision-250615"
+- input (array of objects, required): [ { type: "image_url", image_url: { url: "..." } }, { type: "text", text: "..." } ]
+- encoding_format (string): "float", "base64"
+- dimensions (integer): 1024 or 2048`,
+            parameters: {
+                type: 'object',
+                properties: {
+                    model: { type: 'string', default: 'doubao-embedding-vision-250615' },
+                    input: { type: 'array', items: { type: 'object' } },
+                    encoding_format: { type: 'string', default: 'float' },
+                    dimensions: { type: 'integer' }
+                },
+                required: ['model', 'input']
+            },
+            async execute(toolCallId: string, params: any) {
+                return await callApi('multimodalEmbedding', [
+                    params.model,
+                    params.input,
+                    params.encoding_format,
+                    params.dimensions
+                ]);
+            }
+        },
+        {
+            name: 'las_long_video_understand',
+            label: 'LAS Long Video Understand',
+            description: `Long Video Understanding.
+Parameters in data:
+- video_url (string, required)
+- query (string, required)
+- model_name (string): default "doubao-seed-2-0-lite-260215"
+- ...other params like fps, target_tokens_per_clip`,
+            parameters: {
+                type: 'object',
+                properties: { data: { type: 'object', additionalProperties: true } },
+                required: ['data']
+            },
+            async execute(toolCallId: string, params: any) {
+                return await callApi('submitAndPoll', ['las_long_video_understand', params.data]);
+            }
+        },
+        {
+            name: 'las_pdf_parse_doubao',
+            label: 'LAS PDF Parse Doubao',
+            description: `Parse PDF documents to Markdown.
+Parameters in data:
+- url (string, required): PDF URL or tos://
+- start_page (integer): default 1
+- num_pages (integer): default to end
+- parse_mode (string): "normal" or "detail"`,
+            parameters: {
+                type: 'object',
+                properties: { data: { type: 'object', additionalProperties: true } },
+                required: ['data']
+            },
+            async execute(toolCallId: string, params: any) {
+                return await callApi('submitAndPoll', ['las_pdf_parse_doubao', params.data]);
+            }
+        },
+        {
+            name: 'las_video_resize',
+            label: 'LAS Video Resize',
+            description: `Resize video.
+Parameters in data:
+- video_url (string, required): URL or tos://
+- target_width (integer)
+- target_height (integer)
+- output_dir (string, required): tos://`,
+            parameters: {
+                type: 'object',
+                properties: { data: { type: 'object', additionalProperties: true } },
+                required: ['data']
+            },
+            async execute(toolCallId: string, params: any) {
+                return await callApi('submitAndPoll', ['las_video_resize', params.data]);
+            }
+        }
+    ];
+}

package/src/lib/actions/manage.ts

@@ -1,5 +1,6 @@
 import { createMetadataProvider } from '../../service/metadata/factory';
 import { createStorageProvider } from '../../service/storage/factory';
+import { ContextLakeConfig } from '../../utils/config';
 
 interface ListParams {
   limit?: number;
@@ -10,7 +11,7 @@ interface DeleteParams {
   filter?: string;
 }
 
-export async function listAssets(params: ListParams, config: any, logger?: any) {
+export async function listAssets(params: ListParams, config: ContextLakeConfig, logger?: any) {
   if (logger) {
       logger.info(`[ContextLake-Action] Calling listAssets with params: ${JSON.stringify(params)}`);
   } else {
@@ -20,7 +21,7 @@ export async function listAssets(params: ListParams, config: any, logger?: any)
 
   // Ensure config has default if not provided
   const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
-  const metadataProvider = createMetadataProvider(metaConfig);
+  const metadataProvider = createMetadataProvider(metaConfig as any);
   
   await metadataProvider.connect();
   const docs = await metadataProvider.list(params.limit || 100);
@@ -47,7 +48,7 @@ export async function listAssets(params: ListParams, config: any, logger?: any)
   return JSON.parse(JSON.stringify(Array.from(fileMap.values())));
 }
 
-export async function deleteAssets(params: DeleteParams, config: any, logger?: any) {
+export async function deleteAssets(params: DeleteParams, config: ContextLakeConfig, logger?: any) {
   if (logger) {
       logger.info(`[ContextLake-Action] Calling deleteAssets with params: ${JSON.stringify(params)}`);
   } else {
@@ -56,13 +57,13 @@ export async function deleteAssets(params: DeleteParams, config: any, logger?: a
   }
 
   const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
-  const metadataProvider = createMetadataProvider(metaConfig);
+  const metadataProvider = createMetadataProvider(metaConfig as any);
   
   // file_storage config is optional for deletion (we might not need to delete from storage if inline)
   // Check if file_storage config exists before creating provider
   let storageProvider;
   if (config.file_storage && config.file_storage.type) {
-      storageProvider = createStorageProvider(config.file_storage);
+      storageProvider = createStorageProvider(config.file_storage as any);
   }
   
   await metadataProvider.connect();

package/src/{skills/las-data-profiler/index.ts → lib/actions/profiler.ts}

@@ -19,7 +19,7 @@ export interface ConnectParams {
   sample_rows?: number;
 }
 
-interface ConnectResult {
+export interface ConnectResult {
   status: 'success' | 'error';
   datasource_name: string;
   db_path: string;
@@ -37,7 +37,7 @@ interface ConnectResult {
 // Constants
 // ---------------------------------------------------------------------------
 
-const BASE_DIR = path.join(os.homedir(), '.openclaw', 'las-data-profiler');
+const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
 const PYTHON_DEPS = ['boto3', 'lancedb', 'pyarrow', 'pandas', 'Pillow', 'mutagen', 'pymupdf'];
 
 // ---------------------------------------------------------------------------
@@ -113,8 +113,8 @@ function ensurePythonDeps(): void {
  * Get the path to the bundled Python script.
  */
 function getScriptPath(): string {
-  // The Python script is co-located with this module
-  return path.join(__dirname, 's3_catalog.py');
+  // The Python script is located in the scripts directory
+  return path.join(__dirname, '../scripts', 's3_catalog.py');
 }
 
 // ---------------------------------------------------------------------------
@@ -252,3 +252,20 @@ export async function connectDataSource(
     });
   });
 }
+
+export async function listDataSources(
+  _ctx?: any
+): Promise<{ datasources: string[] }> {
+  try {
+    if (!fs.existsSync(BASE_DIR)) {
+      return { datasources: [] };
+    }
+    const entries = fs.readdirSync(BASE_DIR, { withFileTypes: true });
+    const datasources = entries
+      .filter(entry => entry.isDirectory())
+      .map(entry => entry.name);
+    return { datasources };
+  } catch (error: any) {
+    throw new Error(`Failed to list data sources: ${error.message}`);
+  }
+}