@byted-las/contextlake-openclaw 1.0.3 → 1.0.5

package/dist/src/lib/actions/las-tools.js

@@ -2,8 +2,35 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getLasTools = getLasTools;
 const las_api_1 = require("./las-api");
+const s3_tools_1 = require("./s3-tools");
 function getLasTools(pluginConfig, logger) {
     const apiClient = new las_api_1.LasApiClient(pluginConfig, logger);
+    const processUrl = async (url) => {
+        if (!url)
+            return url;
+        if (url.startsWith('tos://')) {
+            // LAS operators prefer native tos:// paths when supported, leave as is
+            return url;
+        }
+        else if (url.startsWith('oss://') || url.startsWith('s3://') || url.startsWith('cos://') || url.startsWith('file://')) {
+            // Need presigned HTTP url for other vendors
+            logger.info(`[LasTools] Presigning URL for vendor: ${url}`);
+            try {
+                // If it's a file:// we also presign it to file:// which might not be supported by remote LAS, 
+                // but local files typically need to be uploaded to TOS first. We'll leave file:// to fail or be handled elsewhere.
+                if (url.startsWith('file://'))
+                    return url;
+                const urlParts = new URL(url);
+                const key = urlParts.pathname.replace(/^\//, '');
+                return await (0, s3_tools_1.getPresignedUrl)({ url }, key, 3600);
+            }
+            catch (e) {
+                logger.warn(`[LasTools] Failed to presign URL: ${url}`, { error: e.message });
+                return url; // fallback to original
+            }
+        }
+        return url;
+    };
     const callApi = async (method, args) => {
         try {
             // @ts-ignore
@@ -33,6 +60,9 @@ Parameters in data:
                 required: ['data']
             },
             async execute(toolCallId, params) {
+                if (params.data?.image) {
+                    params.data.image = await processUrl(params.data.image);
+                }
                 return await callApi('process', ['las_image_resample', params.data]);
             }
         },
@@ -53,6 +83,9 @@ Parameters in data:
                 required: ['data']
             },
             async execute(toolCallId, params) {
+                if (params.data?.input_path) {
+                    params.data.input_path = await processUrl(params.data.input_path);
+                }
                 return await callApi('process', ['las_audio_extract_and_split', params.data]);
             }
         },
@@ -71,6 +104,9 @@ Parameters in data:
                 required: ['data']
             },
             async execute(toolCallId, params) {
+                if (params.data?.input_path) {
+                    params.data.input_path = await processUrl(params.data.input_path);
+                }
                 return await callApi('process', ['las_audio_convert', params.data]);
             }
         },
@@ -89,6 +125,9 @@ Parameters in data:
                 required: ['data']
             },
             async execute(toolCallId, params) {
+                if (params.data?.audio?.url) {
+                    params.data.audio.url = await processUrl(params.data.audio.url);
+                }
                 return await callApi('submitAndPoll', ['las_asr_pro', params.data]);
             }
         },
@@ -105,6 +144,9 @@ Parameters in data:
                 required: ['data']
             },
             async execute(toolCallId, params) {
+                if (params.data?.audio?.url) {
+                    params.data.audio.url = await processUrl(params.data.audio.url);
+                }
                 return await callApi('submitAndPoll', ['las_seed_2_0', params.data]);
             }
         },
@@ -128,6 +170,13 @@ Parameters:
                 required: ['model', 'input']
             },
             async execute(toolCallId, params) {
+                if (params.input && Array.isArray(params.input)) {
+                    for (const item of params.input) {
+                        if (item.type === 'image_url' && item.image_url?.url) {
+                            item.image_url.url = await processUrl(item.image_url.url);
+                        }
+                    }
+                }
                 return await callApi('multimodalEmbedding', [
                     params.model,
                     params.input,
@@ -151,6 +200,9 @@ Parameters in data:
                 required: ['data']
             },
             async execute(toolCallId, params) {
+                if (params.data?.video_url) {
+                    params.data.video_url = await processUrl(params.data.video_url);
+                }
                 return await callApi('submitAndPoll', ['las_long_video_understand', params.data]);
             }
         },
@@ -169,6 +221,9 @@ Parameters in data:
                 required: ['data']
             },
             async execute(toolCallId, params) {
+                if (params.data?.url) {
+                    params.data.url = await processUrl(params.data.url);
+                }
                 return await callApi('submitAndPoll', ['las_pdf_parse_doubao', params.data]);
             }
         },
@@ -187,6 +242,9 @@ Parameters in data:
                 required: ['data']
             },
             async execute(toolCallId, params) {
+                if (params.data?.video_url) {
+                    params.data.video_url = await processUrl(params.data.video_url);
+                }
                 return await callApi('submitAndPoll', ['las_video_resize', params.data]);
             }
         }

package/dist/src/lib/actions/profiler.d.ts

@@ -1,12 +1,13 @@
 export interface ConnectParams {
     datasource_name: string;
-    vendor: 'volcengine' | 'alibaba' | 'tencent' | 'aws' | 'local';
+    url: string;
+    vendor?: 'volcengine' | 'alibaba' | 'tencent' | 'aws' | 'local';
     endpoint?: string;
     access_key?: string;
     secret_key?: string;
     region?: string;
-    bucket: string;
-    prefix: string;
+    bucket?: string;
+    prefix?: string;
     sample_rows?: number;
 }
 export interface ConnectResult {

package/dist/src/lib/actions/profiler.js

@@ -38,25 +38,16 @@ exports.listDataSources = listDataSources;
 const path = __importStar(require("path"));
 const fs = __importStar(require("fs"));
 const os = __importStar(require("os"));
-const child_process_1 = require("child_process");
-// ---------------------------------------------------------------------------
-// Constants
-// ---------------------------------------------------------------------------
+const s3_tools_1 = require("./s3-tools");
+const lance_tools_1 = require("./lance-tools");
+const mime = __importStar(require("mime-types"));
 const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
-const PYTHON_DEPS = ['boto3', 'lancedb', 'pyarrow', 'pandas', 'Pillow', 'mutagen', 'pymupdf'];
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
 function getDataSourceDir(name) {
     return path.join(BASE_DIR, name);
 }
 function ensureDir(dir) {
     fs.mkdirSync(dir, { recursive: true });
 }
-/**
- * Generate env.sh with all connection parameters for this datasource.
- * This file can be sourced to re-run the profiler or for debugging.
- */
 function writeEnvFile(dir, params) {
     const envPath = path.join(dir, 'env.sh');
     const lines = [
@@ -69,166 +60,190 @@ function writeEnvFile(dir, params) {
         `export LAS_BUCKET="${params.bucket}"`,
         `export LAS_PREFIX="${params.prefix}"`,
     ];
-    if (params.endpoint) {
+    if (params.endpoint)
         lines.push(`export LAS_ENDPOINT="${params.endpoint}"`);
-    }
-    if (params.access_key) {
+    if (params.access_key)
         lines.push(`export LAS_ACCESS_KEY="${params.access_key}"`);
-    }
-    if (params.secret_key) {
+    if (params.secret_key)
         lines.push(`export LAS_SECRET_KEY="${params.secret_key}"`);
-    }
-    if (params.region) {
+    if (params.region)
         lines.push(`export LAS_REGION="${params.region}"`);
-    }
-    if (params.sample_rows) {
+    if (params.sample_rows)
         lines.push(`export LAS_SAMPLE_ROWS="${params.sample_rows}"`);
-    }
     lines.push(`export LAS_DB_PATH="${path.join(dir, 'catalog_db')}"`);
     lines.push(`export LAS_DATASOURCE_NAME="${params.datasource_name}"`);
     lines.push('');
     fs.writeFileSync(envPath, lines.join('\n'), { mode: 0o600 });
     return envPath;
 }
-/**
- * Install Python dependencies if not already available.
- */
-function ensurePythonDeps() {
-    try {
-        (0, child_process_1.execSync)(`python3 -c "import boto3, lancedb, pyarrow, pandas, PIL, mutagen, fitz"`, {
-            stdio: 'pipe',
-        });
-    }
-    catch {
-        console.log('[las-data-profiler] Installing Python dependencies...');
-        (0, child_process_1.execSync)(`pip3 install --user ${PYTHON_DEPS.join(' ')}`, {
-            stdio: 'inherit',
-        });
-    }
-}
-/**
- * Get the path to the bundled Python script.
- */
-function getScriptPath() {
-    // The Python script is located in the scripts directory
-    return path.join(__dirname, '../scripts', 's3_catalog.py');
+function classifyFile(ext) {
+    ext = ext.toLowerCase();
+    const STRUCTURED_EXTS = ['.json', '.jsonl', '.ndjson', '.csv', '.tsv', '.parquet', '.pq'];
+    const IMAGE_EXTS = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.tiff', '.tif', '.svg', '.ico', '.heic', '.heif'];
+    const AUDIO_EXTS = ['.mp3', '.wav', '.flac', '.aac', '.ogg', '.m4a', '.wma', '.opus'];
+    const VIDEO_EXTS = ['.mp4', '.avi', '.mov', '.mkv', '.webm', '.wmv', '.flv', '.m4v', '.3gp'];
+    const PDF_EXTS = ['.pdf'];
+    if (STRUCTURED_EXTS.includes(ext))
+        return { category: 'structured', mediaType: '' };
+    if (IMAGE_EXTS.includes(ext))
+        return { category: 'non-structured', mediaType: 'image' };
+    if (AUDIO_EXTS.includes(ext))
+        return { category: 'non-structured', mediaType: 'audio' };
+    if (VIDEO_EXTS.includes(ext))
+        return { category: 'non-structured', mediaType: 'video' };
+    if (PDF_EXTS.includes(ext))
+        return { category: 'non-structured', mediaType: 'pdf' };
+    return { category: 'non-structured', mediaType: '' };
 }
-// ---------------------------------------------------------------------------
-// Main Entry
-// ---------------------------------------------------------------------------
 async function connectDataSource(params, _ctx) {
-    // Validate required params
-    if (!params.datasource_name) {
+    if (!params.datasource_name)
         throw new Error('datasource_name is required');
+    if (!params.url)
+        throw new Error('url is required (e.g. tos://bucket/prefix)');
+    // Parse URL: tos://bucket/prefix
+    try {
+        if (params.url.startsWith('file://') || params.url.startsWith('/')) {
+            params.vendor = 'local';
+            const localPath = params.url.startsWith('file://') ? params.url.slice(7) : params.url;
+            params.bucket = localPath;
+            params.prefix = '.';
+        }
+        else {
+            const parsedUrl = new URL(params.url);
+            const protocol = parsedUrl.protocol.replace(':', '');
+            if (['tos', 'oss', 'cos', 's3'].includes(protocol)) {
+                if (protocol === 'tos')
+                    params.vendor = 'volcengine';
+                else if (protocol === 'oss')
+                    params.vendor = 'alibaba';
+                else if (protocol === 'cos')
+                    params.vendor = 'tencent';
+                else if (protocol === 's3')
+                    params.vendor = 'aws';
+                params.bucket = parsedUrl.hostname;
+                params.prefix = parsedUrl.pathname.replace(/^\//, ''); // Remove leading slash
+            }
+            else {
+                throw new Error(`Unsupported protocol: ${protocol}`);
+            }
+        }
     }
-    if (!params.vendor) {
-        throw new Error('vendor is required');
-    }
-    if (!params.bucket) {
-        throw new Error('bucket is required');
-    }
-    if (params.prefix === undefined || params.prefix === null) {
-        throw new Error('prefix is required');
+    catch (e) {
+        if (!params.vendor || !params.bucket || params.prefix === undefined) {
+            throw new Error(`Invalid url format: ${e.message}`);
+        }
     }
-    // For non-local vendors, validate credentials
     if (params.vendor !== 'local') {
-        if (!params.endpoint && params.vendor !== 'aws') {
+        if (!params.endpoint && params.vendor !== 'aws')
             throw new Error(`endpoint is required for vendor "${params.vendor}"`);
+        let ak = params.access_key;
+        let sk = params.secret_key;
+        if (!ak || !sk) {
+            try {
+                const { loadCredentials } = require('../../utils/credentials');
+                const creds = loadCredentials();
+                ak = ak || creds.ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY;
+                sk = sk || creds.SECRET_KEY || creds.VOLCENGINE_SECRET_KEY;
+            }
+            catch (e) {
+                // ignore
+            }
         }
-        const ak = params.access_key || process.env.TOS_ACCESS_KEY || process.env.S3_ACCESS_KEY || process.env.AWS_ACCESS_KEY_ID;
-        const sk = params.secret_key || process.env.TOS_SECRET_KEY || process.env.S3_SECRET_KEY || process.env.AWS_SECRET_ACCESS_KEY;
         if (!ak || !sk) {
-            throw new Error('access_key and secret_key are required (via params or env vars TOS_ACCESS_KEY/TOS_SECRET_KEY, S3_ACCESS_KEY/S3_SECRET_KEY, AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY)');
+            throw new Error('access_key and secret_key are required');
         }
-        // Normalise into params so env.sh picks them up
         params.access_key = ak;
         params.secret_key = sk;
     }
     const dsDir = getDataSourceDir(params.datasource_name);
     const dbPath = path.join(dsDir, 'catalog_db');
     ensureDir(dsDir);
-    // 1. Write env.sh
     const envPath = writeEnvFile(dsDir, params);
-    // 2. Ensure Python dependencies
-    ensurePythonDeps();
-    // 3. Build CLI args for the Python script
-    const scriptPath = getScriptPath();
-    const args = [
-        scriptPath,
-        '--vendor', params.vendor,
-        '--bucket', params.bucket,
-        '--prefix', params.prefix,
-        '--db-path', dbPath,
-    ];
-    if (params.endpoint) {
-        args.push('--endpoint', params.endpoint);
-    }
-    if (params.access_key) {
-        args.push('--ak', params.access_key);
-    }
-    if (params.secret_key) {
-        args.push('--sk', params.secret_key);
-    }
-    if (params.region) {
-        args.push('--region', params.region);
-    }
-    if (params.sample_rows) {
-        args.push('--sample-rows', String(params.sample_rows));
-    }
-    // 4. Execute the profiling script
-    return new Promise((resolve) => {
-        let stdout = '';
-        let stderr = '';
-        const proc = (0, child_process_1.spawn)('python3', args, {
-            cwd: dsDir,
-            env: { ...process.env },
-        });
-        proc.stdout.on('data', (data) => {
-            stdout += data.toString();
-        });
-        proc.stderr.on('data', (data) => {
-            stderr += data.toString();
-        });
-        proc.on('close', (code) => {
-            if (code !== 0) {
-                resolve({
-                    status: 'error',
-                    datasource_name: params.datasource_name,
-                    db_path: dbPath,
-                    env_path: envPath,
-                    tables: [],
-                    error: stderr || `Python script exited with code ${code}`,
-                });
-                return;
-            }
-            // Try to parse structured output from the script
-            try {
-                const jsonMatch = stdout.match(/\{[\s\S]*"summary"[\s\S]*\}/);
-                const result = jsonMatch ? JSON.parse(jsonMatch[0]) : {};
-                resolve({
-                    status: 'success',
-                    datasource_name: params.datasource_name,
-                    db_path: dbPath,
-                    env_path: envPath,
-                    tables: ['file_catalog', 'structured_schemas', 'media_metadata'],
-                    summary: result.summary || {
-                        total_files: 0,
-                        structured_files: 0,
-                        media_files: 0,
-                    },
-                });
-            }
-            catch {
-                resolve({
-                    status: 'success',
-                    datasource_name: params.datasource_name,
-                    db_path: dbPath,
-                    env_path: envPath,
-                    tables: ['file_catalog', 'structured_schemas', 'media_metadata'],
+    try {
+        let isTruncated = true;
+        let continuationToken = undefined;
+        let total_files = 0;
+        let structured_files = 0;
+        let media_files = 0;
+        const allRecords = [];
+        const scan_ts = new Date().toISOString() + 'Z';
+        while (isTruncated) {
+            const response = await (0, s3_tools_1.listS3Objects)({
+                vendor: params.vendor,
+                bucket: params.bucket,
+                endpoint: params.endpoint,
+                access_key: params.access_key,
+                secret_key: params.secret_key,
+                region: params.region
+            }, params.prefix || '', 1000, continuationToken);
+            for (const obj of response.Contents) {
+                const key = obj.Key || '';
+                if (key.endsWith('/'))
+                    continue;
+                const name = path.basename(key);
+                const ext = path.extname(name).toLowerCase();
+                const mimeType = mime.lookup(name) || '';
+                const { category, mediaType } = classifyFile(ext);
+                const depth = (key.match(/\//g) || []).length;
+                const parentDir = key.includes('/') ? path.basename(path.dirname(key)) : '';
+                total_files++;
+                if (category === 'structured')
+                    structured_files++;
+                if (mediaType)
+                    media_files++;
+                allRecords.push({
+                    file_path: key,
+                    file_name: name,
+                    extension: ext,
+                    mime_type: mimeType,
+                    category: category,
+                    media_type: mediaType,
+                    size_bytes: obj.Size || 0,
+                    last_modified: obj.LastModified ? String(obj.LastModified) : '',
+                    created_time: obj._created_time ? String(obj._created_time) : '',
+                    etag: (obj.ETag || '').replace(/"/g, ''),
+                    storage_class: obj.StorageClass || '',
+                    is_multipart: (obj.ETag || '').includes('-'),
+                    depth: depth,
+                    parent_dir: parentDir,
+                    vendor: params.vendor,
+                    bucket: params.bucket,
+                    has_schema: false,
+                    has_media_meta: false,
+                    scan_timestamp: scan_ts
                 });
             }
+            isTruncated = response.IsTruncated || false;
+            continuationToken = response.NextContinuationToken;
+        }
+        await (0, lance_tools_1.writeLanceCatalog)({
+            db_path: dbPath,
+            table_name: 'file_catalog',
+            records: allRecords
         });
-    });
+        return {
+            status: 'success',
+            datasource_name: params.datasource_name,
+            db_path: dbPath,
+            env_path: envPath,
+            tables: ['file_catalog'],
+            summary: {
+                total_files,
+                structured_files,
+                media_files
+            }
+        };
+    }
+    catch (error) {
+        return {
+            status: 'error',
+            datasource_name: params.datasource_name,
+            db_path: dbPath,
+            env_path: envPath,
+            tables: [],
+            error: error.message
+        };
+    }
 }
 async function listDataSources(_ctx) {
     try {

package/dist/src/lib/actions/s3-tools.d.ts

@@ -0,0 +1,21 @@
+export interface S3Params {
+    url?: string;
+    vendor?: 'volcengine' | 'alibaba' | 'tencent' | 'aws' | 'local';
+    endpoint?: string;
+    access_key?: string;
+    secret_key?: string;
+    region?: string;
+    bucket?: string;
+}
+export declare function parseS3Url(params: S3Params): S3Params;
+export declare function listS3Objects(params: S3Params, prefix: string, maxKeys?: number, continuationToken?: string): Promise<{
+    Contents: any[];
+    IsTruncated: boolean;
+    NextContinuationToken: undefined;
+} | {
+    Contents: import("@aws-sdk/client-s3")._Object[];
+    IsTruncated: boolean | undefined;
+    NextContinuationToken: string | undefined;
+}>;
+export declare function readS3Object(params: S3Params, key: string, maxBytes?: number): Promise<Buffer>;
+export declare function getPresignedUrl(params: S3Params, key: string, expiresIn?: number): Promise<string>;