@byted-las/contextlake-openclaw 1.0.3 → 1.0.5

package/src/lib/actions/profiler.ts

@@ -1,21 +1,20 @@
 import * as path from 'path';
 import * as fs from 'fs';
 import * as os from 'os';
-import { execSync, spawn } from 'child_process';
-
-// ---------------------------------------------------------------------------
-// Types
-// ---------------------------------------------------------------------------
+import { listS3Objects } from './s3-tools';
+import { writeLanceCatalog } from './lance-tools';
+import * as mime from 'mime-types';
 
 export interface ConnectParams {
   datasource_name: string;
-  vendor: 'volcengine' | 'alibaba' | 'tencent' | 'aws' | 'local';
+  url: string;
+  vendor?: 'volcengine' | 'alibaba' | 'tencent' | 'aws' | 'local';
   endpoint?: string;
   access_key?: string;
   secret_key?: string;
   region?: string;
-  bucket: string;
-  prefix: string;
+  bucket?: string;
+  prefix?: string;
   sample_rows?: number;
 }
 
@@ -33,16 +32,7 @@ export interface ConnectResult {
   error?: string;
 }
 
-// ---------------------------------------------------------------------------
-// Constants
-// ---------------------------------------------------------------------------
-
 const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
-const PYTHON_DEPS = ['boto3', 'lancedb', 'pyarrow', 'pandas', 'Pillow', 'mutagen', 'pymupdf'];
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
 
 function getDataSourceDir(name: string): string {
   return path.join(BASE_DIR, name);
@@ -52,10 +42,6 @@ function ensureDir(dir: string): void {
   fs.mkdirSync(dir, { recursive: true });
 }
 
-/**
- * Generate env.sh with all connection parameters for this datasource.
- * This file can be sourced to re-run the profiler or for debugging.
- */
 function writeEnvFile(dir: string, params: ConnectParams): string {
   const envPath = path.join(dir, 'env.sh');
   const lines: string[] = [
@@ -69,21 +55,11 @@ function writeEnvFile(dir: string, params: ConnectParams): string {
     `export LAS_PREFIX="${params.prefix}"`,
   ];
 
-  if (params.endpoint) {
-    lines.push(`export LAS_ENDPOINT="${params.endpoint}"`);
-  }
-  if (params.access_key) {
-    lines.push(`export LAS_ACCESS_KEY="${params.access_key}"`);
-  }
-  if (params.secret_key) {
-    lines.push(`export LAS_SECRET_KEY="${params.secret_key}"`);
-  }
-  if (params.region) {
-    lines.push(`export LAS_REGION="${params.region}"`);
-  }
-  if (params.sample_rows) {
-    lines.push(`export LAS_SAMPLE_ROWS="${params.sample_rows}"`);
-  }
+  if (params.endpoint) lines.push(`export LAS_ENDPOINT="${params.endpoint}"`);
+  if (params.access_key) lines.push(`export LAS_ACCESS_KEY="${params.access_key}"`);
+  if (params.secret_key) lines.push(`export LAS_SECRET_KEY="${params.secret_key}"`);
+  if (params.region) lines.push(`export LAS_REGION="${params.region}"`);
+  if (params.sample_rows) lines.push(`export LAS_SAMPLE_ROWS="${params.sample_rows}"`);
 
   lines.push(`export LAS_DB_PATH="${path.join(dir, 'catalog_db')}"`);
   lines.push(`export LAS_DATASOURCE_NAME="${params.datasource_name}"`);
@@ -93,65 +69,78 @@ function writeEnvFile(dir: string, params: ConnectParams): string {
   return envPath;
 }
 
-/**
- * Install Python dependencies if not already available.
- */
-function ensurePythonDeps(): void {
-  try {
-    execSync(`python3 -c "import boto3, lancedb, pyarrow, pandas, PIL, mutagen, fitz"`, {
-      stdio: 'pipe',
-    });
-  } catch {
-    console.log('[las-data-profiler] Installing Python dependencies...');
-    execSync(`pip3 install --user ${PYTHON_DEPS.join(' ')}`, {
-      stdio: 'inherit',
-    });
-  }
-}
+function classifyFile(ext: string): { category: string; mediaType: string } {
+    ext = ext.toLowerCase();
+    const STRUCTURED_EXTS = ['.json', '.jsonl', '.ndjson', '.csv', '.tsv', '.parquet', '.pq'];
+    const IMAGE_EXTS = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.tiff', '.tif', '.svg', '.ico', '.heic', '.heif'];
+    const AUDIO_EXTS = ['.mp3', '.wav', '.flac', '.aac', '.ogg', '.m4a', '.wma', '.opus'];
+    const VIDEO_EXTS = ['.mp4', '.avi', '.mov', '.mkv', '.webm', '.wmv', '.flv', '.m4v', '.3gp'];
+    const PDF_EXTS = ['.pdf'];
 
-/**
- * Get the path to the bundled Python script.
- */
-function getScriptPath(): string {
-  // The Python script is located in the scripts directory
-  return path.join(__dirname, '../scripts', 's3_catalog.py');
+    if (STRUCTURED_EXTS.includes(ext)) return { category: 'structured', mediaType: '' };
+    if (IMAGE_EXTS.includes(ext)) return { category: 'non-structured', mediaType: 'image' };
+    if (AUDIO_EXTS.includes(ext)) return { category: 'non-structured', mediaType: 'audio' };
+    if (VIDEO_EXTS.includes(ext)) return { category: 'non-structured', mediaType: 'video' };
+    if (PDF_EXTS.includes(ext)) return { category: 'non-structured', mediaType: 'pdf' };
+    
+    return { category: 'non-structured', mediaType: '' };
 }
 
-// ---------------------------------------------------------------------------
-// Main Entry
-// ---------------------------------------------------------------------------
-
 export async function connectDataSource(
   params: ConnectParams,
   _ctx?: any
 ): Promise<ConnectResult> {
-  // Validate required params
-  if (!params.datasource_name) {
-    throw new Error('datasource_name is required');
-  }
-  if (!params.vendor) {
-    throw new Error('vendor is required');
-  }
-  if (!params.bucket) {
-    throw new Error('bucket is required');
-  }
-  if (params.prefix === undefined || params.prefix === null) {
-    throw new Error('prefix is required');
+  if (!params.datasource_name) throw new Error('datasource_name is required');
+  if (!params.url) throw new Error('url is required (e.g. tos://bucket/prefix)');
+  
+  // Parse URL: tos://bucket/prefix
+  try {
+      if (params.url.startsWith('file://') || params.url.startsWith('/')) {
+          params.vendor = 'local';
+          const localPath = params.url.startsWith('file://') ? params.url.slice(7) : params.url;
+          params.bucket = localPath;
+          params.prefix = '.';
+      } else {
+          const parsedUrl = new URL(params.url);
+          const protocol = parsedUrl.protocol.replace(':', '');
+          
+          if (['tos', 'oss', 'cos', 's3'].includes(protocol)) {
+              if (protocol === 'tos') params.vendor = 'volcengine';
+              else if (protocol === 'oss') params.vendor = 'alibaba';
+              else if (protocol === 'cos') params.vendor = 'tencent';
+              else if (protocol === 's3') params.vendor = 'aws';
+              
+              params.bucket = parsedUrl.hostname;
+              params.prefix = parsedUrl.pathname.replace(/^\//, ''); // Remove leading slash
+          } else {
+              throw new Error(`Unsupported protocol: ${protocol}`);
+          }
+      }
+  } catch (e: any) {
+      if (!params.vendor || !params.bucket || params.prefix === undefined) {
+          throw new Error(`Invalid url format: ${e.message}`);
+      }
   }
 
-  // For non-local vendors, validate credentials
   if (params.vendor !== 'local') {
-    if (!params.endpoint && params.vendor !== 'aws') {
-      throw new Error(`endpoint is required for vendor "${params.vendor}"`);
+    if (!params.endpoint && params.vendor !== 'aws') throw new Error(`endpoint is required for vendor "${params.vendor}"`);
+    let ak = params.access_key;
+    let sk = params.secret_key;
+    
+    if (!ak || !sk) {
+        try {
+            const { loadCredentials } = require('../../utils/credentials');
+            const creds = loadCredentials();
+            ak = ak || creds.ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY;
+            sk = sk || creds.SECRET_KEY || creds.VOLCENGINE_SECRET_KEY;
+        } catch(e) {
+            // ignore
+        }
     }
-    const ak = params.access_key || process.env.TOS_ACCESS_KEY || process.env.S3_ACCESS_KEY || process.env.AWS_ACCESS_KEY_ID;
-    const sk = params.secret_key || process.env.TOS_SECRET_KEY || process.env.S3_SECRET_KEY || process.env.AWS_SECRET_ACCESS_KEY;
+    
     if (!ak || !sk) {
-      throw new Error(
-        'access_key and secret_key are required (via params or env vars TOS_ACCESS_KEY/TOS_SECRET_KEY, S3_ACCESS_KEY/S3_SECRET_KEY, AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY)'
-      );
+      throw new Error('access_key and secret_key are required');
     }
-    // Normalise into params so env.sh picks them up
     params.access_key = ak;
     params.secret_key = sk;
   }
@@ -160,97 +149,99 @@ export async function connectDataSource(
   const dbPath = path.join(dsDir, 'catalog_db');
 
   ensureDir(dsDir);
-
-  // 1. Write env.sh
   const envPath = writeEnvFile(dsDir, params);
 
-  // 2. Ensure Python dependencies
-  ensurePythonDeps();
-
-  // 3. Build CLI args for the Python script
-  const scriptPath = getScriptPath();
-  const args: string[] = [
-    scriptPath,
-    '--vendor', params.vendor,
-    '--bucket', params.bucket,
-    '--prefix', params.prefix,
-    '--db-path', dbPath,
-  ];
-
-  if (params.endpoint) {
-    args.push('--endpoint', params.endpoint);
-  }
-  if (params.access_key) {
-    args.push('--ak', params.access_key);
-  }
-  if (params.secret_key) {
-    args.push('--sk', params.secret_key);
-  }
-  if (params.region) {
-    args.push('--region', params.region);
-  }
-  if (params.sample_rows) {
-    args.push('--sample-rows', String(params.sample_rows));
-  }
+  try {
+      let isTruncated = true;
+      let continuationToken: string | undefined = undefined;
+      let total_files = 0;
+      let structured_files = 0;
+      let media_files = 0;
+      
+      const allRecords: any[] = [];
+      const scan_ts = new Date().toISOString() + 'Z';
 
-  // 4. Execute the profiling script
-  return new Promise<ConnectResult>((resolve) => {
-    let stdout = '';
-    let stderr = '';
+      while (isTruncated) {
+          const response = await listS3Objects({
+              vendor: params.vendor as any,
+              bucket: params.bucket as string,
+              endpoint: params.endpoint,
+              access_key: params.access_key,
+              secret_key: params.secret_key,
+              region: params.region
+          }, params.prefix || '', 1000, continuationToken);
+          
+          for (const obj of response.Contents) {
+              const key = obj.Key || '';
+              if (key.endsWith('/')) continue;
 
-    const proc = spawn('python3', args, {
-      cwd: dsDir,
-      env: { ...process.env },
-    });
+              const name = path.basename(key);
+              const ext = path.extname(name).toLowerCase();
+              const mimeType = mime.lookup(name) || '';
+              const { category, mediaType } = classifyFile(ext);
+              const depth = (key.match(/\//g) || []).length;
+              const parentDir = key.includes('/') ? path.basename(path.dirname(key)) : '';
+              
+              total_files++;
+              if (category === 'structured') structured_files++;
+              if (mediaType) media_files++;
 
-    proc.stdout.on('data', (data: Buffer) => {
-      stdout += data.toString();
-    });
+              allRecords.push({
+                  file_path: key,
+                  file_name: name,
+                  extension: ext,
+                  mime_type: mimeType,
+                  category: category,
+                  media_type: mediaType,
+                  size_bytes: obj.Size || 0,
+                  last_modified: obj.LastModified ? String(obj.LastModified) : '',
+                  created_time: obj._created_time ? String(obj._created_time) : '',
+                  etag: (obj.ETag || '').replace(/"/g, ''),
+                  storage_class: obj.StorageClass || '',
+                  is_multipart: (obj.ETag || '').includes('-'),
+                  depth: depth,
+                  parent_dir: parentDir,
+                  vendor: params.vendor,
+                  bucket: params.bucket,
+                  has_schema: false,
+                  has_media_meta: false,
+                  scan_timestamp: scan_ts
+              });
+          }
 
-    proc.stderr.on('data', (data: Buffer) => {
-      stderr += data.toString();
-    });
+          isTruncated = response.IsTruncated || false;
+          continuationToken = response.NextContinuationToken;
+      }
 
-    proc.on('close', (code: number | null) => {
-      if (code !== 0) {
-        resolve({
-          status: 'error',
-          datasource_name: params.datasource_name,
+      await writeLanceCatalog({
           db_path: dbPath,
-          env_path: envPath,
-          tables: [],
-          error: stderr || `Python script exited with code ${code}`,
-        });
-        return;
-      }
+          table_name: 'file_catalog',
+          records: allRecords
+      });
 
-      // Try to parse structured output from the script
-      try {
-        const jsonMatch = stdout.match(/\{[\s\S]*"summary"[\s\S]*\}/);
-        const result = jsonMatch ? JSON.parse(jsonMatch[0]) : {};
-        resolve({
+      return {
           status: 'success',
           datasource_name: params.datasource_name,
           db_path: dbPath,
           env_path: envPath,
-          tables: ['file_catalog', 'structured_schemas', 'media_metadata'],
-          summary: result.summary || {
-            total_files: 0,
-            structured_files: 0,
-            media_files: 0,
-          },
-        });
-      } catch {
-        resolve({
-          status: 'success',
+          tables: ['file_catalog'],
+          summary: {
+              total_files,
+              structured_files,
+              media_files
+          }
+      };
+
+  } catch (error: any) {
+      return {
+          status: 'error',
           datasource_name: params.datasource_name,
           db_path: dbPath,
           env_path: envPath,
-          tables: ['file_catalog', 'structured_schemas', 'media_metadata'],
-        });
-      }
-    });
-  });
+          tables: [],
+          error: error.message
+      };
+  }
 }
 
 export async function listDataSources(

package/src/lib/actions/s3-tools.ts

@@ -0,0 +1,203 @@
+import { S3Client, ListObjectsV2Command, GetObjectCommand } from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import * as fs from 'fs';
+import * as path from 'path';
+
+export interface S3Params {
+    url?: string;
+    vendor?: 'volcengine' | 'alibaba' | 'tencent' | 'aws' | 'local';
+    endpoint?: string;
+    access_key?: string;
+    secret_key?: string;
+    region?: string;
+    bucket?: string;
+}
+
+export function parseS3Url(params: S3Params): S3Params {
+    if (params.url) {
+        if (params.url.startsWith('file://') || params.url.startsWith('/')) {
+            params.vendor = 'local';
+            const localPath = params.url.startsWith('file://') ? params.url.slice(7) : params.url;
+            params.bucket = localPath;
+        } else {
+            const parsedUrl = new URL(params.url);
+            const protocol = parsedUrl.protocol.replace(':', '');
+            
+            if (['tos', 'oss', 'cos', 's3'].includes(protocol)) {
+                if (protocol === 'tos') params.vendor = 'volcengine';
+                else if (protocol === 'oss') params.vendor = 'alibaba';
+                else if (protocol === 'cos') params.vendor = 'tencent';
+                else if (protocol === 's3') params.vendor = 'aws';
+                
+                params.bucket = parsedUrl.hostname;
+                // Prefix is usually parsed separately or passed explicitly for listing
+            } else {
+                throw new Error(`Unsupported protocol: ${protocol}`);
+            }
+        }
+    }
+    
+    if (!params.vendor || !params.bucket) {
+         throw new Error('Could not determine vendor or bucket. Please provide a valid url or vendor/bucket directly.');
+    }
+    
+    return params;
+}
+
+function createS3Client(params: S3Params): S3Client | null {
+    params = parseS3Url(params);
+    if (params.vendor === 'local') return null;
+
+    let endpoint = params.endpoint;
+    let region = params.region;
+    
+    // Attempt to load credentials if not provided
+    let ak = params.access_key;
+    let sk = params.secret_key;
+    
+    if (!ak || !sk || !region) {
+        try {
+            const { loadCredentials } = require('../../utils/credentials');
+            const creds = loadCredentials();
+            ak = ak || creds.ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY;
+            sk = sk || creds.SECRET_KEY || creds.VOLCENGINE_SECRET_KEY;
+            region = region || creds.REGION || creds.VOLCENGINE_REGION;
+        } catch (e) {
+            // ignore
+        }
+    }
+    
+    ak = ak || '';
+    sk = sk || '';
+
+    if (params.vendor === 'volcengine' && !endpoint) {
+        endpoint = `https://tos-s3-${region || 'cn-beijing'}.volces.com`;
+    } else if (params.vendor === 'alibaba' && !endpoint) {
+        endpoint = `https://s3.oss-${region || 'cn-hangzhou'}.aliyuncs.com`;
+    } else if (params.vendor === 'tencent' && !endpoint) {
+        endpoint = `https://cos.${region || 'ap-beijing'}.myqcloud.com`;
+    }
+
+    return new S3Client({
+        region: region || 'us-east-1',
+        endpoint: endpoint,
+        credentials: {
+            accessKeyId: ak,
+            secretAccessKey: sk
+        },
+        forcePathStyle: false // usually false for virtual hosted style
+    });
+}
+
+export async function listS3Objects(params: S3Params, prefix: string, maxKeys: number = 1000, continuationToken?: string) {
+    params = parseS3Url(params);
+    if (params.vendor === 'local') {
+        const root = params.bucket as string;
+        const prefixPath = prefix && prefix !== '.' ? path.join(root, prefix) : root;
+        const files: any[] = [];
+        
+        function walkSync(currentDirPath: string) {
+            if (!fs.existsSync(currentDirPath)) return;
+            const dirents = fs.readdirSync(currentDirPath, { withFileTypes: true });
+            for (const dirent of dirents) {
+                const res = path.resolve(currentDirPath, dirent.name);
+                if (dirent.isDirectory()) {
+                    walkSync(res);
+                } else {
+                    const stat = fs.statSync(res);
+                    files.push({
+                        Key: path.relative(root, res),
+                        Size: stat.size,
+                        LastModified: stat.mtime,
+                        ETag: '',
+                        StorageClass: 'LOCAL',
+                        _created_time: stat.ctime
+                    });
+                }
+            }
+        }
+        walkSync(prefixPath);
+        return {
+            Contents: files,
+            IsTruncated: false,
+            NextContinuationToken: undefined
+        };
+    }
+
+    const client = createS3Client(params);
+    if (!client) throw new Error('Failed to create S3 client');
+
+    const command = new ListObjectsV2Command({
+        Bucket: params.bucket,
+        Prefix: prefix,
+        MaxKeys: maxKeys,
+        ContinuationToken: continuationToken
+    });
+
+    const response = await client.send(command);
+    return {
+        Contents: response.Contents || [],
+        IsTruncated: response.IsTruncated,
+        NextContinuationToken: response.NextContinuationToken
+    };
+}
+
+export async function readS3Object(params: S3Params, key: string, maxBytes?: number): Promise<Buffer> {
+    params = parseS3Url(params);
+    if (params.vendor === 'local') {
+        const fullPath = path.join(params.bucket as string, key);
+        if (maxBytes) {
+            const fd = fs.openSync(fullPath, 'r');
+            const buffer = Buffer.alloc(maxBytes);
+            const bytesRead = fs.readSync(fd, buffer, 0, maxBytes, 0);
+            fs.closeSync(fd);
+            return buffer.subarray(0, bytesRead);
+        } else {
+            return fs.readFileSync(fullPath);
+        }
+    }
+
+    const client = createS3Client(params);
+    if (!client) throw new Error('Failed to create S3 client');
+
+    const commandInput: any = {
+        Bucket: params.bucket,
+        Key: key
+    };
+
+    if (maxBytes) {
+        commandInput.Range = `bytes=0-${maxBytes - 1}`;
+    }
+
+    const command = new GetObjectCommand(commandInput);
+    const response = await client.send(command);
+    
+    if (response.Body) {
+        // @ts-ignore
+        const chunks = [];
+        for await (const chunk of response.Body as any) {
+            chunks.push(chunk);
+        }
+        return Buffer.concat(chunks);
+    }
+    return Buffer.alloc(0);
+}
+
+export async function getPresignedUrl(params: S3Params, key: string, expiresIn: number = 3600): Promise<string> {
+    params = parseS3Url(params);
+    if (params.vendor === 'local') {
+        const fullPath = path.join(params.bucket as string, key);
+        return `file://${fullPath}`;
+    }
+
+    const client = createS3Client(params);
+    if (!client) throw new Error('Failed to create S3 client');
+
+    const command = new GetObjectCommand({
+        Bucket: params.bucket,
+        Key: key
+    });
+
+    const signedUrl = await getSignedUrl(client, command, { expiresIn });
+    return signedUrl;
+}