@byted-las/contextlake-openclaw 1.0.3 → 1.0.4

package/src/commands/tools.ts

@@ -3,6 +3,8 @@ import { retrieveAssets } from '../lib/actions/retrieve';
 import { listAssets, deleteAssets } from '../lib/actions/manage';
 import { connectDataSource, listDataSources } from '../lib/actions/profiler';
 import { getLasTools } from '../lib/actions/las-tools';
+import { listS3Objects, readS3Object } from '../lib/actions/s3-tools';
+import { writeLanceCatalog } from '../lib/actions/lance-tools';
 import { ContextLakeConfig } from '../utils/config';
 // @ts-ignore
 import type { AnyAgentTool } from 'openclaw/plugin-sdk';
@@ -14,6 +16,9 @@ export function getAgentTools(pluginConfig: ContextLakeConfig, logger: any): {
     deleteTool: AnyAgentTool;
     lasDataProfilerTool: AnyAgentTool;
     listDatasourceTool: AnyAgentTool;
+    listS3ObjectsTool: AnyAgentTool;
+    readS3ObjectTool: AnyAgentTool;
+    writeLanceCatalogTool: AnyAgentTool;
     lasTools: AnyAgentTool[];
 } {
     const lasTools = getLasTools(pluginConfig, logger);
@@ -299,13 +304,95 @@ Example User Queries:
                 } catch (error: any) {
                     logger.error(`[${new Date().toISOString()}] [ContextLake] las-data-profiler skill failed`, { error: error.message, stack: error.stack });
                     return {
-                        content: [{ type: "text", text: String(error.message
-                    ) }],
-                        details: { error: error.message
-                     }
+                        content: [{ type: "text", text: String(error.message) }],
+                        details: { error: error.message }
                     } as any;
                 }
             }
+        },
+        listS3ObjectsTool: {
+            name: 'list-s3-objects',
+            label: 'List S3 Objects',
+            description: 'List objects in an S3-compatible bucket or local directory',
+            parameters: {
+                type: 'object',
+                properties: {
+                    vendor: { type: 'string', enum: ['volcengine', 'alibaba', 'tencent', 'aws', 'local'] },
+                    bucket: { type: 'string' },
+                    prefix: { type: 'string' },
+                    endpoint: { type: 'string' },
+                    access_key: { type: 'string' },
+                    secret_key: { type: 'string' },
+                    region: { type: 'string' },
+                    maxKeys: { type: 'integer' },
+                    continuationToken: { type: 'string' }
+                },
+                required: ['vendor', 'bucket'],
+                additionalProperties: false
+            },
+            async execute(toolCallId: string, params: any) {
+                let actualParams = params.params || params;
+                try {
+                    const result = await listS3Objects(actualParams, actualParams.prefix || '', actualParams.maxKeys, actualParams.continuationToken);
+                    return { content: [{ type: "text", text: JSON.stringify(result) }], details: result } as any;
+                } catch (e: any) {
+                    return { content: [{ type: "text", text: String(e.message) }], details: { error: e.message } } as any;
+                }
+            }
+        },
+        readS3ObjectTool: {
+            name: 'read-s3-object',
+            label: 'Read S3 Object',
+            description: 'Read the contents or headers of an S3 object',
+            parameters: {
+                type: 'object',
+                properties: {
+                    vendor: { type: 'string', enum: ['volcengine', 'alibaba', 'tencent', 'aws', 'local'] },
+                    bucket: { type: 'string' },
+                    key: { type: 'string' },
+                    endpoint: { type: 'string' },
+                    access_key: { type: 'string' },
+                    secret_key: { type: 'string' },
+                    region: { type: 'string' },
+                    maxBytes: { type: 'integer' }
+                },
+                required: ['vendor', 'bucket', 'key'],
+                additionalProperties: false
+            },
+            async execute(toolCallId: string, params: any) {
+                let actualParams = params.params || params;
+                try {
+                    const buf = await readS3Object(actualParams, actualParams.key, actualParams.maxBytes);
+                    // Return as base64 string
+                    return { content: [{ type: "text", text: buf.toString('base64') }], details: { length: buf.length } } as any;
+                } catch (e: any) {
+                    return { content: [{ type: "text", text: String(e.message) }], details: { error: e.message } } as any;
+                }
+            }
+        },
+        writeLanceCatalogTool: {
+            name: 'write-lance-catalog',
+            label: 'Write LanceDB Catalog',
+            description: 'Write an array of file records into a local LanceDB table',
+            parameters: {
+                type: 'object',
+                properties: {
+                    db_path: { type: 'string' },
+                    table_name: { type: 'string' },
+                    records: { type: 'array', items: { type: 'object' } }
+                },
+                required: ['db_path', 'table_name', 'records'],
+                additionalProperties: false
+            },
+            async execute(toolCallId: string, params: any) {
+                let actualParams = params.params || params;
+                try {
+                    await writeLanceCatalog(actualParams);
+                    return { content: [{ type: "text", text: "Successfully wrote records to LanceDB" }], details: { count: actualParams.records.length } } as any;
+                } catch (e: any) {
+                    return { content: [{ type: "text", text: String(e.message) }], details: { error: e.message } } as any;
+                }
+            }
         }
     };
 }

package/src/lib/actions/lance-tools.ts

@@ -0,0 +1,23 @@
+import * as lancedb from '@lancedb/lancedb';
+
+export interface LanceWriteParams {
+    db_path: string;
+    table_name: string;
+    records: any[];
+}
+
+export async function writeLanceCatalog(params: LanceWriteParams) {
+    if (!params.records || params.records.length === 0) {
+        return;
+    }
+
+    const db = await lancedb.connect(params.db_path);
+    const tableNames = await db.tableNames();
+    
+    if (tableNames.includes(params.table_name)) {
+        const table = await db.openTable(params.table_name);
+        await table.add(params.records);
+    } else {
+        await db.createTable(params.table_name, params.records);
+    }
+}

package/src/lib/actions/profiler.ts

@@ -1,11 +1,9 @@
 import * as path from 'path';
 import * as fs from 'fs';
 import * as os from 'os';
-import { execSync, spawn } from 'child_process';
-
-// ---------------------------------------------------------------------------
-// Types
-// ---------------------------------------------------------------------------
+import { listS3Objects } from './s3-tools';
+import { writeLanceCatalog } from './lance-tools';
+import * as mime from 'mime-types';
 
 export interface ConnectParams {
   datasource_name: string;
@@ -33,16 +31,7 @@ export interface ConnectResult {
   error?: string;
 }
 
-// ---------------------------------------------------------------------------
-// Constants
-// ---------------------------------------------------------------------------
-
 const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
-const PYTHON_DEPS = ['boto3', 'lancedb', 'pyarrow', 'pandas', 'Pillow', 'mutagen', 'pymupdf'];
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
 
 function getDataSourceDir(name: string): string {
   return path.join(BASE_DIR, name);
@@ -52,10 +41,6 @@ function ensureDir(dir: string): void {
   fs.mkdirSync(dir, { recursive: true });
 }
 
-/**
- * Generate env.sh with all connection parameters for this datasource.
- * This file can be sourced to re-run the profiler or for debugging.
- */
 function writeEnvFile(dir: string, params: ConnectParams): string {
   const envPath = path.join(dir, 'env.sh');
   const lines: string[] = [
@@ -69,21 +54,11 @@ function writeEnvFile(dir: string, params: ConnectParams): string {
     `export LAS_PREFIX="${params.prefix}"`,
   ];
 
-  if (params.endpoint) {
-    lines.push(`export LAS_ENDPOINT="${params.endpoint}"`);
-  }
-  if (params.access_key) {
-    lines.push(`export LAS_ACCESS_KEY="${params.access_key}"`);
-  }
-  if (params.secret_key) {
-    lines.push(`export LAS_SECRET_KEY="${params.secret_key}"`);
-  }
-  if (params.region) {
-    lines.push(`export LAS_REGION="${params.region}"`);
-  }
-  if (params.sample_rows) {
-    lines.push(`export LAS_SAMPLE_ROWS="${params.sample_rows}"`);
-  }
+  if (params.endpoint) lines.push(`export LAS_ENDPOINT="${params.endpoint}"`);
+  if (params.access_key) lines.push(`export LAS_ACCESS_KEY="${params.access_key}"`);
+  if (params.secret_key) lines.push(`export LAS_SECRET_KEY="${params.secret_key}"`);
+  if (params.region) lines.push(`export LAS_REGION="${params.region}"`);
+  if (params.sample_rows) lines.push(`export LAS_SAMPLE_ROWS="${params.sample_rows}"`);
 
   lines.push(`export LAS_DB_PATH="${path.join(dir, 'catalog_db')}"`);
   lines.push(`export LAS_DATASOURCE_NAME="${params.datasource_name}"`);
@@ -93,65 +68,51 @@ function writeEnvFile(dir: string, params: ConnectParams): string {
   return envPath;
 }
 
-/**
- * Install Python dependencies if not already available.
- */
-function ensurePythonDeps(): void {
-  try {
-    execSync(`python3 -c "import boto3, lancedb, pyarrow, pandas, PIL, mutagen, fitz"`, {
-      stdio: 'pipe',
-    });
-  } catch {
-    console.log('[las-data-profiler] Installing Python dependencies...');
-    execSync(`pip3 install --user ${PYTHON_DEPS.join(' ')}`, {
-      stdio: 'inherit',
-    });
-  }
-}
-
-/**
- * Get the path to the bundled Python script.
- */
-function getScriptPath(): string {
-  // The Python script is located in the scripts directory
-  return path.join(__dirname, '../scripts', 's3_catalog.py');
+function classifyFile(ext: string): { category: string; mediaType: string } {
+    ext = ext.toLowerCase();
+    const STRUCTURED_EXTS = ['.json', '.jsonl', '.ndjson', '.csv', '.tsv', '.parquet', '.pq'];
+    const IMAGE_EXTS = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.tiff', '.tif', '.svg', '.ico', '.heic', '.heif'];
+    const AUDIO_EXTS = ['.mp3', '.wav', '.flac', '.aac', '.ogg', '.m4a', '.wma', '.opus'];
+    const VIDEO_EXTS = ['.mp4', '.avi', '.mov', '.mkv', '.webm', '.wmv', '.flv', '.m4v', '.3gp'];
+    const PDF_EXTS = ['.pdf'];
+
+    if (STRUCTURED_EXTS.includes(ext)) return { category: 'structured', mediaType: '' };
+    if (IMAGE_EXTS.includes(ext)) return { category: 'non-structured', mediaType: 'image' };
+    if (AUDIO_EXTS.includes(ext)) return { category: 'non-structured', mediaType: 'audio' };
+    if (VIDEO_EXTS.includes(ext)) return { category: 'non-structured', mediaType: 'video' };
+    if (PDF_EXTS.includes(ext)) return { category: 'non-structured', mediaType: 'pdf' };
+    
+    return { category: 'non-structured', mediaType: '' };
 }
 
-// ---------------------------------------------------------------------------
-// Main Entry
-// ---------------------------------------------------------------------------
-
 export async function connectDataSource(
   params: ConnectParams,
   _ctx?: any
 ): Promise<ConnectResult> {
-  // Validate required params
-  if (!params.datasource_name) {
-    throw new Error('datasource_name is required');
-  }
-  if (!params.vendor) {
-    throw new Error('vendor is required');
-  }
-  if (!params.bucket) {
-    throw new Error('bucket is required');
-  }
-  if (params.prefix === undefined || params.prefix === null) {
-    throw new Error('prefix is required');
-  }
+  if (!params.datasource_name) throw new Error('datasource_name is required');
+  if (!params.vendor) throw new Error('vendor is required');
+  if (!params.bucket) throw new Error('bucket is required');
+  if (params.prefix === undefined || params.prefix === null) throw new Error('prefix is required');
 
-  // For non-local vendors, validate credentials
   if (params.vendor !== 'local') {
-    if (!params.endpoint && params.vendor !== 'aws') {
-      throw new Error(`endpoint is required for vendor "${params.vendor}"`);
+    if (!params.endpoint && params.vendor !== 'aws') throw new Error(`endpoint is required for vendor "${params.vendor}"`);
+    let ak = params.access_key;
+    let sk = params.secret_key;
+    
+    if (!ak || !sk) {
+        try {
+            const { loadCredentials } = require('../../utils/credentials');
+            const creds = loadCredentials();
+            ak = ak || creds.ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY;
+            sk = sk || creds.SECRET_KEY || creds.VOLCENGINE_SECRET_KEY;
+        } catch(e) {
+            // ignore
+        }
     }
-    const ak = params.access_key || process.env.TOS_ACCESS_KEY || process.env.S3_ACCESS_KEY || process.env.AWS_ACCESS_KEY_ID;
-    const sk = params.secret_key || process.env.TOS_SECRET_KEY || process.env.S3_SECRET_KEY || process.env.AWS_SECRET_ACCESS_KEY;
+    
     if (!ak || !sk) {
-      throw new Error(
-        'access_key and secret_key are required (via params or env vars TOS_ACCESS_KEY/TOS_SECRET_KEY, S3_ACCESS_KEY/S3_SECRET_KEY, AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY)'
-      );
+      throw new Error('access_key and secret_key are required');
     }
-    // Normalise into params so env.sh picks them up
     params.access_key = ak;
     params.secret_key = sk;
   }
@@ -160,97 +121,92 @@ export async function connectDataSource(
   const dbPath = path.join(dsDir, 'catalog_db');
 
   ensureDir(dsDir);
-
-  // 1. Write env.sh
   const envPath = writeEnvFile(dsDir, params);
 
-  // 2. Ensure Python dependencies
-  ensurePythonDeps();
-
-  // 3. Build CLI args for the Python script
-  const scriptPath = getScriptPath();
-  const args: string[] = [
-    scriptPath,
-    '--vendor', params.vendor,
-    '--bucket', params.bucket,
-    '--prefix', params.prefix,
-    '--db-path', dbPath,
-  ];
-
-  if (params.endpoint) {
-    args.push('--endpoint', params.endpoint);
-  }
-  if (params.access_key) {
-    args.push('--ak', params.access_key);
-  }
-  if (params.secret_key) {
-    args.push('--sk', params.secret_key);
-  }
-  if (params.region) {
-    args.push('--region', params.region);
-  }
-  if (params.sample_rows) {
-    args.push('--sample-rows', String(params.sample_rows));
-  }
-
-  // 4. Execute the profiling script
-  return new Promise<ConnectResult>((resolve) => {
-    let stdout = '';
-    let stderr = '';
-
-    const proc = spawn('python3', args, {
-      cwd: dsDir,
-      env: { ...process.env },
-    });
-
-    proc.stdout.on('data', (data: Buffer) => {
-      stdout += data.toString();
-    });
-
-    proc.stderr.on('data', (data: Buffer) => {
-      stderr += data.toString();
-    });
+  try {
+      let isTruncated = true;
+      let continuationToken: string | undefined = undefined;
+      let total_files = 0;
+      let structured_files = 0;
+      let media_files = 0;
+      
+      const allRecords: any[] = [];
+      const scan_ts = new Date().toISOString() + 'Z';
+
+      while (isTruncated) {
+          const response = await listS3Objects(params, params.prefix, 1000, continuationToken);
+          
+          for (const obj of response.Contents) {
+              const key = obj.Key || '';
+              if (key.endsWith('/')) continue;
+
+              const name = path.basename(key);
+              const ext = path.extname(name).toLowerCase();
+              const mimeType = mime.lookup(name) || '';
+              const { category, mediaType } = classifyFile(ext);
+              const depth = (key.match(/\//g) || []).length;
+              const parentDir = key.includes('/') ? path.basename(path.dirname(key)) : '';
+              
+              total_files++;
+              if (category === 'structured') structured_files++;
+              if (mediaType) media_files++;
+
+              allRecords.push({
+                  file_path: key,
+                  file_name: name,
+                  extension: ext,
+                  mime_type: mimeType,
+                  category: category,
+                  media_type: mediaType,
+                  size_bytes: obj.Size || 0,
+                  last_modified: obj.LastModified ? String(obj.LastModified) : '',
+                  created_time: obj._created_time ? String(obj._created_time) : '',
+                  etag: (obj.ETag || '').replace(/"/g, ''),
+                  storage_class: obj.StorageClass || '',
+                  is_multipart: (obj.ETag || '').includes('-'),
+                  depth: depth,
+                  parent_dir: parentDir,
+                  vendor: params.vendor,
+                  bucket: params.bucket,
+                  has_schema: false,
+                  has_media_meta: false,
+                  scan_timestamp: scan_ts
+              });
+          }
+
+          isTruncated = response.IsTruncated || false;
+          continuationToken = response.NextContinuationToken;
+      }
 
-    proc.on('close', (code: number | null) => {
-      if (code !== 0) {
-        resolve({
-          status: 'error',
-          datasource_name: params.datasource_name,
+      await writeLanceCatalog({
           db_path: dbPath,
-          env_path: envPath,
-          tables: [],
-          error: stderr || `Python script exited with code ${code}`,
-        });
-        return;
-      }
+          table_name: 'file_catalog',
+          records: allRecords
+      });
 
-      // Try to parse structured output from the script
-      try {
-        const jsonMatch = stdout.match(/\{[\s\S]*"summary"[\s\S]*\}/);
-        const result = jsonMatch ? JSON.parse(jsonMatch[0]) : {};
-        resolve({
+      return {
           status: 'success',
           datasource_name: params.datasource_name,
           db_path: dbPath,
           env_path: envPath,
-          tables: ['file_catalog', 'structured_schemas', 'media_metadata'],
-          summary: result.summary || {
-            total_files: 0,
-            structured_files: 0,
-            media_files: 0,
-          },
-        });
-      } catch {
-        resolve({
-          status: 'success',
+          tables: ['file_catalog'],
+          summary: {
+              total_files,
+              structured_files,
+              media_files
+          }
+      };
+
+  } catch (error: any) {
+      return {
+          status: 'error',
           datasource_name: params.datasource_name,
           db_path: dbPath,
           env_path: envPath,
-          tables: ['file_catalog', 'structured_schemas', 'media_metadata'],
-        });
-      }
-    });
-  });
+          tables: [],
+          error: error.message
+      };
+  }
 }
 
 export async function listDataSources(

package/src/lib/actions/s3-tools.ts

@@ -0,0 +1,148 @@
+import { S3Client, ListObjectsV2Command, GetObjectCommand } from '@aws-sdk/client-s3';
+import * as fs from 'fs';
+import * as path from 'path';
+
+export interface S3Params {
+    vendor: 'volcengine' | 'alibaba' | 'tencent' | 'aws' | 'local';
+    endpoint?: string;
+    access_key?: string;
+    secret_key?: string;
+    region?: string;
+    bucket: string;
+}
+
+function createS3Client(params: S3Params): S3Client | null {
+    if (params.vendor === 'local') return null;
+
+    let endpoint = params.endpoint;
+    let region = params.region;
+    
+    // Attempt to load credentials if not provided
+    let ak = params.access_key;
+    let sk = params.secret_key;
+    
+    if (!ak || !sk || !region) {
+        try {
+            const { loadCredentials } = require('../../utils/credentials');
+            const creds = loadCredentials();
+            ak = ak || creds.ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY;
+            sk = sk || creds.SECRET_KEY || creds.VOLCENGINE_SECRET_KEY;
+            region = region || creds.REGION || creds.VOLCENGINE_REGION;
+        } catch (e) {
+            // ignore
+        }
+    }
+    
+    ak = ak || '';
+    sk = sk || '';
+
+    if (params.vendor === 'volcengine' && !endpoint) {
+        endpoint = `https://tos-s3-${region || 'cn-beijing'}.volces.com`;
+    } else if (params.vendor === 'alibaba' && !endpoint) {
+        endpoint = `https://s3.oss-${region || 'cn-hangzhou'}.aliyuncs.com`;
+    } else if (params.vendor === 'tencent' && !endpoint) {
+        endpoint = `https://cos.${region || 'ap-beijing'}.myqcloud.com`;
+    }
+
+    return new S3Client({
+        region: region || 'us-east-1',
+        endpoint: endpoint,
+        credentials: {
+            accessKeyId: ak,
+            secretAccessKey: sk
+        },
+        forcePathStyle: false // usually false for virtual hosted style
+    });
+}
+
+export async function listS3Objects(params: S3Params, prefix: string, maxKeys: number = 1000, continuationToken?: string) {
+    if (params.vendor === 'local') {
+        const root = params.bucket;
+        const prefixPath = prefix && prefix !== '.' ? path.join(root, prefix) : root;
+        const files: any[] = [];
+        
+        function walkSync(currentDirPath: string) {
+            if (!fs.existsSync(currentDirPath)) return;
+            const dirents = fs.readdirSync(currentDirPath, { withFileTypes: true });
+            for (const dirent of dirents) {
+                const res = path.resolve(currentDirPath, dirent.name);
+                if (dirent.isDirectory()) {
+                    walkSync(res);
+                } else {
+                    const stat = fs.statSync(res);
+                    files.push({
+                        Key: path.relative(root, res),
+                        Size: stat.size,
+                        LastModified: stat.mtime,
+                        ETag: '',
+                        StorageClass: 'LOCAL',
+                        _created_time: stat.ctime
+                    });
+                }
+            }
+        }
+        walkSync(prefixPath);
+        return {
+            Contents: files,
+            IsTruncated: false,
+            NextContinuationToken: undefined
+        };
+    }
+
+    const client = createS3Client(params);
+    if (!client) throw new Error('Failed to create S3 client');
+
+    const command = new ListObjectsV2Command({
+        Bucket: params.bucket,
+        Prefix: prefix,
+        MaxKeys: maxKeys,
+        ContinuationToken: continuationToken
+    });
+
+    const response = await client.send(command);
+    return {
+        Contents: response.Contents || [],
+        IsTruncated: response.IsTruncated,
+        NextContinuationToken: response.NextContinuationToken
+    };
+}
+
+export async function readS3Object(params: S3Params, key: string, maxBytes?: number): Promise<Buffer> {
+    if (params.vendor === 'local') {
+        const fullPath = path.join(params.bucket, key);
+        if (maxBytes) {
+            const fd = fs.openSync(fullPath, 'r');
+            const buffer = Buffer.alloc(maxBytes);
+            const bytesRead = fs.readSync(fd, buffer, 0, maxBytes, 0);
+            fs.closeSync(fd);
+            return buffer.subarray(0, bytesRead);
+        } else {
+            return fs.readFileSync(fullPath);
+        }
+    }
+
+    const client = createS3Client(params);
+    if (!client) throw new Error('Failed to create S3 client');
+
+    const commandInput: any = {
+        Bucket: params.bucket,
+        Key: key
+    };
+
+    if (maxBytes) {
+        commandInput.Range = `bytes=0-${maxBytes - 1}`;
+    }
+
+    const command = new GetObjectCommand(commandInput);
+    const response = await client.send(command);
+    
+    if (response.Body) {
+        // @ts-ignore
+        const chunks = [];
+        for await (const chunk of response.Body as any) {
+            chunks.push(chunk);
+        }
+        return Buffer.concat(chunks);
+    }
+    return Buffer.alloc(0);
+}