@byted-las/contextlake-openclaw 1.0.0

package/src/skills/las-data-profiler/index.ts

@@ -0,0 +1,254 @@
+import * as path from 'path';
+import * as fs from 'fs';
+import * as os from 'os';
+import { execSync, spawn } from 'child_process';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ConnectParams {
+  datasource_name: string;
+  vendor: 'volcengine' | 'alibaba' | 'tencent' | 'aws' | 'local';
+  endpoint?: string;
+  access_key?: string;
+  secret_key?: string;
+  region?: string;
+  bucket: string;
+  prefix: string;
+  sample_rows?: number;
+}
+
+interface ConnectResult {
+  status: 'success' | 'error';
+  datasource_name: string;
+  db_path: string;
+  env_path: string;
+  tables: string[];
+  summary?: {
+    total_files: number;
+    structured_files: number;
+    media_files: number;
+  };
+  error?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const BASE_DIR = path.join(os.homedir(), '.openclaw', 'las-data-profiler');
+const PYTHON_DEPS = ['boto3', 'lancedb', 'pyarrow', 'pandas', 'Pillow', 'mutagen', 'pymupdf'];
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function getDataSourceDir(name: string): string {
+  return path.join(BASE_DIR, name);
+}
+
+function ensureDir(dir: string): void {
+  fs.mkdirSync(dir, { recursive: true });
+}
+
+/**
+ * Generate env.sh with all connection parameters for this datasource.
+ * This file can be sourced to re-run the profiler or for debugging.
+ */
+function writeEnvFile(dir: string, params: ConnectParams): string {
+  const envPath = path.join(dir, 'env.sh');
+  const lines: string[] = [
+    '#!/usr/bin/env bash',
+    '# Auto-generated by las-data-profiler connect',
+    `# Datasource: ${params.datasource_name}`,
+    `# Created: ${new Date().toISOString()}`,
+    '',
+    `export LAS_VENDOR="${params.vendor}"`,
+    `export LAS_BUCKET="${params.bucket}"`,
+    `export LAS_PREFIX="${params.prefix}"`,
+  ];
+
+  if (params.endpoint) {
+    lines.push(`export LAS_ENDPOINT="${params.endpoint}"`);
+  }
+  if (params.access_key) {
+    lines.push(`export LAS_ACCESS_KEY="${params.access_key}"`);
+  }
+  if (params.secret_key) {
+    lines.push(`export LAS_SECRET_KEY="${params.secret_key}"`);
+  }
+  if (params.region) {
+    lines.push(`export LAS_REGION="${params.region}"`);
+  }
+  if (params.sample_rows) {
+    lines.push(`export LAS_SAMPLE_ROWS="${params.sample_rows}"`);
+  }
+
+  lines.push(`export LAS_DB_PATH="${path.join(dir, 'catalog_db')}"`);
+  lines.push(`export LAS_DATASOURCE_NAME="${params.datasource_name}"`);
+  lines.push('');
+
+  fs.writeFileSync(envPath, lines.join('\n'), { mode: 0o600 });
+  return envPath;
+}
+
+/**
+ * Install Python dependencies if not already available.
+ */
+function ensurePythonDeps(): void {
+  try {
+    execSync(`python3 -c "import boto3, lancedb, pyarrow, pandas, PIL, mutagen, fitz"`, {
+      stdio: 'pipe',
+    });
+  } catch {
+    console.log('[las-data-profiler] Installing Python dependencies...');
+    execSync(`pip3 install --user ${PYTHON_DEPS.join(' ')}`, {
+      stdio: 'inherit',
+    });
+  }
+}
+
+/**
+ * Get the path to the bundled Python script.
+ */
+function getScriptPath(): string {
+  // The Python script is co-located with this module
+  return path.join(__dirname, 's3_catalog.py');
+}
+
+// ---------------------------------------------------------------------------
+// Main Entry
+// ---------------------------------------------------------------------------
+
+export async function connectDataSource(
+  params: ConnectParams,
+  _ctx?: any
+): Promise<ConnectResult> {
+  // Validate required params
+  if (!params.datasource_name) {
+    throw new Error('datasource_name is required');
+  }
+  if (!params.vendor) {
+    throw new Error('vendor is required');
+  }
+  if (!params.bucket) {
+    throw new Error('bucket is required');
+  }
+  if (params.prefix === undefined || params.prefix === null) {
+    throw new Error('prefix is required');
+  }
+
+  // For non-local vendors, validate credentials
+  if (params.vendor !== 'local') {
+    if (!params.endpoint && params.vendor !== 'aws') {
+      throw new Error(`endpoint is required for vendor "${params.vendor}"`);
+    }
+    const ak = params.access_key || process.env.TOS_ACCESS_KEY || process.env.S3_ACCESS_KEY || process.env.AWS_ACCESS_KEY_ID;
+    const sk = params.secret_key || process.env.TOS_SECRET_KEY || process.env.S3_SECRET_KEY || process.env.AWS_SECRET_ACCESS_KEY;
+    if (!ak || !sk) {
+      throw new Error(
+        'access_key and secret_key are required (via params or env vars TOS_ACCESS_KEY/TOS_SECRET_KEY, S3_ACCESS_KEY/S3_SECRET_KEY, AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY)'
+      );
+    }
+    // Normalise into params so env.sh picks them up
+    params.access_key = ak;
+    params.secret_key = sk;
+  }
+
+  const dsDir = getDataSourceDir(params.datasource_name);
+  const dbPath = path.join(dsDir, 'catalog_db');
+
+  ensureDir(dsDir);
+
+  // 1. Write env.sh
+  const envPath = writeEnvFile(dsDir, params);
+
+  // 2. Ensure Python dependencies
+  ensurePythonDeps();
+
+  // 3. Build CLI args for the Python script
+  const scriptPath = getScriptPath();
+  const args: string[] = [
+    scriptPath,
+    '--vendor', params.vendor,
+    '--bucket', params.bucket,
+    '--prefix', params.prefix,
+    '--db-path', dbPath,
+  ];
+
+  if (params.endpoint) {
+    args.push('--endpoint', params.endpoint);
+  }
+  if (params.access_key) {
+    args.push('--ak', params.access_key);
+  }
+  if (params.secret_key) {
+    args.push('--sk', params.secret_key);
+  }
+  if (params.region) {
+    args.push('--region', params.region);
+  }
+  if (params.sample_rows) {
+    args.push('--sample-rows', String(params.sample_rows));
+  }
+
+  // 4. Execute the profiling script
+  return new Promise<ConnectResult>((resolve) => {
+    let stdout = '';
+    let stderr = '';
+
+    const proc = spawn('python3', args, {
+      cwd: dsDir,
+      env: { ...process.env },
+    });
+
+    proc.stdout.on('data', (data: Buffer) => {
+      stdout += data.toString();
+    });
+
+    proc.stderr.on('data', (data: Buffer) => {
+      stderr += data.toString();
+    });
+
+    proc.on('close', (code: number | null) => {
+      if (code !== 0) {
+        resolve({
+          status: 'error',
+          datasource_name: params.datasource_name,
+          db_path: dbPath,
+          env_path: envPath,
+          tables: [],
+          error: stderr || `Python script exited with code ${code}`,
+        });
+        return;
+      }
+
+      // Try to parse structured output from the script
+      try {
+        const jsonMatch = stdout.match(/\{[\s\S]*"summary"[\s\S]*\}/);
+        const result = jsonMatch ? JSON.parse(jsonMatch[0]) : {};
+        resolve({
+          status: 'success',
+          datasource_name: params.datasource_name,
+          db_path: dbPath,
+          env_path: envPath,
+          tables: ['file_catalog', 'structured_schemas', 'media_metadata'],
+          summary: result.summary || {
+            total_files: 0,
+            structured_files: 0,
+            media_files: 0,
+          },
+        });
+      } catch {
+        resolve({
+          status: 'success',
+          datasource_name: params.datasource_name,
+          db_path: dbPath,
+          env_path: envPath,
+          tables: ['file_catalog', 'structured_schemas', 'media_metadata'],
+        });
+      }
+    });
+  });
+}

package/src/skills/las-data-profiler/register.ts

@@ -0,0 +1,19 @@
+// @ts-ignore
+import { PluginContext } from 'openclaw/plugin-sdk';
+import { connectDataSource } from './index';
+
+export function registerLasDataProfilerSkill(ctx: any) {
+  const definition = {
+    name: 'las-data-profiler',
+    description: 'Connect to a data source (TOS/OSS/COS/S3/Local) and profile its structure, schemas, and media metadata into LanceDB',
+    async execute(params: any) {
+      return await connectDataSource(params, ctx);
+    }
+  };
+
+  if (typeof ctx.registerTool === 'function') {
+    ctx.registerTool(definition);
+  } else if (typeof ctx.registerSkill === 'function') {
+    ctx.registerSkill(definition);
+  }
+}