@byted-las/contextlake-openclaw 1.0.0

package/dist/src/skills/las-data-profiler/index.js

@@ -0,0 +1,231 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.connectDataSource = connectDataSource;
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
+const child_process_1 = require("child_process");
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+const BASE_DIR = path.join(os.homedir(), '.openclaw', 'las-data-profiler');
+const PYTHON_DEPS = ['boto3', 'lancedb', 'pyarrow', 'pandas', 'Pillow', 'mutagen', 'pymupdf'];
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function getDataSourceDir(name) {
+    return path.join(BASE_DIR, name);
+}
+function ensureDir(dir) {
+    fs.mkdirSync(dir, { recursive: true });
+}
+/**
+ * Generate env.sh with all connection parameters for this datasource.
+ * This file can be sourced to re-run the profiler or for debugging.
+ */
+function writeEnvFile(dir, params) {
+    const envPath = path.join(dir, 'env.sh');
+    const lines = [
+        '#!/usr/bin/env bash',
+        '# Auto-generated by las-data-profiler connect',
+        `# Datasource: ${params.datasource_name}`,
+        `# Created: ${new Date().toISOString()}`,
+        '',
+        `export LAS_VENDOR="${params.vendor}"`,
+        `export LAS_BUCKET="${params.bucket}"`,
+        `export LAS_PREFIX="${params.prefix}"`,
+    ];
+    if (params.endpoint) {
+        lines.push(`export LAS_ENDPOINT="${params.endpoint}"`);
+    }
+    if (params.access_key) {
+        lines.push(`export LAS_ACCESS_KEY="${params.access_key}"`);
+    }
+    if (params.secret_key) {
+        lines.push(`export LAS_SECRET_KEY="${params.secret_key}"`);
+    }
+    if (params.region) {
+        lines.push(`export LAS_REGION="${params.region}"`);
+    }
+    if (params.sample_rows) {
+        lines.push(`export LAS_SAMPLE_ROWS="${params.sample_rows}"`);
+    }
+    lines.push(`export LAS_DB_PATH="${path.join(dir, 'catalog_db')}"`);
+    lines.push(`export LAS_DATASOURCE_NAME="${params.datasource_name}"`);
+    lines.push('');
+    fs.writeFileSync(envPath, lines.join('\n'), { mode: 0o600 });
+    return envPath;
+}
+/**
+ * Install Python dependencies if not already available.
+ */
+function ensurePythonDeps() {
+    try {
+        (0, child_process_1.execSync)(`python3 -c "import boto3, lancedb, pyarrow, pandas, PIL, mutagen, fitz"`, {
+            stdio: 'pipe',
+        });
+    }
+    catch {
+        console.log('[las-data-profiler] Installing Python dependencies...');
+        (0, child_process_1.execSync)(`pip3 install --user ${PYTHON_DEPS.join(' ')}`, {
+            stdio: 'inherit',
+        });
+    }
+}
+/**
+ * Get the path to the bundled Python script.
+ */
+function getScriptPath() {
+    // The Python script is co-located with this module
+    return path.join(__dirname, 's3_catalog.py');
+}
+// ---------------------------------------------------------------------------
+// Main Entry
+// ---------------------------------------------------------------------------
+async function connectDataSource(params, _ctx) {
+    // Validate required params
+    if (!params.datasource_name) {
+        throw new Error('datasource_name is required');
+    }
+    if (!params.vendor) {
+        throw new Error('vendor is required');
+    }
+    if (!params.bucket) {
+        throw new Error('bucket is required');
+    }
+    if (params.prefix === undefined || params.prefix === null) {
+        throw new Error('prefix is required');
+    }
+    // For non-local vendors, validate credentials
+    if (params.vendor !== 'local') {
+        if (!params.endpoint && params.vendor !== 'aws') {
+            throw new Error(`endpoint is required for vendor "${params.vendor}"`);
+        }
+        const ak = params.access_key || process.env.TOS_ACCESS_KEY || process.env.S3_ACCESS_KEY || process.env.AWS_ACCESS_KEY_ID;
+        const sk = params.secret_key || process.env.TOS_SECRET_KEY || process.env.S3_SECRET_KEY || process.env.AWS_SECRET_ACCESS_KEY;
+        if (!ak || !sk) {
+            throw new Error('access_key and secret_key are required (via params or env vars TOS_ACCESS_KEY/TOS_SECRET_KEY, S3_ACCESS_KEY/S3_SECRET_KEY, AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY)');
+        }
+        // Normalise into params so env.sh picks them up
+        params.access_key = ak;
+        params.secret_key = sk;
+    }
+    const dsDir = getDataSourceDir(params.datasource_name);
+    const dbPath = path.join(dsDir, 'catalog_db');
+    ensureDir(dsDir);
+    // 1. Write env.sh
+    const envPath = writeEnvFile(dsDir, params);
+    // 2. Ensure Python dependencies
+    ensurePythonDeps();
+    // 3. Build CLI args for the Python script
+    const scriptPath = getScriptPath();
+    const args = [
+        scriptPath,
+        '--vendor', params.vendor,
+        '--bucket', params.bucket,
+        '--prefix', params.prefix,
+        '--db-path', dbPath,
+    ];
+    if (params.endpoint) {
+        args.push('--endpoint', params.endpoint);
+    }
+    if (params.access_key) {
+        args.push('--ak', params.access_key);
+    }
+    if (params.secret_key) {
+        args.push('--sk', params.secret_key);
+    }
+    if (params.region) {
+        args.push('--region', params.region);
+    }
+    if (params.sample_rows) {
+        args.push('--sample-rows', String(params.sample_rows));
+    }
+    // 4. Execute the profiling script
+    return new Promise((resolve) => {
+        let stdout = '';
+        let stderr = '';
+        const proc = (0, child_process_1.spawn)('python3', args, {
+            cwd: dsDir,
+            env: { ...process.env },
+        });
+        proc.stdout.on('data', (data) => {
+            stdout += data.toString();
+        });
+        proc.stderr.on('data', (data) => {
+            stderr += data.toString();
+        });
+        proc.on('close', (code) => {
+            if (code !== 0) {
+                resolve({
+                    status: 'error',
+                    datasource_name: params.datasource_name,
+                    db_path: dbPath,
+                    env_path: envPath,
+                    tables: [],
+                    error: stderr || `Python script exited with code ${code}`,
+                });
+                return;
+            }
+            // Try to parse structured output from the script
+            try {
+                const jsonMatch = stdout.match(/\{[\s\S]*"summary"[\s\S]*\}/);
+                const result = jsonMatch ? JSON.parse(jsonMatch[0]) : {};
+                resolve({
+                    status: 'success',
+                    datasource_name: params.datasource_name,
+                    db_path: dbPath,
+                    env_path: envPath,
+                    tables: ['file_catalog', 'structured_schemas', 'media_metadata'],
+                    summary: result.summary || {
+                        total_files: 0,
+                        structured_files: 0,
+                        media_files: 0,
+                    },
+                });
+            }
+            catch {
+                resolve({
+                    status: 'success',
+                    datasource_name: params.datasource_name,
+                    db_path: dbPath,
+                    env_path: envPath,
+                    tables: ['file_catalog', 'structured_schemas', 'media_metadata'],
+                });
+            }
+        });
+    });
+}

package/dist/src/skills/las-data-profiler/register.d.ts

@@ -0,0 +1 @@
+export declare function registerLasDataProfilerSkill(ctx: any): void;

package/dist/src/skills/las-data-profiler/register.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerLasDataProfilerSkill = registerLasDataProfilerSkill;
+const index_1 = require("./index");
+function registerLasDataProfilerSkill(ctx) {
+    const definition = {
+        name: 'las-data-profiler',
+        description: 'Connect to a data source (TOS/OSS/COS/S3/Local) and profile its structure, schemas, and media metadata into LanceDB',
+        async execute(params) {
+            return await (0, index_1.connectDataSource)(params, ctx);
+        }
+    };
+    if (typeof ctx.registerTool === 'function') {
+        ctx.registerTool(definition);
+    }
+    else if (typeof ctx.registerSkill === 'function') {
+        ctx.registerSkill(definition);
+    }
+}

package/dist/src/utils/config.d.ts

@@ -0,0 +1 @@
+export declare function getPluginConfig(ctx: any): any;

package/dist/src/utils/config.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getPluginConfig = getPluginConfig;
+function getPluginConfig(ctx) {
+    return ctx.config?.plugins?.entries?.['contextlake-openclaw']?.config || {
+        metadata_storage: {
+            type: 'local',
+            lancedb_uri: require('path').join(require('os').homedir(), '.openclaw', 'contextlake', 'data'),
+            embedding: {
+                provider: 'local',
+                model_name: 'hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf'
+            }
+        },
+        file_storage: { type: 'local', local_base_dir: require('path').join(require('os').homedir(), '.openclaw', 'contextlake', 'files') }
+    };
+}

package/index.ts

@@ -0,0 +1,78 @@
+// @ts-ignore
+import { PluginContext } from 'openclaw/plugin-sdk';
+import { registerAll } from './src/commands';
+
+const plugin = {
+  id: 'contextlake-openclaw',
+  name: 'ContextLake',
+  version: '1.1.0',
+  description: 'A lightweight knowledge base plugin for OpenClaw using LanceDB and TOS, with data profiling support',
+  configSchema: {
+    type: 'object',
+    properties: {
+      metadata_storage: {
+        type: 'object',
+        properties: {
+          type: { type: 'string', enum: ['local', 'remote'], default: 'local' },
+          lancedb_uri: { type: 'string', default: './data/contextlake' },
+          remote_api_endpoint: { type: 'string' },
+          remote_api_key: { type: 'string' },
+          embedding: {
+            type: 'object',
+            properties: {
+              provider: { type: 'string', enum: ['local', 'remote', 'openai'], default: 'local' },
+              model_name: { type: 'string', default: 'hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf' },
+              api_key: { type: 'string' },
+              api_base: { type: 'string' }
+            },
+            default: { provider: 'local', model_name: 'hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf' }
+          }
+        },
+        default: { type: 'local', lancedb_uri: './data/contextlake' }
+      },
+      file_storage: {
+        type: 'object',
+        properties: {
+          type: { type: 'string', enum: ['local', 'tos'], default: 'local' },
+          local_base_dir: { type: 'string', default: './data/files' },
+          tos: {
+            type: 'object',
+            properties: {
+              access_key: { type: 'string' },
+              secret_key: { type: 'string' },
+              region: { type: 'string' },
+              path: { type: 'string', description: 'TOS path in format tos://bucket/base_path/' },
+              endpoint: { type: 'string' },
+              sts_token: { type: 'string' }
+            }
+          }
+        },
+        default: { type: 'local', local_base_dir: './data/files' }
+      },
+      storage_policy: {
+        type: 'object',
+        properties: {
+          max_inline_size_kb: { type: 'number', default: 1024, description: 'Files smaller than this size (in KB) will be stored directly in LanceDB' }
+        }
+      }
+    }
+  },
+  register(ctx: any) {
+    const logger = ctx.logger || {
+        info: (msg: string, ...args: any[]) => console.log(msg, ...args),
+        warn: (msg: string, ...args: any[]) => console.warn(msg, ...args),
+        error: (msg: string, ...args: any[]) => console.error(msg, ...args),
+        debug: (msg: string, ...args: any[]) => console.debug(msg, ...args),
+    };
+
+    // Add logging
+    logger.info(`[${new Date().toISOString()}] [ContextLake] Plugin register started`);
+
+    // Delegate all registrations to commands/index.ts
+    registerAll(ctx, logger);
+    
+    logger.info(`[${new Date().toISOString()}] [ContextLake] Plugin register completed`);
+  }
+};
+
+export default plugin;

package/openclaw.plugin.json

@@ -0,0 +1,57 @@
+{
+  "id": "contextlake-openclaw",
+  "name": "ContextLake",
+  "version": "1.1.0",
+  "description": "A lightweight knowledge base plugin for OpenClaw using LanceDB and TOS, with data profiling support",
+  "skills": ["./src/skills"],
+  "configSchema": {
+    "type": "object",
+    "properties": {
+      "metadata_storage": {
+        "type": "object",
+        "properties": {
+          "type": { "type": "string", "enum": ["local", "remote"], "default": "local" },
+          "lancedb_uri": { "type": "string", "default": "./data/contextlake" },
+          "remote_api_endpoint": { "type": "string" },
+          "remote_api_key": { "type": "string" },
+          "embedding": {
+            "type": "object",
+            "properties": {
+              "provider": { "type": "string", "enum": ["local", "remote", "openai"], "default": "local" },
+              "model_name": { "type": "string", "default": "hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf" },
+              "api_key": { "type": "string" },
+              "api_base": { "type": "string", "description": "Base URL for remote API (e.g. https://ark.cn-beijing.volces.com/api/v3)" }
+            },
+            "default": { "provider": "local", "model_name": "hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf" }
+          }
+        },
+        "default": { "type": "local", "lancedb_uri": "./data/contextlake" }
+      },
+      "file_storage": {
+        "type": "object",
+        "properties": {
+          "type": { "type": "string", "enum": ["local", "tos"], "default": "local" },
+          "local_base_dir": { "type": "string", "default": "./data/files" },
+          "tos": {
+            "type": "object",
+            "properties": {
+              "access_key": { "type": "string" },
+              "secret_key": { "type": "string" },
+              "region": { "type": "string" },
+              "path": { "type": "string", "description": "TOS path in format tos://bucket/base_path/" },
+              "endpoint": { "type": "string" },
+              "sts_token": { "type": "string" }
+            }
+          }
+        },
+        "default": { "type": "local", "local_base_dir": "./data/files" }
+      },
+      "storage_policy": {
+        "type": "object",
+        "properties": {
+          "max_inline_size_kb": { "type": "number", "default": 1024, "description": "Files smaller than this size (in KB) will be stored directly in LanceDB" }
+        }
+      }
+    }
+  }
+}

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@byted-las/contextlake-openclaw",
+  "version": "1.0.0",
+  "description": "ContextLake OpenClaw Plugin for managing knowledge base",
+  "main": "index.ts",
+  "files": [
+    "dist",
+    "bin",
+    "index.ts",
+    "src",
+    "openclaw.plugin.json"
+  ],
+  "bin": {
+    "contextlake-openclaw": "./bin/contextlake-openclaw.js"
+  },
+  "openclaw": {
+    "extensions": [
+      "./dist/index.js"
+    ]
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest --reporter verbose",
+    "test:local": "npx ts-node scripts/local-test.ts",
+    "test:profiler": "npx ts-node scripts/local-profiler-test.ts",
+    "cli": "npx ts-node scripts/cli.ts"
+  },
+  "keywords": ["openclaw", "contextlake", "plugin"],
+  "author": "byted-las",
+  "license": "ISC",
+  "engines": {
+    "node": ">=20.17.0"
+  },
+  "dependencies": {
+    "@lancedb/lancedb": "^0.26.2",
+    "@volcengine/tos-sdk": "^2.9.0",
+    "commander": "^14.0.3",
+    "mammoth": "^1.12.0",
+    "node-llama-cpp": "^3.16.2",
+    "openclaw": "^2026.3.13",
+    "pdf-parse": "^2.4.5",
+    "uuid": "^13.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.5.0",
+    "@types/pdf-parse": "^1.1.5",
+    "@types/uuid": "^10.0.0",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.0"
+  }
+}

package/src/client/lancedb.ts

@@ -0,0 +1,102 @@
+import * as lancedb from '@lancedb/lancedb';
+import { EmbeddingProvider } from '../service/embedding/interface';
+
+export interface LanceDBConfig {
+  uri: string;
+}
+
+export interface DocumentSchema {
+  id: string;
+  vector: number[];
+  text: string;
+  source: string;
+  file_type: string;
+  storage_type: string;
+  url: string;
+  metadata: string; // JSON string
+  created_at: number; // Unix timestamp
+  binary_data?: Buffer; // Optional direct binary storage
+}
+
+export class ContextLakeLanceDBClient {
+  private db: lancedb.Connection | null = null;
+  private table: lancedb.Table | null = null;
+  private config: LanceDBConfig;
+  private embeddingProvider: EmbeddingProvider;
+
+  constructor(config: LanceDBConfig, embeddingProvider: EmbeddingProvider) {
+    this.config = config;
+    this.embeddingProvider = embeddingProvider;
+  }
+
+  async connect() {
+    if (!this.db) {
+      this.db = await lancedb.connect(this.config.uri);
+    }
+  }
+
+  async getTable(tableName: string = 'documents', dim: number = 0) {
+    if (this.table) return this.table;
+    await this.connect();
+    
+    const tableNames = await this.db!.tableNames();
+    if (tableNames.includes(tableName)) {
+      this.table = await this.db!.openTable(tableName);
+    } else {
+      if (dim <= 0) {
+        // Fallback: use embedding provider to infer dimension only if needed
+        const dummyVec = await this.embeddingProvider.generateEmbedding("init");
+        dim = dummyVec.length;
+      }
+      
+      // @ts-ignore
+      this.table = await this.db!.createTable(tableName, [
+        { 
+          id: 'schema_init', 
+          vector: Array(dim).fill(0), 
+          text: '', 
+          source: '', 
+          file_type: '',
+          storage_type: '',
+          url: '', 
+          metadata: '{}',
+          created_at: 0,
+          binary_data: Buffer.from('')
+        }
+      ]);
+      await this.table.delete('id = "schema_init"');
+    }
+    return this.table;
+  }
+
+  async addAssets(docs: DocumentSchema[]) {
+    const table = await this.getTable();
+    // @ts-ignore
+    await table.add(docs);
+  }
+
+  async search(query: string, limit: number = 5, filter?: string) {
+    const vector = await this.embeddingProvider.generateEmbedding(query);
+    const table = await this.getTable();
+    // @ts-ignore
+    let search = table.vectorSearch(vector).limit(limit);
+    if (filter) {
+      search = search.where(filter);
+    }
+    return await search.toArray();
+  }
+  
+  async delete(filter: string) {
+      const table = await this.getTable();
+      await table.delete(filter);
+  }
+  
+  async list(limit: number = 100, filter?: string) {
+      const table = await this.getTable();
+      let query = table.query().limit(limit);
+      if (filter) {
+        query = query.where(filter);
+      }
+      return await query.toArray();
+  }
+}

package/src/client/tos.ts

@@ -0,0 +1,100 @@
+import { TosClient } from '@volcengine/tos-sdk';
+import { StorageProvider } from '../service/storage/interface';
+
+export interface TosConfig {
+  access_key?: string;
+  secret_key?: string;
+  region: string;
+  endpoint?: string;
+  path: string;
+  sts_token?: string;
+}
+
+export class ContextLakeTosClient implements StorageProvider {
+  private client: TosClient;
+  private bucket: string;
+  private basePath: string;
+
+  constructor(config: TosConfig) {
+    const tosConfig: any = {
+      accessKeyId: config.access_key,
+      accessKeySecret: config.secret_key,
+      region: config.region,
+      endpoint: config.endpoint,
+      securityToken: config.sts_token,
+    };
+    
+    this.client = new TosClient(tosConfig);
+
+    if (!config.path) {
+      throw new Error('TOS configuration requires "path" (e.g. tos://bucket/path/)');
+    }
+
+    if (!config.path.startsWith('tos://')) {
+      throw new Error('TOS path must start with tos://');
+    }
+    const parts = config.path.substring(6).split('/');
+    this.bucket = parts[0];
+    this.basePath = parts.slice(1).join('/');
+    if (this.basePath && !this.basePath.endsWith('/')) {
+      this.basePath += '/';
+    }
+  }
+
+  private parseTosUrl(key: string): string {
+    if (key.startsWith('tos://')) {
+        const pathParts = key.split('/').slice(3);
+        return pathParts.join('/');
+    }
+    return key;
+  }
+
+  async uploadFile(fileName: string, buffer: Buffer): Promise<string> {
+    const key = `${this.basePath}${fileName}`;
+    await this.client.putObject({
+      bucket: this.bucket,
+      key,
+      body: buffer,
+    });
+    return `tos://${this.bucket}/${key}`;
+  }
+
+  async downloadFile(key: string): Promise<Buffer> {
+    const actualKey = this.parseTosUrl(key);
+
+    const result = await this.client.getObject({
+      bucket: this.bucket,
+      key: actualKey,
+    });
+    
+    // Check if result.data is a stream or buffer
+    if (Buffer.isBuffer(result.data)) {
+        return result.data;
+    }
+    
+    // Read stream to buffer
+    // @ts-ignore
+    const stream: any = result.data.content || result.data;
+    if (stream && stream.toArray) {
+        // Optimized path for some stream implementations (like node-fetch/minipass)
+        return Buffer.concat(await stream.toArray());
+    }
+    
+    if (stream && stream[Symbol.asyncIterator]) {
+        const chunks: Buffer[] = [];
+        for await (const chunk of stream) {
+          chunks.push(Buffer.from(chunk));
+        }
+        return Buffer.concat(chunks);
+    }
+    return Buffer.from(result.data as any);
+  }
+
+  async deleteFile(key: string): Promise<void> {
+    const actualKey = this.parseTosUrl(key);
+    await this.client.deleteObject({
+      bucket: this.bucket,
+      key: actualKey,
+    });
+  }
+}