@byted-las/contextlake-openclaw 1.0.0

package/src/service/metadata/local.ts

@@ -0,0 +1,60 @@
+import { MetadataProvider, MetadataConfig } from './interface';
+import { ContextLakeLanceDBClient, DocumentSchema } from '../../client/lancedb';
+import { createEmbeddingProvider } from '../embedding/factory';
+import { EmbeddingProvider } from '../embedding/interface';
+
+export class LocalMetadataProvider implements MetadataProvider {
+  private client: ContextLakeLanceDBClient;
+  private embeddingProvider: EmbeddingProvider;
+
+  constructor(config: MetadataConfig) {
+    if (!config.lancedb_uri) {
+        // Fallback to default if somehow not passed
+        // Use an absolute path or path relative to home to avoid issues when running in different cwds via OpenClaw daemon
+        const os = require('os');
+        const path = require('path');
+        config.lancedb_uri = path.join(os.homedir(), '.openclaw', 'contextlake', 'data');
+    }
+    
+    // Ensure embedding config exists
+    if (!config.embedding) {
+        config.embedding = {
+            provider: 'local',
+            model_name: 'hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf'
+        };
+    }
+
+    if (!config.lancedb_uri || !config.embedding) {
+      throw new Error(`Missing LanceDB configuration: uri=${config.lancedb_uri}, embedding=${!!config.embedding}`);
+    }
+    this.embeddingProvider = createEmbeddingProvider(config.embedding);
+    this.client = new ContextLakeLanceDBClient(
+      { uri: config.lancedb_uri },
+      this.embeddingProvider
+    );
+  }
+
+  async connect(): Promise<void> {
+    await this.client.connect();
+  }
+
+  async addAssets(docs: DocumentSchema[]): Promise<void> {
+    await this.client.addAssets(docs);
+  }
+
+  async search(query: string, limit?: number, filter?: string): Promise<DocumentSchema[]> {
+    return await this.client.search(query, limit, filter) as unknown as DocumentSchema[];
+  }
+
+  async list(limit?: number, filter?: string): Promise<DocumentSchema[]> {
+    return await this.client.list(limit, filter) as unknown as DocumentSchema[];
+  }
+
+  async delete(filter: string): Promise<void> {
+    await this.client.delete(filter);
+  }
+
+  async generateEmbedding(text: string): Promise<number[]> {
+    return await this.embeddingProvider.generateEmbedding(text);
+  }
+}

package/src/service/storage/factory.ts

@@ -0,0 +1,16 @@
+import { StorageProvider, StorageConfig } from './interface';
+import { LocalStorageProvider } from './local';
+import { ContextLakeTosClient } from '../../client/tos';
+
+export function createStorageProvider(config: StorageConfig): StorageProvider {
+  if (config.type === 'local') {
+    return new LocalStorageProvider(config.local_base_dir);
+  } else if (config.type === 'tos') {
+    if (!config.tos || !config.tos.region || !config.tos.path) {
+        throw new Error('Missing TOS configuration: region and path required');
+    }
+    return new ContextLakeTosClient(config.tos as any);
+  } else {
+    throw new Error(`Unsupported storage type: ${config.type}`);
+  }
+}

package/src/service/storage/interface.ts

@@ -0,0 +1,36 @@
+
+export interface StorageProvider {
+  /**
+   * Upload file content
+   * @param fileName - Name of the file
+   * @param buffer - File content
+   * @returns The storage URL (e.g. file:///... or tos://...)
+   */
+  uploadFile(fileName: string, buffer: Buffer): Promise<string>;
+
+  /**
+   * Download file content
+   * @param url - Storage URL
+   * @returns File content buffer
+   */
+  downloadFile(url: string): Promise<Buffer>;
+
+  /**
+   * Delete file
+   * @param url - Storage URL
+   */
+  deleteFile(url: string): Promise<void>;
+}
+
+export interface StorageConfig {
+  type: 'local' | 'tos';
+  local_base_dir?: string;
+  tos?: {
+    access_key?: string;
+    secret_key?: string;
+    region?: string;
+    path?: string; // e.g. tos://bucket/base/path/
+    endpoint?: string;
+    sts_token?: string;
+  };
+}

package/src/service/storage/local.ts

@@ -0,0 +1,42 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { StorageProvider } from './interface';
+
+export class LocalStorageProvider implements StorageProvider {
+  private baseDir: string;
+
+  constructor(baseDir: string = './data/files') {
+    this.baseDir = path.resolve(baseDir);
+    if (!fs.existsSync(this.baseDir)) {
+      fs.mkdirSync(this.baseDir, { recursive: true });
+    }
+  }
+
+  async uploadFile(fileName: string, buffer: Buffer): Promise<string> {
+    if (!fs.existsSync(this.baseDir)) {
+        fs.mkdirSync(this.baseDir, { recursive: true });
+    }
+    const filePath = path.join(this.baseDir, fileName);
+    await fs.promises.writeFile(filePath, buffer);
+    return `file://${filePath}`;
+  }
+
+  async downloadFile(url: string): Promise<Buffer> {
+    const filePath = this.parseUrl(url);
+    return await fs.promises.readFile(filePath);
+  }
+
+  async deleteFile(url: string): Promise<void> {
+    const filePath = this.parseUrl(url);
+    if (fs.existsSync(filePath)) {
+      await fs.promises.unlink(filePath);
+    }
+  }
+
+  private parseUrl(url: string): string {
+    if (!url.startsWith('file://')) {
+      throw new Error(`Invalid local file URL: ${url}`);
+    }
+    return url.replace('file://', '');
+  }
+}

package/src/skills/contextlake-delete/SKILL.md

@@ -0,0 +1,36 @@
+name: contextlake-delete
+description: |
+  Delete documents and assets from the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
+  Use this tool when the user wants to "删除某个文档", "清理知识库", "移除文件", "delete knowledge", or "remove documents".
+  Supports deleting documents by their specific IDs or by applying a SQL-like filter.
+
+  Example User Queries:
+  - "删除 ID 为 12345 的文档"
+  - "Please delete the old architecture document from the knowledge base"
+  - "从知识湖中移除 category 是 finance 的所有记录"
+
+  Example Tool Call (by ID):
+  ```json
+  {
+    "file_ids": ["doc-123", "doc-456"]
+  }
+  ```
+
+  Example Tool Call (by Filter):
+  ```json
+  {
+    "filter": "metadata.category = 'finance'"
+  }
+  ```
+
+parameters:
+  file_ids:
+    type: array
+    items:
+      type: string
+    description: List of specific document IDs to delete.
+    required: false
+  filter:
+    type: string
+    description: SQL-like filter string to identify documents to delete (e.g. "metadata.category = 'obsolete'").
+    required: false

package/src/skills/contextlake-ingest/SKILL.md

@@ -0,0 +1,40 @@
+name: contextlake-ingest
+description: |
+  Upload, ingest, and index documents into the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
+  Use this tool when the user wants to "将知识注入", "上传文件", "入库", "添加文档", "ingest files", or "add knowledge".
+  Supports processing of various file types including PDF, Word, Markdown, and Text.
+  Automatically handles text extraction, cleaning, chunking, embedding generation, and storage.
+
+  Example User Queries:
+  - "帮我把这个文档注入到知识湖中"
+  - "上传这份 PDF 到知识库"
+  - "Please ingest these documents into ContextLake"
+  - "将 /path/to/doc.txt 添加到知识库"
+
+  Example Tool Call:
+  ```json
+  {
+    "files": ["/absolute/path/to/document.pdf"],
+    "metadata": { "category": "tech" }
+  }
+  ```
+
+parameters:
+  files:
+    type: array
+    items:
+      type: string
+    description: List of file paths to ingest
+    required: true
+  metadata:
+    type: object
+    description: Optional JSON metadata to attach to documents
+    required: false
+  chunkSize:
+    type: integer
+    description: Chunk size for text splitting
+    required: false
+  overlap:
+    type: integer
+    description: Overlap size for text splitting
+    required: false

package/src/skills/contextlake-list/SKILL.md

@@ -0,0 +1,22 @@
+name: contextlake-list
+description: |
+  List documents and assets currently in the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
+  Use this tool when the user wants to "列出所有知识", "查看知识库文件", "显示文档列表", "list knowledge", or "show documents".
+
+  Example User Queries:
+  - "知识湖里目前有哪些文件？"
+  - "列出前10个知识库文档"
+  - "Show me all documents in the knowledge base"
+
+  Example Tool Call:
+  ```json
+  {
+    "limit": 50
+  }
+  ```
+
+parameters:
+  limit:
+    type: integer
+    description: Maximum number of documents to return (default 100).
+    required: false

package/src/skills/contextlake-retrieve/SKILL.md

@@ -0,0 +1,37 @@
+name: contextlake-retrieve
+description: |
+  Search, query, and retrieve relevant information from the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
+  Use this tool when the user wants to "搜索知识", "获取信息", "召回文档", "查询知识库", "search knowledge base", or "retrieve documents".
+  Uses vector similarity search to find semantically related document chunks.
+
+  Example User Queries:
+  - "知识库里有关于产品安装的文档吗？"
+  - "帮我从知识湖中召回关于财务报表的资料"
+  - "Search the knowledge base for deployment guides"
+  - "根据知识库内容，回答如何配置网关"
+
+  Example Tool Call:
+  ```json
+  {
+    "query": "产品安装指南",
+    "top_k": 5
+  }
+  ```
+
+parameters:
+  query:
+    type: string
+    description: Search query
+    required: true
+  top_k:
+    type: integer
+    description: Number of results to return
+    required: false
+  filter:
+    type: string
+    description: Filter string
+    required: false
+  include_binary:
+    type: boolean
+    description: Whether to include binary content
+    required: false

package/src/skills/las-data-profiler/SKILL.md

@@ -0,0 +1,174 @@
+name: byted-las-data-profiler
+description: |
+  Volcengine TOS Dataset Profiling Tool. Based on the S3-compatible protocol, it scans the file structure in TOS buckets and catalogs them,
+  performs schema inference and column semantic analysis on structured data (JSONL/CSV/Parquet/JSON),
+  extracts key meta-information for media files (Image/Audio/Video/PDF) by reading only header bytes,
+  and writes all results to a local LanceDB. It is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
+
+## Trigger Scenarios
+Be sure to use this Skill when the user mentions the following scenarios:
+- Need to scan the file structure in a TOS bucket or understand the dataset composition
+- Need to connect to object storage (TOS/OSS/COS/S3) using the S3 protocol
+- Need to scan, traverse, or catalog the file structure of a specific bucket or local directory
+- Need to understand what a batch of data files contains and what their schema looks like
+- Need to extract meta-information such as image resolution, audio/video duration, PDF page count, etc.
+- Need to write the meta-information of object storage or local files into LanceDB
+- Mentions TOS, boto3, or object storage data profiling
+- Mentions keywords like "dataset scanning", "file cataloging", "data catalog", "data profiling", etc.
+- Need to batch identify the type and size of remote/local files and build an index
+- Need to quickly understand the structure of an unfamiliar dataset (what files are there, how the schema is, field meanings)
+- Need to connect/dock a data source for profiling
+- Mentions "connect" data source, docking data source
+
+## Overview
+This Skill is a Dataset Profiling Tool provided by Volcengine TOS. It connects to the TOS bucket via an S3-compatible protocol (boto3), recursively traverses the target path, and accomplishes three things:
+1. **Cataloging**: Records the meta-information (path, size, type, origin, etc.) of all files.
+2. **Understanding Structured Data**: Samples and parses the schema for JSONL / CSV / TSV / Parquet / JSON to infer the semantic role of each column.
+3. **Extracting Media Meta-information**: Reads only the file header (without downloading the full file) for images, audio, video, and PDFs to extract key attributes.
+
+In addition to TOS, this tool is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
+
+## Supported Data Sources
+| Vendor | Service Name | S3 Endpoint Format | Signature | Addressing Style |
+|--------|--------------|--------------------|-----------|------------------|
+| Volcengine | Volcengine TOS | https://tos-s3-{region}.volces.com / ivolces.com | V4 | Virtual-hosted (Required) |
+| Alibaba | Alibaba Cloud OSS | https://s3.oss-{region}.aliyuncs.com / -internal.aliyuncs.com | V2 (boto3 must use signature_version='s3') | Virtual-hosted |
+| Tencent | Tencent Cloud COS | https://cos.{region}.myqcloud.com | V4 | Virtual-hosted |
+| AWS | Amazon S3 | https://s3.{region}.amazonaws.com | V4 | Virtual-hosted |
+| Local | Local File System | Not required | Not required | os.walk |
+
+## Overall Workflow
+1. Confirm data source parameters.
+2. Install dependencies (boto3 lancedb pyarrow pandas Pillow mutagen pymupdf).
+3. Execute the Python script, making three scanning passes:
+   - Pass 1: Traverse all files → file_catalog
+   - Pass 2: Sample structured data → structured_schemas
+   - Pass 3: Read-only header for media files → media_metadata
+4. Report results.
+
+## Parameter Description
+| Parameter | Description | Example |
+|-----------|-------------|---------|
+| datasource_name | The name of the data source, used to identify this connection. LanceDB output directory is ~/.openclaw/las-data-profiler/{datasource_name}/, and configuration file is env.sh in the same directory | my_tos_data |
+| vendor | volcengine / alibaba / tencent / aws / local | volcengine |
+| endpoint | S3 Endpoint URL (not required for local) | https://tos-s3-cn-beijing.volces.com |
+| access_key | AK, can be from parameters or environment variables | - |
+| secret_key | SK, can be from parameters or environment variables | - |
+| region | Region identifier | cn-beijing |
+| bucket | Bucket name (root directory path when local) | my-data-bucket |
+| prefix | Path prefix to limit the scan scope (required to avoid scanning the entire bucket) | datasets/2024/ |
+| sample_rows | [Optional] Number of sampled rows per structured file, default is 100 | 200 |
+
+## Authentication Priority
+Parameters > Environment Variables (TOS_ACCESS_KEY / S3_ACCESS_KEY / AWS_ACCESS_KEY_ID)
+
+## Connection Configuration Differences Across Vendors
+- Volcengine TOS: Does not support PathStyle, must be Virtual-hosted (handled by default by the tool).
+- Alibaba Cloud OSS: boto3 must use V2 signature (signature_version='s3') because V4 will force chunked encoding (handled by default by the tool).
+- Tencent Cloud COS / AWS: Default V4 is sufficient.
+
+## File Classification and Processing Strategy
+| Classification | Extension | Processing |
+|----------------|-----------|------------|
+| structured | .json .jsonl .ndjson .csv .tsv .parquet .pq | Sample parsing schema + column semantic inference |
+| image | .jpg .png .gif .bmp .webp .tiff .svg ... | Read first 64 KB → width, height, format, color mode, EXIF summary |
+| audio | .mp3 .wav .flac .aac .ogg .m4a ... | Read first 512 KB → duration, sample rate, channels, bitrate, codec, tags |
+| video | .mp4 .avi .mov .mkv .webm ... | Read first 2 MB → duration, resolution, container format |
+| pdf | .pdf | Read first 256 KB → page count, title/author, creation time, encryption status, page size |
+| Other | All remaining | Only record meta-information to file_catalog |
+
+Header reading uses S3's Range requests (Range: bytes=0-N) to download only the required bytes, without transferring the full file.
+
+## Understanding Structured Data
+Sample the first N rows of each structured file to infer the semantic_hint for each column:
+
+| semantic_hint | Judgment Logic | Example |
+|---------------|----------------|---------|
+| id | High uniqueness rate + short string | user_id, sample_001 |
+| categorical | Low cardinality (unique count < 50 or uniqueness rate < 20%) | label: cat/dog/bird |
+| numeric | int/float type | score: 0.95 |
+| boolean | bool or only two distinct values | is_valid: true/false |
+| text | Long string (avg > 50 characters) + high uniqueness rate | caption: "A cat sitting..." |
+| file_path | > 50% values match path/URI patterns | images/001.jpg, s3://bucket/file |
+| timestamp | Matches time format | 2024-01-15T10:30:00Z |
+| structured | list / dict type | bbox: [10, 20, 100, 200] |
+| constant | Only 1 distinct value | version: "1.0" |
+
+## LanceDB Three Tables
+### file_catalog — Full file index
+| Column | Description |
+|--------|-------------|
+| file_path, file_name, extension, mime_type | Path and type information |
+| category | structured / non-structured |
+| media_type | image / audio / video / pdf / empty |
+| size_bytes, last_modified, created_time, etag, storage_class | Storage attributes (created_time is only available for local files, empty for S3 objects) |
+| is_multipart | ETag contains multipart flag (-), indicating a large file uploaded in parts |
+| depth | Path level (number of /), helps to understand the directory structure |
+| parent_dir | Nearest parent directory name, convenient for aggregation statistics by directory |
+| vendor, bucket | Data source identifier |
+| has_schema | Whether there are in-depth analysis records in structured_schemas |
+| has_media_meta | Whether there are header meta-information records in media_metadata |
+| scan_timestamp | Scan time |
+
+### structured_schemas — Column-level schema of structured data
+| Column | Description |
+|--------|-------------|
+| file_path, vendor, bucket, format | File identifier |
+| column_name, column_type, non_null_ratio, unique_count | Basic statistics |
+| sample_values | First 3 distinct values (JSON) |
+| semantic_hint, semantic_detail | Semantic tags and supplementary explanations |
+
+### media_metadata — Header meta-information of media files
+| Column | Applicable Type | Description |
+|--------|-----------------|-------------|
+| width, height | Image / Video | Resolution (pixels) |
+| image_format, color_mode | Image | e.g., JPEG, RGB |
+| exif_summary | Image | Camera model, shooting time, GPS, etc. (JSON) |
+| duration_sec | Audio / Video | Duration (seconds) |
+| codec, sample_rate, channels, bitrate | Audio | Encoding parameters |
+| container | Video | Container format (mp4, mkv...) |
+| tags_summary | Audio | Title/artist/album tags, etc. (JSON) |
+| page_count, pdf_title, pdf_author | PDF | Document attributes |
+| creation_date, encrypted | PDF | Creation time, encryption status |
+| page_width_pt, page_height_pt | PDF | First page size (points) |
+| extract_error | All | Error message when extraction fails |
+
+## Output Location
+- LanceDB table storage path: `~/.openclaw/las-data-profiler/{datasource_name}/`
+- Configuration file: `~/.openclaw/las-data-profiler/{datasource_name}/env.sh`
+
+## Usage Examples
+```bash
+# Volcengine TOS (Recommended)
+python s3_catalog.py \
+  --vendor volcengine \
+  --endpoint https://tos-s3-cn-beijing.volces.com \
+  --ak "$TOS_ACCESS_KEY" --sk "$TOS_SECRET_KEY" \
+  --region cn-beijing --bucket my-bucket --prefix data/
+
+# Local Directory
+python s3_catalog.py --vendor local --bucket /path/to/data --prefix .
+
+# Alibaba Cloud OSS
+python s3_catalog.py \
+  --vendor alibaba \
+  --endpoint https://s3.oss-cn-hangzhou.aliyuncs.com \
+  --ak "$ALI_AK" --sk "$ALI_SK" \
+  --region cn-hangzhou --bucket my-bucket --prefix datasets/
+
+# Tencent Cloud COS
+python s3_catalog.py \
+  --vendor tencent \
+  --endpoint https://cos.ap-beijing.myqcloud.com \
+  --ak "$COS_AK" --sk "$COS_SK" \
+  --region ap-beijing --bucket my-bucket --prefix train/
+
+# AWS S3
+python s3_catalog.py --vendor aws --region us-east-1 --bucket my-bucket --prefix train/
+```
+
+## Frequently Asked Questions (FAQ)
+- Volcengine TOS: InvalidPathAccess → TOS does not support PathStyle, ensure addressing_style='virtual' (handled by default by the tool).
+- Alibaba Cloud OSS: InvalidArgument: aws-chunked encoding not supported → V2 signature Config(signature_version='s3') must be used (handled by default by the tool).
+- Media meta-information extraction failed → The file might be corrupted or in a non-standard format. The media_metadata.extract_error column will record the specific error, and other fields will fall back to default values (0 / empty string).
+- Video meta-information is incomplete → The position of the moov box for MP4/MOV is not fixed—if moov is at the end of the file (stream-written video), it might not be found in the first 2MB. In this case, duration/resolution will be 0.