@byted-las/contextlake-openclaw 1.0.3 → 1.0.4

package/src/skills/las-data-profiler/SKILL.md

@@ -1,9 +1,10 @@
 name: byted-las-data-profiler
 description: |
-  Volcengine TOS Dataset Profiling Tool. Based on the S3-compatible protocol, it scans the file structure in TOS buckets and catalogs them,
-  performs schema inference and column semantic analysis on structured data (JSONL/CSV/Parquet/JSON),
-  extracts key meta-information for media files (Image/Audio/Video/PDF) by reading only header bytes,
-  and writes all results to a local LanceDB. It is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
+  Volcengine TOS Dataset Profiling Tool. Based on the S3-compatible protocol, it scans the file structure in TOS buckets and catalogs them.
+  It writes the catalog index to a local LanceDB. It is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
+
+  IMPORTANT RULE: You are STRICTLY FORBIDDEN from writing or executing Python scripts to access S3/TOS or LanceDB.
+  You MUST exclusively use the provided tools (`list-s3-objects`, `read-s3-object`, `write-lance-catalog`, `las-data-profiler`) to accomplish the profiling tasks.
 
 ## Trigger Scenarios
 Be sure to use this Skill when the user mentions the following scenarios:
@@ -13,162 +14,24 @@ Be sure to use this Skill when the user mentions the following scenarios:
 - Need to understand what a batch of data files contains and what their schema looks like
 - Need to extract meta-information such as image resolution, audio/video duration, PDF page count, etc.
 - Need to write the meta-information of object storage or local files into LanceDB
-- Mentions TOS, boto3, or object storage data profiling
 - Mentions keywords like "dataset scanning", "file cataloging", "data catalog", "data profiling", etc.
-- Need to batch identify the type and size of remote/local files and build an index
-- Need to quickly understand the structure of an unfamiliar dataset (what files are there, how the schema is, field meanings)
-- Need to connect/dock a data source for profiling
-- Mentions "connect" data source, docking data source
-
-## Overview
-This Skill is a Dataset Profiling Tool provided by Volcengine TOS. It connects to the TOS bucket via an S3-compatible protocol (boto3), recursively traverses the target path, and accomplishes three things:
-1. **Cataloging**: Records the meta-information (path, size, type, origin, etc.) of all files.
-2. **Understanding Structured Data**: Samples and parses the schema for JSONL / CSV / TSV / Parquet / JSON to infer the semantic role of each column.
-3. **Extracting Media Meta-information**: Reads only the file header (without downloading the full file) for images, audio, video, and PDFs to extract key attributes.
-
-In addition to TOS, this tool is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
-
-## Supported Data Sources
-| Vendor | Service Name | S3 Endpoint Format | Signature | Addressing Style |
-|--------|--------------|--------------------|-----------|------------------|
-| Volcengine | Volcengine TOS | https://tos-s3-{region}.volces.com / ivolces.com | V4 | Virtual-hosted (Required) |
-| Alibaba | Alibaba Cloud OSS | https://s3.oss-{region}.aliyuncs.com / -internal.aliyuncs.com | V2 (boto3 must use signature_version='s3') | Virtual-hosted |
-| Tencent | Tencent Cloud COS | https://cos.{region}.myqcloud.com | V4 | Virtual-hosted |
-| AWS | Amazon S3 | https://s3.{region}.amazonaws.com | V4 | Virtual-hosted |
-| Local | Local File System | Not required | Not required | os.walk |
 
 ## Overall Workflow
-1. Confirm data source parameters.
-2. Install dependencies (boto3 lancedb pyarrow pandas Pillow mutagen pymupdf).
-3. Execute the Python script, making three scanning passes:
-   - Pass 1: Traverse all files → file_catalog
-   - Pass 2: Sample structured data → structured_schemas
-   - Pass 3: Read-only header for media files → media_metadata
-4. Report results.
+When instructed to profile a dataset, you should prefer using the `las-data-profiler` tool directly, which automatically handles the S3 listing and LanceDB writing using internal TypeScript logic.
+If you need to perform custom exploration, you can use `list-s3-objects` to traverse the bucket and `read-s3-object` to read file headers, and `write-lance-catalog` to save results.
 
-## Parameter Description
+## Parameter Description (for `las-data-profiler` tool)
 | Parameter | Description | Example |
 |-----------|-------------|---------|
-| datasource_name | The name of the data source, used to identify this connection. LanceDB output directory is ~/.openclaw/las-data-profiler/{datasource_name}/, and configuration file is env.sh in the same directory | my_tos_data |
+| datasource_name | The name of the data source | my_tos_data |
 | vendor | volcengine / alibaba / tencent / aws / local | volcengine |
 | endpoint | S3 Endpoint URL (not required for local) | https://tos-s3-cn-beijing.volces.com |
-| access_key | AK, can be from parameters or environment variables | - |
-| secret_key | SK, can be from parameters or environment variables | - |
+| access_key | AK | - |
+| secret_key | SK | - |
 | region | Region identifier | cn-beijing |
 | bucket | Bucket name (root directory path when local) | my-data-bucket |
-| prefix | Path prefix to limit the scan scope (required to avoid scanning the entire bucket) | datasets/2024/ |
-| sample_rows | [Optional] Number of sampled rows per structured file, default is 100 | 200 |
-
-## Authentication Priority
-Parameters > Environment Variables (TOS_ACCESS_KEY / S3_ACCESS_KEY / AWS_ACCESS_KEY_ID)
-
-## Connection Configuration Differences Across Vendors
-- Volcengine TOS: Does not support PathStyle, must be Virtual-hosted (handled by default by the tool).
-- Alibaba Cloud OSS: boto3 must use V2 signature (signature_version='s3') because V4 will force chunked encoding (handled by default by the tool).
-- Tencent Cloud COS / AWS: Default V4 is sufficient.
-
-## File Classification and Processing Strategy
-| Classification | Extension | Processing |
-|----------------|-----------|------------|
-| structured | .json .jsonl .ndjson .csv .tsv .parquet .pq | Sample parsing schema + column semantic inference |
-| image | .jpg .png .gif .bmp .webp .tiff .svg ... | Read first 64 KB → width, height, format, color mode, EXIF summary |
-| audio | .mp3 .wav .flac .aac .ogg .m4a ... | Read first 512 KB → duration, sample rate, channels, bitrate, codec, tags |
-| video | .mp4 .avi .mov .mkv .webm ... | Read first 2 MB → duration, resolution, container format |
-| pdf | .pdf | Read first 256 KB → page count, title/author, creation time, encryption status, page size |
-| Other | All remaining | Only record meta-information to file_catalog |
-
-Header reading uses S3's Range requests (Range: bytes=0-N) to download only the required bytes, without transferring the full file.
-
-## Understanding Structured Data
-Sample the first N rows of each structured file to infer the semantic_hint for each column:
-
-| semantic_hint | Judgment Logic | Example |
-|---------------|----------------|---------|
-| id | High uniqueness rate + short string | user_id, sample_001 |
-| categorical | Low cardinality (unique count < 50 or uniqueness rate < 20%) | label: cat/dog/bird |
-| numeric | int/float type | score: 0.95 |
-| boolean | bool or only two distinct values | is_valid: true/false |
-| text | Long string (avg > 50 characters) + high uniqueness rate | caption: "A cat sitting..." |
-| file_path | > 50% values match path/URI patterns | images/001.jpg, s3://bucket/file |
-| timestamp | Matches time format | 2024-01-15T10:30:00Z |
-| structured | list / dict type | bbox: [10, 20, 100, 200] |
-| constant | Only 1 distinct value | version: "1.0" |
-
-## LanceDB Three Tables
-### file_catalog — Full file index
-| Column | Description |
-|--------|-------------|
-| file_path, file_name, extension, mime_type | Path and type information |
-| category | structured / non-structured |
-| media_type | image / audio / video / pdf / empty |
-| size_bytes, last_modified, created_time, etag, storage_class | Storage attributes (created_time is only available for local files, empty for S3 objects) |
-| is_multipart | ETag contains multipart flag (-), indicating a large file uploaded in parts |
-| depth | Path level (number of /), helps to understand the directory structure |
-| parent_dir | Nearest parent directory name, convenient for aggregation statistics by directory |
-| vendor, bucket | Data source identifier |
-| has_schema | Whether there are in-depth analysis records in structured_schemas |
-| has_media_meta | Whether there are header meta-information records in media_metadata |
-| scan_timestamp | Scan time |
-
-### structured_schemas — Column-level schema of structured data
-| Column | Description |
-|--------|-------------|
-| file_path, vendor, bucket, format | File identifier |
-| column_name, column_type, non_null_ratio, unique_count | Basic statistics |
-| sample_values | First 3 distinct values (JSON) |
-| semantic_hint, semantic_detail | Semantic tags and supplementary explanations |
-
-### media_metadata — Header meta-information of media files
-| Column | Applicable Type | Description |
-|--------|-----------------|-------------|
-| width, height | Image / Video | Resolution (pixels) |
-| image_format, color_mode | Image | e.g., JPEG, RGB |
-| exif_summary | Image | Camera model, shooting time, GPS, etc. (JSON) |
-| duration_sec | Audio / Video | Duration (seconds) |
-| codec, sample_rate, channels, bitrate | Audio | Encoding parameters |
-| container | Video | Container format (mp4, mkv...) |
-| tags_summary | Audio | Title/artist/album tags, etc. (JSON) |
-| page_count, pdf_title, pdf_author | PDF | Document attributes |
-| creation_date, encrypted | PDF | Creation time, encryption status |
-| page_width_pt, page_height_pt | PDF | First page size (points) |
-| extract_error | All | Error message when extraction fails |
+| prefix | Path prefix to limit the scan scope | datasets/2024/ |
 
 ## Output Location
-- LanceDB table storage path: `~/.openclaw/las-data-profiler/{datasource_name}/`
-- Configuration file: `~/.openclaw/las-data-profiler/{datasource_name}/env.sh`
-
-## Usage Examples
-```bash
-# Volcengine TOS (Recommended)
-python s3_catalog.py \
-  --vendor volcengine \
-  --endpoint https://tos-s3-cn-beijing.volces.com \
-  --ak "$TOS_ACCESS_KEY" --sk "$TOS_SECRET_KEY" \
-  --region cn-beijing --bucket my-bucket --prefix data/
-
-# Local Directory
-python s3_catalog.py --vendor local --bucket /path/to/data --prefix .
-
-# Alibaba Cloud OSS
-python s3_catalog.py \
-  --vendor alibaba \
-  --endpoint https://s3.oss-cn-hangzhou.aliyuncs.com \
-  --ak "$ALI_AK" --sk "$ALI_SK" \
-  --region cn-hangzhou --bucket my-bucket --prefix datasets/
-
-# Tencent Cloud COS
-python s3_catalog.py \
-  --vendor tencent \
-  --endpoint https://cos.ap-beijing.myqcloud.com \
-  --ak "$COS_AK" --sk "$COS_SK" \
-  --region ap-beijing --bucket my-bucket --prefix train/
-
-# AWS S3
-python s3_catalog.py --vendor aws --region us-east-1 --bucket my-bucket --prefix train/
-```
-
-## Frequently Asked Questions (FAQ)
-- Volcengine TOS: InvalidPathAccess → TOS does not support PathStyle, ensure addressing_style='virtual' (handled by default by the tool).
-- Alibaba Cloud OSS: InvalidArgument: aws-chunked encoding not supported → V2 signature Config(signature_version='s3') must be used (handled by default by the tool).
-- Media meta-information extraction failed → The file might be corrupted or in a non-standard format. The media_metadata.extract_error column will record the specific error, and other fields will fall back to default values (0 / empty string).
-- Video meta-information is incomplete → The position of the moov box for MP4/MOV is not fixed—if moov is at the end of the file (stream-written video), it might not be found in the first 2MB. In this case, duration/resolution will be 0.
+- LanceDB table storage path: `~/.openclaw/contextlake/profiler/{datasource_name}/catalog_db`
+- Configuration file: `~/.openclaw/contextlake/profiler/{datasource_name}/env.sh`

package/src/utils/config.ts

@@ -48,8 +48,8 @@ export function getPluginConfig(ctx: any): ContextLakeConfig {
             embedding: {
                 provider: 'las',
                 model_name: 'doubao-embedding-vision-250615',
-                api_key: process.env.LAS_API_KEY || creds.LAS_API_KEY,
-                api_base: process.env.LAS_BASE_URL || 'https://operator.las.cn-beijing.volces.com',
+                api_key: creds.LAS_API_KEY,
+                api_base: 'https://operator.las.cn-beijing.volces.com',
                 dimensions: 2048,
                 encoding_format: 'float'
             }
@@ -58,8 +58,9 @@ export function getPluginConfig(ctx: any): ContextLakeConfig {
             type: 'local', 
             local_base_dir: require('path').join(require('os').homedir(), '.openclaw', 'contextlake', 'files'),
             tos: {
-                access_key: process.env.VOLCENGINE_ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY,
-                secret_key: process.env.VOLCENGINE_SECRET_KEY || creds.VOLCENGINE_SECRET_KEY
+                access_key: creds.ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY,
+                secret_key: creds.SECRET_KEY || creds.VOLCENGINE_SECRET_KEY,
+                region: creds.REGION || creds.VOLCENGINE_REGION || 'cn-beijing'
             }
         }
     };

package/src/utils/credentials.ts

@@ -8,8 +8,14 @@ const CONFIG_FILE = path.join(CONFIG_DIR, 'credentials.json');
 
 export interface Credentials {
     LAS_API_KEY?: string;
+    ACCESS_KEY?: string;
+    SECRET_KEY?: string;
+    REGION?: string;
+    
+    // Legacy support
     VOLCENGINE_ACCESS_KEY?: string;
     VOLCENGINE_SECRET_KEY?: string;
+    VOLCENGINE_REGION?: string;
 }
 
 export function loadCredentials(): Credentials {