@byted-las/contextlake-openclaw 1.0.3 → 1.0.4

package/dist/src/lib/actions/s3-tools.js

@@ -0,0 +1,167 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.listS3Objects = listS3Objects;
+exports.readS3Object = readS3Object;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+function createS3Client(params) {
+    if (params.vendor === 'local')
+        return null;
+    let endpoint = params.endpoint;
+    let region = params.region;
+    // Attempt to load credentials if not provided
+    let ak = params.access_key;
+    let sk = params.secret_key;
+    if (!ak || !sk || !region) {
+        try {
+            const { loadCredentials } = require('../../utils/credentials');
+            const creds = loadCredentials();
+            ak = ak || creds.ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY;
+            sk = sk || creds.SECRET_KEY || creds.VOLCENGINE_SECRET_KEY;
+            region = region || creds.REGION || creds.VOLCENGINE_REGION;
+        }
+        catch (e) {
+            // ignore
+        }
+    }
+    ak = ak || '';
+    sk = sk || '';
+    if (params.vendor === 'volcengine' && !endpoint) {
+        endpoint = `https://tos-s3-${region || 'cn-beijing'}.volces.com`;
+    }
+    else if (params.vendor === 'alibaba' && !endpoint) {
+        endpoint = `https://s3.oss-${region || 'cn-hangzhou'}.aliyuncs.com`;
+    }
+    else if (params.vendor === 'tencent' && !endpoint) {
+        endpoint = `https://cos.${region || 'ap-beijing'}.myqcloud.com`;
+    }
+    return new client_s3_1.S3Client({
+        region: region || 'us-east-1',
+        endpoint: endpoint,
+        credentials: {
+            accessKeyId: ak,
+            secretAccessKey: sk
+        },
+        forcePathStyle: false // usually false for virtual hosted style
+    });
+}
+async function listS3Objects(params, prefix, maxKeys = 1000, continuationToken) {
+    if (params.vendor === 'local') {
+        const root = params.bucket;
+        const prefixPath = prefix && prefix !== '.' ? path.join(root, prefix) : root;
+        const files = [];
+        function walkSync(currentDirPath) {
+            if (!fs.existsSync(currentDirPath))
+                return;
+            const dirents = fs.readdirSync(currentDirPath, { withFileTypes: true });
+            for (const dirent of dirents) {
+                const res = path.resolve(currentDirPath, dirent.name);
+                if (dirent.isDirectory()) {
+                    walkSync(res);
+                }
+                else {
+                    const stat = fs.statSync(res);
+                    files.push({
+                        Key: path.relative(root, res),
+                        Size: stat.size,
+                        LastModified: stat.mtime,
+                        ETag: '',
+                        StorageClass: 'LOCAL',
+                        _created_time: stat.ctime
+                    });
+                }
+            }
+        }
+        walkSync(prefixPath);
+        return {
+            Contents: files,
+            IsTruncated: false,
+            NextContinuationToken: undefined
+        };
+    }
+    const client = createS3Client(params);
+    if (!client)
+        throw new Error('Failed to create S3 client');
+    const command = new client_s3_1.ListObjectsV2Command({
+        Bucket: params.bucket,
+        Prefix: prefix,
+        MaxKeys: maxKeys,
+        ContinuationToken: continuationToken
+    });
+    const response = await client.send(command);
+    return {
+        Contents: response.Contents || [],
+        IsTruncated: response.IsTruncated,
+        NextContinuationToken: response.NextContinuationToken
+    };
+}
+async function readS3Object(params, key, maxBytes) {
+    if (params.vendor === 'local') {
+        const fullPath = path.join(params.bucket, key);
+        if (maxBytes) {
+            const fd = fs.openSync(fullPath, 'r');
+            const buffer = Buffer.alloc(maxBytes);
+            const bytesRead = fs.readSync(fd, buffer, 0, maxBytes, 0);
+            fs.closeSync(fd);
+            return buffer.subarray(0, bytesRead);
+        }
+        else {
+            return fs.readFileSync(fullPath);
+        }
+    }
+    const client = createS3Client(params);
+    if (!client)
+        throw new Error('Failed to create S3 client');
+    const commandInput = {
+        Bucket: params.bucket,
+        Key: key
+    };
+    if (maxBytes) {
+        commandInput.Range = `bytes=0-${maxBytes - 1}`;
+    }
+    const command = new client_s3_1.GetObjectCommand(commandInput);
+    const response = await client.send(command);
+    if (response.Body) {
+        // @ts-ignore
+        const chunks = [];
+        for await (const chunk of response.Body) {
+            chunks.push(chunk);
+        }
+        return Buffer.concat(chunks);
+    }
+    return Buffer.alloc(0);
+}

package/dist/src/skills/SKILL.md

@@ -1,9 +1,10 @@
 name: byted-las-data-profiler
 description: |
-  Volcengine TOS Dataset Profiling Tool. Based on the S3-compatible protocol, it scans the file structure in TOS buckets and catalogs them,
-  performs schema inference and column semantic analysis on structured data (JSONL/CSV/Parquet/JSON),
-  extracts key meta-information for media files (Image/Audio/Video/PDF) by reading only header bytes,
-  and writes all results to a local LanceDB. It is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
+  Volcengine TOS Dataset Profiling Tool. Based on the S3-compatible protocol, it scans the file structure in TOS buckets and catalogs them.
+  It writes the catalog index to a local LanceDB. It is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
+
+  IMPORTANT RULE: You are STRICTLY FORBIDDEN from writing or executing Python scripts to access S3/TOS or LanceDB.
+  You MUST exclusively use the provided tools (`list-s3-objects`, `read-s3-object`, `write-lance-catalog`, `las-data-profiler`) to accomplish the profiling tasks.
 
 ## Trigger Scenarios
 Be sure to use this Skill when the user mentions the following scenarios:
@@ -13,162 +14,24 @@ Be sure to use this Skill when the user mentions the following scenarios:
 - Need to understand what a batch of data files contains and what their schema looks like
 - Need to extract meta-information such as image resolution, audio/video duration, PDF page count, etc.
 - Need to write the meta-information of object storage or local files into LanceDB
-- Mentions TOS, boto3, or object storage data profiling
 - Mentions keywords like "dataset scanning", "file cataloging", "data catalog", "data profiling", etc.
-- Need to batch identify the type and size of remote/local files and build an index
-- Need to quickly understand the structure of an unfamiliar dataset (what files are there, how the schema is, field meanings)
-- Need to connect/dock a data source for profiling
-- Mentions "connect" data source, docking data source
-
-## Overview
-This Skill is a Dataset Profiling Tool provided by Volcengine TOS. It connects to the TOS bucket via an S3-compatible protocol (boto3), recursively traverses the target path, and accomplishes three things:
-1. **Cataloging**: Records the meta-information (path, size, type, origin, etc.) of all files.
-2. **Understanding Structured Data**: Samples and parses the schema for JSONL / CSV / TSV / Parquet / JSON to infer the semantic role of each column.
-3. **Extracting Media Meta-information**: Reads only the file header (without downloading the full file) for images, audio, video, and PDFs to extract key attributes.
-
-In addition to TOS, this tool is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
-
-## Supported Data Sources
-| Vendor | Service Name | S3 Endpoint Format | Signature | Addressing Style |
-|--------|--------------|--------------------|-----------|------------------|
-| Volcengine | Volcengine TOS | https://tos-s3-{region}.volces.com / ivolces.com | V4 | Virtual-hosted (Required) |
-| Alibaba | Alibaba Cloud OSS | https://s3.oss-{region}.aliyuncs.com / -internal.aliyuncs.com | V2 (boto3 must use signature_version='s3') | Virtual-hosted |
-| Tencent | Tencent Cloud COS | https://cos.{region}.myqcloud.com | V4 | Virtual-hosted |
-| AWS | Amazon S3 | https://s3.{region}.amazonaws.com | V4 | Virtual-hosted |
-| Local | Local File System | Not required | Not required | os.walk |
 
 ## Overall Workflow
-1. Confirm data source parameters.
-2. Install dependencies (boto3 lancedb pyarrow pandas Pillow mutagen pymupdf).
-3. Execute the Python script, making three scanning passes:
-   - Pass 1: Traverse all files → file_catalog
-   - Pass 2: Sample structured data → structured_schemas
-   - Pass 3: Read-only header for media files → media_metadata
-4. Report results.
+When instructed to profile a dataset, you should prefer using the `las-data-profiler` tool directly, which automatically handles the S3 listing and LanceDB writing using internal TypeScript logic.
+If you need to perform custom exploration, you can use `list-s3-objects` to traverse the bucket and `read-s3-object` to read file headers, and `write-lance-catalog` to save results.
 
-## Parameter Description
+## Parameter Description (for `las-data-profiler` tool)
 | Parameter | Description | Example |
 |-----------|-------------|---------|
-| datasource_name | The name of the data source, used to identify this connection. LanceDB output directory is ~/.openclaw/las-data-profiler/{datasource_name}/, and configuration file is env.sh in the same directory | my_tos_data |
+| datasource_name | The name of the data source | my_tos_data |
 | vendor | volcengine / alibaba / tencent / aws / local | volcengine |
 | endpoint | S3 Endpoint URL (not required for local) | https://tos-s3-cn-beijing.volces.com |
-| access_key | AK, can be from parameters or environment variables | - |
-| secret_key | SK, can be from parameters or environment variables | - |
+| access_key | AK | - |
+| secret_key | SK | - |
 | region | Region identifier | cn-beijing |
 | bucket | Bucket name (root directory path when local) | my-data-bucket |
-| prefix | Path prefix to limit the scan scope (required to avoid scanning the entire bucket) | datasets/2024/ |
-| sample_rows | [Optional] Number of sampled rows per structured file, default is 100 | 200 |
-
-## Authentication Priority
-Parameters > Environment Variables (TOS_ACCESS_KEY / S3_ACCESS_KEY / AWS_ACCESS_KEY_ID)
-
-## Connection Configuration Differences Across Vendors
-- Volcengine TOS: Does not support PathStyle, must be Virtual-hosted (handled by default by the tool).
-- Alibaba Cloud OSS: boto3 must use V2 signature (signature_version='s3') because V4 will force chunked encoding (handled by default by the tool).
-- Tencent Cloud COS / AWS: Default V4 is sufficient.
-
-## File Classification and Processing Strategy
-| Classification | Extension | Processing |
-|----------------|-----------|------------|
-| structured | .json .jsonl .ndjson .csv .tsv .parquet .pq | Sample parsing schema + column semantic inference |
-| image | .jpg .png .gif .bmp .webp .tiff .svg ... | Read first 64 KB → width, height, format, color mode, EXIF summary |
-| audio | .mp3 .wav .flac .aac .ogg .m4a ... | Read first 512 KB → duration, sample rate, channels, bitrate, codec, tags |
-| video | .mp4 .avi .mov .mkv .webm ... | Read first 2 MB → duration, resolution, container format |
-| pdf | .pdf | Read first 256 KB → page count, title/author, creation time, encryption status, page size |
-| Other | All remaining | Only record meta-information to file_catalog |
-
-Header reading uses S3's Range requests (Range: bytes=0-N) to download only the required bytes, without transferring the full file.
-
-## Understanding Structured Data
-Sample the first N rows of each structured file to infer the semantic_hint for each column:
-
-| semantic_hint | Judgment Logic | Example |
-|---------------|----------------|---------|
-| id | High uniqueness rate + short string | user_id, sample_001 |
-| categorical | Low cardinality (unique count < 50 or uniqueness rate < 20%) | label: cat/dog/bird |
-| numeric | int/float type | score: 0.95 |
-| boolean | bool or only two distinct values | is_valid: true/false |
-| text | Long string (avg > 50 characters) + high uniqueness rate | caption: "A cat sitting..." |
-| file_path | > 50% values match path/URI patterns | images/001.jpg, s3://bucket/file |
-| timestamp | Matches time format | 2024-01-15T10:30:00Z |
-| structured | list / dict type | bbox: [10, 20, 100, 200] |
-| constant | Only 1 distinct value | version: "1.0" |
-
-## LanceDB Three Tables
-### file_catalog — Full file index
-| Column | Description |
-|--------|-------------|
-| file_path, file_name, extension, mime_type | Path and type information |
-| category | structured / non-structured |
-| media_type | image / audio / video / pdf / empty |
-| size_bytes, last_modified, created_time, etag, storage_class | Storage attributes (created_time is only available for local files, empty for S3 objects) |
-| is_multipart | ETag contains multipart flag (-), indicating a large file uploaded in parts |
-| depth | Path level (number of /), helps to understand the directory structure |
-| parent_dir | Nearest parent directory name, convenient for aggregation statistics by directory |
-| vendor, bucket | Data source identifier |
-| has_schema | Whether there are in-depth analysis records in structured_schemas |
-| has_media_meta | Whether there are header meta-information records in media_metadata |
-| scan_timestamp | Scan time |
-
-### structured_schemas — Column-level schema of structured data
-| Column | Description |
-|--------|-------------|
-| file_path, vendor, bucket, format | File identifier |
-| column_name, column_type, non_null_ratio, unique_count | Basic statistics |
-| sample_values | First 3 distinct values (JSON) |
-| semantic_hint, semantic_detail | Semantic tags and supplementary explanations |
-
-### media_metadata — Header meta-information of media files
-| Column | Applicable Type | Description |
-|--------|-----------------|-------------|
-| width, height | Image / Video | Resolution (pixels) |
-| image_format, color_mode | Image | e.g., JPEG, RGB |
-| exif_summary | Image | Camera model, shooting time, GPS, etc. (JSON) |
-| duration_sec | Audio / Video | Duration (seconds) |
-| codec, sample_rate, channels, bitrate | Audio | Encoding parameters |
-| container | Video | Container format (mp4, mkv...) |
-| tags_summary | Audio | Title/artist/album tags, etc. (JSON) |
-| page_count, pdf_title, pdf_author | PDF | Document attributes |
-| creation_date, encrypted | PDF | Creation time, encryption status |
-| page_width_pt, page_height_pt | PDF | First page size (points) |
-| extract_error | All | Error message when extraction fails |
+| prefix | Path prefix to limit the scan scope | datasets/2024/ |
 
 ## Output Location
-- LanceDB table storage path: `~/.openclaw/las-data-profiler/{datasource_name}/`
-- Configuration file: `~/.openclaw/las-data-profiler/{datasource_name}/env.sh`
-
-## Usage Examples
-```bash
-# Volcengine TOS (Recommended)
-python s3_catalog.py \
-  --vendor volcengine \
-  --endpoint https://tos-s3-cn-beijing.volces.com \
-  --ak "$TOS_ACCESS_KEY" --sk "$TOS_SECRET_KEY" \
-  --region cn-beijing --bucket my-bucket --prefix data/
-
-# Local Directory
-python s3_catalog.py --vendor local --bucket /path/to/data --prefix .
-
-# Alibaba Cloud OSS
-python s3_catalog.py \
-  --vendor alibaba \
-  --endpoint https://s3.oss-cn-hangzhou.aliyuncs.com \
-  --ak "$ALI_AK" --sk "$ALI_SK" \
-  --region cn-hangzhou --bucket my-bucket --prefix datasets/
-
-# Tencent Cloud COS
-python s3_catalog.py \
-  --vendor tencent \
-  --endpoint https://cos.ap-beijing.myqcloud.com \
-  --ak "$COS_AK" --sk "$COS_SK" \
-  --region ap-beijing --bucket my-bucket --prefix train/
-
-# AWS S3
-python s3_catalog.py --vendor aws --region us-east-1 --bucket my-bucket --prefix train/
-```
-
-## Frequently Asked Questions (FAQ)
-- Volcengine TOS: InvalidPathAccess → TOS does not support PathStyle, ensure addressing_style='virtual' (handled by default by the tool).
-- Alibaba Cloud OSS: InvalidArgument: aws-chunked encoding not supported → V2 signature Config(signature_version='s3') must be used (handled by default by the tool).
-- Media meta-information extraction failed → The file might be corrupted or in a non-standard format. The media_metadata.extract_error column will record the specific error, and other fields will fall back to default values (0 / empty string).
-- Video meta-information is incomplete → The position of the moov box for MP4/MOV is not fixed—if moov is at the end of the file (stream-written video), it might not be found in the first 2MB. In this case, duration/resolution will be 0.
+- LanceDB table storage path: `~/.openclaw/contextlake/profiler/{datasource_name}/catalog_db`
+- Configuration file: `~/.openclaw/contextlake/profiler/{datasource_name}/env.sh`

package/dist/src/skills/las-data-profiler/SKILL.md

@@ -1,9 +1,10 @@
 name: byted-las-data-profiler
 description: |
-  Volcengine TOS Dataset Profiling Tool. Based on the S3-compatible protocol, it scans the file structure in TOS buckets and catalogs them,
-  performs schema inference and column semantic analysis on structured data (JSONL/CSV/Parquet/JSON),
-  extracts key meta-information for media files (Image/Audio/Video/PDF) by reading only header bytes,
-  and writes all results to a local LanceDB. It is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
+  Volcengine TOS Dataset Profiling Tool. Based on the S3-compatible protocol, it scans the file structure in TOS buckets and catalogs them.
+  It writes the catalog index to a local LanceDB. It is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
+
+  IMPORTANT RULE: You are STRICTLY FORBIDDEN from writing or executing Python scripts to access S3/TOS or LanceDB.
+  You MUST exclusively use the provided tools (`list-s3-objects`, `read-s3-object`, `write-lance-catalog`, `las-data-profiler`) to accomplish the profiling tasks.
 
 ## Trigger Scenarios
 Be sure to use this Skill when the user mentions the following scenarios:
@@ -13,162 +14,24 @@ Be sure to use this Skill when the user mentions the following scenarios:
 - Need to understand what a batch of data files contains and what their schema looks like
 - Need to extract meta-information such as image resolution, audio/video duration, PDF page count, etc.
 - Need to write the meta-information of object storage or local files into LanceDB
-- Mentions TOS, boto3, or object storage data profiling
 - Mentions keywords like "dataset scanning", "file cataloging", "data catalog", "data profiling", etc.
-- Need to batch identify the type and size of remote/local files and build an index
-- Need to quickly understand the structure of an unfamiliar dataset (what files are there, how the schema is, field meanings)
-- Need to connect/dock a data source for profiling
-- Mentions "connect" data source, docking data source
-
-## Overview
-This Skill is a Dataset Profiling Tool provided by Volcengine TOS. It connects to the TOS bucket via an S3-compatible protocol (boto3), recursively traverses the target path, and accomplishes three things:
-1. **Cataloging**: Records the meta-information (path, size, type, origin, etc.) of all files.
-2. **Understanding Structured Data**: Samples and parses the schema for JSONL / CSV / TSV / Parquet / JSON to infer the semantic role of each column.
-3. **Extracting Media Meta-information**: Reads only the file header (without downloading the full file) for images, audio, video, and PDFs to extract key attributes.
-
-In addition to TOS, this tool is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
-
-## Supported Data Sources
-| Vendor | Service Name | S3 Endpoint Format | Signature | Addressing Style |
-|--------|--------------|--------------------|-----------|------------------|
-| Volcengine | Volcengine TOS | https://tos-s3-{region}.volces.com / ivolces.com | V4 | Virtual-hosted (Required) |
-| Alibaba | Alibaba Cloud OSS | https://s3.oss-{region}.aliyuncs.com / -internal.aliyuncs.com | V2 (boto3 must use signature_version='s3') | Virtual-hosted |
-| Tencent | Tencent Cloud COS | https://cos.{region}.myqcloud.com | V4 | Virtual-hosted |
-| AWS | Amazon S3 | https://s3.{region}.amazonaws.com | V4 | Virtual-hosted |
-| Local | Local File System | Not required | Not required | os.walk |
 
 ## Overall Workflow
-1. Confirm data source parameters.
-2. Install dependencies (boto3 lancedb pyarrow pandas Pillow mutagen pymupdf).
-3. Execute the Python script, making three scanning passes:
-   - Pass 1: Traverse all files → file_catalog
-   - Pass 2: Sample structured data → structured_schemas
-   - Pass 3: Read-only header for media files → media_metadata
-4. Report results.
+When instructed to profile a dataset, you should prefer using the `las-data-profiler` tool directly, which automatically handles the S3 listing and LanceDB writing using internal TypeScript logic.
+If you need to perform custom exploration, you can use `list-s3-objects` to traverse the bucket and `read-s3-object` to read file headers, and `write-lance-catalog` to save results.
 
-## Parameter Description
+## Parameter Description (for `las-data-profiler` tool)
 | Parameter | Description | Example |
 |-----------|-------------|---------|
-| datasource_name | The name of the data source, used to identify this connection. LanceDB output directory is ~/.openclaw/las-data-profiler/{datasource_name}/, and configuration file is env.sh in the same directory | my_tos_data |
+| datasource_name | The name of the data source | my_tos_data |
 | vendor | volcengine / alibaba / tencent / aws / local | volcengine |
 | endpoint | S3 Endpoint URL (not required for local) | https://tos-s3-cn-beijing.volces.com |
-| access_key | AK, can be from parameters or environment variables | - |
-| secret_key | SK, can be from parameters or environment variables | - |
+| access_key | AK | - |
+| secret_key | SK | - |
 | region | Region identifier | cn-beijing |
 | bucket | Bucket name (root directory path when local) | my-data-bucket |
-| prefix | Path prefix to limit the scan scope (required to avoid scanning the entire bucket) | datasets/2024/ |
-| sample_rows | [Optional] Number of sampled rows per structured file, default is 100 | 200 |
-
-## Authentication Priority
-Parameters > Environment Variables (TOS_ACCESS_KEY / S3_ACCESS_KEY / AWS_ACCESS_KEY_ID)
-
-## Connection Configuration Differences Across Vendors
-- Volcengine TOS: Does not support PathStyle, must be Virtual-hosted (handled by default by the tool).
-- Alibaba Cloud OSS: boto3 must use V2 signature (signature_version='s3') because V4 will force chunked encoding (handled by default by the tool).
-- Tencent Cloud COS / AWS: Default V4 is sufficient.
-
-## File Classification and Processing Strategy
-| Classification | Extension | Processing |
-|----------------|-----------|------------|
-| structured | .json .jsonl .ndjson .csv .tsv .parquet .pq | Sample parsing schema + column semantic inference |
-| image | .jpg .png .gif .bmp .webp .tiff .svg ... | Read first 64 KB → width, height, format, color mode, EXIF summary |
-| audio | .mp3 .wav .flac .aac .ogg .m4a ... | Read first 512 KB → duration, sample rate, channels, bitrate, codec, tags |
-| video | .mp4 .avi .mov .mkv .webm ... | Read first 2 MB → duration, resolution, container format |
-| pdf | .pdf | Read first 256 KB → page count, title/author, creation time, encryption status, page size |
-| Other | All remaining | Only record meta-information to file_catalog |
-
-Header reading uses S3's Range requests (Range: bytes=0-N) to download only the required bytes, without transferring the full file.
-
-## Understanding Structured Data
-Sample the first N rows of each structured file to infer the semantic_hint for each column:
-
-| semantic_hint | Judgment Logic | Example |
-|---------------|----------------|---------|
-| id | High uniqueness rate + short string | user_id, sample_001 |
-| categorical | Low cardinality (unique count < 50 or uniqueness rate < 20%) | label: cat/dog/bird |
-| numeric | int/float type | score: 0.95 |
-| boolean | bool or only two distinct values | is_valid: true/false |
-| text | Long string (avg > 50 characters) + high uniqueness rate | caption: "A cat sitting..." |
-| file_path | > 50% values match path/URI patterns | images/001.jpg, s3://bucket/file |
-| timestamp | Matches time format | 2024-01-15T10:30:00Z |
-| structured | list / dict type | bbox: [10, 20, 100, 200] |
-| constant | Only 1 distinct value | version: "1.0" |
-
-## LanceDB Three Tables
-### file_catalog — Full file index
-| Column | Description |
-|--------|-------------|
-| file_path, file_name, extension, mime_type | Path and type information |
-| category | structured / non-structured |
-| media_type | image / audio / video / pdf / empty |
-| size_bytes, last_modified, created_time, etag, storage_class | Storage attributes (created_time is only available for local files, empty for S3 objects) |
-| is_multipart | ETag contains multipart flag (-), indicating a large file uploaded in parts |
-| depth | Path level (number of /), helps to understand the directory structure |
-| parent_dir | Nearest parent directory name, convenient for aggregation statistics by directory |
-| vendor, bucket | Data source identifier |
-| has_schema | Whether there are in-depth analysis records in structured_schemas |
-| has_media_meta | Whether there are header meta-information records in media_metadata |
-| scan_timestamp | Scan time |
-
-### structured_schemas — Column-level schema of structured data
-| Column | Description |
-|--------|-------------|
-| file_path, vendor, bucket, format | File identifier |
-| column_name, column_type, non_null_ratio, unique_count | Basic statistics |
-| sample_values | First 3 distinct values (JSON) |
-| semantic_hint, semantic_detail | Semantic tags and supplementary explanations |
-
-### media_metadata — Header meta-information of media files
-| Column | Applicable Type | Description |
-|--------|-----------------|-------------|
-| width, height | Image / Video | Resolution (pixels) |
-| image_format, color_mode | Image | e.g., JPEG, RGB |
-| exif_summary | Image | Camera model, shooting time, GPS, etc. (JSON) |
-| duration_sec | Audio / Video | Duration (seconds) |
-| codec, sample_rate, channels, bitrate | Audio | Encoding parameters |
-| container | Video | Container format (mp4, mkv...) |
-| tags_summary | Audio | Title/artist/album tags, etc. (JSON) |
-| page_count, pdf_title, pdf_author | PDF | Document attributes |
-| creation_date, encrypted | PDF | Creation time, encryption status |
-| page_width_pt, page_height_pt | PDF | First page size (points) |
-| extract_error | All | Error message when extraction fails |
+| prefix | Path prefix to limit the scan scope | datasets/2024/ |
 
 ## Output Location
-- LanceDB table storage path: `~/.openclaw/las-data-profiler/{datasource_name}/`
-- Configuration file: `~/.openclaw/las-data-profiler/{datasource_name}/env.sh`
-
-## Usage Examples
-```bash
-# Volcengine TOS (Recommended)
-python s3_catalog.py \
-  --vendor volcengine \
-  --endpoint https://tos-s3-cn-beijing.volces.com \
-  --ak "$TOS_ACCESS_KEY" --sk "$TOS_SECRET_KEY" \
-  --region cn-beijing --bucket my-bucket --prefix data/
-
-# Local Directory
-python s3_catalog.py --vendor local --bucket /path/to/data --prefix .
-
-# Alibaba Cloud OSS
-python s3_catalog.py \
-  --vendor alibaba \
-  --endpoint https://s3.oss-cn-hangzhou.aliyuncs.com \
-  --ak "$ALI_AK" --sk "$ALI_SK" \
-  --region cn-hangzhou --bucket my-bucket --prefix datasets/
-
-# Tencent Cloud COS
-python s3_catalog.py \
-  --vendor tencent \
-  --endpoint https://cos.ap-beijing.myqcloud.com \
-  --ak "$COS_AK" --sk "$COS_SK" \
-  --region ap-beijing --bucket my-bucket --prefix train/
-
-# AWS S3
-python s3_catalog.py --vendor aws --region us-east-1 --bucket my-bucket --prefix train/
-```
-
-## Frequently Asked Questions (FAQ)
-- Volcengine TOS: InvalidPathAccess → TOS does not support PathStyle, ensure addressing_style='virtual' (handled by default by the tool).
-- Alibaba Cloud OSS: InvalidArgument: aws-chunked encoding not supported → V2 signature Config(signature_version='s3') must be used (handled by default by the tool).
-- Media meta-information extraction failed → The file might be corrupted or in a non-standard format. The media_metadata.extract_error column will record the specific error, and other fields will fall back to default values (0 / empty string).
-- Video meta-information is incomplete → The position of the moov box for MP4/MOV is not fixed—if moov is at the end of the file (stream-written video), it might not be found in the first 2MB. In this case, duration/resolution will be 0.
+- LanceDB table storage path: `~/.openclaw/contextlake/profiler/{datasource_name}/catalog_db`
+- Configuration file: `~/.openclaw/contextlake/profiler/{datasource_name}/env.sh`

package/dist/src/utils/config.js

@@ -11,8 +11,8 @@ function getPluginConfig(ctx) {
             embedding: {
                 provider: 'las',
                 model_name: 'doubao-embedding-vision-250615',
-                api_key: process.env.LAS_API_KEY || creds.LAS_API_KEY,
-                api_base: process.env.LAS_BASE_URL || 'https://operator.las.cn-beijing.volces.com',
+                api_key: creds.LAS_API_KEY,
+                api_base: 'https://operator.las.cn-beijing.volces.com',
                 dimensions: 2048,
                 encoding_format: 'float'
             }
@@ -21,8 +21,9 @@ function getPluginConfig(ctx) {
             type: 'local',
             local_base_dir: require('path').join(require('os').homedir(), '.openclaw', 'contextlake', 'files'),
             tos: {
-                access_key: process.env.VOLCENGINE_ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY,
-                secret_key: process.env.VOLCENGINE_SECRET_KEY || creds.VOLCENGINE_SECRET_KEY
+                access_key: creds.ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY,
+                secret_key: creds.SECRET_KEY || creds.VOLCENGINE_SECRET_KEY,
+                region: creds.REGION || creds.VOLCENGINE_REGION || 'cn-beijing'
             }
         }
     };

package/dist/src/utils/credentials.d.ts

@@ -1,7 +1,11 @@
 export interface Credentials {
     LAS_API_KEY?: string;
+    ACCESS_KEY?: string;
+    SECRET_KEY?: string;
+    REGION?: string;
     VOLCENGINE_ACCESS_KEY?: string;
     VOLCENGINE_SECRET_KEY?: string;
+    VOLCENGINE_REGION?: string;
 }
 export declare function loadCredentials(): Credentials;
 export declare function saveCredentials(creds: Credentials): void;

package/openclaw.plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "contextlake-openclaw",
   "name": "ContextLake",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "A lightweight knowledge base plugin for OpenClaw using LanceDB and TOS, with data profiling support",
   "skills": ["./src/skills"],
   "configSchema": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@byted-las/contextlake-openclaw",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "ContextLake OpenClaw Plugin for managing knowledge base",
   "main": "index.ts",
   "files": [
@@ -33,6 +33,7 @@
     "node": ">=20.17.0"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.1014.0",
     "@lancedb/lancedb": "^0.26.2",
     "@volcengine/tos-sdk": "^2.9.0",
     "commander": "^14.0.3",

package/src/commands/cli.ts

@@ -147,13 +147,15 @@ export function getCliCommands(pluginConfig: ContextLakeConfig, logger: any) {
                 console.log('Please provide your credentials below. Press enter to keep the current value.');
                 
                 const lasApiKey = await promptForInput('LAS_API_KEY', currentCreds.LAS_API_KEY);
-                const volcengineAccessKey = await promptForInput('VOLCENGINE_ACCESS_KEY', currentCreds.VOLCENGINE_ACCESS_KEY);
-                const volcengineSecretKey = await promptForInput('VOLCENGINE_SECRET_KEY', currentCreds.VOLCENGINE_SECRET_KEY);
+                const accessKey = await promptForInput('ACCESS_KEY', currentCreds.ACCESS_KEY || currentCreds.VOLCENGINE_ACCESS_KEY);
+                const secretKey = await promptForInput('SECRET_KEY', currentCreds.SECRET_KEY || currentCreds.VOLCENGINE_SECRET_KEY);
+                const region = await promptForInput('REGION', currentCreds.REGION || currentCreds.VOLCENGINE_REGION || 'cn-beijing');
                 
                 const newCreds = {
                     LAS_API_KEY: lasApiKey,
-                    VOLCENGINE_ACCESS_KEY: volcengineAccessKey,
-                    VOLCENGINE_SECRET_KEY: volcengineSecretKey
+                    ACCESS_KEY: accessKey,
+                    SECRET_KEY: secretKey,
+                    REGION: region
                 };
                 
                 saveCredentials(newCreds);

package/src/commands/index.ts

@@ -26,6 +26,15 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
         
         ctx.registerTool(tools.lasDataProfilerTool );
         logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.lasDataProfilerTool.name}`);
+
+        ctx.registerTool(tools.listS3ObjectsTool );
+        logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.listS3ObjectsTool.name}`);
+
+        ctx.registerTool(tools.readS3ObjectTool );
+        logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.readS3ObjectTool.name}`);
+
+        ctx.registerTool(tools.writeLanceCatalogTool );
+        logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.writeLanceCatalogTool.name}`);
         
         ctx.registerTool(tools.listDatasourceTool );
         logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.listDatasourceTool.name}`);