@byted-las/contextlake-openclaw 1.0.0

package/src/skills/las-data-profiler/s3_catalog.py

@@ -0,0 +1,608 @@
+#!/usr/bin/env python3
+"""
+s3_catalog.py  --  Data source profiling tool for LanceDB
+
+Three-pass scanning:
+  Pass 1: Walk all files -> file_catalog
+  Pass 2: Sample structured data -> structured_schemas
+  Pass 3: Read media file headers -> media_metadata
+
+Supported sources: volcengine TOS / alibaba OSS / tencent COS / aws S3 / local
+Output: LanceDB tables (file_catalog, structured_schemas, media_metadata)
+"""
+
+import argparse
+import datetime
+import io
+import json
+import mimetypes
+import os
+import re
+import sys
+import time
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+# ---------------------------------------------------------------------------
+# Lazy imports
+# ---------------------------------------------------------------------------
+
+def _import_boto3():
+    try:
+        import boto3
+        from botocore.config import Config as BotoConfig
+        return boto3, BotoConfig
+    except ImportError:
+        print("ERROR: boto3 not installed. Run: pip install boto3", file=sys.stderr)
+        sys.exit(1)
+
+def _import_lancedb():
+    try:
+        import lancedb
+        return lancedb
+    except ImportError:
+        print("ERROR: lancedb not installed. Run: pip install lancedb", file=sys.stderr)
+        sys.exit(1)
+
+def _import_pandas():
+    try:
+        import pandas as pd
+        return pd
+    except ImportError:
+        print("ERROR: pandas not installed. Run: pip install pandas", file=sys.stderr)
+        sys.exit(1)
+
+def _import_pyarrow():
+    try:
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+        return pa, pq
+    except ImportError:
+        print("ERROR: pyarrow not installed. Run: pip install pyarrow", file=sys.stderr)
+        sys.exit(1)
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+STRUCTURED_EXTS = {'.json', '.jsonl', '.ndjson', '.csv', '.tsv', '.parquet', '.pq'}
+IMAGE_EXTS = {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp', '.tiff', '.tif', '.svg', '.ico', '.heic', '.heif'}
+AUDIO_EXTS = {'.mp3', '.wav', '.flac', '.aac', '.ogg', '.m4a', '.wma', '.opus'}
+VIDEO_EXTS = {'.mp4', '.avi', '.mov', '.mkv', '.webm', '.wmv', '.flv', '.m4v', '.3gp'}
+PDF_EXTS = {'.pdf'}
+
+IMAGE_HEAD_BYTES = 64 * 1024
+AUDIO_HEAD_BYTES = 512 * 1024
+VIDEO_HEAD_BYTES = 2 * 1024 * 1024
+PDF_HEAD_BYTES = 256 * 1024
+
+
+def classify_file(ext: str) -> Tuple[str, Optional[str]]:
+    ext = ext.lower()
+    if ext in STRUCTURED_EXTS:
+        return ('structured', None)
+    if ext in IMAGE_EXTS:
+        return ('non-structured', 'image')
+    if ext in AUDIO_EXTS:
+        return ('non-structured', 'audio')
+    if ext in VIDEO_EXTS:
+        return ('non-structured', 'video')
+    if ext in PDF_EXTS:
+        return ('non-structured', 'pdf')
+    return ('non-structured', None)
+
+
+# ===================================================================
+# S3 Client Abstraction
+# ===================================================================
+
+class S3Client:
+    """Thin wrapper around boto3 S3 with vendor-specific configuration."""
+
+    def __init__(self, vendor: str, endpoint: str, credential_id: str, credential_secret: str, region: str):
+        boto3, BotoConfig = _import_boto3()
+        self.vendor = vendor
+
+        config_kwargs: Dict[str, Any] = {}
+        if vendor == 'volcengine':
+            config_kwargs['s3'] = {'addressing_style': 'virtual'}
+        elif vendor == 'alibaba':
+            config_kwargs['signature_version'] = 's3'
+            config_kwargs['s3'] = {'addressing_style': 'virtual'}
+        elif vendor == 'tencent':
+            config_kwargs['s3'] = {'addressing_style': 'virtual'}
+
+        self.client = boto3.client(
+            's3',
+            endpoint_url=endpoint or None,
+            aws_access_key_id=credential_id,
+            aws_secret_access_key=credential_secret,
+            region_name=region,
+            config=BotoConfig(**config_kwargs) if config_kwargs else None,
+        )
+
+    def list_objects(self, bucket: str, prefix: str):
+        paginator = self.client.get_paginator('list_objects_v2')
+        for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+            for obj in page.get('Contents', []):
+                yield obj
+
+    def get_range(self, bucket: str, key: str, start: int, end: int) -> bytes:
+        resp = self.client.get_object(Bucket=bucket, Key=key, Range=f'bytes={start}-{end}')
+        return resp['Body'].read()
+
+    def get_object(self, bucket: str, key: str, max_bytes: Optional[int] = None) -> bytes:
+        kwargs: Dict[str, Any] = {'Bucket': bucket, 'Key': key}
+        if max_bytes:
+            kwargs['Range'] = f'bytes=0-{max_bytes - 1}'
+        resp = self.client.get_object(**kwargs)
+        return resp['Body'].read()
+
+
+# ===================================================================
+# Local FS Abstraction
+# ===================================================================
+
+class LocalClient:
+    def __init__(self, root: str):
+        self.root = root
+
+    def list_objects(self, bucket: str, prefix: str):
+        base = Path(bucket)
+        prefix_path = base / prefix if prefix and prefix != '.' else base
+        for dirpath, _dirs, files in os.walk(prefix_path):
+            for fname in files:
+                full = Path(dirpath) / fname
+                stat = full.stat()
+                key = str(full.relative_to(base))
+                yield {
+                    'Key': key,
+                    'Size': stat.st_size,
+                    'LastModified': datetime.datetime.fromtimestamp(stat.st_mtime),
+                    'ETag': '',
+                    'StorageClass': 'LOCAL',
+                    '_created_time': datetime.datetime.fromtimestamp(stat.st_ctime),
+                }
+
+    def get_range(self, bucket: str, key: str, start: int, end: int) -> bytes:
+        fpath = Path(bucket) / key
+        with open(fpath, 'rb') as f:
+            f.seek(start)
+            return f.read(end - start + 1)
+
+    def get_object(self, bucket: str, key: str, max_bytes: Optional[int] = None) -> bytes:
+        fpath = Path(bucket) / key
+        with open(fpath, 'rb') as f:
+            if max_bytes:
+                return f.read(max_bytes)
+            return f.read()
+
+
+# ===================================================================
+# Pass 1: File Catalog
+# ===================================================================
+
+def build_file_catalog(client, bucket: str, prefix: str, vendor: str) -> List[Dict]:
+    catalog: List[Dict] = []
+    scan_ts = datetime.datetime.utcnow().isoformat() + 'Z'
+
+    for obj in client.list_objects(bucket, prefix):
+        key = obj['Key']
+        if key.endswith('/'):
+            continue
+
+        name = os.path.basename(key)
+        ext = os.path.splitext(name)[1].lower()
+        mime, _ = mimetypes.guess_type(name)
+        category, media_type = classify_file(ext)
+        etag = obj.get('ETag', '').strip('"')
+        depth = key.count('/')
+        parent_dir = os.path.basename(os.path.dirname(key)) if '/' in key else ''
+
+        catalog.append({
+            'file_path': key,
+            'file_name': name,
+            'extension': ext,
+            'mime_type': mime or '',
+            'category': category,
+            'media_type': media_type or '',
+            'size_bytes': obj.get('Size', 0),
+            'last_modified': str(obj.get('LastModified', '')),
+            'created_time': str(obj.get('_created_time', '')),
+            'etag': etag,
+            'storage_class': obj.get('StorageClass', ''),
+            'is_multipart': '-' in etag,
+            'depth': depth,
+            'parent_dir': parent_dir,
+            'vendor': vendor,
+            'bucket': bucket,
+            'has_schema': False,
+            'has_media_meta': False,
+            'scan_timestamp': scan_ts,
+        })
+
+    return catalog
+
+
+# ===================================================================
+# Pass 2: Structured Schema Analysis
+# ===================================================================
+
+def infer_semantic_hint(series) -> Tuple[str, str]:
+    pd = _import_pandas()
+    non_null = series.dropna()
+    n = len(non_null)
+    if n == 0:
+        return ('constant', 'all null')
+
+    unique_count = non_null.nunique()
+    unique_ratio = unique_count / n if n > 0 else 0
+
+    if set(non_null.unique()).issubset({True, False, 0, 1, 'true', 'false', 'True', 'False'}):
+        return ('boolean', f'{unique_count} distinct values')
+
+    if unique_count == 1:
+        return ('constant', f'value: {non_null.iloc[0]}')
+
+    dtype_str = str(series.dtype)
+
+    if 'int' in dtype_str or 'float' in dtype_str:
+        return ('numeric', dtype_str)
+
+    sample_val = non_null.iloc[0]
+    if isinstance(sample_val, (list, dict)):
+        return ('structured', type(sample_val).__name__)
+
+    try:
+        str_vals = non_null.astype(str)
+        avg_len = str_vals.str.len().mean()
+
+        path_pattern = re.compile(r'[\\\\/]|^s3://|^tos://|^gs://|^https?://')
+        path_ratio = str_vals.apply(lambda x: bool(path_pattern.search(x))).mean()
+        if path_ratio > 0.5:
+            return ('file_path', f'{path_ratio:.0%} match path/URI pattern')
+
+        ts_pattern = re.compile(r'\d{4}[-/]\d{2}[-/]\d{2}')
+        ts_ratio = str_vals.apply(lambda x: bool(ts_pattern.search(x))).mean()
+        if ts_ratio > 0.5:
+            return ('timestamp', f'{ts_ratio:.0%} match timestamp pattern')
+
+        if unique_ratio > 0.9 and avg_len < 50:
+            return ('id', f'unique_ratio={unique_ratio:.2f}, avg_len={avg_len:.1f}')
+
+        if unique_count < 50 or unique_ratio < 0.2:
+            return ('categorical', f'{unique_count} categories')
+
+        if avg_len > 50 and unique_ratio > 0.5:
+            return ('text', f'avg_len={avg_len:.1f}')
+
+    except Exception:
+        pass
+
+    return ('text', '')
+
+
+def analyze_structured_file(client, bucket: str, key: str, ext: str, sample_rows: int) -> List[Dict]:
+    pd = _import_pandas()
+    pa, pq = _import_pyarrow()
+
+    schemas: List[Dict] = []
+    max_download = 2 * 1024 * 1024
+
+    try:
+        raw = client.get_object(bucket, key, max_bytes=max_download)
+    except Exception as e:
+        return [{'file_path': key, 'error': str(e)}]
+
+    df = None
+    fmt = ext.lstrip('.')
+
+    try:
+        if ext in ('.parquet', '.pq'):
+            buf = io.BytesIO(raw)
+            table = pq.read_table(buf)
+            df = table.to_pandas().head(sample_rows)
+            fmt = 'parquet'
+        elif ext == '.csv':
+            df = pd.read_csv(io.BytesIO(raw), nrows=sample_rows, on_bad_lines='skip')
+        elif ext == '.tsv':
+            df = pd.read_csv(io.BytesIO(raw), sep='\t', nrows=sample_rows, on_bad_lines='skip')
+        elif ext in ('.jsonl', '.ndjson'):
+            lines = raw.decode('utf-8', errors='replace').strip().split('\n')[:sample_rows]
+            records = [json.loads(line) for line in lines if line.strip()]
+            df = pd.json_normalize(records)
+        elif ext == '.json':
+            data = json.loads(raw.decode('utf-8', errors='replace'))
+            if isinstance(data, list):
+                df = pd.json_normalize(data[:sample_rows])
+            elif isinstance(data, dict):
+                df = pd.json_normalize([data])
+    except Exception as e:
+        return [{'file_path': key, 'error': f'parse error: {e}'}]
+
+    if df is None or df.empty:
+        return []
+
+    for col in df.columns:
+        series = df[col]
+        non_null = series.dropna()
+        unique_count = int(non_null.nunique()) if len(non_null) > 0 else 0
+        non_null_ratio = len(non_null) / len(series) if len(series) > 0 else 0.0
+
+        sample_values = []
+        try:
+            sample_values = [str(v) for v in non_null.unique()[:3]]
+        except Exception:
+            pass
+
+        hint, detail = infer_semantic_hint(series)
+
+        schemas.append({
+            'file_path': key,
+            'vendor': '',
+            'bucket': bucket,
+            'format': fmt,
+            'column_name': str(col),
+            'column_type': str(series.dtype),
+            'non_null_ratio': round(non_null_ratio, 4),
+            'unique_count': unique_count,
+            'sample_values': json.dumps(sample_values, ensure_ascii=False),
+            'semantic_hint': hint,
+            'semantic_detail': detail,
+        })
+
+    return schemas
+
+
+# ===================================================================
+# Pass 3: Media Metadata Extraction
+# ===================================================================
+
+def extract_image_meta(data: bytes) -> Dict:
+    meta: Dict[str, Any] = {
+        'width': 0, 'height': 0, 'image_format': '', 'color_mode': '', 'exif_summary': '{}',
+    }
+    try:
+        from PIL import Image
+        img = Image.open(io.BytesIO(data))
+        meta['width'] = img.width
+        meta['height'] = img.height
+        meta['image_format'] = img.format or ''
+        meta['color_mode'] = img.mode or ''
+
+        exif = {}
+        exif_data = getattr(img, '_getexif', lambda: None)()
+        if exif_data:
+            for tag_id, value in list(exif_data.items())[:10]:
+                try:
+                    from PIL.ExifTags import TAGS
+                    tag_name = TAGS.get(tag_id, str(tag_id))
+                    exif[tag_name] = str(value)[:100]
+                except Exception:
+                    pass
+        meta['exif_summary'] = json.dumps(exif, ensure_ascii=False)
+    except Exception as e:
+        meta['extract_error'] = str(e)
+    return meta
+
+
+def extract_audio_meta(data: bytes) -> Dict:
+    meta: Dict[str, Any] = {
+        'duration_sec': 0.0, 'codec': '', 'sample_rate': 0, 'channels': 0,
+        'bitrate': 0, 'tags_summary': '{}',
+    }
+    try:
+        import mutagen
+        f = mutagen.File(io.BytesIO(data))
+        if f:
+            info = getattr(f, 'info', None)
+            if info:
+                meta['duration_sec'] = round(getattr(info, 'length', 0.0), 2)
+                meta['sample_rate'] = getattr(info, 'sample_rate', 0)
+                meta['channels'] = getattr(info, 'channels', 0)
+                meta['bitrate'] = getattr(info, 'bitrate', 0)
+                meta['codec'] = type(info).__name__
+
+            tags = {}
+            if f.tags:
+                for k in list(f.tags.keys())[:10]:
+                    try:
+                        tags[str(k)] = str(f.tags[k])[:100]
+                    except Exception:
+                        pass
+            meta['tags_summary'] = json.dumps(tags, ensure_ascii=False)
+    except Exception as e:
+        meta['extract_error'] = str(e)
+    return meta
+
+
+def extract_video_meta(data: bytes) -> Dict:
+    meta: Dict[str, Any] = {
+        'width': 0, 'height': 0, 'duration_sec': 0.0, 'container': '',
+    }
+    try:
+        if data[:4] == b'\x1a\x45\xdf\xa3':
+            meta['container'] = 'mkv/webm'
+        elif len(data) > 8 and data[4:8] == b'ftyp':
+            ftyp = data[8:12].decode('ascii', errors='replace').strip()
+            meta['container'] = ftyp
+        elif data[:4] == b'RIFF':
+            meta['container'] = 'avi'
+        elif data[:3] == b'FLV':
+            meta['container'] = 'flv'
+    except Exception as e:
+        meta['extract_error'] = str(e)
+    return meta
+
+
+def extract_pdf_meta(data: bytes) -> Dict:
+    meta: Dict[str, Any] = {
+        'page_count': 0, 'pdf_title': '', 'pdf_author': '',
+        'creation_date': '', 'encrypted': False,
+        'page_width_pt': 0.0, 'page_height_pt': 0.0,
+    }
+    try:
+        import fitz
+        doc = fitz.open(stream=data, filetype='pdf')
+        meta['page_count'] = doc.page_count
+        md = doc.metadata or {}
+        meta['pdf_title'] = md.get('title', '')
+        meta['pdf_author'] = md.get('author', '')
+        meta['creation_date'] = md.get('creationDate', '')
+        meta['encrypted'] = doc.is_encrypted
+
+        if doc.page_count > 0:
+            page = doc[0]
+            rect = page.rect
+            meta['page_width_pt'] = round(rect.width, 2)
+            meta['page_height_pt'] = round(rect.height, 2)
+        doc.close()
+    except Exception as e:
+        meta['extract_error'] = str(e)
+    return meta
+
+
+def extract_media_metadata(client, bucket: str, key: str, media_type: str) -> Dict:
+    head_size = {
+        'image': IMAGE_HEAD_BYTES,
+        'audio': AUDIO_HEAD_BYTES,
+        'video': VIDEO_HEAD_BYTES,
+        'pdf': PDF_HEAD_BYTES,
+    }.get(media_type, IMAGE_HEAD_BYTES)
+
+    try:
+        data = client.get_object(bucket, key, max_bytes=head_size)
+    except Exception as e:
+        return {'extract_error': f'download failed: {e}'}
+
+    if media_type == 'image':
+        return extract_image_meta(data)
+    elif media_type == 'audio':
+        return extract_audio_meta(data)
+    elif media_type == 'video':
+        return extract_video_meta(data)
+    elif media_type == 'pdf':
+        return extract_pdf_meta(data)
+    return {}
+
+
+# ===================================================================
+# LanceDB Writer
+# ===================================================================
+
+def write_to_lancedb(db_path: str, table_name: str, records: List[Dict]):
+    if not records:
+        return
+    lancedb = _import_lancedb()
+    pd = _import_pandas()
+
+    db = lancedb.connect(db_path)
+    df = pd.DataFrame(records)
+
+    table_names = db.table_names()
+    if table_name in table_names:
+        db.drop_table(table_name)
+
+    db.create_table(table_name, data=df)
+    print(f"  [LanceDB] Wrote {len(records)} records to '{table_name}'")
+
+
+# ===================================================================
+# Main
+# ===================================================================
+
+def main():
+    parser = argparse.ArgumentParser(description='S3-compatible data profiler -> LanceDB')
+    parser.add_argument('--vendor', required=True,
+                        choices=['volcengine', 'alibaba', 'tencent', 'aws', 'local'])
+    parser.add_argument('--endpoint', default='')
+    parser.add_argument('--ak', default='', dest='cred_id',
+                        help='Access credential ID')
+    parser.add_argument('--sk', default='', dest='cred_secret',
+                        help='Access credential value')
+    parser.add_argument('--region', default='')
+    parser.add_argument('--bucket', required=True)
+    parser.add_argument('--prefix', required=True)
+    parser.add_argument('--db-path', default='./catalog_db')
+    parser.add_argument('--sample-rows', type=int, default=100)
+    args = parser.parse_args()
+
+    print(f"[las-data-profiler] vendor={args.vendor}, bucket={args.bucket}, prefix={args.prefix}")
+    print(f"[las-data-profiler] db_path={args.db_path}")
+
+    if args.vendor == 'local':
+        client = LocalClient(args.bucket)
+    else:
+        client = S3Client(
+            vendor=args.vendor,
+            endpoint=args.endpoint,
+            credential_id=args.cred_id,
+            credential_secret=args.cred_secret,
+            region=args.region,
+        )
+
+    # ---- Pass 1: File Catalog ----
+    print("\n[Pass 1] Scanning files...")
+    catalog = build_file_catalog(client, args.bucket, args.prefix, args.vendor)
+    print(f"  Found {len(catalog)} files")
+
+    # ---- Pass 2: Structured Schemas ----
+    print("\n[Pass 2] Analyzing structured data...")
+    structured_files = [f for f in catalog if f['category'] == 'structured']
+    all_schemas: List[Dict] = []
+    for i, entry in enumerate(structured_files):
+        key = entry['file_path']
+        ext = entry['extension']
+        print(f"  [{i+1}/{len(structured_files)}] {key}")
+        schemas = analyze_structured_file(client, args.bucket, key, ext, args.sample_rows)
+        for s in schemas:
+            s['vendor'] = args.vendor
+        all_schemas.extend(schemas)
+        entry['has_schema'] = True
+
+    print(f"  Analyzed {len(structured_files)} files, {len(all_schemas)} column records")
+
+    # ---- Pass 3: Media Metadata ----
+    print("\n[Pass 3] Extracting media metadata...")
+    media_files = [f for f in catalog if f['media_type'] in ('image', 'audio', 'video', 'pdf')]
+    all_media_meta: List[Dict] = []
+    for i, entry in enumerate(media_files):
+        key = entry['file_path']
+        media_type = entry['media_type']
+        print(f"  [{i+1}/{len(media_files)}] {key} ({media_type})")
+        meta = extract_media_metadata(client, args.bucket, key, media_type)
+        meta['file_path'] = key
+        meta['vendor'] = args.vendor
+        meta['bucket'] = args.bucket
+        meta['media_type'] = media_type
+        for col in ['width', 'height', 'image_format', 'color_mode', 'exif_summary',
+                     'duration_sec', 'codec', 'sample_rate', 'channels', 'bitrate',
+                     'tags_summary', 'container',
+                     'page_count', 'pdf_title', 'pdf_author', 'creation_date',
+                     'encrypted', 'page_width_pt', 'page_height_pt', 'extract_error']:
+            meta.setdefault(col, '' if isinstance(meta.get(col), str) else 0)
+        all_media_meta.append(meta)
+        entry['has_media_meta'] = True
+
+    print(f"  Extracted metadata for {len(media_files)} media files")
+
+    # ---- Write to LanceDB ----
+    print(f"\n[LanceDB] Writing to {args.db_path}")
+    write_to_lancedb(args.db_path, 'file_catalog', catalog)
+    write_to_lancedb(args.db_path, 'structured_schemas', all_schemas)
+    write_to_lancedb(args.db_path, 'media_metadata', all_media_meta)
+
+    # ---- Summary JSON (stdout, for Node.js to parse) ----
+    summary = {
+        'summary': {
+            'total_files': len(catalog),
+            'structured_files': len(structured_files),
+            'media_files': len(media_files),
+        }
+    }
+    print(f"\n{json.dumps(summary)}")
+    print("\n[las-data-profiler] Done!")
+
+
+if __name__ == '__main__':
+    main()

package/src/utils/config.ts

@@ -0,0 +1,13 @@
+export function getPluginConfig(ctx: any) {
+    return ctx.config?.plugins?.entries?.['contextlake-openclaw']?.config || {
+        metadata_storage: { 
+            type: 'local', 
+            lancedb_uri: require('path').join(require('os').homedir(), '.openclaw', 'contextlake', 'data'),
+            embedding: {
+                provider: 'local',
+                model_name: 'hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf'
+            }
+        },
+        file_storage: { type: 'local', local_base_dir: require('path').join(require('os').homedir(), '.openclaw', 'contextlake', 'files') }
+    };
+}