@byted-las/contextlake-openclaw 1.0.4 → 1.0.6

package/src/lib/actions/s3-tools.ts

@@ -1,17 +1,51 @@
 import { S3Client, ListObjectsV2Command, GetObjectCommand } from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import * as fs from 'fs';
 import * as path from 'path';
 
 export interface S3Params {
-    vendor: 'volcengine' | 'alibaba' | 'tencent' | 'aws' | 'local';
+    url?: string;
+    vendor?: 'volcengine' | 'alibaba' | 'tencent' | 'aws' | 'local';
     endpoint?: string;
     access_key?: string;
     secret_key?: string;
     region?: string;
-    bucket: string;
+    bucket?: string;
+}
+
+export function parseS3Url(params: S3Params): S3Params {
+    if (params.url) {
+        if (params.url.startsWith('file://') || params.url.startsWith('/')) {
+            params.vendor = 'local';
+            const localPath = params.url.startsWith('file://') ? params.url.slice(7) : params.url;
+            params.bucket = localPath;
+        } else {
+            const parsedUrl = new URL(params.url);
+            const protocol = parsedUrl.protocol.replace(':', '');
+            
+            if (['tos', 'oss', 'cos', 's3'].includes(protocol)) {
+                if (protocol === 'tos') params.vendor = 'volcengine';
+                else if (protocol === 'oss') params.vendor = 'alibaba';
+                else if (protocol === 'cos') params.vendor = 'tencent';
+                else if (protocol === 's3') params.vendor = 'aws';
+                
+                params.bucket = parsedUrl.hostname;
+                // Prefix is usually parsed separately or passed explicitly for listing
+            } else {
+                throw new Error(`Unsupported protocol: ${protocol}`);
+            }
+        }
+    }
+    
+    if (!params.vendor || !params.bucket) {
+         throw new Error('Could not determine vendor or bucket. Please provide a valid url or vendor/bucket directly.');
+    }
+    
+    return params;
 }
 
 function createS3Client(params: S3Params): S3Client | null {
+    params = parseS3Url(params);
     if (params.vendor === 'local') return null;
 
     let endpoint = params.endpoint;
@@ -56,8 +90,9 @@ function createS3Client(params: S3Params): S3Client | null {
 }
 
 export async function listS3Objects(params: S3Params, prefix: string, maxKeys: number = 1000, continuationToken?: string) {
+    params = parseS3Url(params);
     if (params.vendor === 'local') {
-        const root = params.bucket;
+        const root = params.bucket as string;
         const prefixPath = prefix && prefix !== '.' ? path.join(root, prefix) : root;
         const files: any[] = [];
         
@@ -108,8 +143,9 @@ export async function listS3Objects(params: S3Params, prefix: string, maxKeys: n
 }
 
 export async function readS3Object(params: S3Params, key: string, maxBytes?: number): Promise<Buffer> {
+    params = parseS3Url(params);
     if (params.vendor === 'local') {
-        const fullPath = path.join(params.bucket, key);
+        const fullPath = path.join(params.bucket as string, key);
         if (maxBytes) {
             const fd = fs.openSync(fullPath, 'r');
             const buffer = Buffer.alloc(maxBytes);
@@ -146,3 +182,22 @@ export async function readS3Object(params: S3Params, key: string, maxBytes?: num
     }
     return Buffer.alloc(0);
 }
+
+export async function getPresignedUrl(params: S3Params, key: string, expiresIn: number = 3600): Promise<string> {
+    params = parseS3Url(params);
+    if (params.vendor === 'local') {
+        const fullPath = path.join(params.bucket as string, key);
+        return `file://${fullPath}`;
+    }
+
+    const client = createS3Client(params);
+    if (!client) throw new Error('Failed to create S3 client');
+
+    const command = new GetObjectCommand({
+        Bucket: params.bucket,
+        Key: key
+    });
+
+    const signedUrl = await getSignedUrl(client, command, { expiresIn });
+    return signedUrl;
+}

package/src/skills/contextlake-delete/SKILL.md

@@ -1,3 +1,4 @@
+---
 name: contextlake-delete
 description: |
   Delete documents and assets from the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
@@ -34,3 +35,4 @@ parameters:
     type: string
     description: SQL-like filter string to identify documents to delete (e.g. "metadata.category = 'obsolete'").
     required: false
+---

package/src/skills/contextlake-ingest/SKILL.md

@@ -1,3 +1,4 @@
+---
 name: contextlake-ingest
 description: |
   Upload, ingest, and index documents into the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
@@ -38,3 +39,4 @@ parameters:
     type: integer
     description: Overlap size for text splitting
     required: false
+---

package/src/skills/contextlake-list/SKILL.md

@@ -1,3 +1,4 @@
+---
 name: contextlake-list
 description: |
   List documents and assets currently in the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
@@ -20,3 +21,4 @@ parameters:
     type: integer
     description: Maximum number of documents to return (default 100).
     required: false
+---

package/src/skills/contextlake-retrieve/SKILL.md

@@ -1,3 +1,4 @@
+---
 name: contextlake-retrieve
 description: |
   Search, query, and retrieve relevant information from the ContextLake Knowledge Base (知识库) / Knowledge Lake (知识湖).
@@ -35,3 +36,4 @@ parameters:
     type: boolean
     description: Whether to include binary content
     required: false
+---

package/src/skills/las-data-profiler/SKILL.md

@@ -1,3 +1,4 @@
+---
 name: byted-las-data-profiler
 description: |
   Volcengine TOS Dataset Profiling Tool. Based on the S3-compatible protocol, it scans the file structure in TOS buckets and catalogs them.
@@ -5,6 +6,7 @@ description: |
 
   IMPORTANT RULE: You are STRICTLY FORBIDDEN from writing or executing Python scripts to access S3/TOS or LanceDB.
   You MUST exclusively use the provided tools (`list-s3-objects`, `read-s3-object`, `write-lance-catalog`, `las-data-profiler`) to accomplish the profiling tasks.
+---
 
 ## Trigger Scenarios
 Be sure to use this Skill when the user mentions the following scenarios:
@@ -34,4 +36,4 @@ If you need to perform custom exploration, you can use `list-s3-objects` to trav
 
 ## Output Location
 - LanceDB table storage path: `~/.openclaw/contextlake/profiler/{datasource_name}/catalog_db`
-- Configuration file: `~/.openclaw/contextlake/profiler/{datasource_name}/env.sh`
+- Configuration file: `~/.openclaw/contextlake/profiler/{datasource_name}/env.sh`