@byted-las/contextlake-openclaw 1.0.0

package/dist/src/processor/loader.js

@@ -0,0 +1,83 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processFile = processFile;
+exports.splitText = splitText;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+// @ts-ignore
+const pdf = __importStar(require("pdf-parse"));
+const mammoth = __importStar(require("mammoth"));
+async function processFile(filePath) {
+    const buffer = fs.readFileSync(filePath);
+    const ext = path.extname(filePath).toLowerCase();
+    let text = '';
+    switch (ext) {
+        case '.pdf':
+            const pdfData = await pdf(buffer);
+            text = pdfData.text;
+            break;
+        case '.docx':
+            const result = await mammoth.extractRawText({ buffer });
+            text = result.value;
+            break;
+        case '.txt':
+        case '.md':
+            text = buffer.toString('utf-8');
+            break;
+        default:
+            text = buffer.toString('utf-8'); // Fallback to text
+    }
+    // Basic cleaning
+    text = text.replace(/\s+/g, ' ').trim();
+    return {
+        buffer,
+        text,
+        type: ext.replace('.', '')
+    };
+}
+function splitText(text, chunkSize = 500, overlap = 50) {
+    const chunks = [];
+    if (!text)
+        return chunks;
+    let start = 0;
+    while (start < text.length) {
+        const end = Math.min(start + chunkSize, text.length);
+        const chunk = text.slice(start, end);
+        chunks.push(chunk);
+        start += chunkSize - overlap;
+    }
+    return chunks;
+}

package/dist/src/service/embedding/factory.d.ts

@@ -0,0 +1,2 @@
+import { EmbeddingProvider, EmbeddingConfig } from './interface';
+export declare function createEmbeddingProvider(config: EmbeddingConfig): EmbeddingProvider;

package/dist/src/service/embedding/factory.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createEmbeddingProvider = createEmbeddingProvider;
+const local_1 = require("./local");
+const remote_1 = require("./remote");
+function createEmbeddingProvider(config) {
+    if (config.provider === 'local') {
+        return new local_1.LocalEmbeddingProvider(config);
+    }
+    else if (config.provider === 'openai' || config.provider === 'remote') {
+        return new remote_1.RemoteEmbeddingProvider(config);
+    }
+    else {
+        throw new Error(`Unsupported embedding provider: ${config.provider}`);
+    }
+}

package/dist/src/service/embedding/interface.d.ts

@@ -0,0 +1,18 @@
+export interface EmbeddingProvider {
+    /**
+     * Generate embedding for text
+     * @param text - Input text
+     */
+    generateEmbedding(text: string): Promise<number[]>;
+    /**
+     * Generate embeddings for multiple texts
+     * @param texts - Array of input texts
+     */
+    generateEmbeddings(texts: string[]): Promise<number[][]>;
+}
+export interface EmbeddingConfig {
+    provider: 'local' | 'remote' | 'openai';
+    model_name: string;
+    api_key?: string;
+    api_base?: string;
+}

package/dist/src/service/embedding/interface.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/src/service/embedding/local.d.ts

@@ -0,0 +1,14 @@
+import { EmbeddingProvider, EmbeddingConfig } from './interface';
+export declare const setNodeLlamaCppImporter: (importer: () => Promise<any>) => void;
+export declare class LocalEmbeddingProvider implements EmbeddingProvider {
+    private llama;
+    private model;
+    private context;
+    private initPromise;
+    private modelPath;
+    constructor(config: EmbeddingConfig);
+    private ensureInitialized;
+    private doInitialize;
+    generateEmbedding(text: string): Promise<number[]>;
+    generateEmbeddings(texts: string[]): Promise<number[][]>;
+}

package/dist/src/service/embedding/local.js

@@ -0,0 +1,104 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalEmbeddingProvider = exports.setNodeLlamaCppImporter = void 0;
+// import type { Llama, LlamaEmbeddingContext, LlamaModel } from 'node-llama-cpp';
+const DEFAULT_LOCAL_MODEL = "hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf";
+let nodeLlamaImportPromise = null;
+const setNodeLlamaCppImporter = (importer) => {
+    nodeLlamaImportPromise = importer();
+};
+exports.setNodeLlamaCppImporter = setNodeLlamaCppImporter;
+const importNodeLlamaCpp = async () => {
+    if (!nodeLlamaImportPromise) {
+        nodeLlamaImportPromise = import("node-llama-cpp");
+    }
+    return nodeLlamaImportPromise;
+};
+class LocalEmbeddingProvider {
+    llama = null;
+    model = null;
+    context = null;
+    initPromise = null;
+    modelPath;
+    constructor(config) {
+        // Override transformers.js default with node-llama-cpp default
+        this.modelPath = config.model_name === 'Xenova/all-MiniLM-L6-v2'
+            ? DEFAULT_LOCAL_MODEL
+            : (config.model_name || DEFAULT_LOCAL_MODEL);
+    }
+    async ensureInitialized() {
+        if (this.context) {
+            return;
+        }
+        if (this.initPromise) {
+            return this.initPromise;
+        }
+        this.initPromise = this.doInitialize();
+        return this.initPromise;
+    }
+    async doInitialize() {
+        try {
+            const { getLlama, resolveModelFile, LlamaLogLevel } = await importNodeLlamaCpp();
+            if (!this.llama) {
+                this.llama = await getLlama({ logLevel: LlamaLogLevel.error });
+            }
+            if (!this.model) {
+                const resolved = await resolveModelFile(this.modelPath);
+                this.model = await this.llama.loadModel({ modelPath: resolved });
+            }
+            if (!this.context) {
+                this.context = await this.model.createEmbeddingContext();
+            }
+        }
+        catch (err) {
+            const detail = err instanceof Error ? err.message : String(err);
+            throw new Error(`Local embeddings unavailable. Reason: ${detail}`, {
+                cause: err,
+            });
+        }
+    }
+    async generateEmbedding(text) {
+        await this.ensureInitialized();
+        const embedding = await this.context.getEmbeddingFor(text);
+        const vector = embedding.vector; // TypedArray
+        // Optimized normalization loop
+        let sumSq = 0;
+        const len = vector.length;
+        // First pass: Calculate magnitude and sanitize (implicitly handled by JS numbers usually, but keeping finite check if needed)
+        // For performance, we assume node-llama-cpp returns valid floats. 
+        // If strict sanitization is needed, it can be combined.
+        for (let i = 0; i < len; i++) {
+            const val = vector[i];
+            if (Number.isFinite(val)) {
+                sumSq += val * val;
+            }
+            else {
+                vector[i] = 0;
+            }
+        }
+        const magnitude = Math.sqrt(sumSq);
+        const result = new Array(len);
+        if (magnitude > 0) {
+            const scale = 1.0 / magnitude;
+            for (let i = 0; i < len; i++) {
+                result[i] = vector[i] * scale;
+            }
+        }
+        else {
+            // Zero vector case
+            for (let i = 0; i < len; i++) {
+                result[i] = vector[i]; // or 0
+            }
+        }
+        return result;
+    }
+    // Optimized batch processing for local embedding
+    async generateEmbeddings(texts) {
+        await this.ensureInitialized();
+        // node-llama-cpp's createEmbeddingContext might not support batch directly yet depending on version,
+        // but we can at least optimize the loop.
+        // If newer version supports batch, we should use it. For now, we parallelize with limit.
+        return Promise.all(texts.map(text => this.generateEmbedding(text)));
+    }
+}
+exports.LocalEmbeddingProvider = LocalEmbeddingProvider;

package/dist/src/service/embedding/remote.d.ts

@@ -0,0 +1,9 @@
+import { EmbeddingProvider, EmbeddingConfig } from './interface';
+export declare class RemoteEmbeddingProvider implements EmbeddingProvider {
+    private apiKey;
+    private modelName;
+    private apiBase;
+    constructor(config: EmbeddingConfig);
+    generateEmbedding(text: string): Promise<number[]>;
+    generateEmbeddings(texts: string[]): Promise<number[][]>;
+}

package/dist/src/service/embedding/remote.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoteEmbeddingProvider = void 0;
+class RemoteEmbeddingProvider {
+    apiKey;
+    modelName;
+    apiBase;
+    constructor(config) {
+        this.apiKey = config.api_key || '';
+        this.modelName = config.model_name;
+        this.apiBase = config.api_base || 'https://api.openai.com/v1';
+    }
+    async generateEmbedding(text) {
+        const embeddings = await this.generateEmbeddings([text]);
+        return embeddings[0];
+    }
+    // Optimized batch embedding generation
+    async generateEmbeddings(texts) {
+        const response = await fetch(`${this.apiBase}/embeddings`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`
+            },
+            body: JSON.stringify({
+                input: texts,
+                model: this.modelName
+            })
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Remote API error: ${response.status} ${error}`);
+        }
+        const data = await response.json();
+        if (data.data && Array.isArray(data.data) && data.data.length > 0) {
+            // Ensure order matches input
+            return data.data.sort((a, b) => a.index - b.index).map((item) => item.embedding);
+        }
+        throw new Error('Unexpected API response format');
+    }
+}
+exports.RemoteEmbeddingProvider = RemoteEmbeddingProvider;

package/dist/src/service/metadata/factory.d.ts

@@ -0,0 +1,13 @@
+import { MetadataProvider, MetadataConfig } from './interface';
+import { DocumentSchema } from '../../client/lancedb';
+export declare class RemoteMetadataProvider implements MetadataProvider {
+    private config;
+    constructor(config: MetadataConfig);
+    connect(): Promise<void>;
+    addAssets(docs: DocumentSchema[]): Promise<void>;
+    search(query: string, limit?: number, filter?: string): Promise<DocumentSchema[]>;
+    list(limit?: number): Promise<DocumentSchema[]>;
+    delete(filter: string): Promise<void>;
+    generateEmbedding(text: string): Promise<number[]>;
+}
+export declare function createMetadataProvider(config: MetadataConfig): MetadataProvider;

package/dist/src/service/metadata/factory.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoteMetadataProvider = void 0;
+exports.createMetadataProvider = createMetadataProvider;
+const local_1 = require("./local");
+class RemoteMetadataProvider {
+    config;
+    constructor(config) {
+        this.config = config;
+        if (!config.remote_api_endpoint) {
+            throw new Error("Missing remote API endpoint configuration");
+        }
+    }
+    async connect() {
+        // TODO: Implement health check or auth verification
+        console.log("Connected to remote metadata service at", this.config.remote_api_endpoint);
+    }
+    async addAssets(docs) {
+        throw new Error('Remote metadata service not implemented yet');
+    }
+    async search(query, limit, filter) {
+        throw new Error('Remote metadata service not implemented yet');
+    }
+    async list(limit) {
+        throw new Error('Remote metadata service not implemented yet');
+    }
+    async delete(filter) {
+        throw new Error('Remote metadata service not implemented yet');
+    }
+    async generateEmbedding(text) {
+        throw new Error('Remote metadata service not implemented yet');
+    }
+}
+exports.RemoteMetadataProvider = RemoteMetadataProvider;
+function createMetadataProvider(config) {
+    if (!config) {
+        throw new Error('Metadata configuration is missing');
+    }
+    if (config.type === 'local' || !config.type) { // Default to local if type is missing but config exists
+        return new local_1.LocalMetadataProvider(config);
+    }
+    else if (config.type === 'remote') {
+        return new RemoteMetadataProvider(config);
+    }
+    else {
+        throw new Error(`Unsupported metadata storage type: ${config.type}`);
+    }
+}

package/dist/src/service/metadata/interface.d.ts

@@ -0,0 +1,17 @@
+import { DocumentSchema } from '../../client/lancedb';
+import { EmbeddingConfig } from '../embedding/interface';
+export interface MetadataProvider {
+    connect(): Promise<void>;
+    addAssets(docs: DocumentSchema[]): Promise<void>;
+    search(query: string, limit?: number, filter?: string): Promise<DocumentSchema[]>;
+    list(limit?: number, filter?: string): Promise<DocumentSchema[]>;
+    delete(filter: string): Promise<void>;
+    generateEmbedding(text: string): Promise<number[]>;
+}
+export interface MetadataConfig {
+    type: 'local' | 'remote';
+    lancedb_uri?: string;
+    remote_api_endpoint?: string;
+    remote_api_key?: string;
+    embedding?: EmbeddingConfig;
+}

package/dist/src/service/metadata/interface.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/src/service/metadata/local.d.ts

@@ -0,0 +1,13 @@
+import { MetadataProvider, MetadataConfig } from './interface';
+import { DocumentSchema } from '../../client/lancedb';
+export declare class LocalMetadataProvider implements MetadataProvider {
+    private client;
+    private embeddingProvider;
+    constructor(config: MetadataConfig);
+    connect(): Promise<void>;
+    addAssets(docs: DocumentSchema[]): Promise<void>;
+    search(query: string, limit?: number, filter?: string): Promise<DocumentSchema[]>;
+    list(limit?: number, filter?: string): Promise<DocumentSchema[]>;
+    delete(filter: string): Promise<void>;
+    generateEmbedding(text: string): Promise<number[]>;
+}

package/dist/src/service/metadata/local.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalMetadataProvider = void 0;
+const lancedb_1 = require("../../client/lancedb");
+const factory_1 = require("../embedding/factory");
+class LocalMetadataProvider {
+    client;
+    embeddingProvider;
+    constructor(config) {
+        if (!config.lancedb_uri) {
+            // Fallback to default if somehow not passed
+            // Use an absolute path or path relative to home to avoid issues when running in different cwds via OpenClaw daemon
+            const os = require('os');
+            const path = require('path');
+            config.lancedb_uri = path.join(os.homedir(), '.openclaw', 'contextlake', 'data');
+        }
+        // Ensure embedding config exists
+        if (!config.embedding) {
+            config.embedding = {
+                provider: 'local',
+                model_name: 'hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf'
+            };
+        }
+        if (!config.lancedb_uri || !config.embedding) {
+            throw new Error(`Missing LanceDB configuration: uri=${config.lancedb_uri}, embedding=${!!config.embedding}`);
+        }
+        this.embeddingProvider = (0, factory_1.createEmbeddingProvider)(config.embedding);
+        this.client = new lancedb_1.ContextLakeLanceDBClient({ uri: config.lancedb_uri }, this.embeddingProvider);
+    }
+    async connect() {
+        await this.client.connect();
+    }
+    async addAssets(docs) {
+        await this.client.addAssets(docs);
+    }
+    async search(query, limit, filter) {
+        return await this.client.search(query, limit, filter);
+    }
+    async list(limit, filter) {
+        return await this.client.list(limit, filter);
+    }
+    async delete(filter) {
+        await this.client.delete(filter);
+    }
+    async generateEmbedding(text) {
+        return await this.embeddingProvider.generateEmbedding(text);
+    }
+}
+exports.LocalMetadataProvider = LocalMetadataProvider;

package/dist/src/service/storage/factory.d.ts

@@ -0,0 +1,2 @@
+import { StorageProvider, StorageConfig } from './interface';
+export declare function createStorageProvider(config: StorageConfig): StorageProvider;

package/dist/src/service/storage/factory.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createStorageProvider = createStorageProvider;
+const local_1 = require("./local");
+const tos_1 = require("../../client/tos");
+function createStorageProvider(config) {
+    if (config.type === 'local') {
+        return new local_1.LocalStorageProvider(config.local_base_dir);
+    }
+    else if (config.type === 'tos') {
+        if (!config.tos || !config.tos.region || !config.tos.path) {
+            throw new Error('Missing TOS configuration: region and path required');
+        }
+        return new tos_1.ContextLakeTosClient(config.tos);
+    }
+    else {
+        throw new Error(`Unsupported storage type: ${config.type}`);
+    }
+}

package/dist/src/service/storage/interface.d.ts

@@ -0,0 +1,32 @@
+export interface StorageProvider {
+    /**
+     * Upload file content
+     * @param fileName - Name of the file
+     * @param buffer - File content
+     * @returns The storage URL (e.g. file:///... or tos://...)
+     */
+    uploadFile(fileName: string, buffer: Buffer): Promise<string>;
+    /**
+     * Download file content
+     * @param url - Storage URL
+     * @returns File content buffer
+     */
+    downloadFile(url: string): Promise<Buffer>;
+    /**
+     * Delete file
+     * @param url - Storage URL
+     */
+    deleteFile(url: string): Promise<void>;
+}
+export interface StorageConfig {
+    type: 'local' | 'tos';
+    local_base_dir?: string;
+    tos?: {
+        access_key?: string;
+        secret_key?: string;
+        region?: string;
+        path?: string;
+        endpoint?: string;
+        sts_token?: string;
+    };
+}

package/dist/src/service/storage/interface.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/src/service/storage/local.d.ts

@@ -0,0 +1,9 @@
+import { StorageProvider } from './interface';
+export declare class LocalStorageProvider implements StorageProvider {
+    private baseDir;
+    constructor(baseDir?: string);
+    uploadFile(fileName: string, buffer: Buffer): Promise<string>;
+    downloadFile(url: string): Promise<Buffer>;
+    deleteFile(url: string): Promise<void>;
+    private parseUrl;
+}

package/dist/src/service/storage/local.js

@@ -0,0 +1,72 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalStorageProvider = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+class LocalStorageProvider {
+    baseDir;
+    constructor(baseDir = './data/files') {
+        this.baseDir = path.resolve(baseDir);
+        if (!fs.existsSync(this.baseDir)) {
+            fs.mkdirSync(this.baseDir, { recursive: true });
+        }
+    }
+    async uploadFile(fileName, buffer) {
+        if (!fs.existsSync(this.baseDir)) {
+            fs.mkdirSync(this.baseDir, { recursive: true });
+        }
+        const filePath = path.join(this.baseDir, fileName);
+        await fs.promises.writeFile(filePath, buffer);
+        return `file://${filePath}`;
+    }
+    async downloadFile(url) {
+        const filePath = this.parseUrl(url);
+        return await fs.promises.readFile(filePath);
+    }
+    async deleteFile(url) {
+        const filePath = this.parseUrl(url);
+        if (fs.existsSync(filePath)) {
+            await fs.promises.unlink(filePath);
+        }
+    }
+    parseUrl(url) {
+        if (!url.startsWith('file://')) {
+            throw new Error(`Invalid local file URL: ${url}`);
+        }
+        return url.replace('file://', '');
+    }
+}
+exports.LocalStorageProvider = LocalStorageProvider;

package/dist/src/skills/las-data-profiler/index.d.ts

@@ -0,0 +1,26 @@
+export interface ConnectParams {
+    datasource_name: string;
+    vendor: 'volcengine' | 'alibaba' | 'tencent' | 'aws' | 'local';
+    endpoint?: string;
+    access_key?: string;
+    secret_key?: string;
+    region?: string;
+    bucket: string;
+    prefix: string;
+    sample_rows?: number;
+}
+interface ConnectResult {
+    status: 'success' | 'error';
+    datasource_name: string;
+    db_path: string;
+    env_path: string;
+    tables: string[];
+    summary?: {
+        total_files: number;
+        structured_files: number;
+        media_files: number;
+    };
+    error?: string;
+}
+export declare function connectDataSource(params: ConnectParams, _ctx?: any): Promise<ConnectResult>;
+export {};