@byted-las/contextlake-openclaw 1.0.6 → 1.0.8

package/dist/src/service/embedding/local.js

@@ -1,107 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.LocalEmbeddingProvider = exports.setNodeLlamaCppImporter = void 0;
-// import type { Llama, LlamaEmbeddingContext, LlamaModel } from 'node-llama-cpp';
-const DEFAULT_LOCAL_MODEL = 'hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf';
-let nodeLlamaImportPromise = null;
-const setNodeLlamaCppImporter = (importer) => {
-    nodeLlamaImportPromise = importer();
-};
-exports.setNodeLlamaCppImporter = setNodeLlamaCppImporter;
-const importNodeLlamaCpp = async () => {
-    if (!nodeLlamaImportPromise) {
-        nodeLlamaImportPromise = import('node-llama-cpp');
-    }
-    return nodeLlamaImportPromise;
-};
-class LocalEmbeddingProvider {
-    llama = null;
-    model = null;
-    context = null;
-    initPromise = null;
-    modelPath;
-    constructor(config) {
-        // Override transformers.js default with node-llama-cpp default
-        this.modelPath = config.model_name === 'Xenova/all-MiniLM-L6-v2'
-            ? DEFAULT_LOCAL_MODEL
-            : (config.model_name || DEFAULT_LOCAL_MODEL);
-    }
-    async ensureInitialized() {
-        if (this.context) {
-            return;
-        }
-        if (this.initPromise) {
-            return this.initPromise;
-        }
-        this.initPromise = this.doInitialize().catch((err) => {
-            this.initPromise = null;
-            this.context = null;
-            this.model = null;
-            this.llama = null;
-            throw err;
-        });
-        return this.initPromise;
-    }
-    async doInitialize() {
-        try {
-            const { getLlama, resolveModelFile, LlamaLogLevel } = await importNodeLlamaCpp();
-            if (!this.llama) {
-                this.llama = await getLlama({ logLevel: LlamaLogLevel.error });
-            }
-            if (!this.model) {
-                const resolved = await resolveModelFile(this.modelPath);
-                this.model = await this.llama.loadModel({ modelPath: resolved });
-            }
-            if (!this.context) {
-                this.context = await this.model.createEmbeddingContext();
-            }
-        }
-        catch (err) {
-            const detail = err instanceof Error ? err.message : String(err);
-            throw new Error(`Local embeddings unavailable. Reason: ${detail}`, {
-                cause: err,
-            });
-        }
-    }
-    async generateEmbedding(text) {
-        if (!text || !text.trim()) {
-            throw new Error('Embedding input text must be a non-empty string');
-        }
-        await this.ensureInitialized();
-        const embedding = await this.context.getEmbeddingFor(text);
-        const vector = embedding.vector; // TypedArray
-        // Optimized normalization loop
-        let sumSq = 0;
-        const len = vector.length;
-        for (let i = 0; i < len; i++) {
-            const val = vector[i];
-            if (Number.isFinite(val)) {
-                sumSq += val * val;
-            }
-            else {
-                vector[i] = 0;
-            }
-        }
-        const magnitude = Math.sqrt(sumSq);
-        const result = new Array(len);
-        if (magnitude > 0) {
-            const scale = 1.0 / magnitude;
-            for (let i = 0; i < len; i++) {
-                result[i] = vector[i] * scale;
-            }
-        }
-        else {
-            for (let i = 0; i < len; i++) {
-                result[i] = vector[i];
-            }
-        }
-        return result;
-    }
-    async generateEmbeddings(texts) {
-        if (!Array.isArray(texts)) {
-            throw new Error('Embedding input must be an array of strings');
-        }
-        return Promise.all(texts.map(text => this.generateEmbedding(text)));
-    }
-}
-exports.LocalEmbeddingProvider = LocalEmbeddingProvider;

package/dist/src/skills/SKILL.md

@@ -1,39 +0,0 @@
----
-name: byted-las-data-profiler
-description: |
-  Volcengine TOS Dataset Profiling Tool. Based on the S3-compatible protocol, it scans the file structure in TOS buckets and catalogs them.
-  It writes the catalog index to a local LanceDB. It is also compatible with Alibaba Cloud OSS, Tencent Cloud COS, AWS S3, and the local file system.
-
-  IMPORTANT RULE: You are STRICTLY FORBIDDEN from writing or executing Python scripts to access S3/TOS or LanceDB.
-  You MUST exclusively use the provided tools (`list-s3-objects`, `read-s3-object`, `write-lance-catalog`, `las-data-profiler`) to accomplish the profiling tasks.
----
-
-## Trigger Scenarios
-Be sure to use this Skill when the user mentions the following scenarios:
-- Need to scan the file structure in a TOS bucket or understand the dataset composition
-- Need to connect to object storage (TOS/OSS/COS/S3) using the S3 protocol
-- Need to scan, traverse, or catalog the file structure of a specific bucket or local directory
-- Need to understand what a batch of data files contains and what their schema looks like
-- Need to extract meta-information such as image resolution, audio/video duration, PDF page count, etc.
-- Need to write the meta-information of object storage or local files into LanceDB
-- Mentions keywords like "dataset scanning", "file cataloging", "data catalog", "data profiling", etc.
-
-## Overall Workflow
-When instructed to profile a dataset, you should prefer using the `las-data-profiler` tool directly, which automatically handles the S3 listing and LanceDB writing using internal TypeScript logic.
-If you need to perform custom exploration, you can use `list-s3-objects` to traverse the bucket and `read-s3-object` to read file headers, and `write-lance-catalog` to save results.
-
-## Parameter Description (for `las-data-profiler` tool)
-| Parameter | Description | Example |
-|-----------|-------------|---------|
-| datasource_name | The name of the data source | my_tos_data |
-| vendor | volcengine / alibaba / tencent / aws / local | volcengine |
-| endpoint | S3 Endpoint URL (not required for local) | https://tos-s3-cn-beijing.volces.com |
-| access_key | AK | - |
-| secret_key | SK | - |
-| region | Region identifier | cn-beijing |
-| bucket | Bucket name (root directory path when local) | my-data-bucket |
-| prefix | Path prefix to limit the scan scope | datasets/2024/ |
-
-## Output Location
-- LanceDB table storage path: `~/.openclaw/contextlake/profiler/{datasource_name}/catalog_db`
-- Configuration file: `~/.openclaw/contextlake/profiler/{datasource_name}/env.sh`