@byted-las/contextlake-openclaw 1.0.0

package/src/lib/actions/ingest.ts

@@ -0,0 +1,103 @@
+import { createStorageProvider } from '../../service/storage/factory';
+import { createMetadataProvider } from '../../service/metadata/factory';
+import { processFile, splitText } from '../../processor/loader';
+import * as path from 'path';
+// @ts-ignore
+import { v4 as uuidv4 } from 'uuid';
+
+interface IngestParams {
+  files: string[];
+  metadata?: Record<string, any>;
+  chunkSize?: number;
+  overlap?: number;
+}
+
+export async function ingestAssets(params: IngestParams, config: any, logger?: any) {
+  if (logger) {
+      logger.info(`[ContextLake-Action] Calling ingestAssets with params: ${JSON.stringify(params)}`);
+  } else {
+      // eslint-disable-next-line no-console
+      console.log(`[ContextLake-Action] Calling ingestAssets with params: ${JSON.stringify(params)}`);
+  }
+
+  const storageConfig = config.file_storage || { type: 'local', local_base_dir: './data/files' };
+  const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
+  
+  const storageProvider = createStorageProvider(storageConfig);
+  const metadataProvider = createMetadataProvider(metaConfig);
+  
+  await metadataProvider.connect();
+  const maxInlineSize = (config.storage_policy?.max_inline_size_kb || 1024) * 1024;
+
+  const results = [];
+
+  for (const filePath of params.files) {
+    try {
+      const fileName = path.basename(filePath);
+      const { buffer, text, type } = await processFile(filePath);
+      const createdAt = Date.now(); // Current timestamp
+      
+      let fileUrl = '';
+      let storageType = '';
+      let binaryData: Buffer = Buffer.alloc(0);
+
+      // Decide storage strategy
+      if (buffer.length <= maxInlineSize) {
+        binaryData = buffer;
+        fileUrl = `inline://${fileName}`; // Virtual URL for inline
+        storageType = 'inline';
+      } else {
+        fileUrl = await storageProvider.uploadFile(fileName, buffer);
+        storageType = storageConfig.type;
+      }
+
+      const chunks = splitText(text, params.chunkSize || 500, params.overlap || 50);
+      const docs = [];
+      
+      // If no text extracted (e.g. image), store one entry with empty text but with metadata/binary
+      if (chunks.length === 0) {
+          const vector = await metadataProvider.generateEmbedding(fileName); // Embed filename as fallback
+           docs.push({
+            id: uuidv4(),
+            vector,
+            text: '',
+            source: fileName,
+            file_type: type,
+            storage_type: storageType,
+            url: fileUrl,
+            metadata: JSON.stringify(params.metadata || {}),
+            created_at: createdAt,
+            binary_data: binaryData
+          });
+      } else {
+           for (const chunk of chunks) {
+            const vector = await metadataProvider.generateEmbedding(chunk);
+            docs.push({
+                id: uuidv4(),
+                vector,
+                text: chunk,
+                source: fileName,
+                file_type: type,
+                storage_type: storageType,
+                url: fileUrl,
+                metadata: JSON.stringify(params.metadata || {}),
+                created_at: createdAt,
+                binary_data: binaryData // Only attach to first chunk
+            });
+            // Clear binary data for subsequent chunks of the same file to avoid duplication
+            binaryData = Buffer.alloc(0); 
+          }
+      }
+
+      await metadataProvider.addAssets(docs);
+      results.push({ file: fileName, status: 'success', chunks: docs.length });
+
+    } catch (error: any) {
+      // @ts-ignore
+      results.push({ file: filePath, status: 'error', message: error.message });
+    }
+  }
+
+  // Ensure plain JSON serialization
+  return JSON.parse(JSON.stringify(results));
+}

package/src/lib/actions/manage.ts

@@ -0,0 +1,107 @@
+import { createMetadataProvider } from '../../service/metadata/factory';
+import { createStorageProvider } from '../../service/storage/factory';
+
+interface ListParams {
+  limit?: number;
+}
+
+interface DeleteParams {
+  file_ids?: string[];
+  filter?: string;
+}
+
+export async function listAssets(params: ListParams, config: any, logger?: any) {
+  if (logger) {
+      logger.info(`[ContextLake-Action] Calling listAssets with params: ${JSON.stringify(params)}`);
+  } else {
+      // eslint-disable-next-line no-console
+      console.log(`[ContextLake-Action] Calling listAssets with params: ${JSON.stringify(params)}`);
+  }
+
+  // Ensure config has default if not provided
+  const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
+  const metadataProvider = createMetadataProvider(metaConfig);
+  
+  await metadataProvider.connect();
+  const docs = await metadataProvider.list(params.limit || 100);
+  
+  // Group by file (since we store chunks)
+  const fileMap = new Map();
+  docs.forEach((doc: any) => {
+      // Basic grouping by source filename. 
+      const key = doc.source; 
+      if (!fileMap.has(key)) {
+          fileMap.set(key, {
+              source: doc.source,
+              file_type: doc.file_type,
+              storage_type: doc.storage_type,
+              url: doc.url,
+              chunks_count: 0,
+              metadata: doc.metadata
+          });
+      }
+      fileMap.get(key).chunks_count++;
+  });
+  
+  // Cleanly serialize before sending back
+  return JSON.parse(JSON.stringify(Array.from(fileMap.values())));
+}
+
+export async function deleteAssets(params: DeleteParams, config: any, logger?: any) {
+  if (logger) {
+      logger.info(`[ContextLake-Action] Calling deleteAssets with params: ${JSON.stringify(params)}`);
+  } else {
+      // eslint-disable-next-line no-console
+      console.log(`[ContextLake-Action] Calling deleteAssets with params: ${JSON.stringify(params)}`);
+  }
+
+  const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
+  const metadataProvider = createMetadataProvider(metaConfig);
+  
+  // file_storage config is optional for deletion (we might not need to delete from storage if inline)
+  // Check if file_storage config exists before creating provider
+  let storageProvider;
+  if (config.file_storage && config.file_storage.type) {
+      storageProvider = createStorageProvider(config.file_storage);
+  }
+  
+  await metadataProvider.connect();
+  
+  let deleteFilter = params.filter || '';
+  if (params.file_ids && params.file_ids.length > 0) {
+      // Construct filter
+      deleteFilter = params.file_ids.map(id => `id = '${id}'`).join(' OR ');
+  }
+  
+  if (!deleteFilter) throw new Error("Delete requires file_ids or filter");
+
+  // Retrieve documents to be deleted to get their URLs for storage cleanup
+  // Note: This might be expensive if deleting many files. 
+  // Optimization: If filter is complex, we might skip storage cleanup or do it asynchronously.
+  const docsToDelete = await metadataProvider.search('', 1000, deleteFilter);
+  
+  // 1. Delete from Metadata
+  await metadataProvider.delete(deleteFilter);
+
+  // 2. Delete from Storage (Best effort)
+  const deletedUrls = new Set<string>();
+  if (storageProvider) {
+      for (const doc of docsToDelete as any[]) {
+          if (doc.url && !deletedUrls.has(doc.url) && !doc.url.startsWith('inline://')) {
+              try {
+                  await storageProvider.deleteFile(doc.url);
+                  deletedUrls.add(doc.url);
+              } catch (e) {
+                  console.error(`Failed to delete file ${doc.url}:`, e);
+              }
+          }
+      }
+  }
+  
+  return { 
+      status: 'success', 
+      message: 'Documents deleted', 
+      deleted_count: docsToDelete.length,
+      storage_cleaned: deletedUrls.size
+  };
+}

package/src/lib/actions/retrieve.ts

@@ -0,0 +1,90 @@
+import { createStorageProvider } from '../../service/storage/factory';
+import { createMetadataProvider } from '../../service/metadata/factory';
+
+interface RetrieveParams {
+  query: string;
+  top_k?: number;
+  filter?: string; // SQL-like filter
+  include_binary?: boolean;
+}
+
+export async function retrieveAssets(params: RetrieveParams, config: any, logger?: any) {
+  if (logger) {
+      logger.info(`[ContextLake-Action] Calling retrieveAssets with params: ${JSON.stringify(params)}`);
+  } else {
+      // eslint-disable-next-line no-console
+      console.log(`[ContextLake-Action] Calling retrieveAssets with params: ${JSON.stringify(params)}`);
+  }
+
+  const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
+  const metadataProvider = createMetadataProvider(metaConfig);
+  
+  let storageProvider: any;
+  if (config.file_storage && config.file_storage.type) {
+      storageProvider = createStorageProvider(config.file_storage);
+  }
+  
+  await metadataProvider.connect();
+
+  const results = await metadataProvider.search(
+    params.query, 
+    params.top_k || 5, 
+    params.filter
+  );
+
+  // Post-process to handle binary data retrieval if requested
+  
+  if (params.include_binary) {
+      const enrichedResults = await Promise.all(results.map(async (doc: any) => {
+          // Remove raw binary_data buffer to avoid clone errors in postMessage
+          const { binary_data, ...rest } = doc;
+          
+          if (rest.vector) {
+              rest.vector = Array.from(rest.vector);
+          }
+
+          try {
+            if (rest.storage_type === 'inline') {
+                // Return inline binary data (Base64)
+                return { 
+                    ...rest, 
+                    binary_content: binary_data ? binary_data.toString('base64') : null 
+                };
+            } else if (rest.storage_type === 'local' || rest.storage_type === 'tos') {
+                if (storageProvider) {
+                    // Download from storage provider
+                    const buffer = await storageProvider.downloadFile(rest.url);
+                    return { 
+                        ...rest, 
+                        binary_content: buffer.toString('base64') 
+                    };
+                } else {
+                    return { ...rest, binary_error: 'Storage provider not configured' };
+                }
+            }
+            return rest;
+          } catch (e) {
+              return { ...rest, binary_error: 'Failed to fetch content' };
+          }
+      }));
+      return enrichedResults;
+  }
+
+  // Remove binary_data from default response to keep it light and serializable
+  return results.map((doc: any) => {
+      const { binary_data, ...rest } = doc;
+      
+      // Ensure all fields are standard serializable types (remove complex objects/buffers)
+      if (rest.vector) {
+          // Keep vector as standard array of numbers if it exists, but some clients might choke on large arrays.
+          // Usually vector is fine, but LanceDB might return custom objects like Float32Array which fail structuredClone in postMessage
+          rest.vector = Array.from(rest.vector);
+      }
+      
+      // LanceDB records might be Proxy objects or contain non-clonable getters.
+      // We stringify and parse to get a clean, plain Javascript Object.
+      const safeDoc = JSON.parse(JSON.stringify(rest));
+      
+      return safeDoc;
+  });
+}

package/src/processor/loader.ts

@@ -0,0 +1,58 @@
+import * as fs from 'fs';
+import * as path from 'path';
+// @ts-ignore
+import * as pdf from 'pdf-parse';
+import * as mammoth from 'mammoth';
+
+export interface FileData {
+  buffer: Buffer;
+  text: string;
+  type: string;
+}
+
+export async function processFile(filePath: string): Promise<FileData> {
+  const buffer = fs.readFileSync(filePath);
+  const ext = path.extname(filePath).toLowerCase();
+  let text = '';
+
+  switch (ext) {
+    case '.pdf':
+      const pdfData = await (pdf as any)(buffer);
+      text = pdfData.text;
+      break;
+    case '.docx':
+      const result = await mammoth.extractRawText({ buffer });
+      text = result.value;
+      break;
+    case '.txt':
+    case '.md':
+      text = buffer.toString('utf-8');
+      break;
+    default:
+      text = buffer.toString('utf-8'); // Fallback to text
+  }
+
+  // Basic cleaning
+  text = text.replace(/\s+/g, ' ').trim();
+
+  return {
+    buffer,
+    text,
+    type: ext.replace('.', '')
+  };
+}
+
+export function splitText(text: string, chunkSize: number = 500, overlap: number = 50): string[] {
+  const chunks: string[] = [];
+  if (!text) return chunks;
+  
+  let start = 0;
+  while (start < text.length) {
+    const end = Math.min(start + chunkSize, text.length);
+    const chunk = text.slice(start, end);
+    chunks.push(chunk);
+    start += chunkSize - overlap;
+  }
+  
+  return chunks;
+}

package/src/service/embedding/factory.ts

@@ -0,0 +1,13 @@
+import { EmbeddingProvider, EmbeddingConfig } from './interface';
+import { LocalEmbeddingProvider } from './local';
+import { RemoteEmbeddingProvider } from './remote';
+
+export function createEmbeddingProvider(config: EmbeddingConfig): EmbeddingProvider {
+    if (config.provider === 'local') {
+        return new LocalEmbeddingProvider(config);
+    } else if (config.provider === 'openai' || config.provider === 'remote') {
+        return new RemoteEmbeddingProvider(config);
+    } else {
+        throw new Error(`Unsupported embedding provider: ${config.provider}`);
+    }
+}

package/src/service/embedding/interface.ts

@@ -0,0 +1,21 @@
+
+export interface EmbeddingProvider {
+  /**
+   * Generate embedding for text
+   * @param text - Input text
+   */
+  generateEmbedding(text: string): Promise<number[]>;
+  
+  /**
+   * Generate embeddings for multiple texts
+   * @param texts - Array of input texts
+   */
+  generateEmbeddings(texts: string[]): Promise<number[][]>;
+}
+
+export interface EmbeddingConfig {
+  provider: 'local' | 'remote' | 'openai'; // Support 'openai' for backward compatibility or clarity
+  model_name: string;
+  api_key?: string;
+  api_base?: string;
+}

package/src/service/embedding/local.ts

@@ -0,0 +1,118 @@
+import { EmbeddingProvider, EmbeddingConfig } from './interface';
+// import type { Llama, LlamaEmbeddingContext, LlamaModel } from 'node-llama-cpp';
+
+const DEFAULT_LOCAL_MODEL = "hf:CompendiumLabs/bge-small-zh-v1.5-gguf/bge-small-zh-v1.5-f16.gguf";
+
+let nodeLlamaImportPromise: Promise<any> | null = null;
+
+export const setNodeLlamaCppImporter = (importer: () => Promise<any>) => {
+  nodeLlamaImportPromise = importer();
+};
+
+const importNodeLlamaCpp = async (): Promise<any> => {
+  if (!nodeLlamaImportPromise) {
+    nodeLlamaImportPromise = import("node-llama-cpp");
+  }
+  return nodeLlamaImportPromise;
+};
+
+export class LocalEmbeddingProvider implements EmbeddingProvider {
+  private llama: any = null;
+  private model: any = null;
+  private context: any = null;
+  private initPromise: Promise<void> | null = null;
+  private modelPath: string;
+
+  constructor(config: EmbeddingConfig) {
+    // Override transformers.js default with node-llama-cpp default
+    this.modelPath = config.model_name === 'Xenova/all-MiniLM-L6-v2' 
+      ? DEFAULT_LOCAL_MODEL 
+      : (config.model_name || DEFAULT_LOCAL_MODEL);
+  }
+
+  private async ensureInitialized(): Promise<void> {
+    if (this.context) {
+      return;
+    }
+    if (this.initPromise) {
+      return this.initPromise;
+    }
+
+    this.initPromise = this.doInitialize();
+    return this.initPromise;
+  }
+
+  private async doInitialize(): Promise<void> {
+    try {
+      const { getLlama, resolveModelFile, LlamaLogLevel } = await importNodeLlamaCpp();
+
+      if (!this.llama) {
+        this.llama = await getLlama({ logLevel: LlamaLogLevel.error });
+      }
+
+      if (!this.model) {
+        const resolved = await resolveModelFile(this.modelPath);
+        this.model = await this.llama.loadModel({ modelPath: resolved });
+      }
+
+      if (!this.context) {
+        this.context = await this.model.createEmbeddingContext();
+      }
+    } catch (err) {
+      const detail = err instanceof Error ? err.message : String(err);
+      throw new Error(`Local embeddings unavailable. Reason: ${detail}`, {
+        cause: err,
+      });
+    }
+  }
+
+  async generateEmbedding(text: string): Promise<number[]> {
+    await this.ensureInitialized();
+
+    const embedding = await this.context!.getEmbeddingFor(text);
+    const vector = embedding.vector; // TypedArray
+
+    // Optimized normalization loop
+    let sumSq = 0;
+    const len = vector.length;
+    
+    // First pass: Calculate magnitude and sanitize (implicitly handled by JS numbers usually, but keeping finite check if needed)
+    // For performance, we assume node-llama-cpp returns valid floats. 
+    // If strict sanitization is needed, it can be combined.
+    for (let i = 0; i < len; i++) {
+        const val = vector[i];
+        if (Number.isFinite(val)) {
+            sumSq += val * val;
+        } else {
+            vector[i] = 0;
+        }
+    }
+
+    const magnitude = Math.sqrt(sumSq);
+    const result = new Array(len);
+
+    if (magnitude > 0) {
+        const scale = 1.0 / magnitude;
+        for (let i = 0; i < len; i++) {
+            result[i] = vector[i] * scale;
+        }
+    } else {
+        // Zero vector case
+        for (let i = 0; i < len; i++) {
+            result[i] = vector[i]; // or 0
+        }
+    }
+
+    return result;
+  }
+
+  // Optimized batch processing for local embedding
+  async generateEmbeddings(texts: string[]): Promise<number[][]> {
+    await this.ensureInitialized();
+    // node-llama-cpp's createEmbeddingContext might not support batch directly yet depending on version,
+    // but we can at least optimize the loop.
+    // If newer version supports batch, we should use it. For now, we parallelize with limit.
+    
+    return Promise.all(texts.map(text => this.generateEmbedding(text)));
+  }
+}

package/src/service/embedding/remote.ts

@@ -0,0 +1,45 @@
+import { EmbeddingProvider, EmbeddingConfig } from './interface';
+
+export class RemoteEmbeddingProvider implements EmbeddingProvider {
+  private apiKey: string;
+  private modelName: string;
+  private apiBase: string;
+
+  constructor(config: EmbeddingConfig) {
+    this.apiKey = config.api_key || '';
+    this.modelName = config.model_name;
+    this.apiBase = config.api_base || 'https://api.openai.com/v1';
+  }
+
+  async generateEmbedding(text: string): Promise<number[]> {
+    const embeddings = await this.generateEmbeddings([text]);
+    return embeddings[0];
+  }
+
+  // Optimized batch embedding generation
+  async generateEmbeddings(texts: string[]): Promise<number[][]> {
+    const response = await fetch(`${this.apiBase}/embeddings`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${this.apiKey}`
+        },
+        body: JSON.stringify({
+            input: texts,
+            model: this.modelName
+        })
+    });
+
+    if (!response.ok) {
+        const error = await response.text();
+        throw new Error(`Remote API error: ${response.status} ${error}`);
+    }
+
+    const data = await response.json();
+    if (data.data && Array.isArray(data.data) && data.data.length > 0) {
+        // Ensure order matches input
+        return data.data.sort((a: any, b: any) => a.index - b.index).map((item: any) => item.embedding);
+    }
+    throw new Error('Unexpected API response format');
+  }
+}

package/src/service/metadata/factory.ts

@@ -0,0 +1,52 @@
+import { MetadataProvider, MetadataConfig } from './interface';
+import { DocumentSchema } from '../../client/lancedb';
+import { LocalMetadataProvider } from './local';
+
+export class RemoteMetadataProvider implements MetadataProvider {
+  private config: MetadataConfig;
+
+  constructor(config: MetadataConfig) {
+    this.config = config;
+    if (!config.remote_api_endpoint) {
+        throw new Error("Missing remote API endpoint configuration");
+    }
+  }
+
+  async connect(): Promise<void> {
+    // TODO: Implement health check or auth verification
+    console.log("Connected to remote metadata service at", this.config.remote_api_endpoint);
+  }
+
+  async addAssets(docs: DocumentSchema[]): Promise<void> {
+    throw new Error('Remote metadata service not implemented yet');
+  }
+
+  async search(query: string, limit?: number, filter?: string): Promise<DocumentSchema[]> {
+    throw new Error('Remote metadata service not implemented yet');
+  }
+
+  async list(limit?: number): Promise<DocumentSchema[]> {
+    throw new Error('Remote metadata service not implemented yet');
+  }
+
+  async delete(filter: string): Promise<void> {
+    throw new Error('Remote metadata service not implemented yet');
+  }
+
+  async generateEmbedding(text: string): Promise<number[]> {
+     throw new Error('Remote metadata service not implemented yet');
+  }
+}
+
+export function createMetadataProvider(config: MetadataConfig): MetadataProvider {
+  if (!config) {
+    throw new Error('Metadata configuration is missing');
+  }
+  if (config.type === 'local' || !config.type) { // Default to local if type is missing but config exists
+    return new LocalMetadataProvider(config);
+  } else if (config.type === 'remote') {
+    return new RemoteMetadataProvider(config);
+  } else {
+    throw new Error(`Unsupported metadata storage type: ${config.type}`);
+  }
+}

package/src/service/metadata/interface.ts

@@ -0,0 +1,19 @@
+import { DocumentSchema } from '../../client/lancedb';
+import { EmbeddingConfig } from '../embedding/interface';
+
+export interface MetadataProvider {
+  connect(): Promise<void>;
+  addAssets(docs: DocumentSchema[]): Promise<void>;
+  search(query: string, limit?: number, filter?: string): Promise<DocumentSchema[]>;
+  list(limit?: number, filter?: string): Promise<DocumentSchema[]>;
+  delete(filter: string): Promise<void>;
+  generateEmbedding(text: string): Promise<number[]>;
+}
+
+export interface MetadataConfig {
+  type: 'local' | 'remote';
+  lancedb_uri?: string;
+  remote_api_endpoint?: string;
+  remote_api_key?: string;
+  embedding?: EmbeddingConfig;
+}