lazlo-ai 1.0.0

package/src/document_loaders/index.ts

@@ -0,0 +1,223 @@
+/**
+ * Document Loaders
+ * 
+ * Load documents from various sources: PDF, text, CSV, JSON, HTML
+ */
+
+import { readFile } from 'fs/promises';
+import { readFileSync } from 'fs';
+import { join, extname } from 'path';
+
+export interface Document {
+  pageContent: string;
+  metadata: Record<string, unknown>;
+}
+
+export interface BaseLoader {
+  load(): Promise<Document[]>;
+  loadSync(): Document[];
+}
+
+/**
+ * Text File Loader
+ */
+export class TextLoader implements BaseLoader {
+  constructor(private filePath: string) {}
+
+  async load(): Promise<Document[]> {
+    const content = await readFile(this.filePath, 'utf-8');
+    return this.parseContent(content);
+  }
+
+  loadSync(): Document[] {
+    const content = readFileSync(this.filePath, 'utf-8');
+    return this.parseContent(content);
+  }
+
+  private parseContent(content: string): Document[] {
+    return [{
+      pageContent: content,
+      metadata: {
+        source: this.filePath,
+        type: 'text',
+      },
+    }];
+  }
+}
+
+/**
+ * CSV File Loader
+ */
+export class CSVLoader implements BaseLoader {
+  constructor(
+    private filePath: string,
+    private options: {
+      delimiter?: string;
+      headers?: string[];
+      skipEmptyLines?: boolean;
+    } = {}
+  ) {}
+
+  async load(): Promise<Document[]> {
+    const content = await readFile(this.filePath, 'utf-8');
+    return this.parseCSV(content);
+  }
+
+  loadSync(): Document[] {
+    const content = readFileSync(this.filePath, 'utf-8');
+    return this.parseCSV(content);
+  }
+
+  private parseCSV(content: string): Document[] {
+    const delimiter = this.options.delimiter || ',';
+    const lines = content.split('\n').filter(line => line.trim());
+    
+    if (lines.length === 0) return [];
+
+    const headers = this.options.headers || this.parseLine(lines[0], delimiter);
+    const documents: Document[] = [];
+
+    const startIndex = this.options.headers ? 0 : 1;
+    
+    for (let i = startIndex; i < lines.length; i++) {
+      const values = this.parseLine(lines[i], delimiter);
+      if (this.options.skipEmptyLines && values.every(v => !v.trim())) continue;
+      
+      const obj: Record<string, string> = {};
+      headers.forEach((header, idx) => {
+        obj[header.trim()] = values[idx] || '';
+      });
+
+      documents.push({
+        pageContent: JSON.stringify(obj),
+        metadata: {
+          source: this.filePath,
+          type: 'csv',
+          row: i - startIndex + 1,
+        },
+      });
+    }
+
+    return documents;
+  }
+
+  private parseLine(line: string, delimiter: string): string[] {
+    const result: string[] = [];
+    let current = '';
+    let inQuotes = false;
+
+    for (const char of line) {
+      if (char === '"') {
+        inQuotes = !inQuotes;
+      } else if (char === delimiter && !inQuotes) {
+        result.push(current);
+        current = '';
+      } else {
+        current += char;
+      }
+    }
+    result.push(current);
+    return result;
+  }
+}
+
+/**
+ * JSON File Loader
+ */
+export class JSONLoader implements BaseLoader {
+  constructor(
+    private filePath: string,
+    private options: {
+      key?: string; // If specified, extract array items from this key
+    } = {}
+  ) {}
+
+  async load(): Promise<Document[]> {
+    const content = await readFile(this.filePath, 'utf-8');
+    return this.parseJSON(content);
+  }
+
+  loadSync(): Document[] {
+    const content = readFileSync(this.filePath, 'utf-8');
+    return this.parseJSON(content);
+  }
+
+  private parseJSON(content: string): Document[] {
+    const data = JSON.parse(content);
+    
+    if (this.options.key) {
+      const array = data[this.options.key];
+      if (Array.isArray(array)) {
+        return array.map((item, idx) => ({
+          pageContent: typeof item === 'string' ? item : JSON.stringify(item),
+          metadata: {
+            source: this.filePath,
+            type: 'json',
+            key: this.options.key,
+            index: idx,
+          },
+        }));
+      }
+    }
+
+    if (Array.isArray(data)) {
+      return data.map((item, idx) => ({
+        pageContent: typeof item === 'string' ? item : JSON.stringify(item),
+        metadata: {
+          source: this.filePath,
+          type: 'json',
+          index: idx,
+        },
+      }));
+    }
+
+    return [{
+      pageContent: JSON.stringify(data),
+      metadata: {
+        source: this.filePath,
+        type: 'json',
+      },
+    }];
+  }
+}
+
+/**
+ * Auto-detect file type and load
+ */
+export async function loadDocument(filePath: string, options?: any): Promise<Document[]> {
+  const ext = extname(filePath).toLowerCase();
+  
+  switch (ext) {
+    case '.txt':
+    case '.md':
+    case '.text':
+      return new TextLoader(filePath).load();
+    
+    case '.csv':
+      return new CSVLoader(filePath, options?.csv).load();
+    
+    case '.json':
+      return new JSONLoader(filePath, options?.json).load();
+    
+    default:
+      // Default to text loader
+      return new TextLoader(filePath).load();
+  }
+}
+
+/**
+ * Load multiple files
+ */
+export async function loadDocuments(
+  filePaths: string[],
+  options?: any
+): Promise<Document[]> {
+  const documents: Document[] = [];
+  
+  for (const filePath of filePaths) {
+    const docs = await loadDocument(filePath, options);
+    documents.push(...docs);
+  }
+  
+  return documents;
+}

package/src/embeddings/google.ts

@@ -0,0 +1,119 @@
+/**
+ * Google Embeddings
+ * 
+ * Google AI Gemini embeddings for text vectorization
+ */
+
+import axios from 'axios';
+import { logger } from '../utils/logger.js';
+
+export interface GoogleEmbeddingsOptions {
+  apiKey?: string;
+  model?: string;
+  taskType?: 'RETRIEVAL_QUERY' | 'RETRIEVAL_DOCUMENT' | 'SEMANTIC_SIMILARITY' | 'CLASSIFICATION' | 'CLUSTERING';
+  title?: string;
+}
+
+export interface EmbeddingResponse {
+  embedding: {
+    values: number[];
+  };
+}
+
+/**
+ * Google Embeddings
+ * 
+ * Uses Google's Generative Language API for embeddings
+ */
+export class GoogleEmbeddings {
+  readonly model: string;
+  readonly taskType: string;
+  readonly title?: string;
+  private apiKey: string;
+  private apiUrl = 'https://generativelanguage.googleapis.com/v1beta';
+
+  constructor(options: GoogleEmbeddingsOptions = {}) {
+    this.apiKey = options.apiKey || (typeof process !== 'undefined' ? (process as any).env?.GEMINI_API_KEY : '') || '';
+    this.model = options.model || 'text-embedding-004';
+    this.taskType = options.taskType || 'RETRIEVAL_DOCUMENT';
+    this.title = options.title;
+
+    if (!this.apiKey) {
+      logger.warn('[GoogleEmbeddings] No API key provided. Set GEMINI_API_KEY env var or pass apiKey option.');
+    }
+  }
+
+  /**
+   * Embed a single text
+   */
+  async embedQuery(text: string): Promise<number[]> {
+    return this.embed([text]).then(res => res[0]);
+  }
+
+  /**
+   * Embed multiple texts
+   */
+  async embedDocuments(texts: string[]): Promise<number[][]> {
+    return this.embed(texts);
+  }
+
+  private async embed(texts: string[]): Promise<number[][]> {
+    if (!this.apiKey) {
+      throw new Error('Google API key required. Set GEMINI_API_KEY or pass apiKey option.');
+    }
+
+    const embeddings: number[][] = [];
+
+    // Google API allows up to 100 inputs per request
+    const batchSize = 100;
+    
+    for (let i = 0; i < texts.length; i += batchSize) {
+      const batch = texts.slice(i, i + batchSize);
+      
+      try {
+        const requests = batch.map((content, idx) => ({
+          model: `models/${this.model}`,
+          content: { role: 'user', parts: [{ text: content }] },
+          taskType: this.taskType,
+          title: this.title,
+        }));
+
+        const response = await axios.post(
+          `${this.apiUrl}/embedContent:batchEmbedContents?key=${this.apiKey}`,
+          { requests },
+          {
+            headers: { 'Content-Type': 'application/json' },
+            timeout: 60000,
+          }
+        );
+
+        const results = response.data.embeddings || [];
+        for (let j = 0; j < results.length; j++) {
+          const result = results[j];
+          if (result.embedding?.values) {
+            embeddings.push(result.embedding.values);
+          } else {
+            // Fallback: generate random embedding
+            logger.warn(`[GoogleEmbeddings] Empty embedding for batch ${Math.floor(i / batchSize)}, index ${j}`);
+            embeddings.push(new Array(768).fill(0));
+          }
+        }
+      } catch (error: any) {
+        logger.error(`[GoogleEmbeddings] Error embedding batch: ${error.message}`);
+        // Return zero embeddings on error
+        for (let j = 0; j < batch.length; j++) {
+          embeddings.push(new Array(768).fill(0));
+        }
+      }
+    }
+
+    return embeddings;
+  }
+}
+
+/**
+ * Factory function
+ */
+export function createGoogleEmbeddings(options?: GoogleEmbeddingsOptions): GoogleEmbeddings {
+  return new GoogleEmbeddings(options);
+}

package/src/embeddings/local.ts

@@ -0,0 +1,118 @@
+/**
+ * Local Embeddings
+ * 
+ * Zero-cost, completely offline embedding generation
+ * Uses a simple hash-based approach for demonstration
+ * For production, use transformers.js or ONNX runtime
+ */
+
+import { logger } from '../utils/logger.js';
+
+/**
+ * Local Embeddings
+ * 
+ * Generates embeddings locally without API calls.
+ * Uses a deterministic hash-based approach for consistent results.
+ * For production use, integrate with transformers.js or ONNX runtime.
+ */
+export class LocalEmbeddings {
+  readonly dimensions: number;
+  private modelName: string;
+
+  constructor(options: { dimensions?: number; modelName?: string } = {}) {
+    this.dimensions = options.dimensions || 384;
+    this.modelName = options.modelName || 'local-hash';
+    logger.info(`[LocalEmbeddings] Initialized with ${this.dimensions} dimensions`);
+  }
+
+  /**
+   * Generate embedding using a simple hash-based approach
+   * This is NOT suitable for production semantic search
+   * but provides consistent results for testing
+   */
+  private generateHashEmbedding(text: string): number[] {
+    const embedding = new Array(this.dimensions).fill(0);
+    
+    // Simple hash to generate consistent embedding
+    for (let i = 0; i < text.length; i++) {
+      const char = text.charCodeAt(i);
+      const idx1 = i % this.dimensions;
+      const idx2 = (i * 31 + char) % this.dimensions;
+      
+      embedding[idx1] += char / text.length;
+      embedding[idx2] += char / text.length;
+    }
+
+    // Normalize
+    const magnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
+    if (magnitude > 0) {
+      for (let i = 0; i < embedding.length; i++) {
+        embedding[i] /= magnitude;
+      }
+    }
+
+    return embedding;
+  }
+
+  /**
+   * Embed a single text
+   */
+  async embedQuery(text: string): Promise<number[]> {
+    return this.generateHashEmbedding(text);
+  }
+
+  /**
+   * Embed multiple texts
+   */
+  async embedDocuments(texts: string[]): Promise<number[][]> {
+    return texts.map(text => this.generateHashEmbedding(text));
+  }
+
+  /**
+   * Batch embed
+   */
+  async embed(texts: string[]): Promise<number[][]> {
+    return this.embedDocuments(texts);
+  }
+}
+
+/**
+ * Factory function
+ */
+export function createLocalEmbeddings(options?: {
+  dimensions?: number;
+  modelName?: string;
+}): LocalEmbeddings {
+  return new LocalEmbeddings(options);
+}
+
+/**
+ * Mock HuggingFace Embeddings (for compatibility)
+ * In production, use @xenova/transformers
+ */
+export class HuggingFaceEmbeddings {
+  readonly model: string;
+  readonly device: string;
+  readonly normalize: boolean;
+
+  constructor(options: {
+    model?: string;
+    device?: string;
+    normalize?: boolean;
+  } = {}) {
+    this.model = options.model || 'sentence-transformers/all-MiniLM-L6-v2';
+    this.device = options.device || 'cpu';
+    this.normalize = options.normalize !== false;
+    
+    logger.info(`[HuggingFaceEmbeddings] Initialized (mock). For real embeddings, install @xenova/transformers`);
+  }
+
+  async embedQuery(text: string): Promise<number[]> {
+    // Mock implementation
+    return new LocalEmbeddings().embedQuery(text);
+  }
+
+  async embedDocuments(texts: string[]): Promise<number[][]> {
+    return new LocalEmbeddings().embedDocuments(texts);
+  }
+}

package/src/evals/judge.ts

@@ -0,0 +1,99 @@
+/**
+ * LLMJudge - Evaluate LLM responses
+ */
+
+import { BaseChatModel, Message } from '../core/types.js';
+import { logger } from '../utils/logger.js';
+
+// ============================================================================
+// Evaluation Criteria
+// ============================================================================
+
+export enum EvalCriteria {
+  FAITHFULNESS = 'faithfulness',
+  RELEVANCE = 'relevance',
+  TOXICITY = 'toxicity',
+  COHERENCE = 'coherence',
+  ACCURACY = 'accuracy',
+}
+
+export interface EvalResult {
+  score: number;
+  explanation: string;
+  passed: boolean;
+}
+
+// ============================================================================
+// LLM Judge
+// ============================================================================
+
+const EVAL_PROMPT = `You are an impartial, strict evaluator. Your job is to grade the Assistant's response 
+based on the following criteria: {criteria}.
+
+Question asked by User: 
+"{question}"
+
+Context provided to Assistant (if any):
+"{context}"
+
+Assistant's Response:
+"{response}"
+
+Evaluate the response. Output strictly in the following JSON schema:
+{{
+    "score": <int from 1 to 10>,
+    "explanation": "<brief reason for the score>",
+    "passed": <boolean true if score >= 7 else false>
+}}
+`;
+
+export class LLMJudge {
+  private judgeLLM: BaseChatModel;
+  
+  constructor(judgeLLM: BaseChatModel) {
+    this.judgeLLM = judgeLLM;
+    logger.info('LLMJudge ready. Ready to evaluate responses.');
+  }
+  
+  async evaluate(
+    question: string,
+    response: string,
+    criteria: EvalCriteria,
+    context: string = ''
+  ): Promise<EvalResult> {
+    const prompt = EVAL_PROMPT
+      .replace('{criteria}', criteria as string)
+      .replace('{question}', question)
+      .replace('{context}', context)
+      .replace('{response}', response);
+    
+    const messages: Message[] = [
+      { role: 'system', content: 'You are a JSON grading API.' },
+      { role: 'user', content: prompt },
+    ];
+    
+    try {
+      const result = await this.judgeLLM.invoke(messages, {
+        responseFormat: { type: 'json_object' }
+      });
+      
+      const content = result.content.trim();
+      const parsed = JSON.parse(content);
+      
+      logger.debug(`[Judge] Score: ${parsed.score}/10 - Passed: ${parsed.passed}`);
+      
+      return {
+        score: parsed.score ?? 0,
+        explanation: parsed.explanation ?? '',
+        passed: parsed.passed ?? false,
+      };
+    } catch (error) {
+      logger.error(`Evaluation failed: ${error}`);
+      return {
+        score: 0,
+        explanation: String(error),
+        passed: false,
+      };
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,121 @@
+/**
+ * Lazlo - The Anti-Framework for JavaScript/TypeScript
+ * 
+ * A production-grade AI framework that prioritizes performance, transparency, and simplicity.
+ */
+
+// ============================================================================
+// Core Types
+// ============================================================================
+
+export * from './core/types.js';
+
+// ============================================================================
+// Providers
+// ============================================================================
+
+export { OpenAI, createOpenAI } from './providers/openai.js';
+export { Ollama, createOllama } from './providers/ollama.js';
+
+// ============================================================================
+// Chains
+// ============================================================================
+
+export { LLMChain, SequentialChain, PromptTemplate } from './chains/llmchain.js';
+export { RetrievalQAChain, createRetrievalChain } from './chains/rag.js';
+
+// ============================================================================
+// Router
+// ============================================================================
+
+export { SmartRouter } from './router/smartrouter.js';
+
+// ============================================================================
+// Cache
+// ============================================================================
+
+export { SemanticCache } from './cache/semantic.js';
+
+// ============================================================================
+// Memory
+// ============================================================================
+
+export { BufferMemory, TokenBudgetMemory } from './memory/buffer.js';
+
+// ============================================================================
+// Tools
+// ============================================================================
+
+export { Tool, tool, createToolDecorator, defineTool, calculator, search, weather, KeiroSearch } from './tools/index.js';
+
+// ============================================================================
+// Tracing
+// ============================================================================
+
+export { tracer, Tracer } from './tracing/tracer.js';
+
+// ============================================================================
+// Evals
+// ============================================================================
+
+export { LLMJudge, EvalCriteria } from './evals/judge.js';
+
+// ============================================================================
+// Text Splitters
+// ============================================================================
+
+export { CharacterTextSplitter, RecursiveCharacterTextSplitter, createCharacterSplitter, createRecursiveSplitter } from './text_splitters/index.js';
+
+// ============================================================================
+// Vector Stores
+// ============================================================================
+
+export { InMemoryVectorStore, ChromaVectorStore, SQLiteVectorStore, createVectorStore, createChromaVectorStore, createSQLiteVectorStore } from './vectorstores/index.js';
+
+// ============================================================================
+// Document Loaders
+// ============================================================================
+
+export { TextLoader, CSVLoader, JSONLoader, loadDocument, loadDocuments } from './document_loaders/index.js';
+
+// ============================================================================
+// Embeddings
+// ============================================================================
+
+export { GoogleEmbeddings, createGoogleEmbeddings } from './embeddings/google.js';
+export { LocalEmbeddings, HuggingFaceEmbeddings, createLocalEmbeddings } from './embeddings/local.js';
+
+// ============================================================================
+// Output Parsers
+// ============================================================================
+
+export { JSONOutputParser, XMLOutputParser, CommaSeparatedListParser, MarkdownCodeBlockParser, ToolCallParser, createJSONParser, createXMLParser, createListParser, createMarkdownParser } from './parsers/output.js';
+
+// ============================================================================
+// Prompt Registry
+// ============================================================================
+
+export { PromptRegistry, SimplePromptTemplate, createPromptRegistry } from './prompts/registry.js';
+
+// ============================================================================
+// Retrievers
+// ============================================================================
+
+export { ContextualCompressionRetriever, ParentDocumentRetriever, EnsembleRetriever, createVectorStoreRetriever } from './retrievers/index.js';
+export type { BaseRetriever } from './retrievers/index.js';
+
+// ============================================================================
+// Utils
+// ============================================================================
+
+export { logger } from './utils/logger.js';
+export { getPricing, setPricing, calculateCost, estimateCost, estimateTokens } from './utils/pricing.js';
+
+// ============================================================================
+// Initialize
+// ============================================================================
+
+import { loadEnvPricing } from './utils/pricing.js';
+loadEnvPricing();
+
+console.log('⚡ Lazlo JS loaded. The Anti-Framework.');