agentlang 0.10.1 → 0.10.3

package/src/runtime/document-retriever.ts

@@ -0,0 +1,311 @@
+import { logger } from './logger.js';
+import { AppConfig } from './state.js';
+import { TextChunker } from './embeddings/chunker.js';
+import { OpenAIEmbeddingProvider } from './embeddings/openai.js';
+import { LanceDBVectorStore } from './resolvers/vector/lancedb-store.js';
+import type { VectorStore } from './resolvers/vector/types.js';
+import crypto from 'crypto';
+import { readFileSync } from 'fs';
+import { resolve as pathResolve } from 'path';
+
+const VECTOR_DIMENSION = 1536;
+
+interface LocalChunk {
+  id: string;
+  content: string;
+  documentTitle: string;
+  chunkIndex: number;
+}
+
+function usePgvector(): boolean {
+  if (AppConfig?.vectorStore?.type === 'pgvector') return true;
+  if (AppConfig?.vectorStore?.type === 'lancedb') return false;
+  return AppConfig?.store?.type === 'postgres';
+}
+
+/**
+ * Local document retriever — embeds documents into pgvector or LanceDB
+ * and retrieves relevant chunks via vector similarity search.
+ */
+class DocumentRetriever {
+  private vectorStore: VectorStore | null = null;
+  private embeddingProvider: OpenAIEmbeddingProvider | null = null;
+  private chunker: TextChunker | null = null;
+  private localChunks: Map<string, LocalChunk> = new Map();
+  private processedDocuments: Set<string> = new Set();
+  private initialized = false;
+
+  private async ensureInit(): Promise<void> {
+    if (this.initialized) return;
+
+    this.chunker = new TextChunker(1000, 200);
+    this.embeddingProvider = new OpenAIEmbeddingProvider({
+      model: 'text-embedding-3-small',
+    });
+
+    if (!usePgvector()) {
+      const dbPath =
+        AppConfig?.vectorStore?.type === 'lancedb'
+          ? (AppConfig.vectorStore as any).dbname || './data/document-vectors.lance'
+          : './data/document-vectors.lance';
+
+      this.vectorStore = new LanceDBVectorStore({
+        moduleName: 'documents',
+        vectorDimension: VECTOR_DIMENSION,
+        dbname: dbPath,
+      });
+      await this.vectorStore.init();
+      logger.info(`[DOCUMENT-RETRIEVER] LanceDB vector store initialized at ${dbPath}`);
+    } else {
+      try {
+        const ag = (globalThis as any).agentlang;
+        if (ag?.rawQuery) {
+          await ag.rawQuery(`
+            CREATE TABLE IF NOT EXISTS document_local_chunks (
+              id TEXT PRIMARY KEY,
+              content TEXT NOT NULL,
+              document_title TEXT NOT NULL,
+              chunk_index INTEGER NOT NULL,
+              embedding vector(${VECTOR_DIMENSION})
+            )
+          `);
+          try {
+            await ag.rawQuery(`
+              CREATE INDEX IF NOT EXISTS idx_document_local_chunks_embedding
+              ON document_local_chunks USING hnsw (embedding vector_cosine_ops)
+            `);
+          } catch {
+            // Index may already exist or pgvector extension not loaded
+          }
+          logger.info('[DOCUMENT-RETRIEVER] pgvector local chunks table initialized');
+        }
+      } catch (err) {
+        logger.warn(`[DOCUMENT-RETRIEVER] Failed to initialize pgvector table: ${err}`);
+      }
+    }
+
+    this.initialized = true;
+  }
+
+  async processDocument(title: string, url: string): Promise<void> {
+    if (this.processedDocuments.has(title)) return;
+
+    await this.ensureInit();
+
+    try {
+      let content: string;
+      if (url.startsWith('http://') || url.startsWith('https://')) {
+        const resp = await fetch(url);
+        if (!resp.ok) {
+          logger.warn(
+            `[DOCUMENT-RETRIEVER] Failed to fetch "${title}" from ${url}: ${resp.status}`
+          );
+          return;
+        }
+        content = await resp.text();
+      } else {
+        const filePath = pathResolve(url);
+        try {
+          content = readFileSync(filePath, 'utf-8');
+        } catch (err) {
+          logger.warn(`[DOCUMENT-RETRIEVER] Failed to read "${title}" from ${filePath}: ${err}`);
+          return;
+        }
+      }
+
+      if (!content || content.trim().length === 0) {
+        logger.debug(`[DOCUMENT-RETRIEVER] Document "${title}" is empty, skipping`);
+        this.processedDocuments.add(title);
+        return;
+      }
+
+      const chunks = this.chunker!.splitText(content);
+      logger.debug(`[DOCUMENT-RETRIEVER] Document "${title}": ${chunks.length} chunks`);
+
+      if (chunks.length === 0) {
+        this.processedDocuments.add(title);
+        return;
+      }
+
+      const embeddings = await this.embeddingProvider!.embedTexts(chunks);
+
+      if (usePgvector()) {
+        await this.storePgvectorChunks(title, chunks, embeddings);
+      } else {
+        await this.storeLanceDBChunks(title, chunks, embeddings);
+      }
+
+      this.processedDocuments.add(title);
+      logger.info(
+        `[DOCUMENT-RETRIEVER] Processed "${title}": ${chunks.length} chunks embedded and stored`
+      );
+    } catch (err) {
+      logger.warn(`[DOCUMENT-RETRIEVER] Error processing "${title}": ${err}`);
+    }
+  }
+
+  private async storePgvectorChunks(
+    title: string,
+    chunks: string[],
+    embeddings: number[][]
+  ): Promise<void> {
+    const ag = (globalThis as any).agentlang;
+    if (!ag?.rawQuery) return;
+
+    for (let i = 0; i < chunks.length; i++) {
+      const id = crypto.randomUUID();
+      const embeddingStr = `[${embeddings[i].join(',')}]`;
+      await ag.rawQuery(
+        `INSERT INTO document_local_chunks (id, content, document_title, chunk_index, embedding)
+         VALUES ($1, $2, $3, $4, $5::vector)
+         ON CONFLICT (id) DO NOTHING`,
+        [id, chunks[i], title, i, embeddingStr]
+      );
+      this.localChunks.set(id, {
+        id,
+        content: chunks[i],
+        documentTitle: title,
+        chunkIndex: i,
+      });
+    }
+  }
+
+  private async storeLanceDBChunks(
+    title: string,
+    chunks: string[],
+    embeddings: number[][]
+  ): Promise<void> {
+    if (!this.vectorStore) return;
+
+    for (let i = 0; i < chunks.length; i++) {
+      const id = crypto.randomUUID();
+      await this.vectorStore.addEmbedding({
+        id,
+        embedding: embeddings[i],
+        documentId: title,
+      });
+      this.localChunks.set(id, {
+        id,
+        content: chunks[i],
+        documentTitle: title,
+        chunkIndex: i,
+      });
+    }
+  }
+
+  async query(queryText: string, documentTitles?: string[], limit: number = 10): Promise<string> {
+    await this.ensureInit();
+
+    try {
+      const results = usePgvector()
+        ? await this.queryPgvector(queryText, documentTitles, limit)
+        : await this.queryLanceDB(queryText, documentTitles, limit);
+
+      if (results.length === 0) return '';
+
+      return results.map(r => r.content).join('\n\n---\n\n');
+    } catch (err) {
+      logger.debug(`[DOCUMENT-RETRIEVER] Query failed: ${err}`);
+      return '';
+    }
+  }
+
+  private async queryPgvector(
+    queryText: string,
+    documentTitles?: string[],
+    limit: number = 10
+  ): Promise<Array<{ id: string; content: string; similarity: number }>> {
+    const ag = (globalThis as any).agentlang;
+    if (!ag?.rawQuery) return [];
+
+    const queryEmbedding = await this.embeddingProvider!.embedText(queryText);
+    const embeddingStr = `[${queryEmbedding.join(',')}]`;
+
+    let sql: string;
+    let params: any[];
+
+    if (documentTitles && documentTitles.length > 0) {
+      const placeholders = documentTitles.map((_, i) => `$${i + 2}`).join(', ');
+      sql = `SELECT id, content, document_title, 1 - (embedding <=> $1::vector) AS similarity
+             FROM document_local_chunks
+             WHERE document_title IN (${placeholders})
+             ORDER BY embedding <=> $1::vector
+             LIMIT ${limit}`;
+      params = [embeddingStr, ...documentTitles];
+    } else {
+      sql = `SELECT id, content, document_title, 1 - (embedding <=> $1::vector) AS similarity
+             FROM document_local_chunks
+             ORDER BY embedding <=> $1::vector
+             LIMIT ${limit}`;
+      params = [embeddingStr];
+    }
+
+    const rows: any[] = await ag.rawQuery(sql, params);
+    return (rows || []).map((r: any) => ({
+      id: r.id,
+      content: r.content,
+      similarity: parseFloat(r.similarity) || 0,
+    }));
+  }
+
+  private async queryLanceDB(
+    queryText: string,
+    documentTitles?: string[],
+    limit: number = 10
+  ): Promise<Array<{ id: string; content: string; similarity: number }>> {
+    if (!this.vectorStore) return [];
+
+    const queryEmbedding = await this.embeddingProvider!.embedText(queryText);
+    const searchResults = await this.vectorStore.search(
+      queryEmbedding,
+      undefined,
+      undefined,
+      limit
+    );
+
+    const results: Array<{ id: string; content: string; similarity: number }> = [];
+
+    for (const sr of searchResults) {
+      const chunk = this.localChunks.get(sr.id);
+      if (!chunk) continue;
+
+      if (documentTitles && documentTitles.length > 0) {
+        if (!documentTitles.includes(chunk.documentTitle)) continue;
+      }
+
+      results.push({
+        id: sr.id,
+        content: chunk.content,
+        similarity: 1 - (sr.distance || 0),
+      });
+    }
+
+    return results.slice(0, limit);
+  }
+
+  async close(): Promise<void> {
+    if (this.vectorStore) {
+      await this.vectorStore.close();
+      this.vectorStore = null;
+    }
+    this.localChunks.clear();
+    this.processedDocuments.clear();
+    this.initialized = false;
+  }
+}
+
+let retrieverInstance: DocumentRetriever | null = null;
+
+export function getDocumentRetriever(): DocumentRetriever {
+  if (!retrieverInstance) {
+    retrieverInstance = new DocumentRetriever();
+  }
+  return retrieverInstance;
+}
+
+export function resetDocumentRetriever(): void {
+  if (retrieverInstance) {
+    retrieverInstance.close().catch(() => {});
+    retrieverInstance = null;
+  }
+}

package/src/runtime/embeddings/chunker.ts

@@ -1,41 +1,79 @@
+/**
+ * Streaming text chunker - yields chunks one at a time without storing all in memory.
+ */
 export class TextChunker {
   private chunkSize: number;
   private chunkOverlap: number;
   private separators: string[] = ['\n\n', '\n', '. ', ' ', ''];
 
   constructor(chunkSize: number = 1000, chunkOverlap: number = 200) {
-    this.chunkSize = chunkSize;
-    this.chunkOverlap = chunkOverlap;
+    // Ensure valid values - overlap must be less than chunk size to avoid infinite loop
+    this.chunkSize = Math.max(100, chunkSize || 1000);
+    // Cap overlap to at most 20% of chunk size to ensure progress
+    this.chunkOverlap = Math.max(
+      0,
+      Math.min(chunkOverlap || 200, Math.floor(this.chunkSize * 0.2))
+    );
   }
 
-  splitText(text: string): string[] {
+  /**
+   * Calculate total chunks without creating them all in memory.
+   * Used for logging/progress tracking.
+   */
+  estimateChunks(text: string): number {
+    if (text.length <= this.chunkSize) {
+      return 1;
+    }
+    // Rough estimate: (text length / effective chunk size) + 1
+    const effectiveChunkSize = this.chunkSize - this.chunkOverlap;
+    return Math.ceil(text.length / effectiveChunkSize);
+  }
+
+  /**
+   * Streaming generator that yields chunks one at a time.
+   * Memory-efficient: doesn't store all chunks in an array.
+   */
+  *streamChunks(text: string): Generator<string, void, unknown> {
     if (text.length <= this.chunkSize) {
-      return [text];
+      yield text;
+      return;
     }
 
-    const chunks: string[] = [];
     let start = 0;
+    const minAdvance = Math.max(50, this.chunkSize - this.chunkOverlap); // Ensure we always advance
 
     while (start < text.length) {
       let end = Math.min(start + this.chunkSize, text.length);
 
       if (end < text.length) {
-        end = this.findBestSplitPoint(text, start, end);
+        // Try to find a good split point, but ensure we advance by at least minAdvance
+        const splitPoint = this.findBestSplitPoint(text, start, end);
+        // Only use split point if it gives us reasonable progress
+        if (splitPoint - start >= minAdvance * 0.5) {
+          end = splitPoint;
+        }
+        // Otherwise use the hard end to ensure progress
       }
 
-      chunks.push(text.substring(start, end));
-      start = end - this.chunkOverlap;
+      yield text.substring(start, end);
 
-      if (start < 0) start = 0;
-      if (start >= text.length - this.chunkOverlap) {
-        if (start < text.length) {
-          chunks.push(text.substring(start));
-        }
+      // Advance by at least minAdvance characters to avoid infinite loops
+      const nextStart = end - this.chunkOverlap;
+      start = Math.max(nextStart, start + minAdvance * 0.5);
+
+      if (start >= text.length) {
         break;
       }
     }
+  }
 
-    return chunks;
+  /**
+   * Legacy method for backwards compatibility.
+   * ⚠️ WARNING: This creates all chunks in memory and can cause OOM on large documents.
+   * Prefer streamChunks() for large documents.
+   */
+  splitText(text: string): string[] {
+    return Array.from(this.streamChunks(text));
   }
 
   private findBestSplitPoint(text: string, start: number, end: number): number {

package/src/runtime/embeddings/openai.ts

@@ -1,6 +1,7 @@
 import { OpenAIEmbeddings } from '@langchain/openai';
 import { EmbeddingProvider, EmbeddingProviderConfig } from './provider.js';
 import { getLocalEnv } from '../auth/defs.js';
+import { logger } from '../logger.js';
 
 export interface OpenAIEmbeddingConfig extends EmbeddingProviderConfig {
   model?: string;
@@ -14,6 +15,9 @@ export class OpenAIEmbeddingProvider extends EmbeddingProvider {
   constructor(config?: EmbeddingProviderConfig) {
     super(config || {});
     this.openaiConfig = (this.config as OpenAIEmbeddingConfig) || {};
+    logger.debug(
+      `[OPENAI-EMBEDDING] Provider created with model: ${this.openaiConfig.model || 'default'}`
+    );
   }
 
   protected createEmbeddings(): OpenAIEmbeddings {
@@ -21,26 +25,40 @@ export class OpenAIEmbeddingProvider extends EmbeddingProvider {
       apiKey: this.resolveApiKey(),
     };
 
-    if (this.openaiConfig.model) {
-      config.model = this.openaiConfig.model;
+    // Use this.config directly since this.openaiConfig is not initialized yet
+    // during parent constructor (super() calls createEmbeddings before child init)
+    const openaiConfig = (this.config as OpenAIEmbeddingConfig) || {};
+
+    if (openaiConfig?.model) {
+      config.model = openaiConfig.model;
     }
 
-    if (this.openaiConfig.dimensions) {
-      config.dimensions = this.openaiConfig.dimensions;
+    if (openaiConfig?.dimensions) {
+      config.dimensions = openaiConfig.dimensions;
     }
 
-    if (this.openaiConfig.maxRetries !== undefined) {
-      config.maxRetries = this.openaiConfig.maxRetries;
+    if (openaiConfig?.maxRetries !== undefined) {
+      config.maxRetries = openaiConfig.maxRetries;
     }
 
     return new OpenAIEmbeddings(config);
   }
 
   protected resolveApiKey(): string {
-    if (this.openaiConfig.apiKey) {
-      return this.openaiConfig.apiKey;
+    // Use this.config directly since this.openaiConfig may not be initialized yet
+    // during constructor (createEmbeddings is called during super())
+    const config = this.openaiConfig || (this.config as OpenAIEmbeddingConfig) || {};
+    if (config.apiKey) {
+      return config.apiKey;
+    }
+    const envKey = process.env.AGENTLANG_OPENAI_KEY || getLocalEnv('AGENTLANG_OPENAI_KEY');
+    if (envKey) {
+      return envKey;
     }
-    return process.env.AGENTLANG_OPENAI_KEY || getLocalEnv('AGENTLANG_OPENAI_KEY') || '';
+    logger.warn(
+      `[OPENAI-EMBEDDING] No API key found! Set AGENTLANG_OPENAI_KEY environment variable.`
+    );
+    return '';
   }
 
   getProviderName(): string {

package/src/runtime/embeddings/provider.ts

@@ -1,4 +1,5 @@
 import { Embeddings } from '@langchain/core/embeddings';
+import { logger } from '../logger.js';
 
 export interface EmbeddingProviderConfig {
   chunkSize?: number;
@@ -23,7 +24,27 @@ export abstract class EmbeddingProvider {
   abstract getProviderName(): string;
 
   async embedText(text: string): Promise<number[]> {
-    return await this.embeddings.embedQuery(text);
+    logger.debug(`[EMBEDDING-PROVIDER] embedText called (${text.length} chars)`);
+    const startTime = Date.now();
+    const result = await this.embeddings.embedQuery(text);
+    const duration = Date.now() - startTime;
+    logger.debug(
+      `[EMBEDDING-PROVIDER] embedText completed in ${duration}ms (${result.length} dimensions)`
+    );
+    return result;
+  }
+
+  async embedTexts(texts: string[]): Promise<number[][]> {
+    if (texts.length === 0) return [];
+    if (texts.length === 1) return [await this.embedText(texts[0])];
+    logger.debug(`[EMBEDDING-PROVIDER] embedTexts called (${texts.length} texts)`);
+    const startTime = Date.now();
+    const results = await this.embeddings.embedDocuments(texts);
+    const duration = Date.now() - startTime;
+    logger.debug(
+      `[EMBEDDING-PROVIDER] embedTexts completed in ${duration}ms (${texts.length} texts, ${results[0]?.length || 0} dimensions)`
+    );
+    return results;
   }
 
   getConfig(): EmbeddingProviderConfig {

package/src/runtime/exec-graph.ts

@@ -304,8 +304,8 @@ export async function executeGraph(execGraph: ExecGraph, env: Environment): Prom
         throw reason;
       } finally {
         if (monitoringEnabled) {
-          if (monitorIncr) env.decrementMonitor();
           env.setMonitorEntryResult(env.getLastResult());
+          if (monitorIncr) env.decrementMonitor();
         }
       }
     }
@@ -425,10 +425,28 @@ export async function executeEvent(
       env.setActiveEvent(eventInstance);
       await executeEventHelper(eventInstance, env);
     } else if (isAgentEventInstance(eventInstance)) {
+      env.setActiveEvent(eventInstance);
+      if (isMonitoringEnabled()) {
+        env.appendEntryToMonitor(
+          `{${eventInstance.getFqName()} {message "${eventInstance.lookup('message')}"}}`
+        );
+      }
       await handleAgentInvocation(eventInstance, env);
+      if (isMonitoringEnabled()) {
+        env.setMonitorEntryResult(env.getLastResult());
+      }
     }
     const r = env.getLastResult();
-    if (continuation) continuation(r);
+    if (continuation) {
+      if (env.getLastPattern() && isAgentEventInstance(eventInstance)) {
+        continuation({
+          result: r.map((res: any) => (res.attributes ? Object.fromEntries(res.attributes) : res)),
+          pattern: env.getLastPattern(),
+        });
+      } else {
+        continuation(r);
+      }
+    }
     return r;
   } catch (err) {
     if (env && env.hasHandlers()) {
@@ -490,6 +508,9 @@ export async function executeEventHelper(eventInstance: Instance, env?: Environm
         );
       }
       await handleAgentInvocation(eventInstance, env);
+      if (isMonitoringEnabled()) {
+        env.setMonitorEntryResult(env.getLastResult());
+      }
     }
     if (isLocalEnv) {
       await env.commitAllTransactions();