agentlang 0.10.3 → 0.10.5

package/out/runtime/resolvers/vector/types.d.ts

@@ -1,32 +0,0 @@
-export interface VectorRecord {
-    id: string;
-    embedding: number[];
-    tenantId?: string;
-    agentId?: string;
-    documentId?: string;
-    metadata?: Record<string, any>;
-}
-export interface SearchResult {
-    id: string;
-    distance: number;
-    tenantId?: string;
-    agentId?: string;
-    documentId?: string;
-    metadata?: Record<string, any>;
-}
-export interface VectorStore {
-    init(): Promise<void>;
-    addEmbedding(record: VectorRecord): Promise<void>;
-    addEmbeddings(records: VectorRecord[]): Promise<void>;
-    search(embedding: number[], tenantId?: string, agentId?: string, limit?: number): Promise<SearchResult[]>;
-    delete(id: string): Promise<void>;
-    exists(id: string): Promise<boolean>;
-    close(): Promise<void>;
-}
-export interface VectorStoreConfig {
-    dbname?: string;
-    moduleName: string;
-    vectorDimension: number;
-    inMemory?: boolean;
-}
-//# sourceMappingURL=types.d.ts.map

package/out/runtime/resolvers/vector/types.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/runtime/resolvers/vector/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,YAAY,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClD,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACtD,MAAM,CACJ,SAAS,EAAE,MAAM,EAAE,EACnB,QAAQ,CAAC,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IAC3B,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACrC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB"}

package/out/runtime/resolvers/vector/types.js

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=types.js.map

package/out/runtime/resolvers/vector/types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/runtime/resolvers/vector/types.ts"],"names":[],"mappings":""}

package/src/runtime/document-retriever.ts

@@ -1,311 +0,0 @@
-import { logger } from './logger.js';
-import { AppConfig } from './state.js';
-import { TextChunker } from './embeddings/chunker.js';
-import { OpenAIEmbeddingProvider } from './embeddings/openai.js';
-import { LanceDBVectorStore } from './resolvers/vector/lancedb-store.js';
-import type { VectorStore } from './resolvers/vector/types.js';
-import crypto from 'crypto';
-import { readFileSync } from 'fs';
-import { resolve as pathResolve } from 'path';
-
-const VECTOR_DIMENSION = 1536;
-
-interface LocalChunk {
-  id: string;
-  content: string;
-  documentTitle: string;
-  chunkIndex: number;
-}
-
-function usePgvector(): boolean {
-  if (AppConfig?.vectorStore?.type === 'pgvector') return true;
-  if (AppConfig?.vectorStore?.type === 'lancedb') return false;
-  return AppConfig?.store?.type === 'postgres';
-}
-
-/**
- * Local document retriever — embeds documents into pgvector or LanceDB
- * and retrieves relevant chunks via vector similarity search.
- */
-class DocumentRetriever {
-  private vectorStore: VectorStore | null = null;
-  private embeddingProvider: OpenAIEmbeddingProvider | null = null;
-  private chunker: TextChunker | null = null;
-  private localChunks: Map<string, LocalChunk> = new Map();
-  private processedDocuments: Set<string> = new Set();
-  private initialized = false;
-
-  private async ensureInit(): Promise<void> {
-    if (this.initialized) return;
-
-    this.chunker = new TextChunker(1000, 200);
-    this.embeddingProvider = new OpenAIEmbeddingProvider({
-      model: 'text-embedding-3-small',
-    });
-
-    if (!usePgvector()) {
-      const dbPath =
-        AppConfig?.vectorStore?.type === 'lancedb'
-          ? (AppConfig.vectorStore as any).dbname || './data/document-vectors.lance'
-          : './data/document-vectors.lance';
-
-      this.vectorStore = new LanceDBVectorStore({
-        moduleName: 'documents',
-        vectorDimension: VECTOR_DIMENSION,
-        dbname: dbPath,
-      });
-      await this.vectorStore.init();
-      logger.info(`[DOCUMENT-RETRIEVER] LanceDB vector store initialized at ${dbPath}`);
-    } else {
-      try {
-        const ag = (globalThis as any).agentlang;
-        if (ag?.rawQuery) {
-          await ag.rawQuery(`
-            CREATE TABLE IF NOT EXISTS document_local_chunks (
-              id TEXT PRIMARY KEY,
-              content TEXT NOT NULL,
-              document_title TEXT NOT NULL,
-              chunk_index INTEGER NOT NULL,
-              embedding vector(${VECTOR_DIMENSION})
-            )
-          `);
-          try {
-            await ag.rawQuery(`
-              CREATE INDEX IF NOT EXISTS idx_document_local_chunks_embedding
-              ON document_local_chunks USING hnsw (embedding vector_cosine_ops)
-            `);
-          } catch {
-            // Index may already exist or pgvector extension not loaded
-          }
-          logger.info('[DOCUMENT-RETRIEVER] pgvector local chunks table initialized');
-        }
-      } catch (err) {
-        logger.warn(`[DOCUMENT-RETRIEVER] Failed to initialize pgvector table: ${err}`);
-      }
-    }
-
-    this.initialized = true;
-  }
-
-  async processDocument(title: string, url: string): Promise<void> {
-    if (this.processedDocuments.has(title)) return;
-
-    await this.ensureInit();
-
-    try {
-      let content: string;
-      if (url.startsWith('http://') || url.startsWith('https://')) {
-        const resp = await fetch(url);
-        if (!resp.ok) {
-          logger.warn(
-            `[DOCUMENT-RETRIEVER] Failed to fetch "${title}" from ${url}: ${resp.status}`
-          );
-          return;
-        }
-        content = await resp.text();
-      } else {
-        const filePath = pathResolve(url);
-        try {
-          content = readFileSync(filePath, 'utf-8');
-        } catch (err) {
-          logger.warn(`[DOCUMENT-RETRIEVER] Failed to read "${title}" from ${filePath}: ${err}`);
-          return;
-        }
-      }
-
-      if (!content || content.trim().length === 0) {
-        logger.debug(`[DOCUMENT-RETRIEVER] Document "${title}" is empty, skipping`);
-        this.processedDocuments.add(title);
-        return;
-      }
-
-      const chunks = this.chunker!.splitText(content);
-      logger.debug(`[DOCUMENT-RETRIEVER] Document "${title}": ${chunks.length} chunks`);
-
-      if (chunks.length === 0) {
-        this.processedDocuments.add(title);
-        return;
-      }
-
-      const embeddings = await this.embeddingProvider!.embedTexts(chunks);
-
-      if (usePgvector()) {
-        await this.storePgvectorChunks(title, chunks, embeddings);
-      } else {
-        await this.storeLanceDBChunks(title, chunks, embeddings);
-      }
-
-      this.processedDocuments.add(title);
-      logger.info(
-        `[DOCUMENT-RETRIEVER] Processed "${title}": ${chunks.length} chunks embedded and stored`
-      );
-    } catch (err) {
-      logger.warn(`[DOCUMENT-RETRIEVER] Error processing "${title}": ${err}`);
-    }
-  }
-
-  private async storePgvectorChunks(
-    title: string,
-    chunks: string[],
-    embeddings: number[][]
-  ): Promise<void> {
-    const ag = (globalThis as any).agentlang;
-    if (!ag?.rawQuery) return;
-
-    for (let i = 0; i < chunks.length; i++) {
-      const id = crypto.randomUUID();
-      const embeddingStr = `[${embeddings[i].join(',')}]`;
-      await ag.rawQuery(
-        `INSERT INTO document_local_chunks (id, content, document_title, chunk_index, embedding)
-         VALUES ($1, $2, $3, $4, $5::vector)
-         ON CONFLICT (id) DO NOTHING`,
-        [id, chunks[i], title, i, embeddingStr]
-      );
-      this.localChunks.set(id, {
-        id,
-        content: chunks[i],
-        documentTitle: title,
-        chunkIndex: i,
-      });
-    }
-  }
-
-  private async storeLanceDBChunks(
-    title: string,
-    chunks: string[],
-    embeddings: number[][]
-  ): Promise<void> {
-    if (!this.vectorStore) return;
-
-    for (let i = 0; i < chunks.length; i++) {
-      const id = crypto.randomUUID();
-      await this.vectorStore.addEmbedding({
-        id,
-        embedding: embeddings[i],
-        documentId: title,
-      });
-      this.localChunks.set(id, {
-        id,
-        content: chunks[i],
-        documentTitle: title,
-        chunkIndex: i,
-      });
-    }
-  }
-
-  async query(queryText: string, documentTitles?: string[], limit: number = 10): Promise<string> {
-    await this.ensureInit();
-
-    try {
-      const results = usePgvector()
-        ? await this.queryPgvector(queryText, documentTitles, limit)
-        : await this.queryLanceDB(queryText, documentTitles, limit);
-
-      if (results.length === 0) return '';
-
-      return results.map(r => r.content).join('\n\n---\n\n');
-    } catch (err) {
-      logger.debug(`[DOCUMENT-RETRIEVER] Query failed: ${err}`);
-      return '';
-    }
-  }
-
-  private async queryPgvector(
-    queryText: string,
-    documentTitles?: string[],
-    limit: number = 10
-  ): Promise<Array<{ id: string; content: string; similarity: number }>> {
-    const ag = (globalThis as any).agentlang;
-    if (!ag?.rawQuery) return [];
-
-    const queryEmbedding = await this.embeddingProvider!.embedText(queryText);
-    const embeddingStr = `[${queryEmbedding.join(',')}]`;
-
-    let sql: string;
-    let params: any[];
-
-    if (documentTitles && documentTitles.length > 0) {
-      const placeholders = documentTitles.map((_, i) => `$${i + 2}`).join(', ');
-      sql = `SELECT id, content, document_title, 1 - (embedding <=> $1::vector) AS similarity
-             FROM document_local_chunks
-             WHERE document_title IN (${placeholders})
-             ORDER BY embedding <=> $1::vector
-             LIMIT ${limit}`;
-      params = [embeddingStr, ...documentTitles];
-    } else {
-      sql = `SELECT id, content, document_title, 1 - (embedding <=> $1::vector) AS similarity
-             FROM document_local_chunks
-             ORDER BY embedding <=> $1::vector
-             LIMIT ${limit}`;
-      params = [embeddingStr];
-    }
-
-    const rows: any[] = await ag.rawQuery(sql, params);
-    return (rows || []).map((r: any) => ({
-      id: r.id,
-      content: r.content,
-      similarity: parseFloat(r.similarity) || 0,
-    }));
-  }
-
-  private async queryLanceDB(
-    queryText: string,
-    documentTitles?: string[],
-    limit: number = 10
-  ): Promise<Array<{ id: string; content: string; similarity: number }>> {
-    if (!this.vectorStore) return [];
-
-    const queryEmbedding = await this.embeddingProvider!.embedText(queryText);
-    const searchResults = await this.vectorStore.search(
-      queryEmbedding,
-      undefined,
-      undefined,
-      limit
-    );
-
-    const results: Array<{ id: string; content: string; similarity: number }> = [];
-
-    for (const sr of searchResults) {
-      const chunk = this.localChunks.get(sr.id);
-      if (!chunk) continue;
-
-      if (documentTitles && documentTitles.length > 0) {
-        if (!documentTitles.includes(chunk.documentTitle)) continue;
-      }
-
-      results.push({
-        id: sr.id,
-        content: chunk.content,
-        similarity: 1 - (sr.distance || 0),
-      });
-    }
-
-    return results.slice(0, limit);
-  }
-
-  async close(): Promise<void> {
-    if (this.vectorStore) {
-      await this.vectorStore.close();
-      this.vectorStore = null;
-    }
-    this.localChunks.clear();
-    this.processedDocuments.clear();
-    this.initialized = false;
-  }
-}
-
-let retrieverInstance: DocumentRetriever | null = null;
-
-export function getDocumentRetriever(): DocumentRetriever {
-  if (!retrieverInstance) {
-    retrieverInstance = new DocumentRetriever();
-  }
-  return retrieverInstance;
-}
-
-export function resetDocumentRetriever(): void {
-  if (retrieverInstance) {
-    retrieverInstance.close().catch(() => {});
-    retrieverInstance = null;
-  }
-}

package/src/runtime/resolvers/vector/lancedb-store.ts

@@ -1,187 +0,0 @@
-import * as lancedb from '@lancedb/lancedb';
-import { Schema, Field, Float32, Utf8, FixedSizeList } from 'apache-arrow';
-import { logger } from '../../logger.js';
-import { VectorStore, VectorRecord, SearchResult, VectorStoreConfig } from './types.js';
-
-export class LanceDBVectorStore implements VectorStore {
-  private db: lancedb.Connection | null = null;
-  private table: lancedb.Table | null = null;
-  private config: VectorStoreConfig;
-
-  constructor(config: VectorStoreConfig) {
-    this.config = config;
-  }
-
-  async init(): Promise<void> {
-    try {
-      const dbPath = this.config.inMemory
-        ? 'memory://'
-        : this.config.dbname || `./data/vector-store/${this.config.moduleName}.lance`;
-
-      this.db = await lancedb.connect(dbPath);
-
-      const tableName = 'embeddings';
-      const tableNames = await this.db.tableNames();
-
-      if (tableNames.includes(tableName)) {
-        this.table = await this.db.openTable(tableName);
-        logger.info(`LanceDB table ${tableName} opened`);
-      } else {
-        const schema = new Schema([
-          new Field('id', new Utf8(), false),
-          new Field(
-            'embedding',
-            new FixedSizeList(this.config.vectorDimension, new Field('item', new Float32())),
-            false
-          ),
-          new Field('tenantId', new Utf8(), true),
-          new Field('agentId', new Utf8(), true),
-          new Field('documentId', new Utf8(), true),
-        ]);
-
-        this.table = await this.db.createEmptyTable(tableName, schema);
-        logger.info(`LanceDB table ${tableName} created`);
-      }
-    } catch (error) {
-      logger.error('Failed to initialize LanceDB vector store:', error);
-      throw error;
-    }
-  }
-
-  async addEmbedding(record: VectorRecord): Promise<void> {
-    if (!this.table) {
-      throw new Error('Vector store not initialized. Call init() first.');
-    }
-
-    try {
-      await this.table.add([
-        {
-          id: record.id,
-          embedding: record.embedding,
-          tenantId: record.tenantId || null,
-          agentId: record.agentId || null,
-          documentId: record.documentId || null,
-        },
-      ]);
-    } catch (error) {
-      logger.error(`Failed to add embedding ${record.id}:`, error);
-      throw error;
-    }
-  }
-
-  async addEmbeddings(records: VectorRecord[]): Promise<void> {
-    if (!this.table) {
-      throw new Error('Vector store not initialized. Call init() first.');
-    }
-
-    try {
-      const data = records.map(record => ({
-        id: record.id,
-        embedding: record.embedding,
-        tenantId: record.tenantId || null,
-        agentId: record.agentId || null,
-        documentId: record.documentId || null,
-      }));
-
-      await this.table.add(data);
-    } catch (error) {
-      logger.error(`Failed to add ${records.length} embeddings:`, error);
-      throw error;
-    }
-  }
-
-  async search(
-    embedding: number[],
-    tenantId?: string,
-    agentId?: string,
-    limit: number = 10
-  ): Promise<SearchResult[]> {
-    if (!this.table) {
-      throw new Error('Vector store not initialized. Call init() first.');
-    }
-
-    try {
-      let query = this.table.vectorSearch(embedding).limit(limit);
-
-      // Build filter conditions for agent-level isolation
-      const filters: string[] = [];
-
-      if (tenantId) {
-        // Use parameterized filtering to prevent SQL injection
-        const escapedTenantId = tenantId.replace(/'/g, "''");
-        filters.push(`tenantId = '${escapedTenantId}'`);
-      }
-
-      if (agentId) {
-        // Add agent-level filtering for strict agent isolation
-        const escapedAgentId = agentId.replace(/'/g, "''");
-        filters.push(`agentId = '${escapedAgentId}'`);
-      }
-
-      if (filters.length > 0) {
-        query = query.where(filters.join(' AND '));
-      }
-
-      const results = await query.toArray();
-
-      return results.map((row: any) => ({
-        id: row.id,
-        distance: row._distance || 0,
-        tenantId: row.tenantId,
-        agentId: row.agentId,
-        documentId: row.documentId,
-      }));
-    } catch (error) {
-      logger.error('Failed to search embeddings:', error);
-      throw error;
-    }
-  }
-
-  async delete(id: string): Promise<void> {
-    if (!this.table) {
-      throw new Error('Vector store not initialized. Call init() first.');
-    }
-
-    try {
-      await this.table.delete(`id = '${id}'`);
-    } catch (error) {
-      logger.error(`Failed to delete embedding ${id}:`, error);
-      throw error;
-    }
-  }
-
-  async exists(id: string): Promise<boolean> {
-    if (!this.table) {
-      throw new Error('Vector store not initialized. Call init() first.');
-    }
-
-    try {
-      const results = await this.table.query().where(`id = '${id}'`).limit(1).toArray();
-      return results.length > 0;
-    } catch (error) {
-      logger.error(`Failed to check existence of ${id}:`, error);
-      throw error;
-    }
-  }
-
-  async close(): Promise<void> {
-    try {
-      if (this.table) {
-        // LanceDB tables don't have explicit close method
-        this.table = null;
-      }
-      if (this.db) {
-        // Note: LanceDB connection doesn't have explicit close method
-        this.db = null;
-      }
-      logger.info('LanceDB vector store closed');
-    } catch (error) {
-      logger.error('Failed to close LanceDB vector store:', error);
-      throw error;
-    }
-  }
-}
-
-export function createLanceDBStore(config: VectorStoreConfig): VectorStore {
-  return new LanceDBVectorStore(config);
-}

package/src/runtime/resolvers/vector/types.ts

@@ -1,39 +0,0 @@
-export interface VectorRecord {
-  id: string;
-  embedding: number[];
-  tenantId?: string;
-  agentId?: string;
-  documentId?: string;
-  metadata?: Record<string, any>;
-}
-
-export interface SearchResult {
-  id: string;
-  distance: number;
-  tenantId?: string;
-  agentId?: string;
-  documentId?: string;
-  metadata?: Record<string, any>;
-}
-
-export interface VectorStore {
-  init(): Promise<void>;
-  addEmbedding(record: VectorRecord): Promise<void>;
-  addEmbeddings(records: VectorRecord[]): Promise<void>;
-  search(
-    embedding: number[],
-    tenantId?: string,
-    agentId?: string,
-    limit?: number
-  ): Promise<SearchResult[]>;
-  delete(id: string): Promise<void>;
-  exists(id: string): Promise<boolean>;
-  close(): Promise<void>;
-}
-
-export interface VectorStoreConfig {
-  dbname?: string;
-  moduleName: string;
-  vectorDimension: number;
-  inMemory?: boolean;
-}