agentlang 0.9.6 → 0.9.8

package/src/runtime/modules/ai.ts

@@ -31,6 +31,7 @@ import {
   newInstanceAttributes,
   Record,
   Retry,
+  resolveDocumentAliases,
 } from '../module.js';
 import { provider } from '../agents/registry.js';
 import {
@@ -60,7 +61,6 @@ import {
   newAgentScenario,
   PlannerInstructions,
 } from '../agents/common.js';
-import { PathAttributeNameQuery } from '../defs.js';
 import { logger } from '../logger.js';
 import { FlowStep } from '../agents/flows.js';
 import Handlebars from 'handlebars';
@@ -74,6 +74,27 @@ export const AgentLearnerType = 'learner';
 
 const AgentEvalType = 'eval';
 
+function buildEmbeddingConfig(): object {
+  const config: any = {
+    provider: process.env.AGENTLANG_EMBEDDING_PROVIDER || 'openai',
+    model: process.env.AGENTLANG_EMBEDDING_MODEL || 'text-embedding-3-small',
+    chunkSize: 1000,
+    chunkOverlap: 200,
+  };
+
+  if (process.env.AGENTLANG_EMBEDDING_CHUNKSIZE) {
+    config.chunkSize = parseInt(process.env.AGENTLANG_EMBEDDING_CHUNKSIZE, 1000);
+  }
+
+  if (process.env.AGENTLANG_EMBEDDING_CHUNKOVERLAP) {
+    config.chunkOverlap = parseInt(process.env.AGENTLANG_EMBEDDING_CHUNKOVERLAP, 200);
+  }
+
+  return config;
+}
+
+const embeddingConfig = JSON.stringify(buildEmbeddingConfig());
+
 export default `module ${CoreAIModuleName}
 
 import "./modules/ai.js" @as ai
@@ -117,7 +138,7 @@ workflow saveAgentChatSession {
 entity Document {
   title String @id,
   content String,
-  @meta {"fullTextSearch": "*"}
+  @meta {"fullTextSearch": "*", "embeddingConfig": ${embeddingConfig}}
 }
 
 event doc {
@@ -980,34 +1001,93 @@ Only return a pure JSON object with no extra text, annotations etc.`;
 
   private async maybeAddRelevantDocuments(message: string, env: Environment): Promise<string> {
     if (this.documents && this.documents.length > 0) {
-      const s = `${message}. Relevant documents are: ${this.documents}`;
-      const result: any[] = await parseHelper(`{${CoreAIModuleName}/Document? "${s}"}`, env);
-      if (result && result.length > 0) {
-        const docs: Instance[] = [];
-        for (let i = 0; i < result.length; ++i) {
-          const v: any = result[i];
-          const r: Instance[] = await parseHelper(
-            `{${CoreAIModuleName}/Document {${PathAttributeNameQuery} "${v.id}"}}`,
+      try {
+        const docNames = this.documents.split(',').map(d => d.trim());
+        const docTitles = resolveDocumentAliases(docNames);
+
+        const searchQuery = message;
+
+        try {
+          const semanticResult: any[] = await parseHelper(
+            `{${CoreAIModuleName}/Document {content? "${searchQuery.replace(/"/g, '\\"')}"}}`,
             env
           );
-          if (r && r.length > 0) {
-            docs.push(r[0]);
+
+          if (semanticResult && semanticResult.length > 0) {
+            const docs: Instance[] = [];
+            for (const doc of semanticResult) {
+              const docTitle = doc.lookup ? doc.lookup('title') : doc.title;
+              if (AgentInstance.docTitlesMatch(docTitle, docTitles)) {
+                docs.push(
+                  doc instanceof Instance
+                    ? doc
+                    : Instance.newWithAttributes(doc, new Map(Object.entries(doc)))
+                );
+              }
+            }
+
+            if (docs.length > 0) {
+              return message.concat('\n\nRelevant context from documents:\n').concat(
+                docs
+                  .map((v: Instance) => {
+                    return `Document: ${v.lookup('title')}\n${v.lookup('content') as string}`;
+                  })
+                  .join('\n\n---\n\n')
+              );
+            }
           }
-        }
-        if (docs.length > 0) {
-          message = message.concat('\nUse the additional information given below:\n').concat(
-            docs
-              .map((v: Instance) => {
-                return v.lookup('content');
-              })
-              .join('\n')
+        } catch (semanticErr) {
+          logger.debug(
+            `Semantic search is not available, falling back to title-based filtering: ${semanticErr}`
           );
         }
+
+        const result: any[] = await parseHelper(`{${CoreAIModuleName}/Document? {}}`, env);
+        if (result && result.length > 0) {
+          const docs: Instance[] = [];
+          for (let i = 0; i < result.length; ++i) {
+            const v: any = result[i];
+            const docTitle: string | undefined = AgentInstance.getDocumentTitle(v);
+
+            if (docTitle && docTitles.includes(docTitle)) {
+              if (v instanceof Instance) {
+                docs.push(v);
+              }
+            }
+          }
+
+          if (docs.length > 0) {
+            return message.concat('\n\nRelevant context from documents:\n').concat(
+              docs
+                .map((v: Instance) => {
+                  return v.lookup('content') as string;
+                })
+                .join('\n\n')
+            );
+          }
+        }
+      } catch (err) {
+        logger.debug(`Error retrieving documents: ${err}`);
       }
     }
     return message;
   }
 
+  private static docTitlesMatch(title: string | undefined, docNames: string[]): boolean {
+    return title !== undefined && docNames.includes(title);
+  }
+
+  private static getDocumentTitle(doc: any): string | undefined {
+    if (typeof doc.lookup === 'function') {
+      return doc.lookup('title') as string | undefined;
+    } else if (doc.attributes) {
+      return doc.attributes.get('title') as string | undefined;
+    } else if (doc.title) {
+      return doc.title;
+    }
+    return undefined;
+  }
+
   private static ToolsCache = new Map<string, string>();
 
   private toolsAsString(): string {

package/src/runtime/resolvers/sqldb/database.ts

@@ -49,6 +49,67 @@ import { AppConfig } from '../../state.js';
 
 export let defaultDataSource: DataSource | undefined;
 
+// Detect browser environment
+function isBrowser(): boolean {
+  // window for DOM pages, self+importScripts for web workers
+  return (
+    (typeof window !== 'undefined' && typeof (window as any).document !== 'undefined') ||
+    (typeof self !== 'undefined' && typeof (self as any).importScripts === 'function')
+  );
+}
+
+// SQLite WASM with built-in sqlite-vec for browsers
+// Loaded from CDN - this has sqlite-vec statically compiled in
+let sqliteVecWasmModule: any = null;
+
+async function loadSqliteVecWasm(): Promise<any> {
+  if (sqliteVecWasmModule) {
+    return sqliteVecWasmModule;
+  }
+
+  try {
+    // Use dynamic import with string to prevent bundlers from analyzing
+    const cdnUrl = 'https://cdn.jsdelivr.net/npm/sqlite-vec-wasm-demo@latest/sqlite3.mjs';
+    const module = await import(/* @vite-ignore */ cdnUrl);
+    sqliteVecWasmModule = await module.default();
+    return sqliteVecWasmModule;
+  } catch (err) {
+    logger.warn('Failed to load sqlite-vec WASM:', err);
+    return null;
+  }
+}
+
+// Helper to load sqlite-vec based on environment
+async function loadSqliteVec(): Promise<any> {
+  // In browser, use WASM version with built-in sqlite-vec
+  // In Node.js, use the npm package
+  if (isBrowser()) {
+    const wasmModule = await loadSqliteVecWasm();
+    if (wasmModule) {
+      // WASM version has sqlite-vec built-in, no need to call load()
+      // Return a compatible interface
+      return {
+        load: () => {
+          // No-op: sqlite-vec is already loaded in WASM version
+          logger.info('sqlite-vec WASM loaded (built-in)');
+        },
+        // Expose the WASM module for direct use if needed
+        _wasmModule: wasmModule,
+      };
+    }
+    return null;
+  }
+
+  // Node.js: use npm package
+  try {
+    // Use variable to prevent bundlers from statically analyzing this import
+    const moduleName = 'sqlite-vec';
+    return await import(/* @vite-ignore */ moduleName);
+  } catch {
+    return null;
+  }
+}
+
 export class DbContext {
   txnId: string | undefined;
   authInfo: ResolverAuthInfo;
@@ -228,9 +289,10 @@ function makeSqliteDataSource(
 ): DataSource {
   const synchronize = needSync();
   //const runMigrations = isRuntimeMode_migration() || isRuntimeMode_undo_migration() || !synchronize;
-  return new DataSource({
+  const dbPath = config?.dbname || mkDbName();
+  const ds = new DataSource({
     type: 'better-sqlite3',
-    database: config?.dbname || mkDbName(),
+    database: dbPath,
     synchronize: synchronize,
     entities: entities,
     migrationsRun: false,
@@ -240,6 +302,25 @@ function makeSqliteDataSource(
       undefined: 'ignore',
     },
   });
+  const originalInit = ds.initialize.bind(ds);
+  ds.initialize = async () => {
+    const res = await originalInit();
+    try {
+      const sqliteVec = await loadSqliteVec();
+      const driver = ds.driver as any;
+      const db = driver.databaseConnection || driver.nativeDatabase;
+      if (db && sqliteVec?.load) {
+        sqliteVec.load(db);
+        logger.info('sqlite-vec extension loaded successfully');
+      }
+    } catch (err: any) {
+      logger.warn(
+        `Failed to load sqlite-vec extension: ${err.message}. Vector operations may not be available.`
+      );
+    }
+    return res;
+  };
+  return ds;
 }
 
 async function execMigrationSql(dataSource: DataSource, sql: string[]) {
@@ -283,14 +364,6 @@ async function maybeHandleMigrations(dataSource: DataSource) {
   }
 }
 
-function isBrowser(): boolean {
-  // window for DOM pages, self+importScripts for web workers
-  return (
-    (typeof window !== 'undefined' && typeof (window as any).document !== 'undefined') ||
-    (typeof self !== 'undefined' && typeof (self as any).importScripts === 'function')
-  );
-}
-
 function defaultLocateFile(file: string): string {
   // Out-of-the-box: use the official CDN in browsers.
   if (isBrowser()) {
@@ -371,9 +444,14 @@ export function isUsingSqljs(): boolean {
   return getDbType(AppConfig?.store) == 'sqljs';
 }
 
-export function isVectorStoreSupported(): boolean {
-  // Only Postgres supports pgvector
-  return getDbType(AppConfig?.store) === 'postgres';
+export async function isVectorStoreSupported(): Promise<boolean> {
+  const dbType = getDbType(AppConfig?.store);
+  if (dbType === 'postgres') return true;
+  if (dbType === 'sqlite') {
+    const sqliteVecModule = await loadSqliteVec();
+    return !!sqliteVecModule;
+  }
+  return false;
 }
 
 export async function initDatabase(config: DatabaseConfig | undefined) {
@@ -442,50 +520,69 @@ export async function addRowForFullTextSearch(
   vect: number[],
   ctx: DbContext
 ) {
-  if (!isVectorStoreSupported()) return;
+  if (!(await isVectorStoreSupported())) return;
   try {
     const vecTableName = tableName + VectorSuffix;
     const qb = getDatasourceForTransaction(ctx.txnId).createQueryBuilder();
     const tenantId = await ctx.getTenantId();
-    const { default: pgvector } = await import('pgvector');
-    await qb
-      .insert()
-      .into(vecTableName)
-      .values([{ id: id, embedding: pgvector.toSql(vect), __tenant__: tenantId }])
-      .execute();
+    const dbType = getDbType(AppConfig?.store);
+    if (dbType === 'postgres') {
+      const { default: pgvector } = await import('pgvector');
+      await qb
+        .insert()
+        .into(vecTableName)
+        .values([{ id: id, embedding: pgvector.toSql(vect), __tenant__: tenantId }])
+        .execute();
+    } else {
+      await qb
+        .insert()
+        .into(vecTableName)
+        .values([{ id: id, embedding: new Float32Array(vect) }])
+        .execute();
+    }
   } catch (err: any) {
     logger.error(`Failed to add row to vector store - ${err}`);
   }
 }
 
 export async function initVectorStore(tableNames: string[], ctx: DbContext) {
-  if (!isVectorStoreSupported()) {
+  if (!(await isVectorStoreSupported())) {
     logger.info(`Vector store not supported for ${getDbType(AppConfig?.store)}, skipping init...`);
     return;
   }
+  const dbType = getDbType(AppConfig?.store);
   let notInited = true;
-  tableNames.forEach(async (vecTableName: string) => {
+  for (const vecTableName of tableNames) {
     const vecRepo = getDatasourceForTransaction(ctx.txnId).getRepository(vecTableName);
-    if (notInited) {
-      let failure = false;
-      try {
-        await vecRepo.query('CREATE EXTENSION IF NOT EXISTS vector');
-      } catch (err: any) {
-        logger.error(`Failed to initialize vector store - ${err}`);
-        failure = true;
+    if (dbType === 'postgres') {
+      if (notInited) {
+        let failure = false;
+        try {
+          await vecRepo.query('CREATE EXTENSION IF NOT EXISTS vector');
+        } catch (err: any) {
+          logger.error(`Failed to initialize vector store - ${err}`);
+          failure = true;
+        }
+        if (failure) continue;
+        notInited = false;
       }
-      if (failure) return;
-      notInited = false;
+      await vecRepo.query(
+        `CREATE TABLE IF NOT EXISTS ${vecTableName} (
+            id varchar PRIMARY KEY,
+            embedding vector(${DefaultVectorDimension}),
+            ${TenantAttributeName} varchar,
+            __is_deleted__ boolean default false
+          )`
+      );
+    } else {
+      // sqlite-vec - vec0 doesn't support type declarations for metadata columns
+      await vecRepo.query(
+        `CREATE VIRTUAL TABLE IF NOT EXISTS ${vecTableName} USING vec0(
+            id TEXT PRIMARY KEY,
+            embedding FLOAT[${DefaultVectorDimension}])`
+      );
     }
-    await vecRepo.query(
-      `CREATE TABLE IF NOT EXISTS ${vecTableName} (
-          id varchar PRIMARY KEY,
-          embedding vector(${DefaultVectorDimension}),
-          ${TenantAttributeName} varchar,
-          __is_deleted__ boolean default false
-        )`
-    );
-  });
+  }
 }
 
 export async function vectorStoreSearch(
@@ -494,7 +591,7 @@ export async function vectorStoreSearch(
   limit: number,
   ctx: DbContext
 ): Promise<any> {
-  if (!isVectorStoreSupported()) {
+  if (!(await isVectorStoreSupported())) {
     // Not supported on sqljs/sqlite
     return [];
   }
@@ -508,18 +605,31 @@ export async function vectorStoreSearch(
     }
     const vecTableName = tableName + VectorSuffix;
     const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
-    const { default: pgvector } = await import('pgvector');
-    let ownersJoinCond: string = '';
+    const dbType = getDbType(AppConfig?.store);
     const tenantId = await ctx.getTenantId();
+    let ownersJoinCond: string = '';
     if (!hasGlobalPerms) {
       const ot = ownersTable(tableName);
       ownersJoinCond = `inner join ${ot} on
-${ot}.path = ${vecTableName}.id and ${ot}.user_id = '${ctx.authInfo.userId}' and ${ot}.r = true
-and ${ot}.${TenantAttributeName} = '${tenantId}' and ${vecTableName}.${TenantAttributeName} = '${tenantId}'`;
+ ${ot}.path = ${vecTableName}.id and ${ot}.user_id = '${ctx.authInfo.userId}' and ${ot}.r = true
+ and ${ot}.${TenantAttributeName} = '${tenantId}'`;
+    }
+    if (dbType === 'postgres') {
+      const { default: pgvector } = await import('pgvector');
+      const sql = `select ${vecTableName}.id from ${vecTableName} ${ownersJoinCond} order by embedding <-> $1 LIMIT ${limit}`;
+      const args = pgvector.toSql(searchVec);
+      return await qb.query(sql, [args]);
+    } else {
+      // sqlite-vec - join with main table to filter by tenant
+      const alias = tableName.toLowerCase();
+      const sql = `SELECT ${vecTableName}.id FROM ${vecTableName}
+        INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
+        ${ownersJoinCond}
+        WHERE ${alias}.${TenantAttributeName} = '${tenantId}' AND ${alias}.${DeletedFlagAttributeName} = false AND ${vecTableName}.embedding MATCH $1
+        LIMIT ${limit}`;
+      const args = new Float32Array(searchVec);
+      return await qb.query(sql, [args]);
     }
-    const sql = `select ${vecTableName}.id from ${vecTableName} ${ownersJoinCond} order by embedding <-> $1 LIMIT ${limit}`;
-    const args = pgvector.toSql(searchVec);
-    return await qb.query(sql, [args]);
   } catch (err: any) {
     logger.error(`Vector store search failed - ${err}`);
     return [];
@@ -531,16 +641,30 @@ export async function vectorStoreSearchEntryExists(
   id: string,
   ctx: DbContext
 ): Promise<boolean> {
-  if (!isVectorStoreSupported()) return false;
+  if (!(await isVectorStoreSupported())) return false;
   try {
     const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
     const vecTableName = tableName + VectorSuffix;
+    const dbType = getDbType(AppConfig?.store);
     const tenantId = await ctx.getTenantId();
-    const result: any[] = await qb.query(
-      `select id from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
-      [id]
-    );
-    return result !== null && result.length > 0;
+
+    if (dbType === 'postgres') {
+      const result: any[] = await qb.query(
+        `select id from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
+        [id]
+      );
+      return result !== null && result.length > 0;
+    } else {
+      // sqlite-vec - join with main table to verify tenant
+      const alias = tableName.toLowerCase();
+      const result: any[] = await qb.query(
+        `SELECT ${vecTableName}.id FROM ${vecTableName}
+         INNER JOIN ${tableName} ${alias} ON ${alias}.${PathAttributeName} = ${vecTableName}.id
+         WHERE ${vecTableName}.id = $1 AND ${alias}.${TenantAttributeName} = '${tenantId}'`,
+        [id]
+      );
+      return result !== null && result.length > 0;
+    }
   } catch (err: any) {
     logger.error(`Vector store search failed - ${err}`);
   }
@@ -548,15 +672,22 @@ export async function vectorStoreSearchEntryExists(
 }
 
 export async function deleteFullTextSearchEntry(tableName: string, id: string, ctx: DbContext) {
-  if (!isVectorStoreSupported()) return;
+  if (!(await isVectorStoreSupported())) return;
   try {
     const qb = getDatasourceForTransaction(ctx.txnId).getRepository(tableName).manager;
     const vecTableName = tableName + VectorSuffix;
+    const dbType = getDbType(AppConfig?.store);
     const tenantId = await ctx.getTenantId();
-    await qb.query(
-      `delete from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
-      [id]
-    );
+
+    if (dbType === 'postgres') {
+      await qb.query(
+        `delete from ${vecTableName} where id = $1 and ${TenantAttributeName} = '${tenantId}'`,
+        [id]
+      );
+    } else {
+      // sqlite-vec - delete just by id (ownership verified by caller)
+      await qb.query(`delete from ${vecTableName} where id = $1`, [id]);
+    }
   } catch (err: any) {
     logger.error(`Vector store delete failed - ${err}`);
   }