agentlang 0.9.7 → 0.9.8

package/src/runtime/module.ts

@@ -31,6 +31,7 @@ import {
   DefaultModules,
   encryptPassword,
   escapeFqName,
+  escapeSpecialChars,
   findAllPrePostTriggerSchema,
   findMetaSchema,
   findUqCompositeAttributes,
@@ -1161,7 +1162,16 @@ export class Agent extends Record {
       if (!skip && value !== null && value !== undefined) {
         let v = value;
         const isf = key == 'flows';
-        if (isf || key == 'tools') {
+        const isDocs = key == 'documents';
+        if (isDocs) {
+          const raw = isString(v) ? v : `${v}`;
+          const items = raw
+            .split(',')
+            .map((entry: string) => entry.trim())
+            .filter((entry: string) => entry.length > 0)
+            .map((entry: string) => entry.replace(/\\/g, '\\\\').replace(/"/g, '\\"'));
+          v = `[${items.map((entry: string) => `"${entry}"`).join(', ')}]`;
+        } else if (isf || key == 'tools') {
           if (isf || v.indexOf(',') > 0 || v.indexOf('/') > 0) v = `[${v}]`;
           else v = `"${v}"`;
         } else if (isString(v)) {
@@ -1885,6 +1895,42 @@ export class GlossaryEntry extends ModuleEntry {
   }
 }
 
+export type ModuleDocument = {
+  title: string;
+  url: string;
+};
+
+export class DocumentEntry extends ModuleEntry {
+  private doc: ModuleDocument;
+
+  constructor(title: string, moduleName: string, url: string) {
+    super(title, moduleName);
+    this.doc = { title, url };
+  }
+
+  getUrl(): string {
+    return this.doc.url;
+  }
+
+  getTitle(): string {
+    return this.doc.title;
+  }
+
+  override toString(): string {
+    const escapedTitle = escapeSpecialChars(this.doc.title);
+    const escapedUrl = escapeSpecialChars(this.doc.url);
+    return `{${DefaultModuleName}.ai/doc {\n    title "${escapedTitle}",\n    url "${escapedUrl}"\n}}`;
+  }
+
+  get url(): string {
+    return this.doc.url;
+  }
+
+  set url(value: string) {
+    this.doc.url = value;
+  }
+}
+
 function statementLabel(stmt: Statement | undefined): string {
   if (stmt === undefined) return '';
   let lbl: string | undefined = undefined;
@@ -2472,6 +2518,48 @@ export class Module {
     return this;
   }
 
+  addDocument(title: string, url: string): Module {
+    if (this.hasEntry(title)) {
+      const existing = this.getEntry(title);
+      if (existing instanceof DocumentEntry) {
+        const doc = existing as DocumentEntry;
+        doc.url = url;
+        return this;
+      }
+    }
+
+    const docEntry = new DocumentEntry(title, this.name, url);
+    this.addEntry(docEntry);
+    return this;
+  }
+
+  getDocument(title: string): string | undefined {
+    if (this.hasEntry(title)) {
+      const entry = this.getEntry(title);
+      if (entry instanceof DocumentEntry) {
+        return (entry as DocumentEntry).url;
+      }
+    }
+    return undefined;
+  }
+
+  getAllDocuments(): DocumentEntry[] {
+    return this.entries.filter((e: ModuleEntry) => {
+      return e instanceof DocumentEntry;
+    }) as DocumentEntry[];
+  }
+
+  removeDocument(title: string): Module {
+    for (let i = 0; i < this.entries.length; ++i) {
+      const entry = this.entries[i];
+      if (entry.name === title && entry instanceof DocumentEntry) {
+        this.entries.splice(i, 1);
+        break;
+      }
+    }
+    return this;
+  }
+
   addRetry(retry: Retry): Module {
     this.addEntry(retry);
     return this;
@@ -2734,6 +2822,17 @@ export class Module {
 
 let GlobalRetries: Array<Retry> | undefined = undefined;
 
+const DocumentAliasMap: Map<string, string> = new Map();
+
+export function registerDocumentAlias(name: string, title: string): void {
+  if (!name || !title) return;
+  DocumentAliasMap.set(name, title);
+}
+
+export function resolveDocumentAliases(names: string[]): string[] {
+  return names.map((name: string) => DocumentAliasMap.get(name) ?? name);
+}
+
 export function addGlobalRetry(r: Retry): Retry {
   if (GlobalRetries === undefined) {
     GlobalRetries = new Array<Retry>();

package/src/runtime/modules/ai.ts

@@ -31,6 +31,7 @@ import {
   newInstanceAttributes,
   Record,
   Retry,
+  resolveDocumentAliases,
 } from '../module.js';
 import { provider } from '../agents/registry.js';
 import {
@@ -73,6 +74,27 @@ export const AgentLearnerType = 'learner';
 
 const AgentEvalType = 'eval';
 
+function buildEmbeddingConfig(): object {
+  const config: any = {
+    provider: process.env.AGENTLANG_EMBEDDING_PROVIDER || 'openai',
+    model: process.env.AGENTLANG_EMBEDDING_MODEL || 'text-embedding-3-small',
+    chunkSize: 1000,
+    chunkOverlap: 200,
+  };
+
+  if (process.env.AGENTLANG_EMBEDDING_CHUNKSIZE) {
+    config.chunkSize = parseInt(process.env.AGENTLANG_EMBEDDING_CHUNKSIZE, 1000);
+  }
+
+  if (process.env.AGENTLANG_EMBEDDING_CHUNKOVERLAP) {
+    config.chunkOverlap = parseInt(process.env.AGENTLANG_EMBEDDING_CHUNKOVERLAP, 200);
+  }
+
+  return config;
+}
+
+const embeddingConfig = JSON.stringify(buildEmbeddingConfig());
+
 export default `module ${CoreAIModuleName}
 
 import "./modules/ai.js" @as ai
@@ -116,7 +138,7 @@ workflow saveAgentChatSession {
 entity Document {
   title String @id,
   content String,
-  @meta {"fullTextSearch": "*"}
+  @meta {"fullTextSearch": "*", "embeddingConfig": ${embeddingConfig}}
 }
 
 event doc {
@@ -981,6 +1003,7 @@ Only return a pure JSON object with no extra text, annotations etc.`;
     if (this.documents && this.documents.length > 0) {
       try {
         const docNames = this.documents.split(',').map(d => d.trim());
+        const docTitles = resolveDocumentAliases(docNames);
 
         const searchQuery = message;
 
@@ -994,7 +1017,7 @@ Only return a pure JSON object with no extra text, annotations etc.`;
             const docs: Instance[] = [];
             for (const doc of semanticResult) {
               const docTitle = doc.lookup ? doc.lookup('title') : doc.title;
-              if (AgentInstance.docTitlesMatch(docTitle, docNames)) {
+              if (AgentInstance.docTitlesMatch(docTitle, docTitles)) {
                 docs.push(
                   doc instanceof Instance
                     ? doc
@@ -1026,7 +1049,7 @@ Only return a pure JSON object with no extra text, annotations etc.`;
             const v: any = result[i];
             const docTitle: string | undefined = AgentInstance.getDocumentTitle(v);
 
-            if (docTitle && docNames.includes(docTitle)) {
+            if (docTitle && docTitles.includes(docTitle)) {
               if (v instanceof Instance) {
                 docs.push(v);
               }

package/src/runtime/resolvers/sqldb/database.ts

@@ -49,6 +49,67 @@ import { AppConfig } from '../../state.js';
 
 export let defaultDataSource: DataSource | undefined;
 
+// Detect browser environment
+function isBrowser(): boolean {
+  // window for DOM pages, self+importScripts for web workers
+  return (
+    (typeof window !== 'undefined' && typeof (window as any).document !== 'undefined') ||
+    (typeof self !== 'undefined' && typeof (self as any).importScripts === 'function')
+  );
+}
+
+// SQLite WASM with built-in sqlite-vec for browsers
+// Loaded from CDN - this has sqlite-vec statically compiled in
+let sqliteVecWasmModule: any = null;
+
+async function loadSqliteVecWasm(): Promise<any> {
+  if (sqliteVecWasmModule) {
+    return sqliteVecWasmModule;
+  }
+
+  try {
+    // Use dynamic import with string to prevent bundlers from analyzing
+    const cdnUrl = 'https://cdn.jsdelivr.net/npm/sqlite-vec-wasm-demo@latest/sqlite3.mjs';
+    const module = await import(/* @vite-ignore */ cdnUrl);
+    sqliteVecWasmModule = await module.default();
+    return sqliteVecWasmModule;
+  } catch (err) {
+    logger.warn('Failed to load sqlite-vec WASM:', err);
+    return null;
+  }
+}
+
+// Helper to load sqlite-vec based on environment
+async function loadSqliteVec(): Promise<any> {
+  // In browser, use WASM version with built-in sqlite-vec
+  // In Node.js, use the npm package
+  if (isBrowser()) {
+    const wasmModule = await loadSqliteVecWasm();
+    if (wasmModule) {
+      // WASM version has sqlite-vec built-in, no need to call load()
+      // Return a compatible interface
+      return {
+        load: () => {
+          // No-op: sqlite-vec is already loaded in WASM version
+          logger.info('sqlite-vec WASM loaded (built-in)');
+        },
+        // Expose the WASM module for direct use if needed
+        _wasmModule: wasmModule,
+      };
+    }
+    return null;
+  }
+
+  // Node.js: use npm package
+  try {
+    // Use variable to prevent bundlers from statically analyzing this import
+    const moduleName = 'sqlite-vec';
+    return await import(/* @vite-ignore */ moduleName);
+  } catch {
+    return null;
+  }
+}
+
 export class DbContext {
   txnId: string | undefined;
   authInfo: ResolverAuthInfo;
@@ -245,11 +306,11 @@ function makeSqliteDataSource(
   ds.initialize = async () => {
     const res = await originalInit();
     try {
-      const { load } = await import('sqlite-vec');
+      const sqliteVec = await loadSqliteVec();
       const driver = ds.driver as any;
       const db = driver.databaseConnection || driver.nativeDatabase;
-      if (db) {
-        load(db);
+      if (db && sqliteVec?.load) {
+        sqliteVec.load(db);
         logger.info('sqlite-vec extension loaded successfully');
       }
     } catch (err: any) {
@@ -303,14 +364,6 @@ async function maybeHandleMigrations(dataSource: DataSource) {
   }
 }
 
-function isBrowser(): boolean {
-  // window for DOM pages, self+importScripts for web workers
-  return (
-    (typeof window !== 'undefined' && typeof (window as any).document !== 'undefined') ||
-    (typeof self !== 'undefined' && typeof (self as any).importScripts === 'function')
-  );
-}
-
 function defaultLocateFile(file: string): string {
   // Out-of-the-box: use the official CDN in browsers.
   if (isBrowser()) {
@@ -395,12 +448,8 @@ export async function isVectorStoreSupported(): Promise<boolean> {
   const dbType = getDbType(AppConfig?.store);
   if (dbType === 'postgres') return true;
   if (dbType === 'sqlite') {
-    try {
-      const sqliteVecModule = await import('sqlite-vec');
-      return !!sqliteVecModule;
-    } catch {
-      return false;
-    }
+    const sqliteVecModule = await loadSqliteVec();
+    return !!sqliteVecModule;
   }
   return false;
 }