agentlang 0.9.6 → 0.9.8

package/src/cli/main.ts

@@ -38,6 +38,7 @@ import {
   setRuntimeMode_init_schema,
   setRuntimeMode_migration,
   setRuntimeMode_prod,
+  setRuntimeMode_test,
   setRuntimeMode_undo_migration,
   updateEndpoints,
 } from '../runtime/defs.js';
@@ -216,8 +217,12 @@ async function internDynamicModule(name: string, definition: string): Promise<st
 setInternDynamicModuleFn(internDynamicModule);
 
 export const runModule = async (fileName: string, releaseDb: boolean = false): Promise<void> => {
-  if (isRuntimeMode_dev() && process.env.NODE_ENV === 'production') {
-    setRuntimeMode_prod();
+  if (isRuntimeMode_dev()) {
+    if (process.env.NODE_ENV === 'production') {
+      setRuntimeMode_prod();
+    } else if (process.env.NODE_ENV === 'test') {
+      setRuntimeMode_test();
+    }
   }
   const r: boolean = await runPreInitTasks();
   if (!r) {

package/src/language/generated/ast.ts

@@ -3,7 +3,7 @@
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 
- 
+/* eslint-disable */
 import * as langium from 'langium';
 
 export const AgentlangTerminals = {

package/src/runtime/docs.ts

@@ -1,5 +1,6 @@
-import { getFileSystem } from '../utils/fs-utils.js';
 import { logger } from './logger.js';
+import { isNodeEnv } from '../utils/runtime.js';
+import { getFileSystem } from '../utils/fs-utils.js';
 
 const DocFetchers = new Map<string, Function>();
 
@@ -25,6 +26,39 @@ export async function fetchDoc(url: string): Promise<string | undefined> {
   else return undefined;
 }
 
+let PDFParse: any = null;
+let pdfWorkerSet = false;
+
+async function getPDFParse() {
+  if (!PDFParse) {
+    const pdfModule = await import('pdf-parse');
+    PDFParse = pdfModule.PDFParse;
+
+    // Set up web worker for browser
+    if (!isNodeEnv && !pdfWorkerSet && PDFParse.setWorker) {
+      // Worker is served from public/ directory
+      PDFParse.setWorker('/pdf.worker.mjs');
+      pdfWorkerSet = true;
+    }
+  }
+  return PDFParse;
+}
+
+async function parsePdfBuffer(buffer: Uint8Array): Promise<string> {
+  try {
+    const PDFParseClass = await getPDFParse();
+    const parser = new PDFParseClass({
+      data: buffer,
+      verbosity: 0,
+    });
+    const data = await parser.getText();
+    return data.text;
+  } catch (error: any) {
+    logger.error(`Failed to parse PDF: ${error.message}`);
+    throw new Error(`PDF parsing failed: ${error.message}`);
+  }
+}
+
 async function httpFetcher(url: string): Promise<string | undefined> {
   try {
     const response = await fetch(url, {
@@ -32,12 +66,24 @@ async function httpFetcher(url: string): Promise<string | undefined> {
     });
 
     if (!response.ok) {
-      logger.error(
-        `Failed to fetch document ${url}, HTTP error! status: ${response.status} ${response.text} ${response.statusText}`
-      );
+      logger.error(`Failed to fetch document ${url}, HTTP error! status: ${response.status}`);
       return undefined;
     }
-    return await response.text();
+
+    const contentType = response.headers.get('content-type') || '';
+    const content = await response.arrayBuffer();
+
+    const lowerUrl = url.toLowerCase();
+
+    // Process based on content type or file extension
+    if (contentType.includes('application/pdf') || lowerUrl.endsWith('.pdf')) {
+      return await parsePdfBuffer(new Uint8Array(content));
+    } else if (contentType.includes('text/markdown') || lowerUrl.endsWith('.md')) {
+      return new TextDecoder().decode(content);
+    } else {
+      // Default to text
+      return new TextDecoder().decode(content);
+    }
   } catch (reason: any) {
     logger.error(`Failed to fetch document ${url}: ${reason}`);
   }
@@ -45,11 +91,76 @@ async function httpFetcher(url: string): Promise<string | undefined> {
 }
 
 async function fetchFile(path: string): Promise<string> {
-  const fs = await getFileSystem();
-  if (path.startsWith('.')) {
-    path = `${process.cwd()}${path.substring(1)}`;
+  const lowerPath = path.toLowerCase();
+
+  if (lowerPath.endsWith('.pdf')) {
+    return await fetchPdfFile(path);
+  } else if (
+    lowerPath.endsWith('.md') ||
+    lowerPath.endsWith('.markdown') ||
+    lowerPath.endsWith('.mdown')
+  ) {
+    return await fetchMarkdownFile(path);
+  } else {
+    // Default: plain text
+    return await fetchTextFile(path);
+  }
+}
+
+async function fetchPdfFile(path: string): Promise<string> {
+  try {
+    const fs = await getFileSystem();
+
+    if (isNodeEnv && path.startsWith('.')) {
+      path = `${process.cwd()}${path.substring(1)}`;
+    }
+
+    const content = await fs.readFile(path);
+
+    let buffer: Uint8Array;
+    if (typeof content === 'string') {
+      buffer = new TextEncoder().encode(content);
+    } else if (Buffer.isBuffer(content)) {
+      buffer = new Uint8Array(content);
+    } else {
+      buffer = new Uint8Array(Buffer.from(content));
+    }
+
+    return await parsePdfBuffer(buffer);
+  } catch (error: any) {
+    logger.error(`Failed to read PDF file ${path}: ${error.message}`);
+    throw error;
+  }
+}
+
+async function fetchMarkdownFile(path: string): Promise<string> {
+  try {
+    const fs = await getFileSystem();
+
+    if (isNodeEnv && path.startsWith('.')) {
+      path = `${process.cwd()}${path.substring(1)}`;
+    }
+
+    return await fs.readFile(path);
+  } catch (error: any) {
+    logger.error(`Failed to read Markdown file ${path}: ${error.message}`);
+    throw error;
+  }
+}
+
+async function fetchTextFile(path: string): Promise<string> {
+  try {
+    const fs = await getFileSystem();
+
+    if (isNodeEnv && path.startsWith('.')) {
+      path = `${process.cwd()}${path.substring(1)}`;
+    }
+
+    return await fs.readFile(path);
+  } catch (error: any) {
+    logger.error(`Failed to read text file ${path}: ${error.message}`);
+    throw error;
   }
-  return fs.readFile(path);
 }
 
 registerDocFetcher('http', httpFetcher);

package/src/runtime/embeddings/chunker.ts

@@ -0,0 +1,50 @@
+export class TextChunker {
+  private chunkSize: number;
+  private chunkOverlap: number;
+  private separators: string[] = ['\n\n', '\n', '. ', ' ', ''];
+
+  constructor(chunkSize: number = 1000, chunkOverlap: number = 200) {
+    this.chunkSize = chunkSize;
+    this.chunkOverlap = chunkOverlap;
+  }
+
+  splitText(text: string): string[] {
+    if (text.length <= this.chunkSize) {
+      return [text];
+    }
+
+    const chunks: string[] = [];
+    let start = 0;
+
+    while (start < text.length) {
+      let end = Math.min(start + this.chunkSize, text.length);
+
+      if (end < text.length) {
+        end = this.findBestSplitPoint(text, start, end);
+      }
+
+      chunks.push(text.substring(start, end));
+      start = end - this.chunkOverlap;
+
+      if (start < 0) start = 0;
+      if (start >= text.length - this.chunkOverlap) {
+        if (start < text.length) {
+          chunks.push(text.substring(start));
+        }
+        break;
+      }
+    }
+
+    return chunks;
+  }
+
+  private findBestSplitPoint(text: string, start: number, end: number): number {
+    for (const sep of this.separators) {
+      const lastSep = text.lastIndexOf(sep, end);
+      if (lastSep > start) {
+        return Math.min(lastSep + sep.length, end);
+      }
+    }
+    return end;
+  }
+}

package/src/runtime/embeddings/index.ts

@@ -0,0 +1,5 @@
+export { EmbeddingProvider, EmbeddingProviderConfig } from './provider.js';
+export { EmbeddingService } from '../resolvers/sqldb/impl.js';
+export { TextChunker } from './chunker.js';
+export { embeddingProvider, getDefaultEmbeddingProvider } from './registry.js';
+export { OpenAIEmbeddingProvider, OpenAIEmbeddingConfig } from './openai.js';

package/src/runtime/embeddings/openai.ts

@@ -0,0 +1,49 @@
+import { OpenAIEmbeddings } from '@langchain/openai';
+import { EmbeddingProvider, EmbeddingProviderConfig } from './provider.js';
+import { getLocalEnv } from '../auth/defs.js';
+
+export interface OpenAIEmbeddingConfig extends EmbeddingProviderConfig {
+  model?: string;
+  dimensions?: number;
+  maxRetries?: number;
+}
+
+export class OpenAIEmbeddingProvider extends EmbeddingProvider {
+  private openaiConfig: OpenAIEmbeddingConfig;
+
+  constructor(config?: EmbeddingProviderConfig) {
+    super(config || {});
+    this.openaiConfig = (this.config as OpenAIEmbeddingConfig) || {};
+  }
+
+  protected createEmbeddings(): OpenAIEmbeddings {
+    const config: any = {
+      apiKey: this.resolveApiKey(),
+    };
+
+    if (this.openaiConfig.model) {
+      config.model = this.openaiConfig.model;
+    }
+
+    if (this.openaiConfig.dimensions) {
+      config.dimensions = this.openaiConfig.dimensions;
+    }
+
+    if (this.openaiConfig.maxRetries !== undefined) {
+      config.maxRetries = this.openaiConfig.maxRetries;
+    }
+
+    return new OpenAIEmbeddings(config);
+  }
+
+  protected resolveApiKey(): string {
+    if (this.openaiConfig.apiKey) {
+      return this.openaiConfig.apiKey;
+    }
+    return process.env.AGENTLANG_OPENAI_KEY || getLocalEnv('AGENTLANG_OPENAI_KEY') || '';
+  }
+
+  getProviderName(): string {
+    return 'openai';
+  }
+}

package/src/runtime/embeddings/provider.ts

@@ -0,0 +1,37 @@
+import { Embeddings } from '@langchain/core/embeddings';
+
+export interface EmbeddingProviderConfig {
+  chunkSize?: number;
+  chunkOverlap?: number;
+  apiKey?: string;
+  model?: string;
+  [key: string]: any;
+}
+
+export abstract class EmbeddingProvider {
+  protected config: EmbeddingProviderConfig;
+  protected embeddings: Embeddings;
+
+  constructor(config: EmbeddingProviderConfig = {}) {
+    this.config = config;
+    this.embeddings = this.createEmbeddings();
+  }
+
+  protected abstract createEmbeddings(): Embeddings;
+  protected abstract resolveApiKey(): string;
+
+  abstract getProviderName(): string;
+
+  async embedText(text: string): Promise<number[]> {
+    return await this.embeddings.embedQuery(text);
+  }
+
+  getConfig(): EmbeddingProviderConfig {
+    return { ...this.config };
+  }
+
+  updateConfig(newConfig: Partial<EmbeddingProviderConfig>): void {
+    this.config = { ...this.config, ...newConfig };
+    this.embeddings = this.createEmbeddings();
+  }
+}

package/src/runtime/embeddings/registry.ts

@@ -0,0 +1,17 @@
+import { OpenAIEmbeddingProvider } from './openai.js';
+
+const EmbeddingProviders = new Map().set('openai', OpenAIEmbeddingProvider);
+
+export function embeddingProvider(service: string): any {
+  const requestedService = service.toLowerCase();
+  const provider = EmbeddingProviders.get(requestedService);
+  if (provider) {
+    return provider;
+  } else {
+    throw new Error(`No embedding provider found for ${service}`);
+  }
+}
+
+export function getDefaultEmbeddingProvider(): string {
+  return 'openai';
+}

package/src/runtime/interpreter.ts

@@ -32,8 +32,8 @@ import {
   WhereSpec,
 } from '../language/generated/ast.js';
 import {
-  maybeInstanceAsString,
   defineAgentEvent,
+  Event,
   getOneOfRef,
   getRelationship,
   getWorkflow,
@@ -46,15 +46,15 @@ import {
   isEntityInstance,
   isEventInstance,
   isInstanceOfType,
+  isOneToOneBetweenRelationship,
   isTimer,
   makeInstance,
+  maybeInstanceAsString,
   newInstanceAttributes,
   PlaceholderRecordEntry,
   Relationship,
-  Workflow,
   setMetaAttributes,
-  Event,
-  isOneToOneBetweenRelationship,
+  Workflow,
 } from './module.js';
 import { JoinInfo, Resolver, WhereClause } from './resolvers/interface.js';
 import { ResolverAuthInfo } from './resolvers/authinfo.js';
@@ -65,26 +65,26 @@ import {
   escapeFqName,
   escapeQueryName,
   fqNameFromPath,
+  isCoreModule,
   isFqName,
   isPath,
   isString,
   makeCoreModuleName,
   makeFqName,
+  nameToPath,
   Path,
+  preprocessRawConfig,
   QuerySuffix,
   restoreSpecialChars,
-  nameToPath,
   splitRefs,
-  isCoreModule,
-  preprocessRawConfig,
 } from './util.js';
 import { getResolver, getResolverNameForPath } from './resolvers/registry.js';
 import { parseStatement, parseWorkflow } from '../language/parser.js';
 import { ActiveSessionInfo, AdminSession, AdminUserId } from './auth/defs.js';
 import {
-  AgentInstance,
   AgentEntityName,
   AgentFqName,
+  AgentInstance,
   findAgentByName,
   normalizeGeneratedCode,
 } from './modules/ai.js';
@@ -1755,11 +1755,15 @@ async function handleDocEvent(inst: Instance, env: Environment): Promise<void> {
   const s = await fetchDoc(inst.lookup('url'));
   if (s) {
     const title = inst.lookup('title');
-    await parseAndEvaluateStatement(
-      `{${CoreAIModuleName}/Document {title "${title}", content "${s}"}}`,
-      undefined,
-      env
+    const doc = makeInstance(
+      CoreAIModuleName,
+      'Document',
+      newInstanceAttributes().set('title', title).set('content', s)
     );
+    await computeExprAttributes(doc, undefined, undefined, env);
+    await setMetaAttributes(doc.attributes, env);
+    const res: Resolver = await getResolverForPath('Document', CoreAIModuleName, env);
+    await res.createInstance(doc);
   }
 }
 

package/src/runtime/module.ts

@@ -1,57 +1,58 @@
 import chalk from 'chalk';
 import {
-  Statement,
-  KvPair,
-  Literal,
+  AttributeDefinition,
+  Expr,
+  FlowDefinition,
+  FlowEntry,
   FnCall,
-  RelNodes,
+  isLiteral,
   isRelNodes,
-  AttributeDefinition,
-  PropertyDefinition,
-  NodeDefinition,
-  RecordSchemaDefinition,
+  KvPair,
+  Literal,
   MapEntry,
-  isLiteral,
   MetaDefinition,
+  NodeDefinition,
   PrePostTriggerDefinition,
-  TriggerEntry,
-  Expr,
-  RbacSpecEntry,
-  RbacSpecEntries,
+  PropertyDefinition,
   RbacOpr,
-  WorkflowHeader,
-  FlowDefinition,
-  FlowEntry,
+  RbacSpecEntries,
+  RbacSpecEntry,
+  RecordSchemaDefinition,
+  RelNodes,
+  Statement,
+  TriggerEntry,
   WorkflowDirectives,
+  WorkflowHeader,
 } from '../language/generated/ast.js';
 import {
-  Path,
-  nameToPath,
-  isString,
-  isNumber,
-  isBoolean,
-  isFqName,
-  makeFqName,
+  asCrudType,
+  CrudType,
   DefaultModuleName,
   DefaultModules,
-  joinStatements,
-  isMinusZero,
-  now,
-  findMetaSchema,
+  encryptPassword,
+  escapeFqName,
+  escapeSpecialChars,
   findAllPrePostTriggerSchema,
-  CrudType,
-  asCrudType,
-  isPath,
+  findMetaSchema,
   findUqCompositeAttributes,
-  escapeFqName,
-  encryptPassword,
-  splitFqName,
-  splitRefs,
   forceAsFqName,
-  validateIdFormat,
+  isBoolean,
+  isFqName,
+  isMinusZero,
+  isNumber,
+  isPath,
+  isString,
+  joinStatements,
+  makeFqName,
   nameContainsSepEscape,
+  nameToPath,
+  now,
+  Path,
   registerInitFunction,
   ScratchModuleName,
+  splitFqName,
+  splitRefs,
+  validateIdFormat,
 } from './util.js';
 import { parseStatement } from '../language/parser.js';
 import { ActiveSessionInfo, AdminSession } from './auth/defs.js';
@@ -513,7 +514,10 @@ export class Record extends ModuleEntry {
   }
 
   getFullTextSearchAttributes(): string[] | undefined {
-    const fts: string[] | string | undefined = this.getMeta('fullTextSearch');
+    let fts: string[] | string | undefined = this.getMeta('fullTextSearchAttributes');
+    if (!fts) {
+      fts = this.getMeta('fullTextSearch');
+    }
     if (fts) {
       if (fts instanceof Array) {
         return fts as string[];
@@ -527,6 +531,14 @@ export class Record extends ModuleEntry {
     }
   }
 
+  getEmbeddingConfig(): { [key: string]: any } | undefined {
+    const config = this.getMeta('embeddingConfig');
+    if (config && typeof config === 'object') {
+      return config as { [key: string]: any };
+    }
+    return undefined;
+  }
+
   private resetUserAttrs() {
     this.userAttrNames = undefined;
     this.userAttrsSchema = undefined;
@@ -1150,7 +1162,16 @@ export class Agent extends Record {
       if (!skip && value !== null && value !== undefined) {
         let v = value;
         const isf = key == 'flows';
-        if (isf || key == 'tools') {
+        const isDocs = key == 'documents';
+        if (isDocs) {
+          const raw = isString(v) ? v : `${v}`;
+          const items = raw
+            .split(',')
+            .map((entry: string) => entry.trim())
+            .filter((entry: string) => entry.length > 0)
+            .map((entry: string) => entry.replace(/\\/g, '\\\\').replace(/"/g, '\\"'));
+          v = `[${items.map((entry: string) => `"${entry}"`).join(', ')}]`;
+        } else if (isf || key == 'tools') {
           if (isf || v.indexOf(',') > 0 || v.indexOf('/') > 0) v = `[${v}]`;
           else v = `"${v}"`;
         } else if (isString(v)) {
@@ -1874,6 +1895,42 @@ export class GlossaryEntry extends ModuleEntry {
   }
 }
 
+export type ModuleDocument = {
+  title: string;
+  url: string;
+};
+
+export class DocumentEntry extends ModuleEntry {
+  private doc: ModuleDocument;
+
+  constructor(title: string, moduleName: string, url: string) {
+    super(title, moduleName);
+    this.doc = { title, url };
+  }
+
+  getUrl(): string {
+    return this.doc.url;
+  }
+
+  getTitle(): string {
+    return this.doc.title;
+  }
+
+  override toString(): string {
+    const escapedTitle = escapeSpecialChars(this.doc.title);
+    const escapedUrl = escapeSpecialChars(this.doc.url);
+    return `{${DefaultModuleName}.ai/doc {\n    title "${escapedTitle}",\n    url "${escapedUrl}"\n}}`;
+  }
+
+  get url(): string {
+    return this.doc.url;
+  }
+
+  set url(value: string) {
+    this.doc.url = value;
+  }
+}
+
 function statementLabel(stmt: Statement | undefined): string {
   if (stmt === undefined) return '';
   let lbl: string | undefined = undefined;
@@ -2461,6 +2518,48 @@ export class Module {
     return this;
   }
 
+  addDocument(title: string, url: string): Module {
+    if (this.hasEntry(title)) {
+      const existing = this.getEntry(title);
+      if (existing instanceof DocumentEntry) {
+        const doc = existing as DocumentEntry;
+        doc.url = url;
+        return this;
+      }
+    }
+
+    const docEntry = new DocumentEntry(title, this.name, url);
+    this.addEntry(docEntry);
+    return this;
+  }
+
+  getDocument(title: string): string | undefined {
+    if (this.hasEntry(title)) {
+      const entry = this.getEntry(title);
+      if (entry instanceof DocumentEntry) {
+        return (entry as DocumentEntry).url;
+      }
+    }
+    return undefined;
+  }
+
+  getAllDocuments(): DocumentEntry[] {
+    return this.entries.filter((e: ModuleEntry) => {
+      return e instanceof DocumentEntry;
+    }) as DocumentEntry[];
+  }
+
+  removeDocument(title: string): Module {
+    for (let i = 0; i < this.entries.length; ++i) {
+      const entry = this.entries[i];
+      if (entry.name === title && entry instanceof DocumentEntry) {
+        this.entries.splice(i, 1);
+        break;
+      }
+    }
+    return this;
+  }
+
   addRetry(retry: Retry): Module {
     this.addEntry(retry);
     return this;
@@ -2496,7 +2595,7 @@ export class Module {
 
   private getEntryIndex(entryName: string): number {
     return this.entries.findIndex((v: ModuleEntry) => {
-      return v.name == entryName;
+      return v.name === entryName;
     });
   }
 
@@ -2723,6 +2822,17 @@ export class Module {
 
 let GlobalRetries: Array<Retry> | undefined = undefined;
 
+const DocumentAliasMap: Map<string, string> = new Map();
+
+export function registerDocumentAlias(name: string, title: string): void {
+  if (!name || !title) return;
+  DocumentAliasMap.set(name, title);
+}
+
+export function resolveDocumentAliases(names: string[]): string[] {
+  return names.map((name: string) => DocumentAliasMap.get(name) ?? name);
+}
+
 export function addGlobalRetry(r: Retry): Retry {
   if (GlobalRetries === undefined) {
     GlobalRetries = new Array<Retry>();
@@ -2782,8 +2892,6 @@ export function removeModule(name: string): boolean {
   return false;
 }
 
-addModule(DefaultModuleName);
-addRecord('env', DefaultModuleName);
 addModule(ScratchModuleName);
 
 export function getModuleNames(): string[] {