agentlang 0.9.6 → 0.9.7

package/src/cli/main.ts

@@ -38,6 +38,7 @@ import {
   setRuntimeMode_init_schema,
   setRuntimeMode_migration,
   setRuntimeMode_prod,
+  setRuntimeMode_test,
   setRuntimeMode_undo_migration,
   updateEndpoints,
 } from '../runtime/defs.js';
@@ -216,8 +217,12 @@ async function internDynamicModule(name: string, definition: string): Promise<st
 setInternDynamicModuleFn(internDynamicModule);
 
 export const runModule = async (fileName: string, releaseDb: boolean = false): Promise<void> => {
-  if (isRuntimeMode_dev() && process.env.NODE_ENV === 'production') {
-    setRuntimeMode_prod();
+  if (isRuntimeMode_dev()) {
+    if (process.env.NODE_ENV === 'production') {
+      setRuntimeMode_prod();
+    } else if (process.env.NODE_ENV === 'test') {
+      setRuntimeMode_test();
+    }
   }
   const r: boolean = await runPreInitTasks();
   if (!r) {

package/src/language/generated/ast.ts

@@ -3,7 +3,7 @@
  * DO NOT EDIT MANUALLY!
  ******************************************************************************/
 
- 
+/* eslint-disable */
 import * as langium from 'langium';
 
 export const AgentlangTerminals = {

package/src/runtime/docs.ts

@@ -1,5 +1,6 @@
-import { getFileSystem } from '../utils/fs-utils.js';
 import { logger } from './logger.js';
+import { isNodeEnv } from '../utils/runtime.js';
+import { getFileSystem } from '../utils/fs-utils.js';
 
 const DocFetchers = new Map<string, Function>();
 
@@ -25,6 +26,39 @@ export async function fetchDoc(url: string): Promise<string | undefined> {
   else return undefined;
 }
 
+let PDFParse: any = null;
+let pdfWorkerSet = false;
+
+async function getPDFParse() {
+  if (!PDFParse) {
+    const pdfModule = await import('pdf-parse');
+    PDFParse = pdfModule.PDFParse;
+
+    // Set up web worker for browser
+    if (!isNodeEnv && !pdfWorkerSet && PDFParse.setWorker) {
+      // Worker is served from public/ directory
+      PDFParse.setWorker('/pdf.worker.mjs');
+      pdfWorkerSet = true;
+    }
+  }
+  return PDFParse;
+}
+
+async function parsePdfBuffer(buffer: Uint8Array): Promise<string> {
+  try {
+    const PDFParseClass = await getPDFParse();
+    const parser = new PDFParseClass({
+      data: buffer,
+      verbosity: 0,
+    });
+    const data = await parser.getText();
+    return data.text;
+  } catch (error: any) {
+    logger.error(`Failed to parse PDF: ${error.message}`);
+    throw new Error(`PDF parsing failed: ${error.message}`);
+  }
+}
+
 async function httpFetcher(url: string): Promise<string | undefined> {
   try {
     const response = await fetch(url, {
@@ -32,12 +66,24 @@ async function httpFetcher(url: string): Promise<string | undefined> {
     });
 
     if (!response.ok) {
-      logger.error(
-        `Failed to fetch document ${url}, HTTP error! status: ${response.status} ${response.text} ${response.statusText}`
-      );
+      logger.error(`Failed to fetch document ${url}, HTTP error! status: ${response.status}`);
       return undefined;
     }
-    return await response.text();
+
+    const contentType = response.headers.get('content-type') || '';
+    const content = await response.arrayBuffer();
+
+    const lowerUrl = url.toLowerCase();
+
+    // Process based on content type or file extension
+    if (contentType.includes('application/pdf') || lowerUrl.endsWith('.pdf')) {
+      return await parsePdfBuffer(new Uint8Array(content));
+    } else if (contentType.includes('text/markdown') || lowerUrl.endsWith('.md')) {
+      return new TextDecoder().decode(content);
+    } else {
+      // Default to text
+      return new TextDecoder().decode(content);
+    }
   } catch (reason: any) {
     logger.error(`Failed to fetch document ${url}: ${reason}`);
   }
@@ -45,11 +91,76 @@ async function httpFetcher(url: string): Promise<string | undefined> {
 }
 
 async function fetchFile(path: string): Promise<string> {
-  const fs = await getFileSystem();
-  if (path.startsWith('.')) {
-    path = `${process.cwd()}${path.substring(1)}`;
+  const lowerPath = path.toLowerCase();
+
+  if (lowerPath.endsWith('.pdf')) {
+    return await fetchPdfFile(path);
+  } else if (
+    lowerPath.endsWith('.md') ||
+    lowerPath.endsWith('.markdown') ||
+    lowerPath.endsWith('.mdown')
+  ) {
+    return await fetchMarkdownFile(path);
+  } else {
+    // Default: plain text
+    return await fetchTextFile(path);
+  }
+}
+
+async function fetchPdfFile(path: string): Promise<string> {
+  try {
+    const fs = await getFileSystem();
+
+    if (isNodeEnv && path.startsWith('.')) {
+      path = `${process.cwd()}${path.substring(1)}`;
+    }
+
+    const content = await fs.readFile(path);
+
+    let buffer: Uint8Array;
+    if (typeof content === 'string') {
+      buffer = new TextEncoder().encode(content);
+    } else if (Buffer.isBuffer(content)) {
+      buffer = new Uint8Array(content);
+    } else {
+      buffer = new Uint8Array(Buffer.from(content));
+    }
+
+    return await parsePdfBuffer(buffer);
+  } catch (error: any) {
+    logger.error(`Failed to read PDF file ${path}: ${error.message}`);
+    throw error;
+  }
+}
+
+async function fetchMarkdownFile(path: string): Promise<string> {
+  try {
+    const fs = await getFileSystem();
+
+    if (isNodeEnv && path.startsWith('.')) {
+      path = `${process.cwd()}${path.substring(1)}`;
+    }
+
+    return await fs.readFile(path);
+  } catch (error: any) {
+    logger.error(`Failed to read Markdown file ${path}: ${error.message}`);
+    throw error;
+  }
+}
+
+async function fetchTextFile(path: string): Promise<string> {
+  try {
+    const fs = await getFileSystem();
+
+    if (isNodeEnv && path.startsWith('.')) {
+      path = `${process.cwd()}${path.substring(1)}`;
+    }
+
+    return await fs.readFile(path);
+  } catch (error: any) {
+    logger.error(`Failed to read text file ${path}: ${error.message}`);
+    throw error;
   }
-  return fs.readFile(path);
 }
 
 registerDocFetcher('http', httpFetcher);

package/src/runtime/embeddings/chunker.ts

@@ -0,0 +1,50 @@
+export class TextChunker {
+  private chunkSize: number;
+  private chunkOverlap: number;
+  private separators: string[] = ['\n\n', '\n', '. ', ' ', ''];
+
+  constructor(chunkSize: number = 1000, chunkOverlap: number = 200) {
+    this.chunkSize = chunkSize;
+    this.chunkOverlap = chunkOverlap;
+  }
+
+  splitText(text: string): string[] {
+    if (text.length <= this.chunkSize) {
+      return [text];
+    }
+
+    const chunks: string[] = [];
+    let start = 0;
+
+    while (start < text.length) {
+      let end = Math.min(start + this.chunkSize, text.length);
+
+      if (end < text.length) {
+        end = this.findBestSplitPoint(text, start, end);
+      }
+
+      chunks.push(text.substring(start, end));
+      start = end - this.chunkOverlap;
+
+      if (start < 0) start = 0;
+      if (start >= text.length - this.chunkOverlap) {
+        if (start < text.length) {
+          chunks.push(text.substring(start));
+        }
+        break;
+      }
+    }
+
+    return chunks;
+  }
+
+  private findBestSplitPoint(text: string, start: number, end: number): number {
+    for (const sep of this.separators) {
+      const lastSep = text.lastIndexOf(sep, end);
+      if (lastSep > start) {
+        return Math.min(lastSep + sep.length, end);
+      }
+    }
+    return end;
+  }
+}

package/src/runtime/embeddings/index.ts

@@ -0,0 +1,5 @@
+export { EmbeddingProvider, EmbeddingProviderConfig } from './provider.js';
+export { EmbeddingService } from '../resolvers/sqldb/impl.js';
+export { TextChunker } from './chunker.js';
+export { embeddingProvider, getDefaultEmbeddingProvider } from './registry.js';
+export { OpenAIEmbeddingProvider, OpenAIEmbeddingConfig } from './openai.js';

package/src/runtime/embeddings/openai.ts

@@ -0,0 +1,49 @@
+import { OpenAIEmbeddings } from '@langchain/openai';
+import { EmbeddingProvider, EmbeddingProviderConfig } from './provider.js';
+import { getLocalEnv } from '../auth/defs.js';
+
+export interface OpenAIEmbeddingConfig extends EmbeddingProviderConfig {
+  model?: string;
+  dimensions?: number;
+  maxRetries?: number;
+}
+
+export class OpenAIEmbeddingProvider extends EmbeddingProvider {
+  private openaiConfig: OpenAIEmbeddingConfig;
+
+  constructor(config?: EmbeddingProviderConfig) {
+    super(config || {});
+    this.openaiConfig = (this.config as OpenAIEmbeddingConfig) || {};
+  }
+
+  protected createEmbeddings(): OpenAIEmbeddings {
+    const config: any = {
+      apiKey: this.resolveApiKey(),
+    };
+
+    if (this.openaiConfig.model) {
+      config.model = this.openaiConfig.model;
+    }
+
+    if (this.openaiConfig.dimensions) {
+      config.dimensions = this.openaiConfig.dimensions;
+    }
+
+    if (this.openaiConfig.maxRetries !== undefined) {
+      config.maxRetries = this.openaiConfig.maxRetries;
+    }
+
+    return new OpenAIEmbeddings(config);
+  }
+
+  protected resolveApiKey(): string {
+    if (this.openaiConfig.apiKey) {
+      return this.openaiConfig.apiKey;
+    }
+    return process.env.AGENTLANG_OPENAI_KEY || getLocalEnv('AGENTLANG_OPENAI_KEY') || '';
+  }
+
+  getProviderName(): string {
+    return 'openai';
+  }
+}

package/src/runtime/embeddings/provider.ts

@@ -0,0 +1,37 @@
+import { Embeddings } from '@langchain/core/embeddings';
+
+export interface EmbeddingProviderConfig {
+  chunkSize?: number;
+  chunkOverlap?: number;
+  apiKey?: string;
+  model?: string;
+  [key: string]: any;
+}
+
+export abstract class EmbeddingProvider {
+  protected config: EmbeddingProviderConfig;
+  protected embeddings: Embeddings;
+
+  constructor(config: EmbeddingProviderConfig = {}) {
+    this.config = config;
+    this.embeddings = this.createEmbeddings();
+  }
+
+  protected abstract createEmbeddings(): Embeddings;
+  protected abstract resolveApiKey(): string;
+
+  abstract getProviderName(): string;
+
+  async embedText(text: string): Promise<number[]> {
+    return await this.embeddings.embedQuery(text);
+  }
+
+  getConfig(): EmbeddingProviderConfig {
+    return { ...this.config };
+  }
+
+  updateConfig(newConfig: Partial<EmbeddingProviderConfig>): void {
+    this.config = { ...this.config, ...newConfig };
+    this.embeddings = this.createEmbeddings();
+  }
+}

package/src/runtime/embeddings/registry.ts

@@ -0,0 +1,17 @@
+import { OpenAIEmbeddingProvider } from './openai.js';
+
+const EmbeddingProviders = new Map().set('openai', OpenAIEmbeddingProvider);
+
+export function embeddingProvider(service: string): any {
+  const requestedService = service.toLowerCase();
+  const provider = EmbeddingProviders.get(requestedService);
+  if (provider) {
+    return provider;
+  } else {
+    throw new Error(`No embedding provider found for ${service}`);
+  }
+}
+
+export function getDefaultEmbeddingProvider(): string {
+  return 'openai';
+}

package/src/runtime/interpreter.ts

@@ -32,8 +32,8 @@ import {
   WhereSpec,
 } from '../language/generated/ast.js';
 import {
-  maybeInstanceAsString,
   defineAgentEvent,
+  Event,
   getOneOfRef,
   getRelationship,
   getWorkflow,
@@ -46,15 +46,15 @@ import {
   isEntityInstance,
   isEventInstance,
   isInstanceOfType,
+  isOneToOneBetweenRelationship,
   isTimer,
   makeInstance,
+  maybeInstanceAsString,
   newInstanceAttributes,
   PlaceholderRecordEntry,
   Relationship,
-  Workflow,
   setMetaAttributes,
-  Event,
-  isOneToOneBetweenRelationship,
+  Workflow,
 } from './module.js';
 import { JoinInfo, Resolver, WhereClause } from './resolvers/interface.js';
 import { ResolverAuthInfo } from './resolvers/authinfo.js';
@@ -65,26 +65,26 @@ import {
   escapeFqName,
   escapeQueryName,
   fqNameFromPath,
+  isCoreModule,
   isFqName,
   isPath,
   isString,
   makeCoreModuleName,
   makeFqName,
+  nameToPath,
   Path,
+  preprocessRawConfig,
   QuerySuffix,
   restoreSpecialChars,
-  nameToPath,
   splitRefs,
-  isCoreModule,
-  preprocessRawConfig,
 } from './util.js';
 import { getResolver, getResolverNameForPath } from './resolvers/registry.js';
 import { parseStatement, parseWorkflow } from '../language/parser.js';
 import { ActiveSessionInfo, AdminSession, AdminUserId } from './auth/defs.js';
 import {
-  AgentInstance,
   AgentEntityName,
   AgentFqName,
+  AgentInstance,
   findAgentByName,
   normalizeGeneratedCode,
 } from './modules/ai.js';
@@ -1755,11 +1755,15 @@ async function handleDocEvent(inst: Instance, env: Environment): Promise<void> {
   const s = await fetchDoc(inst.lookup('url'));
   if (s) {
     const title = inst.lookup('title');
-    await parseAndEvaluateStatement(
-      `{${CoreAIModuleName}/Document {title "${title}", content "${s}"}}`,
-      undefined,
-      env
+    const doc = makeInstance(
+      CoreAIModuleName,
+      'Document',
+      newInstanceAttributes().set('title', title).set('content', s)
     );
+    await computeExprAttributes(doc, undefined, undefined, env);
+    await setMetaAttributes(doc.attributes, env);
+    const res: Resolver = await getResolverForPath('Document', CoreAIModuleName, env);
+    await res.createInstance(doc);
   }
 }
 

package/src/runtime/module.ts

@@ -1,57 +1,57 @@
 import chalk from 'chalk';
 import {
-  Statement,
-  KvPair,
-  Literal,
+  AttributeDefinition,
+  Expr,
+  FlowDefinition,
+  FlowEntry,
   FnCall,
-  RelNodes,
+  isLiteral,
   isRelNodes,
-  AttributeDefinition,
-  PropertyDefinition,
-  NodeDefinition,
-  RecordSchemaDefinition,
+  KvPair,
+  Literal,
   MapEntry,
-  isLiteral,
   MetaDefinition,
+  NodeDefinition,
   PrePostTriggerDefinition,
-  TriggerEntry,
-  Expr,
-  RbacSpecEntry,
-  RbacSpecEntries,
+  PropertyDefinition,
   RbacOpr,
-  WorkflowHeader,
-  FlowDefinition,
-  FlowEntry,
+  RbacSpecEntries,
+  RbacSpecEntry,
+  RecordSchemaDefinition,
+  RelNodes,
+  Statement,
+  TriggerEntry,
   WorkflowDirectives,
+  WorkflowHeader,
 } from '../language/generated/ast.js';
 import {
-  Path,
-  nameToPath,
-  isString,
-  isNumber,
-  isBoolean,
-  isFqName,
-  makeFqName,
+  asCrudType,
+  CrudType,
   DefaultModuleName,
   DefaultModules,
-  joinStatements,
-  isMinusZero,
-  now,
-  findMetaSchema,
+  encryptPassword,
+  escapeFqName,
   findAllPrePostTriggerSchema,
-  CrudType,
-  asCrudType,
-  isPath,
+  findMetaSchema,
   findUqCompositeAttributes,
-  escapeFqName,
-  encryptPassword,
-  splitFqName,
-  splitRefs,
   forceAsFqName,
-  validateIdFormat,
+  isBoolean,
+  isFqName,
+  isMinusZero,
+  isNumber,
+  isPath,
+  isString,
+  joinStatements,
+  makeFqName,
   nameContainsSepEscape,
+  nameToPath,
+  now,
+  Path,
   registerInitFunction,
   ScratchModuleName,
+  splitFqName,
+  splitRefs,
+  validateIdFormat,
 } from './util.js';
 import { parseStatement } from '../language/parser.js';
 import { ActiveSessionInfo, AdminSession } from './auth/defs.js';
@@ -513,7 +513,10 @@ export class Record extends ModuleEntry {
   }
 
   getFullTextSearchAttributes(): string[] | undefined {
-    const fts: string[] | string | undefined = this.getMeta('fullTextSearch');
+    let fts: string[] | string | undefined = this.getMeta('fullTextSearchAttributes');
+    if (!fts) {
+      fts = this.getMeta('fullTextSearch');
+    }
     if (fts) {
       if (fts instanceof Array) {
         return fts as string[];
@@ -527,6 +530,14 @@ export class Record extends ModuleEntry {
     }
   }
 
+  getEmbeddingConfig(): { [key: string]: any } | undefined {
+    const config = this.getMeta('embeddingConfig');
+    if (config && typeof config === 'object') {
+      return config as { [key: string]: any };
+    }
+    return undefined;
+  }
+
   private resetUserAttrs() {
     this.userAttrNames = undefined;
     this.userAttrsSchema = undefined;
@@ -2496,7 +2507,7 @@ export class Module {
 
   private getEntryIndex(entryName: string): number {
     return this.entries.findIndex((v: ModuleEntry) => {
-      return v.name == entryName;
+      return v.name === entryName;
     });
   }
 
@@ -2782,8 +2793,6 @@ export function removeModule(name: string): boolean {
   return false;
 }
 
-addModule(DefaultModuleName);
-addRecord('env', DefaultModuleName);
 addModule(ScratchModuleName);
 
 export function getModuleNames(): string[] {

package/src/runtime/modules/ai.ts

@@ -60,7 +60,6 @@ import {
   newAgentScenario,
   PlannerInstructions,
 } from '../agents/common.js';
-import { PathAttributeNameQuery } from '../defs.js';
 import { logger } from '../logger.js';
 import { FlowStep } from '../agents/flows.js';
 import Handlebars from 'handlebars';
@@ -980,34 +979,92 @@ Only return a pure JSON object with no extra text, annotations etc.`;
 
   private async maybeAddRelevantDocuments(message: string, env: Environment): Promise<string> {
     if (this.documents && this.documents.length > 0) {
-      const s = `${message}. Relevant documents are: ${this.documents}`;
-      const result: any[] = await parseHelper(`{${CoreAIModuleName}/Document? "${s}"}`, env);
-      if (result && result.length > 0) {
-        const docs: Instance[] = [];
-        for (let i = 0; i < result.length; ++i) {
-          const v: any = result[i];
-          const r: Instance[] = await parseHelper(
-            `{${CoreAIModuleName}/Document {${PathAttributeNameQuery} "${v.id}"}}`,
+      try {
+        const docNames = this.documents.split(',').map(d => d.trim());
+
+        const searchQuery = message;
+
+        try {
+          const semanticResult: any[] = await parseHelper(
+            `{${CoreAIModuleName}/Document {content? "${searchQuery.replace(/"/g, '\\"')}"}}`,
             env
           );
-          if (r && r.length > 0) {
-            docs.push(r[0]);
+
+          if (semanticResult && semanticResult.length > 0) {
+            const docs: Instance[] = [];
+            for (const doc of semanticResult) {
+              const docTitle = doc.lookup ? doc.lookup('title') : doc.title;
+              if (AgentInstance.docTitlesMatch(docTitle, docNames)) {
+                docs.push(
+                  doc instanceof Instance
+                    ? doc
+                    : Instance.newWithAttributes(doc, new Map(Object.entries(doc)))
+                );
+              }
+            }
+
+            if (docs.length > 0) {
+              return message.concat('\n\nRelevant context from documents:\n').concat(
+                docs
+                  .map((v: Instance) => {
+                    return `Document: ${v.lookup('title')}\n${v.lookup('content') as string}`;
+                  })
+                  .join('\n\n---\n\n')
+              );
+            }
           }
-        }
-        if (docs.length > 0) {
-          message = message.concat('\nUse the additional information given below:\n').concat(
-            docs
-              .map((v: Instance) => {
-                return v.lookup('content');
-              })
-              .join('\n')
+        } catch (semanticErr) {
+          logger.debug(
+            `Semantic search is not available, falling back to title-based filtering: ${semanticErr}`
           );
         }
+
+        const result: any[] = await parseHelper(`{${CoreAIModuleName}/Document? {}}`, env);
+        if (result && result.length > 0) {
+          const docs: Instance[] = [];
+          for (let i = 0; i < result.length; ++i) {
+            const v: any = result[i];
+            const docTitle: string | undefined = AgentInstance.getDocumentTitle(v);
+
+            if (docTitle && docNames.includes(docTitle)) {
+              if (v instanceof Instance) {
+                docs.push(v);
+              }
+            }
+          }
+
+          if (docs.length > 0) {
+            return message.concat('\n\nRelevant context from documents:\n').concat(
+              docs
+                .map((v: Instance) => {
+                  return v.lookup('content') as string;
+                })
+                .join('\n\n')
+            );
+          }
+        }
+      } catch (err) {
+        logger.debug(`Error retrieving documents: ${err}`);
       }
     }
     return message;
   }
 
+  private static docTitlesMatch(title: string | undefined, docNames: string[]): boolean {
+    return title !== undefined && docNames.includes(title);
+  }
+
+  private static getDocumentTitle(doc: any): string | undefined {
+    if (typeof doc.lookup === 'function') {
+      return doc.lookup('title') as string | undefined;
+    } else if (doc.attributes) {
+      return doc.attributes.get('title') as string | undefined;
+    } else if (doc.title) {
+      return doc.title;
+    }
+    return undefined;
+  }
+
   private static ToolsCache = new Map<string, string>();
 
   private toolsAsString(): string {