agentlang 0.10.0 → 0.10.1

package/src/language/parser.ts

@@ -8,6 +8,7 @@ import {
   ForEach,
   FullTextSearch,
   Group,
+  GroupByClause,
   Handler,
   If,
   isExpr,
@@ -24,14 +25,17 @@ import {
   ModuleDefinition,
   NegExpr,
   NotExpr,
+  OrderByClause,
   Pattern,
   PrimExpr,
+  QueryOption,
   RelationshipPattern,
   Return,
   SelectIntoEntry,
   SelectIntoSpec,
   SetAttribute,
   Statement,
+  WhereSpec,
   WorkflowDefinition,
 } from './generated/ast.js';
 import { firstAliasSpec, firstCatchSpec, isString, QuerySuffix } from '../runtime/util.js';
@@ -269,8 +273,9 @@ function introspectPattern(pat: Pattern): BasePattern {
     } else {
       r = introspectCreatePattern(pat.crudMap);
     }
-    if (pat.crudMap.into) {
-      r = introspectInto(pat.crudMap.into, r as CrudPattern);
+    const opts = extractQueryOptions(pat.crudMap);
+    if (opts.into) {
+      r = introspectInto(opts.into, r as CrudPattern);
     }
   } else if (pat.expr) {
     r = introspectExpression(pat.expr);
@@ -360,10 +365,11 @@ function introspectQueryPattern(crudMap: CrudMap): CrudPattern {
     crudMap.body?.attributes.forEach((sa: SetAttribute) => {
       cp.addAttribute(sa.name, introspectExpression(sa.value), sa.op);
     });
-    crudMap.relationships.forEach((rp: RelationshipPattern) => {
+    const opts = extractQueryOptions(crudMap);
+    crudMap.relationships?.forEach((rp: RelationshipPattern) => {
       cp.addRelationship(rp.name, introspectPattern(rp.pattern) as CrudPattern | CrudPattern[]);
     });
-    crudMap.joins.forEach((js: JoinSpec) => {
+    opts.joins?.forEach((js: JoinSpec) => {
       const jp: JoinPattern = {
         type: js.type,
         targetEntity: js.name,
@@ -381,6 +387,49 @@ function introspectQueryPattern(crudMap: CrudMap): CrudPattern {
   throw new Error(`Failed to introspect query-pattern: ${crudMap}`);
 }
 
+export type ExtractedQueryOptions = {
+  joins: JoinSpec[] | undefined;
+  into: SelectIntoSpec | undefined;
+  where: WhereSpec | undefined;
+  groupByClause: GroupByClause | undefined;
+  orderByClause: OrderByClause | undefined;
+  upsert: '@upsert' | undefined;
+  distinct: '@distinct' | undefined;
+};
+
+export function extractQueryOptions(crudMap: CrudMap): ExtractedQueryOptions {
+  const r: ExtractedQueryOptions = {
+    joins: undefined,
+    into: undefined,
+    where: undefined,
+    groupByClause: undefined,
+    orderByClause: undefined,
+    upsert: undefined,
+    distinct: undefined,
+  };
+  crudMap.queryOptions.forEach((qo: QueryOption) => {
+    if (qo.join) {
+      if (r.joins === undefined) {
+        r.joins = new Array<JoinSpec>();
+      }
+      r.joins.push(qo.join);
+    } else if (qo.into) {
+      r.into = qo.into;
+    } else if (qo.where) {
+      r.where = qo.where;
+    } else if (qo.groupByClause) {
+      r.groupByClause = qo.groupByClause;
+    } else if (qo.orderByClause) {
+      r.orderByClause = qo.orderByClause;
+    } else if (qo.upsert) {
+      r.upsert = qo.upsert;
+    } else if (qo.distinct) {
+      r.distinct = qo.distinct;
+    }
+  });
+  return r;
+}
+
 function introspectCreatePattern(crudMap: CrudMap): CrudPattern {
   if (crudMap) {
     const cp: CrudPattern = new CrudPattern(crudMap.name);
@@ -393,7 +442,7 @@ function introspectCreatePattern(crudMap: CrudMap): CrudPattern {
       }
       cp.addAttribute(sa.name, introspectExpression(sa.value), sa.op);
     });
-    crudMap.relationships.forEach((rp: RelationshipPattern) => {
+    crudMap.relationships?.forEach((rp: RelationshipPattern) => {
       cp.addRelationship(rp.name, introspectPattern(rp.pattern) as CrudPattern | CrudPattern[]);
     });
     cp.isQueryUpdate = qup;

package/src/runtime/agents/common.ts

@@ -209,7 +209,7 @@ A pattern may execute asynchronously and its eventual result can be handled by p
 If you are instructed that a particular event will be called asynchronously, always provide the patterns that follows in its '@then' clause. You must add the 
 '@then' clause only if an event's documentation or instruction explicitly requires to do so.
 
-Earlier we discussed teh concept of 'between' relationship. Here's the 'CreateEmployee' workflow updated to create the Employee with the his/her Profile attached:
+Earlier we discussed the concept of 'between' relationship. Here's the 'CreateEmployee' workflow updated to create the Employee with the his/her Profile attached:
 
 workflow CreateEmployee {
     {Erp/Employee {firstName CreateEmployee.firstName,

package/src/runtime/interpreter.ts

@@ -80,7 +80,12 @@ import {
   splitRefs,
 } from './util.js';
 import { getResolver, getResolverNameForPath } from './resolvers/registry.js';
-import { parseStatement, parseWorkflow } from '../language/parser.js';
+import {
+  ExtractedQueryOptions,
+  extractQueryOptions,
+  parseStatement,
+  parseWorkflow,
+} from '../language/parser.js';
 import { ActiveSessionInfo, AdminSession, AdminUserId } from './auth/defs.js';
 import {
   AgentEntityName,
@@ -1497,17 +1502,18 @@ async function maybeValidateOneOfRefs(inst: Instance, env: Environment) {
   }
 }
 
-function maybeSetQueryClauses(inst: Instance, crud: CrudMap) {
-  if (crud.groupByClause) {
-    inst.setGroupBy(crud.groupByClause.colNames);
+function maybeSetQueryClauses(inst: Instance, qopts: ExtractedQueryOptions) {
+  if (qopts.groupByClause) {
+    inst.setGroupBy(qopts.groupByClause.colNames);
   }
-  if (crud.orderByClause) {
-    inst.setOrderBy(crud.orderByClause.colNames, crud.orderByClause.order === '@desc');
+  if (qopts.orderByClause) {
+    inst.setOrderBy(qopts.orderByClause.colNames, qopts.orderByClause.order === '@desc');
   }
 }
 
 async function evaluateCrudMap(crud: CrudMap, env: Environment): Promise<void> {
-  if (!env.isInUpsertMode() && crud.upsert.length > 0) {
+  const qopts = extractQueryOptions(crud);
+  if (!env.isInUpsertMode() && qopts.upsert !== undefined) {
     return await evaluateUpsert(crud, env);
   }
   const inst: Instance = crud.source
@@ -1519,12 +1525,12 @@ async function evaluateCrudMap(crud: CrudMap, env: Environment): Promise<void> {
   const qattrs = inst.queryAttributes;
   const onlyAggregates = inst.aggregates !== undefined && qattrs === undefined;
   const isQueryAll = onlyAggregates || crud.name.endsWith(QuerySuffix);
-  const distinct: boolean = crud.distinct.length > 0;
-  maybeSetQueryClauses(inst, crud);
+  const distinct: boolean = qopts.distinct !== undefined;
+  maybeSetQueryClauses(inst, qopts);
   if (attrs.size > 0) {
     await maybeValidateOneOfRefs(inst, env);
   }
-  if (crud.into) {
+  if (qopts.into) {
     if (attrs.size > 0) {
       throw new Error(
         `Query pattern for ${entryName} with 'into' clause cannot be used to update attributes`
@@ -1533,10 +1539,16 @@ async function evaluateCrudMap(crud: CrudMap, env: Environment): Promise<void> {
     if (qattrs === undefined && !isQueryAll) {
       throw new Error(`Pattern for ${entryName} with 'into' clause must be a query`);
     }
-    if (crud.joins.length > 0) {
-      await evaluateJoinQuery(crud.joins, crud.into, crud.where, inst, distinct, env);
+    if (qopts.joins && qopts.joins.length > 0) {
+      await evaluateJoinQuery(qopts.joins, qopts.into, qopts.where, inst, distinct, env);
     } else {
-      await evaluateJoinQueryWithRelationships(crud.into, inst, crud.relationships, distinct, env);
+      await evaluateJoinQueryWithRelationships(
+        qopts.into,
+        inst,
+        crud.relationships || [],
+        distinct,
+        env
+      );
     }
     return;
   }

package/src/runtime/modules/ai.ts

@@ -1,6 +1,10 @@
 import {
   DefaultModuleName,
   escapeSpecialChars,
+  extractAndRemoveAllXmlTaggedText,
+  ExtractedCode,
+  ExtractedText,
+  extractFencedCodeBlocks,
   isFqName,
   isString,
   makeCoreModuleName,
@@ -580,7 +584,7 @@ export class AgentInstance {
     }
   }
 
-  private async getFullInstructions(
+  private async getFullInstructionsHelper(
     env: Environment,
     activator: AgentInstructionActivator
   ): Promise<string> {
@@ -652,6 +656,20 @@ Only return a pure JSON object with no extra text, annotations etc.`;
     }
   }
 
+  private static UserTag = 'user';
+
+  private async getFullInstructions(
+    env: Environment,
+    activator: AgentInstructionActivator
+  ): Promise<ExtractedText> {
+    const finalInstruction = await this.getFullInstructionsHelper(env, activator);
+    if (finalInstruction.indexOf(`<${AgentInstance.UserTag}>`) > 0) {
+      return extractAndRemoveAllXmlTaggedText(finalInstruction, AgentInstance.UserTag);
+    } else {
+      return { extracted: undefined, updatedText: finalInstruction };
+    }
+  }
+
   private static maybeRewriteTemplatePatterns(
     scratchPad: any,
     instruction: string,
@@ -820,6 +838,7 @@ Only return a pure JSON object with no extra text, annotations etc.`;
     const sess: Instance | null = this.withSession ? await findAgentChatSession(chatId, env) : null;
     let msgs: BaseMessage[] | undefined;
     let cachedMsg: string | undefined = undefined;
+    let extractedText: ExtractedText | undefined;
     const activator: AgentInstructionActivator = {
       provider: p,
       userMessage: message,
@@ -829,7 +848,8 @@ Only return a pure JSON object with no extra text, annotations etc.`;
     if (sess) {
       msgs = sess.lookup('messages');
     } else {
-      cachedMsg = await this.getFullInstructions(env, activator);
+      extractedText = await this.getFullInstructions(env, activator);
+      cachedMsg = extractedText.updatedText;
       msgs = [systemMessage(cachedMsg || '')];
     }
     if (msgs) {
@@ -844,12 +864,21 @@ Only return a pure JSON object with no extra text, annotations etc.`;
                 ? EvalInstructions
                 : LearnerAgentInstructions;
           const ts = this.toolsAsString();
-          const msg = `${s}\n${ts}\n${cachedMsg || (await this.getFullInstructions(env, activator))}`;
+          let tmpMsg = cachedMsg;
+          if (!tmpMsg) {
+            extractedText = await this.getFullInstructions(env, activator);
+            tmpMsg = extractedText.updatedText;
+          }
+          const msg = `${s}\n${ts}\n${tmpMsg}`;
           const newSysMsg = systemMessage(msg);
           msgs[0] = newSysMsg;
         }
+        let tmpMsg = message;
+        if (extractedText?.extracted) {
+          tmpMsg = `${tmpMsg}\n${extractedText.extracted.join('\n')}`;
+        }
         const hmsg = await this.maybeAddRelevantDocuments(
-          this.maybeAddFlowContext(message, env),
+          this.maybeAddFlowContext(tmpMsg, env),
           env
         );
         if (hmsg.length > 0) {
@@ -1299,26 +1328,6 @@ function processScenarioResponse(resp: string): string {
   return resp;
 }
 
-type ExtractedCode = {
-  language: string | null;
-  code: string;
-};
-
-export function extractFencedCodeBlocks(markdown: string): ExtractedCode[] {
-  const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
-  const blocks: ExtractedCode[] = [];
-  let match;
-
-  while ((match = codeBlockRegex.exec(markdown)) !== null) {
-    blocks.push({
-      language: match[1] || null,
-      code: match[2],
-    });
-  }
-
-  return blocks;
-}
-
 export function normalizeGeneratedCode(code: string | undefined): string {
   if (code !== undefined) {
     const blocks = extractFencedCodeBlocks(code);

package/src/runtime/services/documentFetcher.ts

@@ -1,5 +1,5 @@
 import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
-import { readFile } from 'node:fs/promises';
+import { readFile } from '../../utils/fs-utils.js';
 import { logger } from '../logger.js';
 import { parseAndEvaluateStatement } from '../interpreter.js';
 import { CoreAIModuleName } from '../modules/ai.js';
@@ -443,7 +443,7 @@ class DocumentFetcherService {
 
   private async fetchFromLocal(filePath: string): Promise<string> {
     try {
-      const content = await readFile(filePath, 'utf-8');
+      const content = await readFile(filePath);
       const lowerPath = filePath.toLowerCase();
       const isMarkdown = lowerPath.endsWith('.md') || lowerPath.endsWith('.markdown');
 

package/src/runtime/util.ts

@@ -1,4 +1,4 @@
-import { isNodeEnv } from '../utils/runtime.js';
+import { isNodeEnv, path } from '../utils/runtime.js';
 import {
   AliasSpec,
   CatchSpec,
@@ -15,7 +15,6 @@ import {
 } from '../language/generated/ast.js';
 import { readFile } from '../utils/fs-utils.js';
 import bcrypt from 'bcryptjs';
-import path from 'node:path';
 
 export const QuerySuffix = '?';
 
@@ -665,3 +664,47 @@ export function objectAsString(obj: any, keyAsString: boolean = false) {
   });
   return `{${entries.join(', ')}}`;
 }
+
+export type ExtractedText = {
+  extracted: string[] | undefined;
+  updatedText: string;
+};
+
+// extract all data between a given xml tag from within an arbitray text.
+export function extractAndRemoveAllXmlTaggedText(text: string, tagName: string): ExtractedText {
+  const pattern = `<${tagName}\\b[^>]*>([\\s\\S]*?)</${tagName}>`;
+  const regex = new RegExp(pattern, 'gi');
+
+  const extracted = [];
+  let updatedText = text;
+
+  let match;
+  while ((match = regex.exec(text)) !== null) {
+    extracted.push(match[1]);
+  }
+
+  updatedText = text.replace(regex, '');
+
+  return { extracted, updatedText };
+}
+
+export type ExtractedCode = {
+  language: string | null;
+  code: string;
+};
+
+// extract tick-quoted code from markdown-formatted text.
+export function extractFencedCodeBlocks(markdown: string): ExtractedCode[] {
+  const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
+  const blocks: ExtractedCode[] = [];
+  let match;
+
+  while ((match = codeBlockRegex.exec(markdown)) !== null) {
+    blocks.push({
+      language: match[1] || null,
+      code: match[2],
+    });
+  }
+
+  return blocks;
+}

package/src/utils/fs/index.ts

@@ -4,25 +4,49 @@
 
 export * from './interfaces.js';
 import { ExtendedFileSystem } from './interfaces.js';
-import { createNodeFS } from './node-fs.js';
-import { createLightningFS } from './lightning-fs.js';
 
 /**
  * Create the appropriate filesystem implementation based on environment
+ * Uses dynamic imports to avoid bundling Node.js-specific code in browser builds
  * @returns Promise resolving to appropriate filesystem implementation
  */
 export async function createFS(options?: any): Promise<ExtendedFileSystem> {
   // Check if we're in a browser or Node environment
   if (typeof window === 'undefined') {
-    // Node.js environment
+    // Node.js environment - use dynamic import to avoid bundling in browser
+    const { createNodeFS } = await import('./node-fs.js');
     return createNodeFS();
   } else {
     // Browser environment - use Lightning FS
+    const { createLightningFS } = await import('./lightning-fs.js');
     return createLightningFS(options);
   }
 }
 
-// Export the specific filesystem implementations
-export { createNodeFS } from './node-fs.js';
-export { createLightningFS } from './lightning-fs.js';
+// Re-export interface types (these are safe for browser)
 export * from './interfaces.js';
+
+// Export factory functions that use dynamic imports internally
+// These are async to support dynamic loading based on environment
+
+/**
+ * Create Node.js filesystem - only works in Node.js environment
+ * @returns Promise resolving to NodeFileSystem instance
+ */
+export async function createNodeFS(): Promise<ExtendedFileSystem> {
+  if (typeof window !== 'undefined') {
+    throw new Error('createNodeFS is only available in Node.js environment');
+  }
+  const module = await import('./node-fs.js');
+  return module.createNodeFS();
+}
+
+/**
+ * Create Lightning FS - works in browser environment
+ * @param options Optional configuration for Lightning FS
+ * @returns Promise resolving to LightningFileSystem instance
+ */
+export async function createLightningFS(options?: any): Promise<ExtendedFileSystem> {
+  const module = await import('./lightning-fs.js');
+  return module.createLightningFS(options);
+}