agentlang 0.9.11 → 0.10.1

package/src/language/parser.ts

@@ -8,6 +8,7 @@ import {
   ForEach,
   FullTextSearch,
   Group,
+  GroupByClause,
   Handler,
   If,
   isExpr,
@@ -17,20 +18,24 @@ import {
   isNotExpr,
   isPrimExpr,
   isWorkflowDefinition,
+  JoinSpec,
   Literal,
   MapEntry,
   MapLiteral,
   ModuleDefinition,
   NegExpr,
   NotExpr,
+  OrderByClause,
   Pattern,
   PrimExpr,
+  QueryOption,
   RelationshipPattern,
   Return,
   SelectIntoEntry,
   SelectIntoSpec,
   SetAttribute,
   Statement,
+  WhereSpec,
   WorkflowDefinition,
 } from './generated/ast.js';
 import { firstAliasSpec, firstCatchSpec, isString, QuerySuffix } from '../runtime/util.js';
@@ -44,6 +49,7 @@ import {
   FunctionCallPattern,
   GroupExpressionPattern,
   IfPattern,
+  JoinPattern,
   LiteralPattern,
   NegExpressionPattern,
   NotExpressionPattern,
@@ -267,8 +273,9 @@ function introspectPattern(pat: Pattern): BasePattern {
     } else {
       r = introspectCreatePattern(pat.crudMap);
     }
-    if (pat.crudMap.into) {
-      r = introspectInto(pat.crudMap.into, r as CrudPattern);
+    const opts = extractQueryOptions(pat.crudMap);
+    if (opts.into) {
+      r = introspectInto(opts.into, r as CrudPattern);
     }
   } else if (pat.expr) {
     r = introspectExpression(pat.expr);
@@ -358,9 +365,20 @@ function introspectQueryPattern(crudMap: CrudMap): CrudPattern {
     crudMap.body?.attributes.forEach((sa: SetAttribute) => {
       cp.addAttribute(sa.name, introspectExpression(sa.value), sa.op);
     });
-    crudMap.relationships.forEach((rp: RelationshipPattern) => {
+    const opts = extractQueryOptions(crudMap);
+    crudMap.relationships?.forEach((rp: RelationshipPattern) => {
       cp.addRelationship(rp.name, introspectPattern(rp.pattern) as CrudPattern | CrudPattern[]);
     });
+    opts.joins?.forEach((js: JoinSpec) => {
+      const jp: JoinPattern = {
+        type: js.type,
+        targetEntity: js.name,
+        conditionLhs: js.lhs,
+        conditionOperator: js.op ? js.op : '=',
+        conditionRhs: js.rhs,
+      };
+      cp.joins.push(jp);
+    });
     cp.isCreate = false;
     cp.isQueryUpdate = false;
     cp.isQuery = true;
@@ -369,6 +387,49 @@ function introspectQueryPattern(crudMap: CrudMap): CrudPattern {
   throw new Error(`Failed to introspect query-pattern: ${crudMap}`);
 }
 
+export type ExtractedQueryOptions = {
+  joins: JoinSpec[] | undefined;
+  into: SelectIntoSpec | undefined;
+  where: WhereSpec | undefined;
+  groupByClause: GroupByClause | undefined;
+  orderByClause: OrderByClause | undefined;
+  upsert: '@upsert' | undefined;
+  distinct: '@distinct' | undefined;
+};
+
+export function extractQueryOptions(crudMap: CrudMap): ExtractedQueryOptions {
+  const r: ExtractedQueryOptions = {
+    joins: undefined,
+    into: undefined,
+    where: undefined,
+    groupByClause: undefined,
+    orderByClause: undefined,
+    upsert: undefined,
+    distinct: undefined,
+  };
+  crudMap.queryOptions.forEach((qo: QueryOption) => {
+    if (qo.join) {
+      if (r.joins === undefined) {
+        r.joins = new Array<JoinSpec>();
+      }
+      r.joins.push(qo.join);
+    } else if (qo.into) {
+      r.into = qo.into;
+    } else if (qo.where) {
+      r.where = qo.where;
+    } else if (qo.groupByClause) {
+      r.groupByClause = qo.groupByClause;
+    } else if (qo.orderByClause) {
+      r.orderByClause = qo.orderByClause;
+    } else if (qo.upsert) {
+      r.upsert = qo.upsert;
+    } else if (qo.distinct) {
+      r.distinct = qo.distinct;
+    }
+  });
+  return r;
+}
+
 function introspectCreatePattern(crudMap: CrudMap): CrudPattern {
   if (crudMap) {
     const cp: CrudPattern = new CrudPattern(crudMap.name);
@@ -381,7 +442,7 @@ function introspectCreatePattern(crudMap: CrudMap): CrudPattern {
       }
       cp.addAttribute(sa.name, introspectExpression(sa.value), sa.op);
     });
-    crudMap.relationships.forEach((rp: RelationshipPattern) => {
+    crudMap.relationships?.forEach((rp: RelationshipPattern) => {
       cp.addRelationship(rp.name, introspectPattern(rp.pattern) as CrudPattern | CrudPattern[]);
     });
     cp.isQueryUpdate = qup;

package/src/language/syntax.ts

@@ -373,10 +373,24 @@ export type AttributePattern = {
   value: BasePattern;
 };
 
+export type JoinPattern = {
+  type: '@join' | '@inner_join' | '@left_join' | '@right_join' | '@full_join';
+  targetEntity: string;
+  conditionLhs: string;
+  conditionOperator: string;
+  conditionRhs: string;
+};
+
+function joinPatternToString(jp: JoinPattern): string {
+  const opr = jp.conditionOperator === '=' ? '' : jp.conditionOperator;
+  return `${jp.type} ${jp.targetEntity} {${jp.conditionLhs}${opr} ${jp.conditionRhs}}`;
+}
+
 export class CrudPattern extends BasePattern {
   recordName: string;
   attributes: Array<AttributePattern>;
   relationships: Map<string, CrudPattern[] | CrudPattern> | undefined;
+  joins: JoinPattern[];
   into: Map<string, string> | undefined;
   isQuery: boolean = false;
   isQueryUpdate: boolean = false;
@@ -391,6 +405,7 @@ export class CrudPattern extends BasePattern {
     } else {
       this.isCreate = true;
     }
+    this.joins = new Array<JoinPattern>();
   }
 
   addAttribute(n: string, p: BasePattern, op?: string): CrudPattern {
@@ -521,11 +536,15 @@ export class CrudPattern extends BasePattern {
     let s = `{${this.recordName} ${this.attributesAsString()}`;
     const rs = this.relationshipsAsString();
     if (rs) {
-      s = s.concat(`,${rs}`);
+      s = s.concat(`,\n${rs}`);
+    }
+    if (this.joins.length > 0) {
+      const js = this.joins.map(joinPatternToString);
+      s = s.concat(`,\n${js.join(',\n')}`);
     }
     const ins = this.intoAsString();
     if (ins) {
-      s = s.concat(`,${ins}`);
+      s = s.concat(`,\n${ins}`);
     }
     return s.concat('}', this.hintsAsString());
   }

package/src/runtime/agents/common.ts

@@ -209,7 +209,7 @@ A pattern may execute asynchronously and its eventual result can be handled by p
 If you are instructed that a particular event will be called asynchronously, always provide the patterns that follows in its '@then' clause. You must add the 
 '@then' clause only if an event's documentation or instruction explicitly requires to do so.
 
-Earlier we discussed teh concept of 'between' relationship. Here's the 'CreateEmployee' workflow updated to create the Employee with the his/her Profile attached:
+Earlier we discussed the concept of 'between' relationship. Here's the 'CreateEmployee' workflow updated to create the Employee with the his/her Profile attached:
 
 workflow CreateEmployee {
     {Erp/Employee {firstName CreateEmployee.firstName,

package/src/runtime/interpreter.ts

@@ -32,6 +32,7 @@ import {
   WhereSpec,
 } from '../language/generated/ast.js';
 import {
+  Agent,
   defineAgentEvent,
   Event,
   getOneOfRef,
@@ -79,7 +80,12 @@ import {
   splitRefs,
 } from './util.js';
 import { getResolver, getResolverNameForPath } from './resolvers/registry.js';
-import { parseStatement, parseWorkflow } from '../language/parser.js';
+import {
+  ExtractedQueryOptions,
+  extractQueryOptions,
+  parseStatement,
+  parseWorkflow,
+} from '../language/parser.js';
 import { ActiveSessionInfo, AdminSession, AdminUserId } from './auth/defs.js';
 import {
   AgentEntityName,
@@ -1496,17 +1502,18 @@ async function maybeValidateOneOfRefs(inst: Instance, env: Environment) {
   }
 }
 
-function maybeSetQueryClauses(inst: Instance, crud: CrudMap) {
-  if (crud.groupByClause) {
-    inst.setGroupBy(crud.groupByClause.colNames);
+function maybeSetQueryClauses(inst: Instance, qopts: ExtractedQueryOptions) {
+  if (qopts.groupByClause) {
+    inst.setGroupBy(qopts.groupByClause.colNames);
   }
-  if (crud.orderByClause) {
-    inst.setOrderBy(crud.orderByClause.colNames, crud.orderByClause.order === '@desc');
+  if (qopts.orderByClause) {
+    inst.setOrderBy(qopts.orderByClause.colNames, qopts.orderByClause.order === '@desc');
   }
 }
 
 async function evaluateCrudMap(crud: CrudMap, env: Environment): Promise<void> {
-  if (!env.isInUpsertMode() && crud.upsert.length > 0) {
+  const qopts = extractQueryOptions(crud);
+  if (!env.isInUpsertMode() && qopts.upsert !== undefined) {
     return await evaluateUpsert(crud, env);
   }
   const inst: Instance = crud.source
@@ -1518,12 +1525,12 @@ async function evaluateCrudMap(crud: CrudMap, env: Environment): Promise<void> {
   const qattrs = inst.queryAttributes;
   const onlyAggregates = inst.aggregates !== undefined && qattrs === undefined;
   const isQueryAll = onlyAggregates || crud.name.endsWith(QuerySuffix);
-  const distinct: boolean = crud.distinct.length > 0;
-  maybeSetQueryClauses(inst, crud);
+  const distinct: boolean = qopts.distinct !== undefined;
+  maybeSetQueryClauses(inst, qopts);
   if (attrs.size > 0) {
     await maybeValidateOneOfRefs(inst, env);
   }
-  if (crud.into) {
+  if (qopts.into) {
     if (attrs.size > 0) {
       throw new Error(
         `Query pattern for ${entryName} with 'into' clause cannot be used to update attributes`
@@ -1532,10 +1539,16 @@ async function evaluateCrudMap(crud: CrudMap, env: Environment): Promise<void> {
     if (qattrs === undefined && !isQueryAll) {
       throw new Error(`Pattern for ${entryName} with 'into' clause must be a query`);
     }
-    if (crud.joins.length > 0) {
-      await evaluateJoinQuery(crud.joins, crud.into, crud.where, inst, distinct, env);
+    if (qopts.joins && qopts.joins.length > 0) {
+      await evaluateJoinQuery(qopts.joins, qopts.into, qopts.where, inst, distinct, env);
     } else {
-      await evaluateJoinQueryWithRelationships(crud.into, inst, crud.relationships, distinct, env);
+      await evaluateJoinQueryWithRelationships(
+        qopts.into,
+        inst,
+        crud.relationships || [],
+        distinct,
+        env
+      );
     }
     return;
   }
@@ -2081,7 +2094,7 @@ async function agentInvoke(agent: AgentInstance, msg: string, env: Environment):
           const obj = agent.maybeValidateJsonResponse(result);
           if (obj !== undefined) {
             env.setLastResult(obj);
-            env.addToScratchPad(agent.getFqName(), obj);
+            env.addToScratchPad(Agent.NormalizeName(agent.getFqName()), obj);
           }
           break;
         } catch (err: any) {
@@ -2187,6 +2200,7 @@ export async function restartFlow(
 }
 
 const MaxFlowSteps = 25;
+const MaxFlowRetries = 10;
 
 async function iterateOnFlow(
   flow: FlowSpec,
@@ -2195,88 +2209,101 @@ async function iterateOnFlow(
   env: Environment
 ): Promise<void> {
   rootAgent.disableSession();
-  const initContext = msg;
-  const s = `Now consider the following flowchart and return the next step:\n${flow}\n
+  const chatId = env.getActiveEventInstance()?.lookup('chatId');
+  const iterId = chatId || crypto.randomUUID();
+  let step = '';
+  let fullFlowRetries = 0;
+  while (true) {
+    try {
+      const initContext = msg;
+      const s = `Now consider the following flowchart and return the next step:\n${flow}\n
   If you understand from the context that a step with no further possible steps has been evaluated,
   terminate the flowchart by returning DONE. Never return to the top or root step of the flowchart, instead return DONE.
   Important: Return only the next flow-step or DONE. Do not return any additional description, like your thinking process.\n`;
-  env.setFlowContext(initContext);
-  await agentInvoke(rootAgent, s, env);
-  const rootModuleName = rootAgent.moduleName;
-  let preprocResult = await preprocessStep(env.getLastResult(), rootModuleName, env);
-  let step = preprocResult.step;
-  let needAgentProcessing = preprocResult.needAgentProcessing;
-  let context = initContext;
-  let stepc = 0;
-  const chatId = env.getActiveEventInstance()?.lookup('chatId');
-  const iterId = chatId || crypto.randomUUID();
-  console.debug(`Starting iteration ${iterId} on flow: ${flow}`);
-  const executedSteps = new Set<string>();
-  const monitoringEnabled = isMonitoringEnabled();
-  if (monitoringEnabled) {
-    env.flagMonitorEntryAsFlow().incrementMonitor();
-  }
-  let isfxc = false;
-  try {
-    while (step != 'DONE' && !executedSteps.has(step)) {
-      if (stepc > MaxFlowSteps) {
-        throw new Error(`Flow execution exceeded maximum steps limit`);
+      env.setFlowContext(initContext);
+      await agentInvoke(rootAgent, s, env);
+      const rootModuleName = rootAgent.moduleName;
+      let preprocResult = await preprocessStep(env.getLastResult(), rootModuleName, env);
+      step = preprocResult.step;
+      let needAgentProcessing = preprocResult.needAgentProcessing;
+      let context = initContext;
+      let stepc = 0;
+      console.debug(`Starting iteration ${iterId} on flow: ${flow}`);
+      const executedSteps = new Set<string>();
+      const monitoringEnabled = isMonitoringEnabled();
+      let isfxc = false;
+      if (monitoringEnabled) {
+        env.flagMonitorEntryAsFlow().incrementMonitor();
       }
-      executedSteps.add(step);
-      ++stepc;
-      const agent = needAgentProcessing
-        ? AgentInstance.FromFlowStep(step, rootAgent, context)
-        : undefined;
-      if (agent) {
-        console.debug(`\n---------------------------------------------------\n`);
+      while (step != 'DONE' && !executedSteps.has(step)) {
+        if (stepc > MaxFlowSteps) {
+          throw new Error(`Flow execution exceeded maximum steps limit`);
+        }
+        executedSteps.add(step);
+        ++stepc;
+        const agent = needAgentProcessing
+          ? AgentInstance.FromFlowStep(step, rootAgent, context)
+          : undefined;
+        if (agent) {
+          console.debug(
+            `Starting to execute flow step ${step} with agent ${agent.name} with iteration ID ${iterId} and context: \n${context}`
+          );
+          isfxc = agent.isFlowExecutor();
+          const isdec = agent.isDecisionExecutor();
+          if (isfxc || isdec) env.setFlowContext(context);
+          else env.setFlowContext(initContext);
+          if (monitoringEnabled) {
+            env.appendEntryToMonitor(step);
+          }
+          const inst = agent.swapInstruction('');
+          await agentInvoke(agent, inst, env);
+        } else {
+          rootAgent.maybeAddScratchData(env);
+        }
+        if (monitoringEnabled) env.setMonitorEntryResult(env.getLastResult());
+        if (env.isSuspended()) {
+          console.debug(`${iterId} suspending iteration on step ${step}`);
+          await saveFlowSuspension(rootAgent, context, step, env);
+          env.releaseSuspension();
+          return;
+        }
+        const r = env.getLastResult();
+        const rs = maybeInstanceAsString(r);
         console.debug(
-          `Starting to execute flow step ${step} with agent ${agent.name} with iteration ID ${iterId} and context: \n${context}`
+          `\n----> Completed execution of step ${step}, iteration id ${iterId} with result:\n${rs}`
         );
-        isfxc = agent.isFlowExecutor();
-        const isdec = agent.isDecisionExecutor();
-        if (isfxc || isdec) env.setFlowContext(context);
-        else env.setFlowContext(initContext);
-        if (monitoringEnabled) {
-          env.appendEntryToMonitor(step);
+        context = `${context}\n${step} --> ${rs}\n`;
+        if (chatId) {
+          const suspEnv = new Environment(env.name, env);
+          suspEnv.softSuspend();
+          await saveFlowSuspension(rootAgent, context, step, suspEnv);
+          await saveFlowStepResult(chatId, step, rs, suspEnv.getSuspensionId(), env);
         }
-        const inst = agent.swapInstruction('');
-        await agentInvoke(agent, inst, env);
-      } else {
-        rootAgent.maybeAddScratchData(env);
-      }
-      if (monitoringEnabled) env.setMonitorEntryResult(env.getLastResult());
-      if (env.isSuspended()) {
-        console.debug(`${iterId} suspending iteration on step ${step}`);
-        await saveFlowSuspension(rootAgent, context, step, env);
-        env.releaseSuspension();
-        return;
-      }
-      const r = env.getLastResult();
-      const rs = maybeInstanceAsString(r);
-      console.debug(
-        `\n----> Completed execution of step ${step}, iteration id ${iterId} with result:\n${rs}`
-      );
-      context = `${context}\n${step} --> ${rs}\n`;
-      if (chatId) {
-        const suspEnv = new Environment(env.name, env);
-        suspEnv.softSuspend();
-        await saveFlowSuspension(rootAgent, context, step, suspEnv);
-        await saveFlowStepResult(chatId, step, rs, suspEnv.getSuspensionId(), env);
+        if (isfxc) {
+          preprocResult = await preprocessStep(rs, rootModuleName, env);
+        } else {
+          env.setFlowContext(context);
+          await agentInvoke(rootAgent, `${s}\n${context}`, env);
+          preprocResult = await preprocessStep(env.getLastResult(), rootModuleName, env);
+        }
+        step = preprocResult.step;
+        needAgentProcessing = preprocResult.needAgentProcessing;
       }
-      if (isfxc) {
-        preprocResult = await preprocessStep(rs, rootModuleName, env);
+    } catch (reason: any) {
+      if (fullFlowRetries < MaxFlowRetries) {
+        msg = `The previous attempt failed at step ${step} with the error ${reason}. Restart the flow the appropriate step
+(maybe even from the first step) and try to fix the issue.`;
+        ++fullFlowRetries;
+        continue;
       } else {
-        env.setFlowContext(context);
-        await agentInvoke(rootAgent, `${s}\n${context}`, env);
-        preprocResult = await preprocessStep(env.getLastResult(), rootModuleName, env);
+        throw new Error(reason);
       }
-      step = preprocResult.step;
-      needAgentProcessing = preprocResult.needAgentProcessing;
+    } finally {
+      env.decrementMonitor().revokeLastResult().setMonitorFlowResult();
     }
-  } finally {
-    env.decrementMonitor().revokeLastResult().setMonitorFlowResult();
+    console.debug(`No more flow steps, completed iteration ${iterId} on flow:\n${flow}`);
+    break;
   }
-  console.debug(`No more flow steps, completed iteration ${iterId} on flow:\n${flow}`);
 }
 
 type PreprocStepResult = {

package/src/runtime/modules/ai.ts

@@ -1,6 +1,10 @@
 import {
   DefaultModuleName,
   escapeSpecialChars,
+  extractAndRemoveAllXmlTaggedText,
+  ExtractedCode,
+  ExtractedText,
+  extractFencedCodeBlocks,
   isFqName,
   isString,
   makeCoreModuleName,
@@ -17,6 +21,7 @@ import {
   parseAndEvaluateStatement,
 } from '../interpreter.js';
 import {
+  Agent,
   AgentEvaluator,
   asJSONSchema,
   Decision,
@@ -579,7 +584,7 @@ export class AgentInstance {
     }
   }
 
-  private async getFullInstructions(
+  private async getFullInstructionsHelper(
     env: Environment,
     activator: AgentInstructionActivator
   ): Promise<string> {
@@ -651,6 +656,20 @@ Only return a pure JSON object with no extra text, annotations etc.`;
     }
   }
 
+  private static UserTag = 'user';
+
+  private async getFullInstructions(
+    env: Environment,
+    activator: AgentInstructionActivator
+  ): Promise<ExtractedText> {
+    const finalInstruction = await this.getFullInstructionsHelper(env, activator);
+    if (finalInstruction.indexOf(`<${AgentInstance.UserTag}>`) > 0) {
+      return extractAndRemoveAllXmlTaggedText(finalInstruction, AgentInstance.UserTag);
+    } else {
+      return { extracted: undefined, updatedText: finalInstruction };
+    }
+  }
+
   private static maybeRewriteTemplatePatterns(
     scratchPad: any,
     instruction: string,
@@ -709,7 +728,7 @@ Only return a pure JSON object with no extra text, annotations etc.`;
     ) {
       r = obj;
     } else {
-      env.addToScratchPad(this.name, obj);
+      env.addToScratchPad(Agent.NormalizeName(this.name), obj);
       return this;
     }
     const scratchNames = this.getScratchNames();
@@ -819,6 +838,7 @@ Only return a pure JSON object with no extra text, annotations etc.`;
     const sess: Instance | null = this.withSession ? await findAgentChatSession(chatId, env) : null;
     let msgs: BaseMessage[] | undefined;
     let cachedMsg: string | undefined = undefined;
+    let extractedText: ExtractedText | undefined;
     const activator: AgentInstructionActivator = {
       provider: p,
       userMessage: message,
@@ -828,7 +848,8 @@ Only return a pure JSON object with no extra text, annotations etc.`;
     if (sess) {
       msgs = sess.lookup('messages');
     } else {
-      cachedMsg = await this.getFullInstructions(env, activator);
+      extractedText = await this.getFullInstructions(env, activator);
+      cachedMsg = extractedText.updatedText;
       msgs = [systemMessage(cachedMsg || '')];
     }
     if (msgs) {
@@ -843,12 +864,21 @@ Only return a pure JSON object with no extra text, annotations etc.`;
                 ? EvalInstructions
                 : LearnerAgentInstructions;
           const ts = this.toolsAsString();
-          const msg = `${s}\n${ts}\n${cachedMsg || (await this.getFullInstructions(env, activator))}`;
+          let tmpMsg = cachedMsg;
+          if (!tmpMsg) {
+            extractedText = await this.getFullInstructions(env, activator);
+            tmpMsg = extractedText.updatedText;
+          }
+          const msg = `${s}\n${ts}\n${tmpMsg}`;
           const newSysMsg = systemMessage(msg);
           msgs[0] = newSysMsg;
         }
+        let tmpMsg = message;
+        if (extractedText?.extracted) {
+          tmpMsg = `${tmpMsg}\n${extractedText.extracted.join('\n')}`;
+        }
         const hmsg = await this.maybeAddRelevantDocuments(
-          this.maybeAddFlowContext(message, env),
+          this.maybeAddFlowContext(tmpMsg, env),
           env
         );
         if (hmsg.length > 0) {
@@ -1298,26 +1328,6 @@ function processScenarioResponse(resp: string): string {
   return resp;
 }
 
-type ExtractedCode = {
-  language: string | null;
-  code: string;
-};
-
-export function extractFencedCodeBlocks(markdown: string): ExtractedCode[] {
-  const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
-  const blocks: ExtractedCode[] = [];
-  let match;
-
-  while ((match = codeBlockRegex.exec(markdown)) !== null) {
-    blocks.push({
-      language: match[1] || null,
-      code: match[2],
-    });
-  }
-
-  return blocks;
-}
-
 export function normalizeGeneratedCode(code: string | undefined): string {
   if (code !== undefined) {
     const blocks = extractFencedCodeBlocks(code);

package/src/runtime/services/documentFetcher.ts

@@ -1,5 +1,5 @@
 import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
-import { readFile } from 'node:fs/promises';
+import { readFile } from '../../utils/fs-utils.js';
 import { logger } from '../logger.js';
 import { parseAndEvaluateStatement } from '../interpreter.js';
 import { CoreAIModuleName } from '../modules/ai.js';
@@ -443,7 +443,7 @@ class DocumentFetcherService {
 
   private async fetchFromLocal(filePath: string): Promise<string> {
     try {
-      const content = await readFile(filePath, 'utf-8');
+      const content = await readFile(filePath);
       const lowerPath = filePath.toLowerCase();
       const isMarkdown = lowerPath.endsWith('.md') || lowerPath.endsWith('.markdown');
 

package/src/runtime/util.ts

@@ -1,4 +1,4 @@
-import { isNodeEnv } from '../utils/runtime.js';
+import { isNodeEnv, path } from '../utils/runtime.js';
 import {
   AliasSpec,
   CatchSpec,
@@ -15,7 +15,6 @@ import {
 } from '../language/generated/ast.js';
 import { readFile } from '../utils/fs-utils.js';
 import bcrypt from 'bcryptjs';
-import path from 'node:path';
 
 export const QuerySuffix = '?';
 
@@ -665,3 +664,47 @@ export function objectAsString(obj: any, keyAsString: boolean = false) {
   });
   return `{${entries.join(', ')}}`;
 }
+
+export type ExtractedText = {
+  extracted: string[] | undefined;
+  updatedText: string;
+};
+
+// extract all data between a given xml tag from within an arbitray text.
+export function extractAndRemoveAllXmlTaggedText(text: string, tagName: string): ExtractedText {
+  const pattern = `<${tagName}\\b[^>]*>([\\s\\S]*?)</${tagName}>`;
+  const regex = new RegExp(pattern, 'gi');
+
+  const extracted = [];
+  let updatedText = text;
+
+  let match;
+  while ((match = regex.exec(text)) !== null) {
+    extracted.push(match[1]);
+  }
+
+  updatedText = text.replace(regex, '');
+
+  return { extracted, updatedText };
+}
+
+export type ExtractedCode = {
+  language: string | null;
+  code: string;
+};
+
+// extract tick-quoted code from markdown-formatted text.
+export function extractFencedCodeBlocks(markdown: string): ExtractedCode[] {
+  const codeBlockRegex = /```(\w+)?\n([\s\S]*?)```/g;
+  const blocks: ExtractedCode[] = [];
+  let match;
+
+  while ((match = codeBlockRegex.exec(markdown)) !== null) {
+    blocks.push({
+      language: match[1] || null,
+      code: match[2],
+    });
+  }
+
+  return blocks;
+}