agentlang 0.10.1 → 0.10.2

package/src/runtime/modules/ai.ts

@@ -70,7 +70,6 @@ import {
 } from '../agents/common.js';
 import { logger } from '../logger.js';
 import { FlowStep } from '../agents/flows.js';
-import Handlebars from 'handlebars';
 import { Statement } from '../../language/generated/ast.js';
 import { isMonitoringEnabled, TtlCache } from '../state.js';
 import { isNodeEnv } from '../../utils/runtime.js';
@@ -107,6 +106,7 @@ entity ${AgentEntityName} {
     flows String @optional,
     validate String @optional,
     retry String @optional,
+    saveResponseAs String @optional,
     llm String
 }
 
@@ -323,6 +323,7 @@ export class AgentInstance {
   flows: string | undefined;
   validate: string | undefined;
   retry: string | undefined;
+  saveResponseAs: string | undefined;
   stateless: boolean = false;
   private toolsArray: string[] | undefined = undefined;
   private hasModuleTools = false;
@@ -645,7 +646,7 @@ Only return a pure JSON object with no extra text, annotations etc.`;
     const spad = env.getScratchPad();
     if (spad !== undefined && Object.keys(spad).length > 0) {
       if (finalInstruction.indexOf('{{') > 0) {
-        return AgentInstance.maybeRewriteTemplatePatterns(spad, finalInstruction, env);
+        return env.maybeRewriteTemplatePatterns(finalInstruction, spad);
       } else {
         const ctx = JSON.stringify(spad);
         return `${finalInstruction}\nSome additional context:\n${ctx}`;
@@ -657,6 +658,7 @@ Only return a pure JSON object with no extra text, annotations etc.`;
   }
 
   private static UserTag = 'user';
+  private static FileTag = 'file';
 
   private async getFullInstructions(
     env: Environment,
@@ -670,13 +672,21 @@ Only return a pure JSON object with no extra text, annotations etc.`;
     }
   }
 
-  private static maybeRewriteTemplatePatterns(
-    scratchPad: any,
-    instruction: string,
-    env: Environment
-  ): string {
-    const templ = Handlebars.compile(env.rewriteTemplateMappings(instruction));
-    return templ(scratchPad);
+  private async maybeFillInFileContents(instructions: string, env: Environment): Promise<string> {
+    const chatId = env.getActiveChatId();
+    const FC = '$FC';
+    if (chatId && instructions.indexOf(`<${AgentInstance.FileTag}>`) > 0) {
+      const ext = extractAndRemoveAllXmlTaggedText(instructions, AgentInstance.FileTag, FC);
+      if (ext.extracted) {
+        instructions = ext.updatedText;
+        for (let i = 0; i < ext.extracted.length; ++i) {
+          const fileName = ext.extracted[i];
+          const contents = await loadLocalAgentResult(chatId, fileName);
+          if (contents) instructions = instructions.replace(FC, contents);
+        }
+      }
+    }
+    return instructions;
   }
 
   maybeValidateJsonResponse(response: string | undefined): object | undefined {
@@ -875,7 +885,8 @@ Only return a pure JSON object with no extra text, annotations etc.`;
         }
         let tmpMsg = message;
         if (extractedText?.extracted) {
-          tmpMsg = `${tmpMsg}\n${extractedText.extracted.join('\n')}`;
+          const s = await this.maybeFillInFileContents(extractedText.extracted.join('\n'), env);
+          tmpMsg = `${tmpMsg}\n${s}`;
         }
         const hmsg = await this.maybeAddRelevantDocuments(
           this.maybeAddFlowContext(tmpMsg, env),
@@ -927,6 +938,13 @@ Only return a pure JSON object with no extra text, annotations etc.`;
           await saveAgentChatSession(chatId, msgs, env);
         }
         if (monitoringEnabled) env.setMonitorEntryLlmResponse(response.content);
+        if (monitoringEnabled && response.sysMsg.usage_metadata) {
+          const u = response.sysMsg.usage_metadata;
+          env.setMonitorEntryLlmTokenUsage(u.input_tokens, u.output_tokens, u.total_tokens);
+        }
+        if (this.saveResponseAs) {
+          await saveAgentResponse(this.saveResponseAs, response.content, env);
+        }
         env.setLastResult(response.content);
       } catch (err: any) {
         logger.error(`Error while invoking ${agentName} - ${err}`);
@@ -1448,15 +1466,9 @@ export async function saveFlowStepResultLocally(
   chatId: string,
   step: string,
   result: string,
-  suspensionId: string
-): Promise<Instance | undefined> {
+  suspensionId?: string | undefined
+): Promise<string> {
   const fs = await getFileSystem();
-  const attrs = newInstanceAttributes()
-    .set('chatId', chatId)
-    .set('step', step)
-    .set('result', result)
-    .set('suspensionId', suspensionId);
-  const inst = makeInstance(CoreAIModuleName, AgentFlowStep, attrs);
   const rootDirName = LocalFlowStepsRootDirName;
   if (!(await fs.exists(rootDirName))) {
     await fs.mkdir(rootDirName);
@@ -1469,8 +1481,18 @@ export async function saveFlowStepResultLocally(
   if (await fs.exists(fileName)) {
     await fs.unlink(fileName);
   }
-  await fs.writeFile(fileName, JSON.stringify(inst.attributesAsObject(true)));
-  return inst;
+  let s = result;
+  if (suspensionId) {
+    const attrs = newInstanceAttributes()
+      .set('chatId', chatId)
+      .set('step', step)
+      .set('result', result)
+      .set('suspensionId', suspensionId);
+    const inst = makeInstance(CoreAIModuleName, AgentFlowStep, attrs);
+    s = JSON.stringify(inst.attributesAsObject(true));
+  }
+  await fs.writeFile(fileName, s);
+  return s;
 }
 
 export async function saveFlowStepResult(
@@ -1479,9 +1501,10 @@ export async function saveFlowStepResult(
   result: string,
   suspensionId: string,
   env: Environment
-): Promise<Instance | undefined> {
+): Promise<boolean> {
   if (LocalAgentgFlow) {
-    return await saveFlowStepResultLocally(chatId, step, result, suspensionId);
+    await saveFlowStepResultLocally(chatId, step, result, suspensionId);
+    return true;
   } else {
     const t = `${CoreAIModuleName}/${AgentFlowStep}`;
     try {
@@ -1495,11 +1518,11 @@ export async function saveFlowStepResult(
         undefined,
         env
       );
-      if (isInstanceOfType(inst, t)) return inst;
-      else return undefined;
+      if (isInstanceOfType(inst, t)) return true;
+      else return false;
     } catch (reason: any) {
       logger.error(`failed to save flow result for step ${step} - ${reason}`);
-      return undefined;
+      return false;
     }
   }
 }
@@ -1537,10 +1560,10 @@ export async function loadFlowStepResults(chatId: string): Promise<Instance[]> {
   }
 }
 
-export async function loadLocalFlowStep(
+export async function loadLocalAgentResult(
   chatId: string,
   step: string
-): Promise<Instance | undefined> {
+): Promise<string | undefined> {
   const fs = await getFileSystem();
   const dirName = `${LocalFlowStepsRootDirName}/${chatId}`;
   if (await fs.exists(dirName)) {
@@ -1548,10 +1571,7 @@ export async function loadLocalFlowStep(
     for (let i = 0; i < fileNames.length; ++i) {
       const fileName = fileNames[i];
       if (fileName === step) {
-        const attrs = objectToInstanceAttributes(
-          JSON.parse(await fs.readFile(`${dirName}/${fileName}`))
-        );
-        return makeInstance(CoreAIModuleName, AgentFlowStep, attrs);
+        return await fs.readFile(`${dirName}/${fileName}`);
       }
     }
     return undefined;
@@ -1560,6 +1580,19 @@ export async function loadLocalFlowStep(
   }
 }
 
+export async function loadLocalFlowStep(
+  chatId: string,
+  step: string
+): Promise<Instance | undefined> {
+  const contents = await loadLocalAgentResult(chatId, step);
+  if (contents) {
+    const attrs = objectToInstanceAttributes(JSON.parse(contents));
+    return makeInstance(CoreAIModuleName, AgentFlowStep, attrs);
+  } else {
+    return undefined;
+  }
+}
+
 export async function loadFlowStep(chatId: string, step: string): Promise<Instance | undefined> {
   if (LocalAgentgFlow) {
     return await loadLocalFlowStep(chatId, step);
@@ -1639,3 +1672,17 @@ export async function fetchAndCreateDocument(
     throw new Error(`Failed to fetch document: ${title} from ${url}`);
   }
 }
+
+export async function saveAgentResponse(
+  fileName: string,
+  response: string,
+  env: Environment
+): Promise<string> {
+  const chatId = env.getActiveChatId();
+  if (chatId) {
+    await saveFlowStepResultLocally(chatId, fileName, response);
+  } else {
+    logger.warn(`No chatId set, ${fileName} was not saved.`);
+  }
+  return response;
+}

package/src/runtime/modules/core.ts

@@ -749,5 +749,9 @@ export function migrationDowns(inst: Instance): string[] | undefined {
 
 export async function doRawQuery(q: any): Promise<any> {
   const qs = objectToQueryPattern(q);
-  return await parseAndEvaluateStatement(qs);
+  const result = await parseAndEvaluateStatement(qs);
+  return {
+    query: qs,
+    result: result.map((res: any) => (res.attributes ? Object.fromEntries(res.attributes) : res)),
+  };
 }

package/src/runtime/monitor.ts

@@ -10,6 +10,9 @@ export class MonitorEntry {
   private llm: boolean = false;
   private llmPrompt: string | undefined;
   private llmResponse: string | undefined;
+  private llmInputTokens: number | undefined;
+  private llmOutputTokens: number | undefined;
+  private llmTotalTokens: number | undefined;
   private planner: boolean = false;
   private flowStep: boolean = false;
   private flow: boolean = false;
@@ -25,6 +28,10 @@ export class MonitorEntry {
     return this.input;
   }
 
+  getTimestamp(): number {
+    return this.timestamp;
+  }
+
   setResult(result: any): MonitorEntry {
     if (this.result === undefined) {
       this.result = result;
@@ -68,6 +75,13 @@ export class MonitorEntry {
     return this;
   }
 
+  setLlmTokenUsage(input: number, output: number, total: number): MonitorEntry {
+    this.llmInputTokens = input;
+    this.llmOutputTokens = output;
+    this.llmTotalTokens = total;
+    return this;
+  }
+
   flagAsPlanner(): MonitorEntry {
     this.llm = true;
     if (this.flowStep || this.flow || this.decision) {
@@ -137,6 +151,11 @@ export class MonitorEntry {
       llmObj.isFlowStep = this.flowStep;
       llmObj.isDecision = this.decision;
       llmObj.isFlow = this.flow;
+      if (this.llmInputTokens !== undefined) {
+        llmObj.inputTokens = this.llmInputTokens;
+        llmObj.outputTokens = this.llmOutputTokens;
+        llmObj.totalTokens = this.llmTotalTokens;
+      }
       obj.llm = llmObj;
     }
     obj.label = this.input;
@@ -269,9 +288,16 @@ export class Monitor {
     return this;
   }
 
+  setEntryLlmTokenUsage(input: number, output: number, total: number): Monitor {
+    if (this.lastEntry !== undefined) {
+      this.lastEntry.setLlmTokenUsage(input, output, total);
+    }
+    return this;
+  }
+
   private finalizeLastEntry(): void {
     if (this.lastEntry) {
-      const ms = Date.now() - this.lastEntrySetAtMs;
+      const ms = Date.now() - this.lastEntry.getTimestamp();
       this.lastEntry.setLatencyMs(ms);
       if (MonitoringCallback !== undefined) {
         MonitoringCallback(this.lastEntry);

package/src/runtime/relgraph.ts

@@ -51,8 +51,11 @@ export class RelationshipGraph {
     node: RelationshipGraphNode,
     onNode: Function,
     onContainsRelationship: Function,
-    onBetweenRelationship: Function
+    onBetweenRelationship: Function,
+    visited: Set<RelationshipGraphNode>
   ) {
+    if (visited.has(node)) return;
+    visited.add(node);
     const n = node.entity.asFqName();
     onNode(n, node.edges);
     node.edges.forEach((edge: RelationshipGraphEdge) => {
@@ -60,13 +63,14 @@ export class RelationshipGraph {
         ? onContainsRelationship
         : onBetweenRelationship;
       rf(n, edge.node.entity.asFqName(), edge.relationship);
-      this.walkEdges(edge.node, onNode, onContainsRelationship, onBetweenRelationship);
+      this.walkEdges(edge.node, onNode, onContainsRelationship, onBetweenRelationship, visited);
     });
   }
 
   walk(onNode: Function, onContainsRelationship: Function, onBetweenRelationship: Function) {
+    const visited = new Set<RelationshipGraphNode>();
     this.nodes.forEach((node: RelationshipGraphNode) => {
-      this.walkEdges(node, onNode, onContainsRelationship, onBetweenRelationship);
+      this.walkEdges(node, onNode, onContainsRelationship, onBetweenRelationship, visited);
     });
   }
 }

package/src/runtime/resolvers/interface.ts

@@ -121,13 +121,15 @@ export class Resolver {
   /**
    * Return all instances connected to connectedInstance via the given between-relationship
    * @param relationship Between relationship
-   * @param connectedInstance The instance to traveres the relationship from
+   * @param connectedInstance The instance to traverse the relationship from
    * @param inst Target instance with query attributes
+   * @param connectedAlias For self-referencing relationships, the alias of the connected instance's role
    */
   public async queryConnectedInstances(
     relationship: Relationship,
     connectedInstance: Instance,
-    inst: Instance
+    inst: Instance,
+    connectedAlias?: string
   ): Promise<any> {
     return this.notImpl(`queryConnectedInstances(${relationship}, ${connectedInstance}, ${inst})`);
   }

package/src/runtime/resolvers/sqldb/dbutil.ts

@@ -188,13 +188,27 @@ function ormSchemaFromRecordSchema(
   const fqName = makeFqName(moduleName, entityName);
   getAllOneToOneRelationshipsForEntity(moduleName, entityName, allBetRels).forEach(
     (re: Relationship) => {
-      const colName = re.getInverseAliasForName(fqName);
-      if (cols.has(colName)) {
-        throw new Error(
-          `Cannot establish relationship ${re.name}, ${entityName}.${colName} already exists`
-        );
+      if (re.isSelfReferencing()) {
+        // Self-referencing one-to-one: add columns for both aliases
+        // so each side of the relationship can be stored on the entity.
+        // Both must be nullable since they're set after entity creation.
+        [re.node1.alias, re.node2.alias].forEach((colName: string) => {
+          if (cols.has(colName)) {
+            throw new Error(
+              `Cannot establish relationship ${re.name}, ${entityName}.${colName} already exists`
+            );
+          }
+          cols.set(colName, { type: 'varchar', unique: true });
+        });
+      } else {
+        const colName = re.getInverseAliasForName(fqName);
+        if (cols.has(colName)) {
+          throw new Error(
+            `Cannot establish relationship ${re.name}, ${entityName}.${colName} already exists`
+          );
+        }
+        cols.set(colName, { type: 'varchar', unique: true });
       }
-      cols.set(colName, { type: 'varchar', unique: true });
     }
   );
   if (relsSpec.size > 0) {

package/src/runtime/resolvers/sqldb/impl.ts

@@ -480,16 +480,17 @@ export class SqlDbResolver extends Resolver {
   public override async queryConnectedInstances(
     relationship: Relationship,
     connectedInstance: Instance,
-    inst: Instance
+    inst: Instance,
+    connectedAlias?: string
   ): Promise<Instance[]> {
     let result = SqlDbResolver.EmptyResultSet;
     if (relationship.isOneToOne()) {
-      const col = relationship.getAliasFor(connectedInstance);
+      const col = relationship.getAliasForConnected(connectedInstance, connectedAlias);
       inst.addQuery(col, '=', connectedInstance.lookup(PathAttributeName));
       return await this.queryInstances(inst, false);
     } else {
-      const from = relationship.getAliasFor(connectedInstance);
-      const to = relationship.getInverseAliasFor(connectedInstance);
+      const from = relationship.getAliasForConnected(connectedInstance, connectedAlias);
+      const to = relationship.getInverseAliasForConnected(connectedInstance, connectedAlias);
       await getAllConnected(
         asTableReference(inst.moduleName, inst.name),
         inst.queryAttributesAsObject(),
@@ -797,9 +798,18 @@ function ensureOneToOneAttributes(inst: Instance) {
   const betRels = getAllBetweenRelationships();
   getAllOneToOneRelationshipsForEntity(inst.moduleName, inst.name, betRels).forEach(
     (re: Relationship) => {
-      const n = re.getInverseAliasFor(inst);
-      if (!inst.attributes.has(n)) {
-        inst.attributes.set(n, crypto.randomUUID());
+      if (re.isSelfReferencing()) {
+        // Self-referencing one-to-one: ensure both alias columns have placeholder values
+        [re.node1.alias, re.node2.alias].forEach((n: string) => {
+          if (!inst.attributes.has(n)) {
+            inst.attributes.set(n, crypto.randomUUID());
+          }
+        });
+      } else {
+        const n = re.getInverseAliasFor(inst);
+        if (!inst.attributes.has(n)) {
+          inst.attributes.set(n, crypto.randomUUID());
+        }
       }
     }
   );

package/src/runtime/util.ts

@@ -2,6 +2,7 @@ import { isNodeEnv, path } from '../utils/runtime.js';
 import {
   AliasSpec,
   CatchSpec,
+  EmptySpec,
   ExtendsClause,
   isLiteral,
   MapEntry,
@@ -441,6 +442,18 @@ export function firstCatchSpec(stmt: Statement): CatchSpec | undefined {
   return undefined;
 }
 
+export function firstEmptySpec(stmt: Statement): EmptySpec | undefined {
+  if (stmt.hints) {
+    for (let i = 0; i < stmt.hints.length; ++i) {
+      const rh = stmt.hints[i];
+      if (rh.emptySpec) {
+        return rh.emptySpec;
+      }
+    }
+  }
+  return undefined;
+}
+
 function maybeExtractEntryName(n: string): string {
   const i = n.indexOf('$');
   if (i > 0) {
@@ -671,7 +684,11 @@ export type ExtractedText = {
 };
 
 // extract all data between a given xml tag from within an arbitray text.
-export function extractAndRemoveAllXmlTaggedText(text: string, tagName: string): ExtractedText {
+export function extractAndRemoveAllXmlTaggedText(
+  text: string,
+  tagName: string,
+  replaceWith: string = ''
+): ExtractedText {
   const pattern = `<${tagName}\\b[^>]*>([\\s\\S]*?)</${tagName}>`;
   const regex = new RegExp(pattern, 'gi');
 
@@ -683,7 +700,7 @@ export function extractAndRemoveAllXmlTaggedText(text: string, tagName: string):
     extracted.push(match[1]);
   }
 
-  updatedText = text.replace(regex, '');
+  updatedText = text.replace(regex, replaceWith);
 
   return { extracted, updatedText };
 }

package/src/syntaxes/agentlang.monarch.ts

@@ -1,7 +1,7 @@
 // Monarch syntax highlighting for the agentlang language.
 export default {
     keywords: [
-        '@actions','@after','@as','@asc','@async','@before','@catch','@desc','@distinct','@enum','@expr','@from','@full_join','@groupBy','@inner_join','@into','@join','@left_join','@meta','@oneof','@orderBy','@public','@rbac','@ref','@right_join','@then','@upsert','@where','@withRole','@with_unique','agent','agentlang/retry','allow','and','attempts','await','backoff','between','case','commitTransaction','contains','create','decision','delete','directive','else','entity','error','eval','event','extends','false','flow','for','glossaryEntry','if','import','in','like','module','not','not_found','onSubscription','or','purge','query','read','record','relationship','resolver','return','roles','rollbackTransaction','scenario','startTransaction','subscribe','throw','true','update','upsert','where','workflow'
+        '@actions','@after','@as','@asc','@async','@before','@catch','@desc','@distinct','@empty','@enum','@expr','@from','@full_join','@groupBy','@inner_join','@into','@join','@left_join','@meta','@oneof','@orderBy','@public','@rbac','@ref','@right_join','@then','@upsert','@where','@withRole','@with_unique','agent','agentlang/retry','allow','and','attempts','await','backoff','between','case','commitTransaction','contains','create','decision','delete','directive','else','entity','error','eval','event','extends','false','flow','for','glossaryEntry','if','import','in','like','module','not','not_found','onSubscription','or','purge','query','read','record','relationship','resolver','return','roles','rollbackTransaction','scenario','startTransaction','subscribe','throw','true','update','upsert','where','workflow'
     ],
     operators: [
         '!=','*','+',',','-','-->','.','/',':',';','<','<=','<>','=','==','>','>=','?','@'