@ax-llm/ax 10.0.41 → 10.0.43

package/index.js

@@ -3488,9 +3488,6 @@ var assertStreamingAssertions = (asserts, values, xstate, content, final) => {
   }
 };
 
-// dsp/extract.ts
-import JSON5 from "json5";
-
 // dsp/datetime.ts
 import moment from "moment-timezone";
 
@@ -3521,104 +3518,143 @@ import { createHash } from "crypto";
 var SignatureParser = class {
   input;
   position;
+  currentFieldName = null;
   constructor(input) {
     this.input = input;
     this.position = 0;
   }
   parse() {
-    this.skipWhitespace();
-    const optionalDesc = this.parseParsedString();
-    this.skipWhitespace();
-    const inputs = this.parseInputParsedFieldList();
-    this.skipWhitespace();
-    this.expect("->");
-    this.skipWhitespace();
-    const outputs = this.parseOutputParsedFieldList();
-    return {
-      desc: optionalDesc?.trim(),
-      inputs,
-      outputs
-    };
-  }
-  parseInputParsedFieldList() {
-    const fields = [];
-    fields.push(this.parseInputParsedField());
-    while (this.match(",")) {
+    try {
       this.skipWhitespace();
-      fields.push(this.parseInputParsedField());
-    }
-    return fields;
-  }
-  parseOutputParsedFieldList() {
-    const fields = [];
-    fields.push(this.parseOutputParsedField());
-    while (this.match(",")) {
+      const optionalDesc = this.parseParsedString();
       this.skipWhitespace();
-      fields.push(this.parseOutputParsedField());
-    }
-    return fields;
-  }
-  parseInputParsedField() {
-    this.skipWhitespace();
-    const name = this.parseParsedIdentifier();
-    const isOptional = this.match("?");
-    let type;
-    if (this.match(":")) {
+      const inputs = this.parseFieldList(this.parseField.bind(this), "input");
+      this.skipWhitespace();
+      if (this.position >= this.input.length) {
+        throw new Error(
+          'Incomplete signature: Missing output section. Expected "->" followed by output fields'
+        );
+      }
+      this.expect("->");
       this.skipWhitespace();
-      const typeName = this.parseTypeNotClass();
-      const isArray = this.match("[]");
-      type = { name: typeName, isArray };
+      if (this.position >= this.input.length) {
+        throw new Error(
+          'Incomplete signature: No output fields specified after "->"'
+        );
+      }
+      const outputs = this.parseFieldList(this.parseField.bind(this), "output");
+      return {
+        desc: optionalDesc?.trim(),
+        inputs,
+        outputs
+      };
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : "Unknown error";
+      const context = this.getErrorContext();
+      throw new Error(`${errorMessage}
+${context}`);
+    }
+  }
+  getErrorContext() {
+    const start = Math.max(0, this.position - 20);
+    const end = Math.min(this.input.length, this.position + 20);
+    const before = this.input.slice(start, this.position);
+    const after = this.input.slice(this.position, end);
+    const pointer = " ".repeat(before.length) + "^";
+    return `Near position ${this.position}:
+${before}${after}
+${pointer}`;
+  }
+  parseFieldList(parseFieldFn, section) {
+    const fields = [];
+    this.skipWhitespace();
+    if (this.position >= this.input.length) {
+      throw new Error(`Empty ${section} section: Expected at least one field`);
+    }
+    try {
+      fields.push(parseFieldFn());
+    } catch (error) {
+      throw new Error(
+        `Invalid first ${section} field: ${error instanceof Error ? error.message : "Unknown error"}`
+      );
     }
     this.skipWhitespace();
-    const desc = this.parseParsedString();
-    return {
-      name,
-      desc: desc?.trim(),
-      type,
-      isOptional
-    };
+    while (this.position < this.input.length) {
+      if (this.input[this.position] === "-" && this.input[this.position + 1] === ">") {
+        break;
+      }
+      if (this.match(",")) {
+        this.skipWhitespace();
+        if (this.position >= this.input.length) {
+          throw new Error(
+            `Unexpected end of input after comma in ${section} section`
+          );
+        }
+        try {
+          fields.push(parseFieldFn());
+        } catch (error) {
+          throw new Error(
+            `Invalid ${section} field after comma: ${error instanceof Error ? error.message : "Unknown error"}`
+          );
+        }
+        this.skipWhitespace();
+      } else {
+        break;
+      }
+    }
+    return fields;
   }
-  parseOutputParsedField() {
+  parseField() {
     this.skipWhitespace();
     const name = this.parseParsedIdentifier();
+    this.currentFieldName = name;
     const isOptional = this.match("?");
+    let type;
     this.skipWhitespace();
     if (this.match(":")) {
       this.skipWhitespace();
       if (this.match("class")) {
         const isArray = this.match("[]");
         this.skipWhitespace();
-        const desc = this.parseParsedString();
-        if (!desc) {
+        const desc2 = this.parseParsedString();
+        if (!desc2) {
           throw new Error(
-            "Expected description containing class names after type 'class'"
+            `Field "${name}": Expected class names in quotes after "class" type. Example: class "MyClass1, MyClass2"`
           );
         }
-        const classNames = desc.split(",").map((s) => s.trim());
-        return {
-          name,
-          type: { name: "class", isArray, classes: classNames },
-          isOptional
-        };
+        const classes = desc2.split(/[,\s]+/).map((s) => s.trim()).filter((s) => s.length > 0);
+        if (classes.length === 0) {
+          throw new Error(
+            `Field "${name}": Empty class list provided. At least one class name is required`
+          );
+        }
+        type = { name: "class", isArray, classes };
       } else {
-        const typeName = this.parseTypeNotClass();
-        const isArray = this.match("[]");
-        this.skipWhitespace();
-        const desc = this.parseParsedString();
-        return {
-          name,
-          desc: desc?.trim(),
-          type: { name: typeName, isArray },
-          isOptional
-        };
+        try {
+          const typeName = this.parseTypeNotClass();
+          const isArray = this.match("[]");
+          type = { name: typeName, isArray };
+        } catch (error) {
+          throw new Error(
+            `Field "${name}": ${error instanceof Error ? error.message : "Unknown error"}`
+          );
+        }
       }
+    }
+    this.skipWhitespace();
+    const desc = this.parseParsedString();
+    if (type?.name === "class") {
+      return {
+        name,
+        desc: desc?.trim(),
+        type,
+        isOptional
+      };
     } else {
-      this.skipWhitespace();
-      const desc = this.parseParsedString();
       return {
         name,
         desc: desc?.trim(),
-        type: void 0,
+        type,
         isOptional
       };
     }
@@ -3634,14 +3670,17 @@ var SignatureParser = class {
       "datetime",
       "date"
     ];
-    for (const type of types) {
-      if (this.match(type)) {
-        return type;
-      }
+    const foundType = types.find((type) => this.match(type));
+    if (!foundType) {
+      const currentWord = this.input.slice(this.position).match(/^\w+/)?.[0] || "empty";
+      throw new Error(
+        `Invalid type "${currentWord}". Expected one of: ${types.join(", ")}`
+      );
     }
-    throw new Error(`Expected one of ${types.join(", ")}`);
+    return foundType;
   }
   parseParsedIdentifier() {
+    this.skipWhitespace();
     const match = /^[a-zA-Z_][a-zA-Z_0-9]*/.exec(
       this.input.slice(this.position)
     );
@@ -3649,40 +3688,69 @@ var SignatureParser = class {
       this.position += match[0].length;
       return match[0];
     }
-    throw new Error("Expected identifier");
+    const invalidMatch = /^\S+/.exec(this.input.slice(this.position));
+    const invalidId = invalidMatch ? invalidMatch[0] : "empty";
+    throw new Error(
+      `Invalid identifier "${invalidId}". Identifiers must start with a letter or underscore and contain only letters, numbers, or underscores`
+    );
   }
   parseParsedString() {
-    if (this.match("'")) {
-      const endQuote = this.input.indexOf("'", this.position);
-      if (endQuote === -1) throw new Error("Unterminated string");
-      const content = this.input.slice(this.position, endQuote);
-      this.position = endQuote + 1;
-      return content;
-    } else if (this.match('"')) {
-      const endQuote = this.input.indexOf('"', this.position);
-      if (endQuote === -1) throw new Error("Unterminated string");
-      const content = this.input.slice(this.position, endQuote);
-      this.position = endQuote + 1;
-      return content;
+    const quoteChars = ["'", '"'];
+    for (const quoteChar of quoteChars) {
+      if (this.match(quoteChar)) {
+        let content = "";
+        let escaped = false;
+        let startPos = this.position;
+        while (this.position < this.input.length) {
+          const char = this.input[this.position];
+          this.position++;
+          if (escaped) {
+            content += char;
+            escaped = false;
+          } else if (char === "\\") {
+            escaped = true;
+          } else if (char === quoteChar) {
+            return content;
+          } else {
+            content += char;
+          }
+        }
+        const partialString = this.input.slice(startPos, this.position);
+        throw new Error(
+          `Unterminated string starting at position ${startPos}: "${partialString}..."`
+        );
+      }
     }
     return void 0;
   }
   skipWhitespace() {
-    const match = /^[ \t\r\n]+/.exec(this.input.slice(this.position));
+    const match = /^[\s\t\r\n]+/.exec(this.input.slice(this.position));
     if (match) {
       this.position += match[0].length;
     }
   }
-  match(str) {
-    if (this.input.startsWith(str, this.position)) {
-      this.position += str.length;
-      return true;
+  match(strOrRegex) {
+    let match;
+    if (typeof strOrRegex === "string") {
+      if (this.input.startsWith(strOrRegex, this.position)) {
+        this.position += strOrRegex.length;
+        return true;
+      }
+    } else {
+      match = strOrRegex.exec(this.input.slice(this.position));
+      if (match) {
+        this.position += match[0].length;
+        return true;
+      }
     }
     return false;
   }
   expect(str) {
     if (!this.match(str)) {
-      throw new Error(`Expected "${str}"`);
+      const found = this.input.slice(this.position, this.position + 10);
+      throw new Error(
+        `Expected "${str}" but found "${found}..." at position ${this.position}`
+      );
     }
   }
 };
@@ -4783,9 +4851,6 @@ var checkMissingRequiredFields = (xstate, values, currentIndex) => {
   }
 };
 var streamingExtractValues = (sig, values, xstate, content) => {
-  if (content.endsWith("\n")) {
-    return true;
-  }
   const fields = sig.getOutputFields();
   for (const [index, field] of fields.entries()) {
     if (field.name in values) {
@@ -4801,10 +4866,6 @@ var streamingExtractValues = (sig, values, xstate, content) => {
         return true;
     }
     let prefixLen = prefix.length;
-    if (e - 1 >= 0 && content[e - 1] === "\n") {
-      e -= 1;
-      prefixLen += 1;
-    }
     if (xstate.currField) {
       const val = content.substring(xstate.s, e).trim();
       const parsedValue = validateAndParseFieldValue(xstate.currField, val);
@@ -4871,7 +4932,7 @@ var convertValueToType = (field, val) => {
       return val;
   }
 };
-function* streamValues(sig, values, xstate, content) {
+function* streamValues(sig, values, xstate, content, final = false) {
   if (!xstate.currField) {
     return;
   }
@@ -4879,13 +4940,17 @@ function* streamValues(sig, values, xstate, content) {
   if (!xstate.streamedIndex) {
     xstate.streamedIndex = { [fieldName]: 0 };
   }
-  if (!xstate.currField.type || !xstate.currField.type.isArray && xstate.currField.type.name === "string") {
-    const pos = xstate.streamedIndex[fieldName] ?? 0;
-    const s = xstate.s + pos;
-    const v = content.substring(s);
-    yield { [fieldName]: pos === 0 ? v.trimStart() : v };
-    xstate.streamedIndex[fieldName] = pos + v.length;
-    return;
+  if (!final) {
+    if (!xstate.currField.type || !xstate.currField.type.isArray && xstate.currField.type.name === "string") {
+      const pos = xstate.streamedIndex[fieldName] ?? 0;
+      const s = xstate.s + pos;
+      const v = content.substring(s);
+      const v1 = v.replace(/[\s\n\t]+$/, "");
+      const v2 = pos === 0 ? v1.trimStart() : v1;
+      yield { [fieldName]: v2 };
+      xstate.streamedIndex[fieldName] = pos + v1.length;
+      return;
+    }
   }
   for (const key of Object.keys(values)) {
     const value = values[key];
@@ -4919,7 +4984,7 @@ function validateAndParseFieldValue(field, fieldValue) {
   if (field.type?.name === "json") {
     try {
       const text = extractBlock(fieldValue);
-      value = JSON5.parse(text);
+      value = JSON.parse(text);
       return value;
     } catch (e) {
       throw new ValidationError({
@@ -4932,7 +4997,7 @@ function validateAndParseFieldValue(field, fieldValue) {
   if (field.type?.isArray) {
     try {
       try {
-        value = JSON5.parse(fieldValue);
+        value = JSON.parse(fieldValue);
       } catch {
         value = parseMarkdownList(fieldValue);
       }
@@ -4985,9 +5050,6 @@ var extractBlock = (input) => {
   return input;
 };
 
-// dsp/functions.ts
-import JSON52 from "json5";
-
 // dsp/jsonschema.ts
 var validateJSONSchema = (schema) => {
   const errors = [];
@@ -5045,7 +5107,7 @@ var AxFunctionProcessor = class {
   executeFunction = async (fnSpec, func, options) => {
     let args;
     if (typeof func.args === "string" && func.args.length > 0) {
-      args = JSON52.parse(func.args);
+      args = JSON.parse(func.args);
     } else {
       args = func.args;
     }
@@ -5311,6 +5373,7 @@ var AxGen = class extends AxProgramWithSignature {
       );
       this.functionsExecuted = /* @__PURE__ */ new Set([...this.functionsExecuted, ...fx]);
     }
+    streamingExtractFinalValue(this.signature, values, xstate, content);
     assertStreamingAssertions(
       this.streamingAsserts,
       values,
@@ -5318,9 +5381,8 @@ var AxGen = class extends AxProgramWithSignature {
       content,
       true
     );
-    streamingExtractFinalValue(this.signature, values, xstate, content);
     assertAssertions(this.asserts, values);
-    yield* streamValues(this.signature, values, xstate, content);
+    yield* streamValues(this.signature, values, xstate, content, true);
   }
   async processResponse({
     ai,
@@ -5332,37 +5394,35 @@ var AxGen = class extends AxProgramWithSignature {
     functions
   }) {
     const values = {};
-    const result = res.results[0];
-    if (!result) {
-      throw new Error("No result found");
-    }
-    if (res.modelUsage) {
-      this.usage.push({ ...usageInfo, ...res.modelUsage });
-    }
-    mem.addResult(result, sessionId);
-    if (result.content) {
-      extractValues(this.signature, values, result.content);
-      assertAssertions(this.asserts, values);
-    }
-    if (result.functionCalls) {
-      const funcs = parseFunctionCalls(ai, result.functionCalls, values);
-      if (funcs) {
-        if (!functions) {
-          throw new Error("Functions are not defined");
+    for (const result of res.results ?? []) {
+      if (res.modelUsage) {
+        this.usage.push({ ...usageInfo, ...res.modelUsage });
+      }
+      mem.addResult(result, sessionId);
+      if (result.content) {
+        extractValues(this.signature, values, result.content);
+        assertAssertions(this.asserts, values);
+      }
+      if (result.functionCalls) {
+        const funcs = parseFunctionCalls(ai, result.functionCalls, values);
+        if (funcs) {
+          if (!functions) {
+            throw new Error("Functions are not defined");
+          }
+          const fx = await processFunctions(
+            ai,
+            functions,
+            funcs,
+            mem,
+            sessionId,
+            traceId
+          );
+          this.functionsExecuted = /* @__PURE__ */ new Set([...this.functionsExecuted, ...fx]);
         }
-        const fx = await processFunctions(
-          ai,
-          functions,
-          funcs,
-          mem,
-          sessionId,
-          traceId
-        );
-        this.functionsExecuted = /* @__PURE__ */ new Set([...this.functionsExecuted, ...fx]);
       }
-    }
-    if (result.finishReason === "length") {
-      throw new Error("Max tokens reached before completion");
+      if (result.finishReason === "length") {
+        throw new Error("Max tokens reached before completion");
+      }
     }
     return { ...values };
   }