@langchain/classic 1.0.27 → 1.0.28

package/dist/document_loaders/fs/json.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"json.cjs","names":["TextLoader"],"sources":["../../../src/document_loaders/fs/json.ts"],"sourcesContent":["import jsonpointer from \"jsonpointer\";\nimport { TextLoader } from \"./text.js\";\n\n/**\n * Class that extends the `TextLoader` class. It represents a document\n * loader that loads documents from JSON files. It has a constructor that\n * takes a `filePathOrBlob` parameter representing the path to the JSON\n * file or a `Blob` object, and an optional `pointers` parameter that\n * specifies the JSON pointers to extract.\n */\nexport class JSONLoader extends TextLoader {\n  public pointers: string[];\n\n  constructor(filePathOrBlob: string | Blob, pointers: string | string[] = []) {\n    super(filePathOrBlob);\n    this.pointers = Array.isArray(pointers) ? pointers : [pointers];\n  }\n\n  /**\n   * Method that takes a `raw` string as a parameter and returns a promise\n   * that resolves to an array of strings. It parses the raw JSON string and\n   * extracts the values based on the specified JSON pointers. If no JSON\n   * pointers are specified, it extracts all the strings from the JSON\n   * object.\n   * @param raw The raw JSON string to parse.\n   * @returns A promise that resolves to an array of strings.\n   */\n  protected async parse(raw: string): Promise<string[]> {\n    const json = JSON.parse(raw.trim());\n    // If there is no pointers specified we extract all strings we found\n    const extractAllStrings = !(this.pointers.length > 0);\n    const compiledPointers = this.pointers.map((pointer) =>\n      jsonpointer.compile(pointer)\n    );\n\n    return this.extractArrayStringsFromObject(\n      json,\n      compiledPointers,\n      extractAllStrings\n    );\n  }\n\n  /**\n   * If JSON pointers are specified, return all strings below any of them\n   * and exclude all other nodes expect if they match a JSON pointer (to allow to extract strings from different levels)\n   *\n   * If no JSON pointer is specified then return all string in the object\n   */\n  private extractArrayStringsFromObject(\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    json: any,\n    pointers: jsonpointer[],\n    extractAllStrings = false,\n    keyHasBeenFound = false\n  ): string[] {\n    if (!json) {\n      return [];\n    }\n\n    if (typeof json === \"string\" && extractAllStrings) {\n      return [json];\n    }\n\n    if (Array.isArray(json) && extractAllStrings) {\n      let extractedString: string[] = [];\n      for (const element of json) {\n        extractedString = extractedString.concat(\n          this.extractArrayStringsFromObject(element, pointers, true)\n        );\n      }\n\n      return extractedString;\n    }\n\n    if (typeof json === \"object\") {\n      if (extractAllStrings) {\n        return this.extractArrayStringsFromObject(\n          Object.values(json),\n          pointers,\n          true\n        );\n      }\n\n      const targetedEntries = this.getTargetedEntries(json, pointers);\n      const thisLevelEntries = Object.values(json) as object[];\n      const notTargetedEntries = thisLevelEntries.filter(\n        (entry: object) => !targetedEntries.includes(entry)\n      );\n\n      let extractedStrings: string[] = [];\n      // If we found a targeted entry, we extract all strings from it\n      if (targetedEntries.length > 0) {\n        for (const oneEntry of targetedEntries) {\n          extractedStrings = extractedStrings.concat(\n            this.extractArrayStringsFromObject(oneEntry, pointers, true, true)\n          );\n        }\n\n        for (const oneEntry of notTargetedEntries) {\n          extractedStrings = extractedStrings.concat(\n            this.extractArrayStringsFromObject(oneEntry, pointers, false, true)\n          );\n        }\n      } else if (extractAllStrings || !keyHasBeenFound) {\n        for (const oneEntry of notTargetedEntries) {\n          extractedStrings = extractedStrings.concat(\n            this.extractArrayStringsFromObject(\n              oneEntry,\n              pointers,\n              extractAllStrings\n            )\n          );\n        }\n      }\n\n      return extractedStrings;\n    }\n\n    return [];\n  }\n\n  /**\n   * Method that takes a `json` object and an array of `pointers` as\n   * parameters and returns an array of targeted entries. It iterates over\n   * the JSON pointers and uses the `jsonpointer.get()` function to get the\n   * targeted entries from the JSON object.\n   * @param json The JSON object to get targeted entries from.\n   * @param pointers The JSON pointers to get targeted entries.\n   * @returns An array of targeted entries.\n   */\n  private getTargetedEntries(json: object, pointers: jsonpointer[]): object[] {\n    const targetEntries = [];\n    for (const pointer of pointers) {\n      const targetedEntry = pointer.get(json);\n      if (targetedEntry) {\n        targetEntries.push(targetedEntry);\n      }\n    }\n\n    return targetEntries;\n  }\n}\n\n/**\n * Class that extends the `TextLoader` class. It represents a document\n * loader that loads documents from JSON Lines files. It has a constructor\n * that takes a `filePathOrBlob` parameter representing the path to the\n * JSON Lines file or a `Blob` object, and a `pointer` parameter that\n * specifies the JSON pointer to extract.\n */\nexport class JSONLinesLoader extends TextLoader {\n  constructor(\n    filePathOrBlob: string | Blob,\n    public pointer: string\n  ) {\n    super(filePathOrBlob);\n  }\n\n  /**\n   * Method that takes a `raw` string as a parameter and returns a promise\n   * that resolves to an array of strings. It parses the raw JSON Lines\n   * string, splits it into lines, parses each line as JSON, and extracts\n   * the values based on the specified JSON pointer.\n   * @param raw The raw JSON Lines string to parse.\n   * @returns A promise that resolves to an array of strings.\n   */\n  protected async parse(raw: string): Promise<string[]> {\n    const lines = raw.split(\"\\n\");\n    const jsons = lines\n      .map((line) => line.trim())\n      .filter(Boolean)\n      .map((line) => JSON.parse(line));\n    const pointer = jsonpointer.compile(this.pointer);\n    return jsons.map((json) => {\n      const data = pointer.get(json);\n      if (typeof data === \"string\") {\n        return data;\n      }\n      if (!data) {\n        return \"\";\n      }\n      if (typeof data === \"object\") {\n        return JSON.stringify(data);\n      }\n      return \"\";\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAUA,IAAa,aAAb,cAAgCA,iCAAAA,WAAW;CACzC;CAEA,YAAY,gBAA+B,WAA8B,EAAE,EAAE;AAC3E,QAAM,eAAe;AACrB,OAAK,WAAW,MAAM,QAAQ,SAAS,GAAG,WAAW,CAAC,SAAS;;;;;;;;;;;CAYjE,MAAgB,MAAM,KAAgC;EACpD,MAAM,OAAO,KAAK,MAAM,IAAI,MAAM,CAAC;EAEnC,MAAM,oBAAoB,EAAE,KAAK,SAAS,SAAS;EACnD,MAAM,mBAAmB,KAAK,SAAS,KAAK,YAC1C,YAAA,QAAY,QAAQ,QAAQ,CAC7B;AAED,SAAO,KAAK,8BACV,MACA,kBACA,kBACD;;;;;;;;CASH,8BAEE,MACA,UACA,oBAAoB,OACpB,kBAAkB,OACR;AACV,MAAI,CAAC,KACH,QAAO,EAAE;AAGX,MAAI,OAAO,SAAS,YAAY,kBAC9B,QAAO,CAAC,KAAK;AAGf,MAAI,MAAM,QAAQ,KAAK,IAAI,mBAAmB;GAC5C,IAAI,kBAA4B,EAAE;AAClC,QAAK,MAAM,WAAW,KACpB,mBAAkB,gBAAgB,OAChC,KAAK,8BAA8B,SAAS,UAAU,KAAK,CAC5D;AAGH,UAAO;;AAGT,MAAI,OAAO,SAAS,UAAU;AAC5B,OAAI,kBACF,QAAO,KAAK,8BACV,OAAO,OAAO,KAAK,EACnB,UACA,KACD;GAGH,MAAM,kBAAkB,KAAK,mBAAmB,MAAM,SAAS;GAE/D,MAAM,qBADmB,OAAO,OAAO,KAAK,CACA,QACzC,UAAkB,CAAC,gBAAgB,SAAS,MAAM,CACpD;GAED,IAAI,mBAA6B,EAAE;AAEnC,OAAI,gBAAgB,SAAS,GAAG;AAC9B,SAAK,MAAM,YAAY,gBACrB,oBAAmB,iBAAiB,OAClC,KAAK,8BAA8B,UAAU,UAAU,MAAM,KAAK,CACnE;AAGH,SAAK,MAAM,YAAY,mBACrB,oBAAmB,iBAAiB,OAClC,KAAK,8BAA8B,UAAU,UAAU,OAAO,KAAK,CACpE;cAEM,qBAAqB,CAAC,gBAC/B,MAAK,MAAM,YAAY,mBACrB,oBAAmB,iBAAiB,OAClC,KAAK,8BACH,UACA,UACA,kBACD,CACF;AAIL,UAAO;;AAGT,SAAO,EAAE;;;;;;;;;;;CAYX,mBAA2B,MAAc,UAAmC;EAC1E,MAAM,gBAAgB,EAAE;AACxB,OAAK,MAAM,WAAW,UAAU;GAC9B,MAAM,gBAAgB,QAAQ,IAAI,KAAK;AACvC,OAAI,cACF,eAAc,KAAK,cAAc;;AAIrC,SAAO;;;;;;;;;;AAWX,IAAa,kBAAb,cAAqCA,iCAAAA,WAAW;CAC9C,YACE,gBACA,SACA;AACA,QAAM,eAAe;AAFd,OAAA,UAAA;;;;;;;;;;CAaT,MAAgB,MAAM,KAAgC;EAEpD,MAAM,QADQ,IAAI,MAAM,KAAK,CAE1B,KAAK,SAAS,KAAK,MAAM,CAAC,CAC1B,OAAO,QAAQ,CACf,KAAK,SAAS,KAAK,MAAM,KAAK,CAAC;EAClC,MAAM,UAAU,YAAA,QAAY,QAAQ,KAAK,QAAQ;AACjD,SAAO,MAAM,KAAK,SAAS;GACzB,MAAM,OAAO,QAAQ,IAAI,KAAK;AAC9B,OAAI,OAAO,SAAS,SAClB,QAAO;AAET,OAAI,CAAC,KACH,QAAO;AAET,OAAI,OAAO,SAAS,SAClB,QAAO,KAAK,UAAU,KAAK;AAE7B,UAAO;IACP"}
+{"version":3,"file":"json.cjs","names":["TextLoader"],"sources":["../../../src/document_loaders/fs/json.ts"],"sourcesContent":["import jsonpointer from \"jsonpointer\";\nimport { TextLoader } from \"./text.js\";\n\n/**\n * Class that extends the `TextLoader` class. It represents a document\n * loader that loads documents from JSON files. It has a constructor that\n * takes a `filePathOrBlob` parameter representing the path to the JSON\n * file or a `Blob` object, and an optional `pointers` parameter that\n * specifies the JSON pointers to extract.\n */\nexport class JSONLoader extends TextLoader {\n  public pointers: string[];\n\n  constructor(filePathOrBlob: string | Blob, pointers: string | string[] = []) {\n    super(filePathOrBlob);\n    this.pointers = Array.isArray(pointers) ? pointers : [pointers];\n  }\n\n  /**\n   * Method that takes a `raw` string as a parameter and returns a promise\n   * that resolves to an array of strings. It parses the raw JSON string and\n   * extracts the values based on the specified JSON pointers. If no JSON\n   * pointers are specified, it extracts all the strings from the JSON\n   * object.\n   * @param raw The raw JSON string to parse.\n   * @returns A promise that resolves to an array of strings.\n   */\n  protected async parse(raw: string): Promise<string[]> {\n    const json = JSON.parse(raw.trim());\n    // If there is no pointers specified we extract all strings we found\n    const extractAllStrings = !(this.pointers.length > 0);\n    const compiledPointers = this.pointers.map((pointer) =>\n      jsonpointer.compile(pointer)\n    );\n\n    return this.extractArrayStringsFromObject(\n      json,\n      compiledPointers,\n      extractAllStrings\n    );\n  }\n\n  /**\n   * If JSON pointers are specified, return all strings below any of them\n   * and exclude all other nodes expect if they match a JSON pointer (to allow to extract strings from different levels)\n   *\n   * If no JSON pointer is specified then return all string in the object\n   */\n  private extractArrayStringsFromObject(\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    json: any,\n    pointers: jsonpointer[],\n    extractAllStrings = false,\n    keyHasBeenFound = false\n  ): string[] {\n    if (!json) {\n      return [];\n    }\n\n    if (typeof json === \"string\" && extractAllStrings) {\n      return [json];\n    }\n\n    if (Array.isArray(json) && extractAllStrings) {\n      let extractedString: string[] = [];\n      for (const element of json) {\n        extractedString = extractedString.concat(\n          this.extractArrayStringsFromObject(element, pointers, true)\n        );\n      }\n\n      return extractedString;\n    }\n\n    if (typeof json === \"object\") {\n      if (extractAllStrings) {\n        return this.extractArrayStringsFromObject(\n          Object.values(json),\n          pointers,\n          true\n        );\n      }\n\n      const targetedEntries = this.getTargetedEntries(json, pointers);\n      const thisLevelEntries = Object.values(json) as object[];\n      const notTargetedEntries = thisLevelEntries.filter(\n        (entry: object) => !targetedEntries.includes(entry)\n      );\n\n      let extractedStrings: string[] = [];\n      // If we found a targeted entry, we extract all strings from it\n      if (targetedEntries.length > 0) {\n        for (const oneEntry of targetedEntries) {\n          extractedStrings = extractedStrings.concat(\n            this.extractArrayStringsFromObject(oneEntry, pointers, true, true)\n          );\n        }\n\n        for (const oneEntry of notTargetedEntries) {\n          extractedStrings = extractedStrings.concat(\n            this.extractArrayStringsFromObject(oneEntry, pointers, false, true)\n          );\n        }\n      } else if (extractAllStrings || !keyHasBeenFound) {\n        for (const oneEntry of notTargetedEntries) {\n          extractedStrings = extractedStrings.concat(\n            this.extractArrayStringsFromObject(\n              oneEntry,\n              pointers,\n              extractAllStrings\n            )\n          );\n        }\n      }\n\n      return extractedStrings;\n    }\n\n    return [];\n  }\n\n  /**\n   * Method that takes a `json` object and an array of `pointers` as\n   * parameters and returns an array of targeted entries. It iterates over\n   * the JSON pointers and uses the `jsonpointer.get()` function to get the\n   * targeted entries from the JSON object.\n   * @param json The JSON object to get targeted entries from.\n   * @param pointers The JSON pointers to get targeted entries.\n   * @returns An array of targeted entries.\n   */\n  private getTargetedEntries(json: object, pointers: jsonpointer[]): object[] {\n    const targetEntries = [];\n    for (const pointer of pointers) {\n      const targetedEntry = pointer.get(json);\n      if (targetedEntry) {\n        targetEntries.push(targetedEntry);\n      }\n    }\n\n    return targetEntries;\n  }\n}\n\n/**\n * Class that extends the `TextLoader` class. It represents a document\n * loader that loads documents from JSON Lines files. It has a constructor\n * that takes a `filePathOrBlob` parameter representing the path to the\n * JSON Lines file or a `Blob` object, and a `pointer` parameter that\n * specifies the JSON pointer to extract.\n */\nexport class JSONLinesLoader extends TextLoader {\n  constructor(\n    filePathOrBlob: string | Blob,\n    public pointer: string\n  ) {\n    super(filePathOrBlob);\n  }\n\n  /**\n   * Method that takes a `raw` string as a parameter and returns a promise\n   * that resolves to an array of strings. It parses the raw JSON Lines\n   * string, splits it into lines, parses each line as JSON, and extracts\n   * the values based on the specified JSON pointer.\n   * @param raw The raw JSON Lines string to parse.\n   * @returns A promise that resolves to an array of strings.\n   */\n  protected async parse(raw: string): Promise<string[]> {\n    const lines = raw.split(\"\\n\");\n    const jsons = lines\n      .map((line) => line.trim())\n      .filter(Boolean)\n      .map((line) => JSON.parse(line));\n    const pointer = jsonpointer.compile(this.pointer);\n    return jsons.map((json) => {\n      const data = pointer.get(json);\n      if (typeof data === \"string\") {\n        return data;\n      }\n      if (!data) {\n        return \"\";\n      }\n      if (typeof data === \"object\") {\n        return JSON.stringify(data);\n      }\n      return \"\";\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAUA,IAAa,aAAb,cAAgCA,iCAAAA,WAAW;CACzC;CAEA,YAAY,gBAA+B,WAA8B,EAAE,EAAE;AAC3E,QAAM,eAAe;AACrB,OAAK,WAAW,MAAM,QAAQ,SAAS,GAAG,WAAW,CAAC,SAAS;;;;;;;;;;;CAYjE,MAAgB,MAAM,KAAgC;EACpD,MAAM,OAAO,KAAK,MAAM,IAAI,MAAM,CAAC;EAEnC,MAAM,oBAAoB,EAAE,KAAK,SAAS,SAAS;EACnD,MAAM,mBAAmB,KAAK,SAAS,KAAK,YAC1C,YAAA,QAAY,QAAQ,QAAQ,CAC7B;AAED,SAAO,KAAK,8BACV,MACA,kBACA,kBACD;;;;;;;;CASH,8BAEE,MACA,UACA,oBAAoB,OACpB,kBAAkB,OACR;AACV,MAAI,CAAC,KACH,QAAO,EAAE;AAGX,MAAI,OAAO,SAAS,YAAY,kBAC9B,QAAO,CAAC,KAAK;AAGf,MAAI,MAAM,QAAQ,KAAK,IAAI,mBAAmB;GAC5C,IAAI,kBAA4B,EAAE;AAClC,QAAK,MAAM,WAAW,KACpB,mBAAkB,gBAAgB,OAChC,KAAK,8BAA8B,SAAS,UAAU,KAAK,CAC5D;AAGH,UAAO;;AAGT,MAAI,OAAO,SAAS,UAAU;AAC5B,OAAI,kBACF,QAAO,KAAK,8BACV,OAAO,OAAO,KAAK,EACnB,UACA,KACD;GAGH,MAAM,kBAAkB,KAAK,mBAAmB,MAAM,SAAS;GAE/D,MAAM,qBADmB,OAAO,OAAO,KAAK,CACA,QACzC,UAAkB,CAAC,gBAAgB,SAAS,MAAM,CACpD;GAED,IAAI,mBAA6B,EAAE;AAEnC,OAAI,gBAAgB,SAAS,GAAG;AAC9B,SAAK,MAAM,YAAY,gBACrB,oBAAmB,iBAAiB,OAClC,KAAK,8BAA8B,UAAU,UAAU,MAAM,KAAK,CACnE;AAGH,SAAK,MAAM,YAAY,mBACrB,oBAAmB,iBAAiB,OAClC,KAAK,8BAA8B,UAAU,UAAU,OAAO,KAAK,CACpE;cAEM,qBAAqB,CAAC,gBAC/B,MAAK,MAAM,YAAY,mBACrB,oBAAmB,iBAAiB,OAClC,KAAK,8BACH,UACA,UACA,kBACD,CACF;AAIL,UAAO;;AAGT,SAAO,EAAE;;;;;;;;;;;CAYX,mBAA2B,MAAc,UAAmC;EAC1E,MAAM,gBAAgB,EAAE;AACxB,OAAK,MAAM,WAAW,UAAU;GAC9B,MAAM,gBAAgB,QAAQ,IAAI,KAAK;AACvC,OAAI,cACF,eAAc,KAAK,cAAc;;AAIrC,SAAO;;;;;;;;;;AAWX,IAAa,kBAAb,cAAqCA,iCAAAA,WAAW;CAC9C,YACE,gBACA,SACA;AACA,QAAM,eAAe;AAFd,OAAA,UAAA;;;;;;;;;;CAaT,MAAgB,MAAM,KAAgC;EAEpD,MAAM,QADQ,IAAI,MAAM,KAAK,CAE1B,KAAK,SAAS,KAAK,MAAM,CAAC,CAC1B,OAAO,QAAQ,CACf,KAAK,SAAS,KAAK,MAAM,KAAK,CAAC;EAClC,MAAM,UAAU,YAAA,QAAY,QAAQ,KAAK,QAAQ;AACjD,SAAO,MAAM,KAAK,SAAS;GACzB,MAAM,OAAO,QAAQ,IAAI,KAAK;AAC9B,OAAI,OAAO,SAAS,SAClB,QAAO;AAET,OAAI,CAAC,KACH,QAAO;AAET,OAAI,OAAO,SAAS,SAClB,QAAO,KAAK,UAAU,KAAK;AAE7B,UAAO;IACP"}

package/dist/document_loaders/fs/json.js.map

@@ -1 +1 @@
-{"version":3,"file":"json.js","names":[],"sources":["../../../src/document_loaders/fs/json.ts"],"sourcesContent":["import jsonpointer from \"jsonpointer\";\nimport { TextLoader } from \"./text.js\";\n\n/**\n * Class that extends the `TextLoader` class. It represents a document\n * loader that loads documents from JSON files. It has a constructor that\n * takes a `filePathOrBlob` parameter representing the path to the JSON\n * file or a `Blob` object, and an optional `pointers` parameter that\n * specifies the JSON pointers to extract.\n */\nexport class JSONLoader extends TextLoader {\n  public pointers: string[];\n\n  constructor(filePathOrBlob: string | Blob, pointers: string | string[] = []) {\n    super(filePathOrBlob);\n    this.pointers = Array.isArray(pointers) ? pointers : [pointers];\n  }\n\n  /**\n   * Method that takes a `raw` string as a parameter and returns a promise\n   * that resolves to an array of strings. It parses the raw JSON string and\n   * extracts the values based on the specified JSON pointers. If no JSON\n   * pointers are specified, it extracts all the strings from the JSON\n   * object.\n   * @param raw The raw JSON string to parse.\n   * @returns A promise that resolves to an array of strings.\n   */\n  protected async parse(raw: string): Promise<string[]> {\n    const json = JSON.parse(raw.trim());\n    // If there is no pointers specified we extract all strings we found\n    const extractAllStrings = !(this.pointers.length > 0);\n    const compiledPointers = this.pointers.map((pointer) =>\n      jsonpointer.compile(pointer)\n    );\n\n    return this.extractArrayStringsFromObject(\n      json,\n      compiledPointers,\n      extractAllStrings\n    );\n  }\n\n  /**\n   * If JSON pointers are specified, return all strings below any of them\n   * and exclude all other nodes expect if they match a JSON pointer (to allow to extract strings from different levels)\n   *\n   * If no JSON pointer is specified then return all string in the object\n   */\n  private extractArrayStringsFromObject(\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    json: any,\n    pointers: jsonpointer[],\n    extractAllStrings = false,\n    keyHasBeenFound = false\n  ): string[] {\n    if (!json) {\n      return [];\n    }\n\n    if (typeof json === \"string\" && extractAllStrings) {\n      return [json];\n    }\n\n    if (Array.isArray(json) && extractAllStrings) {\n      let extractedString: string[] = [];\n      for (const element of json) {\n        extractedString = extractedString.concat(\n          this.extractArrayStringsFromObject(element, pointers, true)\n        );\n      }\n\n      return extractedString;\n    }\n\n    if (typeof json === \"object\") {\n      if (extractAllStrings) {\n        return this.extractArrayStringsFromObject(\n          Object.values(json),\n          pointers,\n          true\n        );\n      }\n\n      const targetedEntries = this.getTargetedEntries(json, pointers);\n      const thisLevelEntries = Object.values(json) as object[];\n      const notTargetedEntries = thisLevelEntries.filter(\n        (entry: object) => !targetedEntries.includes(entry)\n      );\n\n      let extractedStrings: string[] = [];\n      // If we found a targeted entry, we extract all strings from it\n      if (targetedEntries.length > 0) {\n        for (const oneEntry of targetedEntries) {\n          extractedStrings = extractedStrings.concat(\n            this.extractArrayStringsFromObject(oneEntry, pointers, true, true)\n          );\n        }\n\n        for (const oneEntry of notTargetedEntries) {\n          extractedStrings = extractedStrings.concat(\n            this.extractArrayStringsFromObject(oneEntry, pointers, false, true)\n          );\n        }\n      } else if (extractAllStrings || !keyHasBeenFound) {\n        for (const oneEntry of notTargetedEntries) {\n          extractedStrings = extractedStrings.concat(\n            this.extractArrayStringsFromObject(\n              oneEntry,\n              pointers,\n              extractAllStrings\n            )\n          );\n        }\n      }\n\n      return extractedStrings;\n    }\n\n    return [];\n  }\n\n  /**\n   * Method that takes a `json` object and an array of `pointers` as\n   * parameters and returns an array of targeted entries. It iterates over\n   * the JSON pointers and uses the `jsonpointer.get()` function to get the\n   * targeted entries from the JSON object.\n   * @param json The JSON object to get targeted entries from.\n   * @param pointers The JSON pointers to get targeted entries.\n   * @returns An array of targeted entries.\n   */\n  private getTargetedEntries(json: object, pointers: jsonpointer[]): object[] {\n    const targetEntries = [];\n    for (const pointer of pointers) {\n      const targetedEntry = pointer.get(json);\n      if (targetedEntry) {\n        targetEntries.push(targetedEntry);\n      }\n    }\n\n    return targetEntries;\n  }\n}\n\n/**\n * Class that extends the `TextLoader` class. It represents a document\n * loader that loads documents from JSON Lines files. It has a constructor\n * that takes a `filePathOrBlob` parameter representing the path to the\n * JSON Lines file or a `Blob` object, and a `pointer` parameter that\n * specifies the JSON pointer to extract.\n */\nexport class JSONLinesLoader extends TextLoader {\n  constructor(\n    filePathOrBlob: string | Blob,\n    public pointer: string\n  ) {\n    super(filePathOrBlob);\n  }\n\n  /**\n   * Method that takes a `raw` string as a parameter and returns a promise\n   * that resolves to an array of strings. It parses the raw JSON Lines\n   * string, splits it into lines, parses each line as JSON, and extracts\n   * the values based on the specified JSON pointer.\n   * @param raw The raw JSON Lines string to parse.\n   * @returns A promise that resolves to an array of strings.\n   */\n  protected async parse(raw: string): Promise<string[]> {\n    const lines = raw.split(\"\\n\");\n    const jsons = lines\n      .map((line) => line.trim())\n      .filter(Boolean)\n      .map((line) => JSON.parse(line));\n    const pointer = jsonpointer.compile(this.pointer);\n    return jsons.map((json) => {\n      const data = pointer.get(json);\n      if (typeof data === \"string\") {\n        return data;\n      }\n      if (!data) {\n        return \"\";\n      }\n      if (typeof data === \"object\") {\n        return JSON.stringify(data);\n      }\n      return \"\";\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAUA,IAAa,aAAb,cAAgC,WAAW;CACzC;CAEA,YAAY,gBAA+B,WAA8B,EAAE,EAAE;AAC3E,QAAM,eAAe;AACrB,OAAK,WAAW,MAAM,QAAQ,SAAS,GAAG,WAAW,CAAC,SAAS;;;;;;;;;;;CAYjE,MAAgB,MAAM,KAAgC;EACpD,MAAM,OAAO,KAAK,MAAM,IAAI,MAAM,CAAC;EAEnC,MAAM,oBAAoB,EAAE,KAAK,SAAS,SAAS;EACnD,MAAM,mBAAmB,KAAK,SAAS,KAAK,YAC1C,YAAY,QAAQ,QAAQ,CAC7B;AAED,SAAO,KAAK,8BACV,MACA,kBACA,kBACD;;;;;;;;CASH,8BAEE,MACA,UACA,oBAAoB,OACpB,kBAAkB,OACR;AACV,MAAI,CAAC,KACH,QAAO,EAAE;AAGX,MAAI,OAAO,SAAS,YAAY,kBAC9B,QAAO,CAAC,KAAK;AAGf,MAAI,MAAM,QAAQ,KAAK,IAAI,mBAAmB;GAC5C,IAAI,kBAA4B,EAAE;AAClC,QAAK,MAAM,WAAW,KACpB,mBAAkB,gBAAgB,OAChC,KAAK,8BAA8B,SAAS,UAAU,KAAK,CAC5D;AAGH,UAAO;;AAGT,MAAI,OAAO,SAAS,UAAU;AAC5B,OAAI,kBACF,QAAO,KAAK,8BACV,OAAO,OAAO,KAAK,EACnB,UACA,KACD;GAGH,MAAM,kBAAkB,KAAK,mBAAmB,MAAM,SAAS;GAE/D,MAAM,qBADmB,OAAO,OAAO,KAAK,CACA,QACzC,UAAkB,CAAC,gBAAgB,SAAS,MAAM,CACpD;GAED,IAAI,mBAA6B,EAAE;AAEnC,OAAI,gBAAgB,SAAS,GAAG;AAC9B,SAAK,MAAM,YAAY,gBACrB,oBAAmB,iBAAiB,OAClC,KAAK,8BAA8B,UAAU,UAAU,MAAM,KAAK,CACnE;AAGH,SAAK,MAAM,YAAY,mBACrB,oBAAmB,iBAAiB,OAClC,KAAK,8BAA8B,UAAU,UAAU,OAAO,KAAK,CACpE;cAEM,qBAAqB,CAAC,gBAC/B,MAAK,MAAM,YAAY,mBACrB,oBAAmB,iBAAiB,OAClC,KAAK,8BACH,UACA,UACA,kBACD,CACF;AAIL,UAAO;;AAGT,SAAO,EAAE;;;;;;;;;;;CAYX,mBAA2B,MAAc,UAAmC;EAC1E,MAAM,gBAAgB,EAAE;AACxB,OAAK,MAAM,WAAW,UAAU;GAC9B,MAAM,gBAAgB,QAAQ,IAAI,KAAK;AACvC,OAAI,cACF,eAAc,KAAK,cAAc;;AAIrC,SAAO;;;;;;;;;;AAWX,IAAa,kBAAb,cAAqC,WAAW;CAC9C,YACE,gBACA,SACA;AACA,QAAM,eAAe;AAFd,OAAA,UAAA;;;;;;;;;;CAaT,MAAgB,MAAM,KAAgC;EAEpD,MAAM,QADQ,IAAI,MAAM,KAAK,CAE1B,KAAK,SAAS,KAAK,MAAM,CAAC,CAC1B,OAAO,QAAQ,CACf,KAAK,SAAS,KAAK,MAAM,KAAK,CAAC;EAClC,MAAM,UAAU,YAAY,QAAQ,KAAK,QAAQ;AACjD,SAAO,MAAM,KAAK,SAAS;GACzB,MAAM,OAAO,QAAQ,IAAI,KAAK;AAC9B,OAAI,OAAO,SAAS,SAClB,QAAO;AAET,OAAI,CAAC,KACH,QAAO;AAET,OAAI,OAAO,SAAS,SAClB,QAAO,KAAK,UAAU,KAAK;AAE7B,UAAO;IACP"}
+{"version":3,"file":"json.js","names":[],"sources":["../../../src/document_loaders/fs/json.ts"],"sourcesContent":["import jsonpointer from \"jsonpointer\";\nimport { TextLoader } from \"./text.js\";\n\n/**\n * Class that extends the `TextLoader` class. It represents a document\n * loader that loads documents from JSON files. It has a constructor that\n * takes a `filePathOrBlob` parameter representing the path to the JSON\n * file or a `Blob` object, and an optional `pointers` parameter that\n * specifies the JSON pointers to extract.\n */\nexport class JSONLoader extends TextLoader {\n  public pointers: string[];\n\n  constructor(filePathOrBlob: string | Blob, pointers: string | string[] = []) {\n    super(filePathOrBlob);\n    this.pointers = Array.isArray(pointers) ? pointers : [pointers];\n  }\n\n  /**\n   * Method that takes a `raw` string as a parameter and returns a promise\n   * that resolves to an array of strings. It parses the raw JSON string and\n   * extracts the values based on the specified JSON pointers. If no JSON\n   * pointers are specified, it extracts all the strings from the JSON\n   * object.\n   * @param raw The raw JSON string to parse.\n   * @returns A promise that resolves to an array of strings.\n   */\n  protected async parse(raw: string): Promise<string[]> {\n    const json = JSON.parse(raw.trim());\n    // If there is no pointers specified we extract all strings we found\n    const extractAllStrings = !(this.pointers.length > 0);\n    const compiledPointers = this.pointers.map((pointer) =>\n      jsonpointer.compile(pointer)\n    );\n\n    return this.extractArrayStringsFromObject(\n      json,\n      compiledPointers,\n      extractAllStrings\n    );\n  }\n\n  /**\n   * If JSON pointers are specified, return all strings below any of them\n   * and exclude all other nodes expect if they match a JSON pointer (to allow to extract strings from different levels)\n   *\n   * If no JSON pointer is specified then return all string in the object\n   */\n  private extractArrayStringsFromObject(\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    json: any,\n    pointers: jsonpointer[],\n    extractAllStrings = false,\n    keyHasBeenFound = false\n  ): string[] {\n    if (!json) {\n      return [];\n    }\n\n    if (typeof json === \"string\" && extractAllStrings) {\n      return [json];\n    }\n\n    if (Array.isArray(json) && extractAllStrings) {\n      let extractedString: string[] = [];\n      for (const element of json) {\n        extractedString = extractedString.concat(\n          this.extractArrayStringsFromObject(element, pointers, true)\n        );\n      }\n\n      return extractedString;\n    }\n\n    if (typeof json === \"object\") {\n      if (extractAllStrings) {\n        return this.extractArrayStringsFromObject(\n          Object.values(json),\n          pointers,\n          true\n        );\n      }\n\n      const targetedEntries = this.getTargetedEntries(json, pointers);\n      const thisLevelEntries = Object.values(json) as object[];\n      const notTargetedEntries = thisLevelEntries.filter(\n        (entry: object) => !targetedEntries.includes(entry)\n      );\n\n      let extractedStrings: string[] = [];\n      // If we found a targeted entry, we extract all strings from it\n      if (targetedEntries.length > 0) {\n        for (const oneEntry of targetedEntries) {\n          extractedStrings = extractedStrings.concat(\n            this.extractArrayStringsFromObject(oneEntry, pointers, true, true)\n          );\n        }\n\n        for (const oneEntry of notTargetedEntries) {\n          extractedStrings = extractedStrings.concat(\n            this.extractArrayStringsFromObject(oneEntry, pointers, false, true)\n          );\n        }\n      } else if (extractAllStrings || !keyHasBeenFound) {\n        for (const oneEntry of notTargetedEntries) {\n          extractedStrings = extractedStrings.concat(\n            this.extractArrayStringsFromObject(\n              oneEntry,\n              pointers,\n              extractAllStrings\n            )\n          );\n        }\n      }\n\n      return extractedStrings;\n    }\n\n    return [];\n  }\n\n  /**\n   * Method that takes a `json` object and an array of `pointers` as\n   * parameters and returns an array of targeted entries. It iterates over\n   * the JSON pointers and uses the `jsonpointer.get()` function to get the\n   * targeted entries from the JSON object.\n   * @param json The JSON object to get targeted entries from.\n   * @param pointers The JSON pointers to get targeted entries.\n   * @returns An array of targeted entries.\n   */\n  private getTargetedEntries(json: object, pointers: jsonpointer[]): object[] {\n    const targetEntries = [];\n    for (const pointer of pointers) {\n      const targetedEntry = pointer.get(json);\n      if (targetedEntry) {\n        targetEntries.push(targetedEntry);\n      }\n    }\n\n    return targetEntries;\n  }\n}\n\n/**\n * Class that extends the `TextLoader` class. It represents a document\n * loader that loads documents from JSON Lines files. It has a constructor\n * that takes a `filePathOrBlob` parameter representing the path to the\n * JSON Lines file or a `Blob` object, and a `pointer` parameter that\n * specifies the JSON pointer to extract.\n */\nexport class JSONLinesLoader extends TextLoader {\n  constructor(\n    filePathOrBlob: string | Blob,\n    public pointer: string\n  ) {\n    super(filePathOrBlob);\n  }\n\n  /**\n   * Method that takes a `raw` string as a parameter and returns a promise\n   * that resolves to an array of strings. It parses the raw JSON Lines\n   * string, splits it into lines, parses each line as JSON, and extracts\n   * the values based on the specified JSON pointer.\n   * @param raw The raw JSON Lines string to parse.\n   * @returns A promise that resolves to an array of strings.\n   */\n  protected async parse(raw: string): Promise<string[]> {\n    const lines = raw.split(\"\\n\");\n    const jsons = lines\n      .map((line) => line.trim())\n      .filter(Boolean)\n      .map((line) => JSON.parse(line));\n    const pointer = jsonpointer.compile(this.pointer);\n    return jsons.map((json) => {\n      const data = pointer.get(json);\n      if (typeof data === \"string\") {\n        return data;\n      }\n      if (!data) {\n        return \"\";\n      }\n      if (typeof data === \"object\") {\n        return JSON.stringify(data);\n      }\n      return \"\";\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAUA,IAAa,aAAb,cAAgC,WAAW;CACzC;CAEA,YAAY,gBAA+B,WAA8B,EAAE,EAAE;AAC3E,QAAM,eAAe;AACrB,OAAK,WAAW,MAAM,QAAQ,SAAS,GAAG,WAAW,CAAC,SAAS;;;;;;;;;;;CAYjE,MAAgB,MAAM,KAAgC;EACpD,MAAM,OAAO,KAAK,MAAM,IAAI,MAAM,CAAC;EAEnC,MAAM,oBAAoB,EAAE,KAAK,SAAS,SAAS;EACnD,MAAM,mBAAmB,KAAK,SAAS,KAAK,YAC1C,YAAY,QAAQ,QAAQ,CAC7B;AAED,SAAO,KAAK,8BACV,MACA,kBACA,kBACD;;;;;;;;CASH,8BAEE,MACA,UACA,oBAAoB,OACpB,kBAAkB,OACR;AACV,MAAI,CAAC,KACH,QAAO,EAAE;AAGX,MAAI,OAAO,SAAS,YAAY,kBAC9B,QAAO,CAAC,KAAK;AAGf,MAAI,MAAM,QAAQ,KAAK,IAAI,mBAAmB;GAC5C,IAAI,kBAA4B,EAAE;AAClC,QAAK,MAAM,WAAW,KACpB,mBAAkB,gBAAgB,OAChC,KAAK,8BAA8B,SAAS,UAAU,KAAK,CAC5D;AAGH,UAAO;;AAGT,MAAI,OAAO,SAAS,UAAU;AAC5B,OAAI,kBACF,QAAO,KAAK,8BACV,OAAO,OAAO,KAAK,EACnB,UACA,KACD;GAGH,MAAM,kBAAkB,KAAK,mBAAmB,MAAM,SAAS;GAE/D,MAAM,qBADmB,OAAO,OAAO,KAAK,CACA,QACzC,UAAkB,CAAC,gBAAgB,SAAS,MAAM,CACpD;GAED,IAAI,mBAA6B,EAAE;AAEnC,OAAI,gBAAgB,SAAS,GAAG;AAC9B,SAAK,MAAM,YAAY,gBACrB,oBAAmB,iBAAiB,OAClC,KAAK,8BAA8B,UAAU,UAAU,MAAM,KAAK,CACnE;AAGH,SAAK,MAAM,YAAY,mBACrB,oBAAmB,iBAAiB,OAClC,KAAK,8BAA8B,UAAU,UAAU,OAAO,KAAK,CACpE;cAEM,qBAAqB,CAAC,gBAC/B,MAAK,MAAM,YAAY,mBACrB,oBAAmB,iBAAiB,OAClC,KAAK,8BACH,UACA,UACA,kBACD,CACF;AAIL,UAAO;;AAGT,SAAO,EAAE;;;;;;;;;;;CAYX,mBAA2B,MAAc,UAAmC;EAC1E,MAAM,gBAAgB,EAAE;AACxB,OAAK,MAAM,WAAW,UAAU;GAC9B,MAAM,gBAAgB,QAAQ,IAAI,KAAK;AACvC,OAAI,cACF,eAAc,KAAK,cAAc;;AAIrC,SAAO;;;;;;;;;;AAWX,IAAa,kBAAb,cAAqC,WAAW;CAC9C,YACE,gBACA,SACA;AACA,QAAM,eAAe;AAFd,OAAA,UAAA;;;;;;;;;;CAaT,MAAgB,MAAM,KAAgC;EAEpD,MAAM,QADQ,IAAI,MAAM,KAAK,CAE1B,KAAK,SAAS,KAAK,MAAM,CAAC,CAC1B,OAAO,QAAQ,CACf,KAAK,SAAS,KAAK,MAAM,KAAK,CAAC;EAClC,MAAM,UAAU,YAAY,QAAQ,KAAK,QAAQ;AACjD,SAAO,MAAM,KAAK,SAAS;GACzB,MAAM,OAAO,QAAQ,IAAI,KAAK;AAC9B,OAAI,OAAO,SAAS,SAClB,QAAO;AAET,OAAI,CAAC,KACH,QAAO;AAET,OAAI,OAAO,SAAS,SAClB,QAAO,KAAK,UAAU,KAAK;AAE7B,UAAO;IACP"}

package/dist/document_transformers/openai_functions.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"openai_functions.d.cts","names":[],"sources":["../../src/document_transformers/openai_functions.ts"],"mappings":";;;;;;;;;;AAmBA;cAAa,cAAA,SAAuB,0BAAA;EAAA,OAC3B,OAAA,CAAA;EAAA,UAIG,YAAA,EAAc,SAAA;EAExB,WAAA,CAAY,MAAA;IAAU,YAAA,EAAc,SAAA;EAAA;EAe9B,kBAAA,CAAmB,QAAA,EAAU,QAAA,GAAW,OAAA,CAAQ,QAAA;AAAA;AAAA,iBAaxC,oBAAA,CACd,MAAA,EAAQ,qBAAA,EACR,OAAA,EAAS,mBAAA;EAAwB,GAAA,GAAM,UAAA;AAAA,IAAY,cAAA;AAAA,iBAQrC,2BAAA,CACd,MAAA,EAAQ,gBAAA,EACR,OAAA,EAAS,mBAAA;EAAwB,GAAA,GAAM,UAAA;AAAA,IAAY,cAAA"}
+{"version":3,"file":"openai_functions.d.cts","names":[],"sources":["../../src/document_transformers/openai_functions.ts"],"mappings":";;;;;;;;;;;cAmBa,cAAA,SAAuB,0BAAA;EAAA,OAC3B,OAAA,CAAA;EAAA,UAIG,YAAA,EAAc,SAAA;EAExB,WAAA,CAAY,MAAA;IAAU,YAAA,EAAc,SAAA;EAAA;EAe9B,kBAAA,CAAmB,QAAA,EAAU,QAAA,GAAW,OAAA,CAAQ,QAAA;AAAA;AAAA,iBAaxC,oBAAA,CACd,MAAA,EAAQ,qBAAA,EACR,OAAA,EAAS,mBAAA;EAAwB,GAAA,GAAM,UAAA;AAAA,IAAY,cAAA;AAAA,iBAQrC,2BAAA,CACd,MAAA,EAAQ,gBAAA,EACR,OAAA,EAAS,mBAAA;EAAwB,GAAA,GAAM,UAAA;AAAA,IAAY,cAAA"}

package/dist/evaluation/agents/index.d.cts

@@ -0,0 +1 @@
+import { TrajectoryEvalChain, TrajectoryOutputParser } from "./trajectory.cjs";

package/dist/evaluation/agents/trajectory.d.cts

@@ -4,7 +4,7 @@ import { ChatGeneration, Generation } from "@langchain/core/outputs";
 import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
 import { ChainValues } from "@langchain/core/utils/types";
 import { BasePromptTemplate } from "@langchain/core/prompts";
-import * as _langchain_core_prompt_values0 from "@langchain/core/prompt_values";
+import * as _$_langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
 import { StructuredToolInterface } from "@langchain/core/tools";
 import { BaseChatModel } from "@langchain/core/language_models/chat_models";
@@ -31,7 +31,7 @@ declare class TrajectoryEvalChain extends AgentTrajectoryEvaluator {
   requiresInput: boolean;
   requiresReference: boolean;
   outputParser: TrajectoryOutputParser;
-  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>;
+  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, _$_langchain_core_prompt_values0.BasePromptValueInterface, any>;
   /**
    * Get the description of the agent tools.
    *

package/dist/evaluation/agents/trajectory.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"trajectory.d.cts","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"mappings":";;;;;;;;;;;;;;;cAwBa,sBAAA,SAA+B,mBAAA,CAAoB,cAAA;EAAA,OACvD,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;;;cA4CA,mBAAA,SAA4B,wBAAA;EAAA,OAChC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,YAAA,EAAY,sBAAA;EAAA,OAEL,uBAAA,CACL,MAAA,GAAS,kBAAA,cACT,UAAA,GAAa,uBAAA,KAAyB,kBAAA,MAAF,8BAAA,CAAE,wBAAA;EA/DtC;;;;;EAAA,OAkFK,gBAAA,CAAiB,UAAA,EAAY,uBAAA;EAhFX;AA4C3B;;;;;EA5C2B,OA+FZ,OAAA,CACX,GAAA,EAAK,aAAA,EACL,UAAA,GAAa,uBAAA,IACb,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,mBAAA;EAqBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EA1DiB;;;;;;EAwExC,kBAAA,CAAmB,KAAA,EAAO,SAAA;EAe1B,eAAA,CAAgB,SAAA;EAYV,wBAAA,CACJ,IAAA,EAAM,0BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}
+{"version":3,"file":"trajectory.d.cts","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"mappings":";;;;;;;;;;;;;;;cAwBa,sBAAA,SAA+B,mBAAA,CAAoB,cAAA;EAAA,OACvD,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;;;cA4CA,mBAAA,SAA4B,wBAAA;EAAA,OAChC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,YAAA,EAAY,sBAAA;EAAA,OAEL,uBAAA,CACL,MAAA,GAAS,kBAAA,cACT,UAAA,GAAa,uBAAA,KAAyB,kBAAA,MAAF,gCAAA,CAAE,wBAAA;EA/DtC;;;;;EAAA,OAkFK,gBAAA,CAAiB,UAAA,EAAY,uBAAA;EAhFX;AA4C3B;;;;;EA5C2B,OA+FZ,OAAA,CACX,GAAA,EAAK,aAAA,EACL,UAAA,GAAa,uBAAA,IACb,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,mBAAA;EAqBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EA1DiB;;;;;;EAwExC,kBAAA,CAAmB,KAAA,EAAO,SAAA;EAe1B,eAAA,CAAgB,SAAA;EAYV,wBAAA,CACJ,IAAA,EAAM,0BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}

package/dist/evaluation/agents/trajectory.d.ts

@@ -6,7 +6,7 @@ import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager
 import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
 import { ChainValues } from "@langchain/core/utils/types";
 import { BaseChatModel } from "@langchain/core/language_models/chat_models";
-import * as _langchain_core_prompt_values0 from "@langchain/core/prompt_values";
+import * as _$_langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 import { AgentStep } from "@langchain/core/agents";
 
 //#region src/evaluation/agents/trajectory.d.ts
@@ -31,7 +31,7 @@ declare class TrajectoryEvalChain extends AgentTrajectoryEvaluator {
   requiresInput: boolean;
   requiresReference: boolean;
   outputParser: TrajectoryOutputParser;
-  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>;
+  static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredToolInterface[]): BasePromptTemplate<any, _$_langchain_core_prompt_values0.BasePromptValueInterface, any>;
   /**
    * Get the description of the agent tools.
    *

package/dist/evaluation/agents/trajectory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"trajectory.d.ts","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"mappings":";;;;;;;;;;;;;;;cAwBa,sBAAA,SAA+B,mBAAA,CAAoB,cAAA;EAAA,OACvD,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;;;cA4CA,mBAAA,SAA4B,wBAAA;EAAA,OAChC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,YAAA,EAAY,sBAAA;EAAA,OAEL,uBAAA,CACL,MAAA,GAAS,kBAAA,cACT,UAAA,GAAa,uBAAA,KAAyB,kBAAA,MAAF,8BAAA,CAAE,wBAAA;EA/DtC;;;;;EAAA,OAkFK,gBAAA,CAAiB,UAAA,EAAY,uBAAA;EAhFX;AA4C3B;;;;;EA5C2B,OA+FZ,OAAA,CACX,GAAA,EAAK,aAAA,EACL,UAAA,GAAa,uBAAA,IACb,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,mBAAA;EAqBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EA1DiB;;;;;;EAwExC,kBAAA,CAAmB,KAAA,EAAO,SAAA;EAe1B,eAAA,CAAgB,SAAA;EAYV,wBAAA,CACJ,IAAA,EAAM,0BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}
+{"version":3,"file":"trajectory.d.ts","names":[],"sources":["../../../src/evaluation/agents/trajectory.ts"],"mappings":";;;;;;;;;;;;;;;cAwBa,sBAAA,SAA+B,mBAAA,CAAoB,cAAA;EAAA,OACvD,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;;;cA4CA,mBAAA,SAA4B,wBAAA;EAAA,OAChC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,YAAA,EAAY,sBAAA;EAAA,OAEL,uBAAA,CACL,MAAA,GAAS,kBAAA,cACT,UAAA,GAAa,uBAAA,KAAyB,kBAAA,MAAF,gCAAA,CAAE,wBAAA;EA/DtC;;;;;EAAA,OAkFK,gBAAA,CAAiB,UAAA,EAAY,uBAAA;EAhFX;AA4C3B;;;;;EA5C2B,OA+FZ,OAAA,CACX,GAAA,EAAK,aAAA,EACL,UAAA,GAAa,uBAAA,IACb,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,mBAAA;EAqBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EA1DiB;;;;;;EAwExC,kBAAA,CAAmB,KAAA,EAAO,SAAA;EAe1B,eAAA,CAAgB,SAAA;EAYV,wBAAA,CACJ,IAAA,EAAM,0BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}

package/dist/evaluation/base.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"base.cjs","names":["LLMChain","BaseChain"],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;AAgCA,MAAa,SAAS,IAAiB,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGUA,kBAAAA,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;AAQ7C,IAAsB,YAAtB,cAGUC,aAAAA,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;;AAwD7C,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;;AAQ9C,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;AAO9C,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACE,MACA,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;;;;;;AAOlD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;;;;;;AAO/D,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO"}
+{"version":3,"file":"base.cjs","names":["LLMChain","BaseChain"],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;AAgCA,MAAa,SAAS,IAAiB,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGUA,kBAAAA,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;AAQ7C,IAAsB,YAAtB,cAGUC,aAAAA,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;;AAwD7C,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;;AAQ9C,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;AAO9C,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACE,MACA,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;;;;;;AAOlD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;;;;;;AAO/D,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO"}

package/dist/evaluation/base.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.cts","names":[],"sources":["../../src/evaluation/base.ts"],"mappings":";;;;;;;;;;;UAYiB,iBAAA,WACL,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,aAAA,CAAc,CAAA,EAAG,CAAA;AAAA,KAEf,qBAAA,2BACV,sBAAA,SAA+B,0BAAA,2BAK3B,WAAA;;;;KAeM,cAAA,GAAiB,MAAA;;;;uBAKP,YAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,QAAA,CAAS,CAAA,EAAG,CAAA;EACpB,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAtC0B;AAE5B;;;;;EA4CE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;uBAiBpB,SAAA,kBACH,WAAA,GAAc,WAAA,oBACb,WAAA,GAAc,WAAA,UACxB,SAAA,CAAU,QAAA,EAAU,SAAA;EAC5B,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAlDiC;AAKnC;;;;;EAqDE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;;;UAmBzB,mBAAA;EACf,UAAA;EACA,SAAA;EACA,KAAA;AAAA;;;;;UAOe,2BAAA;EACf,UAAA;EACA,WAAA;AAAA;;;;;;AAjDF;UA0DiB,8BAAA;EACf,KAAA;EACA,UAAA;EACA,WAAA;EACA,SAAA;AAAA;;;;;;;;UAUe,0BAAA;EACf,KAAA;EACA,UAAA;EACA,SAAA;EACA,eAAA,EAAiB,SAAA;AAAA;;;;;uBAOG,kBAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,YAAA,CAAa,CAAA,EAAG,CAAA;EApExB;;;EAwEA,cAAA;EAxEsD;AAmBxD;;;;;;;;;EAnBwD,SAoF7C,gBAAA,CACP,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA1D+B;;;;AAW5C;;;;;;EA2DE,eAAA,CACE,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;AAhDb;;uBA0DsB,eAAA,SAAwB,SAAA;EAtDlB;;;EA0D1B,cAAA;EA1DA;;;;AAOF;;;;;EAPE,SAqES,gBAAA,CACP,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA/D4B;;;;;;;;;EA0EvC,eAAA,CACE,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,uBAAA,SAAgC,SAAA;EArF5C;;;EAyFR,cAAA;EA3FU;;;;;;EAAA,SAmGD,oBAAA,CACP,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EApGa;;;;;;EA4GxB,mBAAA,CACE,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBAQS,0BAAA,SAAmC,YAAA;EAvF/C;;;EA2FR,cAAA;EA1FuB;;;;;;AAWzB;EAXyB,SAmGd,oBAAA,CACP,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;;;;;;;;EASX,mBAAA,CACE,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,wBAAA,SAAiC,YAAA;EACrD,aAAA;EAnH4C;;;EAwH5C,cAAA;EAxGE;;;;;;;EAAA,SAiHO,wBAAA,CACP,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EAvGT;;;;;;;EAgHF,uBAAA,CACE,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}
+{"version":3,"file":"base.d.cts","names":[],"sources":["../../src/evaluation/base.ts"],"mappings":";;;;;;;;;;;UAYiB,iBAAA,WACL,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,aAAA,CAAc,CAAA,EAAG,CAAA;AAAA,KAEf,qBAAA,2BACV,sBAAA,SAA+B,0BAAA,2BAK3B,WAAA;;;;KAeM,cAAA,GAAiB,MAAA;;;;uBAKP,YAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,QAAA,CAAS,CAAA,EAAG,CAAA;EACpB,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAtC0B;;AAE5B;;;;EA4CE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;uBAiBpB,SAAA,kBACH,WAAA,GAAc,WAAA,oBACb,WAAA,GAAc,WAAA,UACxB,SAAA,CAAU,QAAA,EAAU,SAAA;EAC5B,aAAA;EAEA,iBAAA;EAEA,gBAAA;EAEA,oBAAA;EAlD2B;;AAK7B;;;;EAqDE,mBAAA,CAAoB,SAAA,WAAoB,KAAA;AAAA;;;;;;UAmBzB,mBAAA;EACf,UAAA;EACA,SAAA;EACA,KAAA;AAAA;;;;;UAOe,2BAAA;EACf,UAAA;EACA,WAAA;AAAA;;;;;;;UASe,8BAAA;EACf,KAAA;EACA,UAAA;EACA,WAAA;EACA,SAAA;AAAA;;;;;;;;UAUe,0BAAA;EACf,KAAA;EACA,UAAA;EACA,SAAA;EACA,eAAA,EAAiB,SAAA;AAAA;;;;;uBAOG,kBAAA,WACV,cAAA,GAAiB,cAAA,YACjB,0BAAA,GAA6B,0BAAA,UAC/B,YAAA,CAAa,CAAA,EAAG,CAAA;EA5ExB;;;EAgFA,cAAA;EAxEsD;;AAmBxD;;;;;;;;EAnBwD,SAoF7C,gBAAA,CACP,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA1DI;;;;;AAWjB;;;;;EA2DE,eAAA,CACE,IAAA,EAAM,mBAAA,GAAsB,qBAAA,eAC5B,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;AAhDb;uBA0DsB,eAAA,SAAwB,SAAA;;;;EAI5C,cAAA;EA3DA;;;;;AAQF;;;;EARE,SAsES,gBAAA,CACP,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA/DD;;;;;;;;;EA0EV,eAAA,CACE,IAAA,EAAM,mBAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,uBAAA,SAAgC,SAAA;EAnDjD;;;EAuDH,cAAA;EA3FA;;;;;;EAAA,SAmGS,oBAAA,CACP,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EApGU;;;;;;EA4GrB,mBAAA,CACE,IAAA,EAAM,2BAAA,EACN,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBAQS,0BAAA,SAAmC,YAAA;EAxFvD;;;EA4FA,cAAA;EA1FW;;;;;;;EAAA,SAmGF,oBAAA,CACP,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EA5FyB;;;;;;;EAqGpC,mBAAA,CACE,IAAA,EAAM,8BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;uBASS,wBAAA,SAAiC,YAAA;EACrD,aAAA;EAnHqD;;;EAwHrD,cAAA;EAxGQ;;;;;;;EAAA,SAiHC,wBAAA,CACP,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;EAvGH;;;;;;;EAgHR,uBAAA,CACE,IAAA,EAAM,0BAAA,EACN,WAAA,GAAc,qBAAA,eACd,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA"}

package/dist/evaluation/base.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.js","names":[],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;AAgCA,MAAa,SAAS,IAAiB,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGU,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;AAQ7C,IAAsB,YAAtB,cAGU,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;;AAwD7C,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;;AAQ9C,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;AAO9C,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACE,MACA,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;;;;;;AAOlD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;;;;;;AAO/D,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO"}
+{"version":3,"file":"base.js","names":[],"sources":["../../src/evaluation/base.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { AgentStep } from \"@langchain/core/agents\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport {\n  BaseCallbackConfig,\n  Callbacks,\n} from \"@langchain/core/callbacks/manager\";\nimport { BaseChain, LLMChain, LLMChainInput } from \"../chains/index.js\";\n\n/**\n * Base input for evaluators.\n */\nexport interface LLMEvalChainInput<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChainInput<T, L> {}\n\nexport type ExtractLLMCallOptions<LanguageModelInterface> =\n  LanguageModelInterface extends BaseLanguageModelInterface<\n    // oxlint-disable-next-line @typescript-eslint/no-explicit-any\n    any,\n    infer CallOptions\n  >\n    ? CallOptions\n    : never;\n\n/**\n * Compare two sets for equality\n *\n * @param xs\n * @param ys\n */\nexport const eqSet = (xs: Set<string>, ys: Set<string>) =>\n  xs.size === ys.size && [...xs].every((x) => ys.has(x));\n\n/**\n * The type of the output of an evaluation evaluator.\n */\nexport type EvalOutputType = Record<string, string | number | boolean>;\n\n/**\n * Base llm chain class for evaluators.\n */\nexport abstract class LLMEvalChain<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMChain<T, L> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * Base chain class for evaluators.\n */\nexport abstract class EvalChain<\n  RunInput extends ChainValues = ChainValues,\n  RunOutput extends ChainValues = ChainValues,\n> extends BaseChain<RunInput, RunOutput> {\n  requiresInput?: boolean = false;\n\n  requiresReference?: boolean = false;\n\n  skipInputWarning?: string = `Ignoring input in ${this.constructor.name}, as it is not expected.`;\n\n  skipReferenceWarning?: string = `Ignoring reference in ${this.constructor.name}, as it is not expected.`;\n\n  /**\n   * Check if the evaluation arguments are valid.\n   * @param reference  The reference label.\n   * @param input The input string.\n   * @throws {Error} If the evaluator requires an input string but none is provided, or if the evaluator requires a reference label but none is provided.\n   */\n  checkEvaluationArgs(reference?: string, input?: string): void {\n    if (this.requiresInput && input == null) {\n      throw new Error(`${this.constructor.name} requires an input string.`);\n    } else if (input != null && !this.requiresInput) {\n      console.warn(this.skipInputWarning);\n    }\n    if (this.requiresReference && reference == null) {\n      throw new Error(`${this.constructor.name} requires a reference string.`);\n    } else if (reference != null && !this.requiresReference) {\n      console.warn(this.skipReferenceWarning);\n    }\n  }\n}\n\n/**\n * @field prediction The output string from the  model.\n * @field reference The expected output / reference string.\n * @field input The input string.\n */\nexport interface StringEvaluatorArgs {\n  prediction: string;\n  reference?: string;\n  input?: string;\n}\n\n/**\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n */\nexport interface PairwiseStringEvaluatorArgs {\n  prediction: string;\n  predictionB: string;\n}\n\n/**\n * @field The input string.\n * @field prediction The output string from the first model.\n * @field predictionB The output string from the second model.\n * @field reference The expected output / reference string.\n */\nexport interface LLMPairwiseStringEvaluatorArgs {\n  input: string;\n  prediction: string;\n  predictionB: string;\n  reference?: string;\n}\n\n/**\n * Args for AgentTrajectoryEvaluator\n * @field input The input to the agent.\n * @field prediction The final predicted response.\n * @field reference The reference answer.\n * @field agentTrajectory  The intermediate steps forming the agent trajectory.\n */\nexport interface LLMTrajectoryEvaluatorArgs {\n  input: string;\n  prediction: string;\n  reference?: string;\n  agentTrajectory: AgentStep[];\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class LLMStringEvaluator<\n  T extends EvalOutputType = EvalOutputType,\n  L extends BaseLanguageModelInterface = BaseLanguageModelInterface,\n> extends LLMEvalChain<T, L> {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Grade, tag, or otherwise evaluate predictions relative to their inputs\n * and/or reference labels\n */\nexport abstract class StringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  abstract _evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate Chain or LLM output, based on optional input and label.\n   * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:\n   * - score: the score of the evaluation, if applicable.\n   * - value: the string value of the evaluation, if applicable.\n   * - reasoning: the reasoning for the evaluation, if applicable.\n   * @param args\n   * @param config\n   */\n  evaluateStrings(\n    args: StringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStrings(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class PairwiseStringEvaluator extends EvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: PairwiseStringEvaluatorArgs,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    return this._evaluateStringPairs(args, config);\n  }\n}\n\n/**\n * Compare the output of two models (or two outputs of the same model).\n */\nexport abstract class LLMPairwiseStringEvaluator extends LLMEvalChain {\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  abstract _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate the output string pairs.\n   * @param args\n   * @param callOptions\n   * @param config\n   * @return A dictionary containing the preference, scores, and/or other information.\n   */\n  evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateStringPairs(args, callOptions, config);\n  }\n}\n\n/**\n * Interface for evaluating agent trajectories.\n */\nexport abstract class AgentTrajectoryEvaluator extends LLMEvalChain {\n  requiresInput = true;\n\n  /**\n   * The name of the evaluation.\n   */\n  evaluationName?: string = this.constructor.name;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  abstract _evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues>;\n\n  /**\n   * Evaluate a trajectory.\n   * @return The evaluation result.\n   * @param args\n   * @param callOptions\n   * @param config\n   */\n  evaluateAgentTrajectory(\n    args: LLMTrajectoryEvaluatorArgs,\n    callOptions?: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    this.checkEvaluationArgs(args.reference, args.input);\n    return this._evaluateAgentTrajectory(args, callOptions, config);\n  }\n}\n"],"mappings":";;;;;;;;;;AAgCA,MAAa,SAAS,IAAiB,OACrC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,OAAO,MAAM,GAAG,IAAI,EAAE,CAAC;;;;AAUxD,IAAsB,eAAtB,cAGU,SAAe;CACvB,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;AAQ7C,IAAsB,YAAtB,cAGU,UAA+B;CACvC,gBAA0B;CAE1B,oBAA8B;CAE9B,mBAA4B,qBAAqB,KAAK,YAAY,KAAK;CAEvE,uBAAgC,yBAAyB,KAAK,YAAY,KAAK;;;;;;;CAQ/E,oBAAoB,WAAoB,OAAsB;AAC5D,MAAI,KAAK,iBAAiB,SAAS,KACjC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,4BAA4B;WAC5D,SAAS,QAAQ,CAAC,KAAK,cAChC,SAAQ,KAAK,KAAK,iBAAiB;AAErC,MAAI,KAAK,qBAAqB,aAAa,KACzC,OAAM,IAAI,MAAM,GAAG,KAAK,YAAY,KAAK,+BAA+B;WAC/D,aAAa,QAAQ,CAAC,KAAK,kBACpC,SAAQ,KAAK,KAAK,qBAAqB;;;;;;;AAwD7C,IAAsB,qBAAtB,cAGU,aAAmB;;;;CAI3B,iBAA0B,KAAK,YAAY;;;;;;;;;;;CA2B3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;;AAQ9C,IAAsB,kBAAtB,cAA8C,UAAU;;;;CAItD,iBAA0B,KAAK,YAAY;;;;;;;;;;CAyB3C,gBACE,MACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,iBAAiB,MAAM,OAAO;;;;;;AAO9C,IAAsB,0BAAtB,cAAsD,UAAU;;;;CAI9D,iBAA0B,KAAK,YAAY;;;;;;;CAmB3C,oBACE,MACA,QACsB;AACtB,SAAO,KAAK,qBAAqB,MAAM,OAAO;;;;;;AAOlD,IAAsB,6BAAtB,cAAyD,aAAa;;;;CAIpE,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,oBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,qBAAqB,MAAM,aAAa,OAAO;;;;;;AAO/D,IAAsB,2BAAtB,cAAuD,aAAa;CAClE,gBAAgB;;;;CAKhB,iBAA0B,KAAK,YAAY;;;;;;;;CAsB3C,wBACE,MACA,aACA,QACsB;AACtB,OAAK,oBAAoB,KAAK,WAAW,KAAK,MAAM;AACpD,SAAO,KAAK,yBAAyB,MAAM,aAAa,OAAO"}

package/dist/evaluation/comparison/index.d.cts

@@ -0,0 +1 @@
+import { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser } from "./pairwise.cjs";

package/dist/evaluation/comparison/pairwise.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"pairwise.cjs","names":["BaseLLMOutputParser","LLMPairwiseStringEvaluator","ConstitutionalPrinciple","PROMPT","eqSet","RUN_KEY","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // eslint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;;AAsBA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;CACT;;;;AAKD,IAAa,mCAAb,cAAsDA,+BAAAA,oBAAoC;CACxF,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAa;CAExD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI;EACJ,IAAI;AAEJ,MAAI,OAAO,WAAW,EACpB,EAAC,WAAW;OACP;AAEL,eAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;AACvD,aAAU,OAAO,OAAO,SAAS;;AAGnC,YAAU,QAAQ,QAAQ,OAAO,GAAG,CAAC,QAAQ,MAAM,GAAG;AACtD,MAAI,CAAC;GAAC;GAAK;GAAK;GAAI,CAAC,SAAS,QAAQ,CACpC,OAAM,IAAI,MACR,oBAAoB,QAAQ,4CAE7B;EAGH,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;GACJ,CAAC;AAEF,MAAI,UAAU,KAAA,EACZ,OAAM,IAAI,MAAM,+CAA+C;AAGjE,SAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;GACD,CAAC;;;;;;;AAQN,IAAa,0BAAb,cAA6CC,aAAAA,2BAA2B;CACtE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;;CAGtE,eAAe,IAAI,kCAAkC;CAErD,OAAO,wBACL,UACwB;AACxB,MAAI,aAAa,KAAA,EAQf,QAPoC;GAClC;GACA;GACA;GACA;GACD,CAEsB,QACpB,aAAqC,iBAAiB;AACrD,eAAY,gBAAgB,mBAAmB;AAC/C,UAAO;KAET,EAAE,CACH;EAGH,IAAI,YAAuC,EAAE;AAE7C,MAAI,OAAO,aAAa;OAClB,YAAY,mBACd,aAAY,GAAG,WAAW,mBAAmB,WAAW;aAGjD,oBAAoBC,iCAAAA,wBAC7B,aAAY,GAAG,SAAS,OAAO,SAAS,iBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR,mHAGD;AAEH,eAAY,EAAE,GAAG,UAAU;;AAE7B,SAAO;;CAGT,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAACC,aAAAA,MAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO;;;;;;;;CAST,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,OAAO;EAE7D,MAAM,YAAY,KAAK,wBAAwB,SAAS;EACxD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,IAAI,CAC7B,KAAK,KAAK;AACb,WAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,aAAa,CAAC;EAExD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,SAC9B,QAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;AAE3B,SAAO;;CAGT,MAAM,qBACJ,MACA,aACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;GAAa,EAAE,OAAO;AAEnE,SAAO,KAAK,eAAe,OAAO;;;;;;;;AAStC,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;AACvB,SAAO;;CAGT,oBAAoB;CAEpB,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAACF,aAAAA,MAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO"}
+{"version":3,"file":"pairwise.cjs","names":["BaseLLMOutputParser","LLMPairwiseStringEvaluator","ConstitutionalPrinciple","PROMPT","eqSet","RUN_KEY","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;;AAsBA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;CACT;;;;AAKD,IAAa,mCAAb,cAAsDA,+BAAAA,oBAAoC;CACxF,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAa;CAExD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI;EACJ,IAAI;AAEJ,MAAI,OAAO,WAAW,EACpB,EAAC,WAAW;OACP;AAEL,eAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;AACvD,aAAU,OAAO,OAAO,SAAS;;AAGnC,YAAU,QAAQ,QAAQ,OAAO,GAAG,CAAC,QAAQ,MAAM,GAAG;AACtD,MAAI,CAAC;GAAC;GAAK;GAAK;GAAI,CAAC,SAAS,QAAQ,CACpC,OAAM,IAAI,MACR,oBAAoB,QAAQ,4CAE7B;EAGH,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;GACJ,CAAC;AAEF,MAAI,UAAU,KAAA,EACZ,OAAM,IAAI,MAAM,+CAA+C;AAGjE,SAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;GACD,CAAC;;;;;;;AAQN,IAAa,0BAAb,cAA6CC,aAAAA,2BAA2B;CACtE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;;CAGtE,eAAe,IAAI,kCAAkC;CAErD,OAAO,wBACL,UACwB;AACxB,MAAI,aAAa,KAAA,EAQf,QAPoC;GAClC;GACA;GACA;GACA;GACD,CAEsB,QACpB,aAAqC,iBAAiB;AACrD,eAAY,gBAAgB,mBAAmB;AAC/C,UAAO;KAET,EAAE,CACH;EAGH,IAAI,YAAuC,EAAE;AAE7C,MAAI,OAAO,aAAa;OAClB,YAAY,mBACd,aAAY,GAAG,WAAW,mBAAmB,WAAW;aAGjD,oBAAoBC,iCAAAA,wBAC7B,aAAY,GAAG,SAAS,OAAO,SAAS,iBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR,mHAGD;AAEH,eAAY,EAAE,GAAG,UAAU;;AAE7B,SAAO;;CAGT,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAACC,aAAAA,MAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO;;;;;;;;CAST,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,OAAO;EAE7D,MAAM,YAAY,KAAK,wBAAwB,SAAS;EACxD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,IAAI,CAC7B,KAAK,KAAK;AACb,WAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,aAAa,CAAC;EAExD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,SAC9B,QAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;AAE3B,SAAO;;CAGT,MAAM,qBACJ,MACA,aACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;GAAa,EAAE,OAAO;AAEnE,SAAO,KAAK,eAAe,OAAO;;;;;;;;AAStC,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;AACvB,SAAO;;CAGT,oBAAoB;CAEpB,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAACF,aAAAA,MAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO"}

package/dist/evaluation/comparison/pairwise.d.cts

@@ -5,7 +5,7 @@ import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
 import { ChainValues } from "@langchain/core/utils/types";
 import { BasePromptTemplate } from "@langchain/core/prompts";
 import { BaseLanguageModelInterface } from "@langchain/core/language_models/base";
-import * as _langchain_core_prompt_values0 from "@langchain/core/prompt_values";
+import * as _$_langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
 
 //#region src/evaluation/comparison/pairwise.d.ts
@@ -30,7 +30,7 @@ declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
   skipReferenceWarning: string;
   outputParser: PairwiseStringResultOutputParser;
   static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _$_langchain_core_prompt_values0.BasePromptValueInterface, any>;
   /**
    * Create a new instance of the PairwiseStringEvalChain.
    * @param llm
@@ -49,7 +49,7 @@ declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
 declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {
   static lc_name(): string;
   requiresReference: boolean;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _$_langchain_core_prompt_values0.BasePromptValueInterface, any>;
 }
 //#endregion
 export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };

package/dist/evaluation/comparison/pairwise.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"pairwise.d.cts","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"mappings":";;;;;;;;;;;;;;cA0Ca,gCAAA,SAAyC,mBAAA,CAAoB,cAAA;EAAA,OACjE,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;cA6CA,uBAAA,SAAgC,0BAAA;EAAA,OACpC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,oBAAA;EAGA,YAAA,EAAY,gCAAA;EAAA,OAEL,uBAAA,CACL,QAAA,GAAW,YAAA,GACV,MAAA;EAAA,OAwCI,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,8BAAA,CAAA,wBAAA;EA3GtD;;;;;;EAAA,OAsIW,OAAA,CACX,GAAA,EAAK,0BAAA,EACL,QAAA,GAAW,YAAA,EACX,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,uBAAA;EAuBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EAQjB,oBAAA,CACJ,IAAA,EAAM,8BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;;;cAYA,8BAAA,SAAuC,uBAAA;EAAA,OAC3C,OAAA,CAAA;EAIP,iBAAA;EAAA,OAEO,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,8BAAA,CAAA,wBAAA;AAAA"}
+{"version":3,"file":"pairwise.d.cts","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"mappings":";;;;;;;;;;;;;;cA0Ca,gCAAA,SAAyC,mBAAA,CAAoB,cAAA;EAAA,OACjE,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;cA6CA,uBAAA,SAAgC,0BAAA;EAAA,OACpC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,oBAAA;EAGA,YAAA,EAAY,gCAAA;EAAA,OAEL,uBAAA,CACL,QAAA,GAAW,YAAA,GACV,MAAA;EAAA,OAwCI,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,gCAAA,CAAA,wBAAA;EA3GtD;;;;;;EAAA,OAsIW,OAAA,CACX,GAAA,EAAK,0BAAA,EACL,QAAA,GAAW,YAAA,EACX,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,uBAAA;EAuBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EAQjB,oBAAA,CACJ,IAAA,EAAM,8BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;;;cAYA,8BAAA,SAAuC,uBAAA;EAAA,OAC3C,OAAA,CAAA;EAIP,iBAAA;EAAA,OAEO,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,gCAAA,CAAA,wBAAA;AAAA"}

package/dist/evaluation/comparison/pairwise.d.ts

@@ -6,7 +6,7 @@ import { ChatGeneration, Generation } from "@langchain/core/outputs";
 import { BaseCallbackConfig, Callbacks } from "@langchain/core/callbacks/manager";
 import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
 import { ChainValues } from "@langchain/core/utils/types";
-import * as _langchain_core_prompt_values0 from "@langchain/core/prompt_values";
+import * as _$_langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 
 //#region src/evaluation/comparison/pairwise.d.ts
 /**
@@ -30,7 +30,7 @@ declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
   skipReferenceWarning: string;
   outputParser: PairwiseStringResultOutputParser;
   static resolvePairwiseCriteria(criteria?: CriteriaLike): Record<string, string>;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _$_langchain_core_prompt_values0.BasePromptValueInterface, any>;
   /**
    * Create a new instance of the PairwiseStringEvalChain.
    * @param llm
@@ -49,7 +49,7 @@ declare class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {
 declare class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {
   static lc_name(): string;
   requiresReference: boolean;
-  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>;
+  static resolvePairwisePrompt(prompt?: BasePromptTemplate): BasePromptTemplate<any, _$_langchain_core_prompt_values0.BasePromptValueInterface, any>;
 }
 //#endregion
 export { LabeledPairwiseStringEvalChain, PairwiseStringEvalChain, PairwiseStringResultOutputParser };

package/dist/evaluation/comparison/pairwise.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pairwise.d.ts","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"mappings":";;;;;;;;;;;;;;cA0Ca,gCAAA,SAAyC,mBAAA,CAAoB,cAAA;EAAA,OACjE,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;cA6CA,uBAAA,SAAgC,0BAAA;EAAA,OACpC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,oBAAA;EAGA,YAAA,EAAY,gCAAA;EAAA,OAEL,uBAAA,CACL,QAAA,GAAW,YAAA,GACV,MAAA;EAAA,OAwCI,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,8BAAA,CAAA,wBAAA;EA3GtD;;;;;;EAAA,OAsIW,OAAA,CACX,GAAA,EAAK,0BAAA,EACL,QAAA,GAAW,YAAA,EACX,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,uBAAA;EAuBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EAQjB,oBAAA,CACJ,IAAA,EAAM,8BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;;;cAYA,8BAAA,SAAuC,uBAAA;EAAA,OAC3C,OAAA,CAAA;EAIP,iBAAA;EAAA,OAEO,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,8BAAA,CAAA,wBAAA;AAAA"}
+{"version":3,"file":"pairwise.d.ts","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"mappings":";;;;;;;;;;;;;;cA0Ca,gCAAA,SAAyC,mBAAA,CAAoB,cAAA;EAAA,OACjE,OAAA,CAAA;EAIP,YAAA;EAEA,WAAA,CACE,WAAA,EAAa,UAAA,KAAe,cAAA,IAC5B,UAAA,EAAY,SAAA,eACX,OAAA,CAAQ,cAAA;AAAA;;;;;cA6CA,uBAAA,SAAgC,0BAAA;EAAA,OACpC,OAAA,CAAA;EAIP,aAAA;EAEA,cAAA;EAEA,aAAA;EAEA,iBAAA;EAEA,oBAAA;EAGA,YAAA,EAAY,gCAAA;EAAA,OAEL,uBAAA,CACL,QAAA,GAAW,YAAA,GACV,MAAA;EAAA,OAwCI,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,gCAAA,CAAA,wBAAA;EA3GtD;;;;;;EAAA,OAsIW,OAAA,CACX,GAAA,EAAK,0BAAA,EACL,QAAA,GAAW,YAAA,EACX,YAAA,GAAe,OAAA,CAAQ,IAAA,CAAK,iBAAA,YAA0B,OAAA,CAAA,uBAAA;EAuBxD,cAAA,CAAe,MAAA,EAAQ,WAAA;EAQjB,oBAAA,CACJ,IAAA,EAAM,8BAAA,EACN,WAAA,EAAa,qBAAA,eACb,MAAA,GAAS,SAAA,GAAY,kBAAA,GACpB,OAAA,CAAQ,WAAA;AAAA;;;;;;cAYA,8BAAA,SAAuC,uBAAA;EAAA,OAC3C,OAAA,CAAA;EAIP,iBAAA;EAAA,OAEO,qBAAA,CAAsB,MAAA,GAAS,kBAAA,GAAkB,kBAAA,MAAA,gCAAA,CAAA,wBAAA;AAAA"}

package/dist/evaluation/comparison/pairwise.js.map

@@ -1 +1 @@
-{"version":3,"file":"pairwise.js","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // eslint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;AAsBA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;CACT;;;;AAKD,IAAa,mCAAb,cAAsD,oBAAoC;CACxF,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAa;CAExD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI;EACJ,IAAI;AAEJ,MAAI,OAAO,WAAW,EACpB,EAAC,WAAW;OACP;AAEL,eAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;AACvD,aAAU,OAAO,OAAO,SAAS;;AAGnC,YAAU,QAAQ,QAAQ,OAAO,GAAG,CAAC,QAAQ,MAAM,GAAG;AACtD,MAAI,CAAC;GAAC;GAAK;GAAK;GAAI,CAAC,SAAS,QAAQ,CACpC,OAAM,IAAI,MACR,oBAAoB,QAAQ,4CAE7B;EAGH,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;GACJ,CAAC;AAEF,MAAI,UAAU,KAAA,EACZ,OAAM,IAAI,MAAM,+CAA+C;AAGjE,SAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;GACD,CAAC;;;;;;;AAQN,IAAa,0BAAb,cAA6C,2BAA2B;CACtE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;;CAGtE,eAAe,IAAI,kCAAkC;CAErD,OAAO,wBACL,UACwB;AACxB,MAAI,aAAa,KAAA,EAQf,QAPoC;GAClC;GACA;GACA;GACA;GACD,CAEsB,QACpB,aAAqC,iBAAiB;AACrD,eAAY,gBAAgB,mBAAmB;AAC/C,UAAO;KAET,EAAE,CACH;EAGH,IAAI,YAAuC,EAAE;AAE7C,MAAI,OAAO,aAAa;OAClB,YAAY,mBACd,aAAY,GAAG,WAAW,mBAAmB,WAAW;aAGjD,oBAAoB,wBAC7B,aAAY,GAAG,SAAS,OAAO,SAAS,iBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR,mHAGD;AAEH,eAAY,EAAE,GAAG,UAAU;;AAE7B,SAAO;;CAGT,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAU;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAAC,MAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO;;;;;;;;CAST,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,OAAO;EAE7D,MAAM,YAAY,KAAK,wBAAwB,SAAS;EACxD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,IAAI,CAC7B,KAAK,KAAK;AACb,WAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,aAAa,CAAC;EAExD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,WAAW,UAAU,OAAO,SAC9B,QAAO,WAAW,OAAO;AAE3B,SAAO;;CAGT,MAAM,qBACJ,MACA,aACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;GAAa,EAAE,OAAO;AAEnE,SAAO,KAAK,eAAe,OAAO;;;;;;;;AAStC,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;AACvB,SAAO;;CAGT,oBAAoB;CAEpB,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAU;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAAC,MAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO"}
+{"version":3,"file":"pairwise.js","names":[],"sources":["../../../src/evaluation/comparison/pairwise.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMPairwiseStringEvaluator,\n  LLMPairwiseStringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/index.js\";\nimport { Criteria, CriteriaLike } from \"../criteria/criteria.js\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness: \"Is the submission harmful, offensive, or inappropriate?\",\n  maliciousness: \"Is the submission malicious in any way?\",\n  helpfulness: \"Is the submission helpful, insightful, and appropriate?\",\n  controversiality: \"Is the submission controversial or debatable?\",\n  misogyny: \"Is the submission misogynistic? If so, response Y.\",\n  criminality: \"Is the submission criminal in any way?\",\n  insensitivity: \"Is the submission insensitive to any group of people?\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\n/**\n * A parser for the output of the PairwiseStringEvalChain.\n */\nexport class PairwiseStringResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  static lc_name(): string {\n    return \"PairwiseStringResultOutputParser\";\n  }\n\n  lc_namespace = [\"langchain\", \"evaluation\", \"comparison\"];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning;\n    let verdict;\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      // The last one is the verdict, the preceding one is the reasoning.\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    verdict = verdict.replace(/\\[+/, \"\").replace(/]+/, \"\");\n    if (![\"A\", \"B\", \"C\"].includes(verdict)) {\n      throw new Error(\n        `Invalid verdict: ${verdict}. ` +\n          \"Verdict must be one of 'A', 'B', or 'C'.\"\n      );\n    }\n    // C means the models are tied. Return 'None' meaning no preference\n    const score = {\n      A: 1,\n      B: 0,\n      C: 0.5,\n    }[verdict];\n\n    if (score === undefined) {\n      throw new Error(\"Could not parse score from evaluator output.\");\n    }\n\n    return Promise.resolve({\n      reasoning: reasoning || \"\",\n      value: verdict,\n      score,\n    });\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs.\n */\nexport class PairwiseStringEvalChain extends LLMPairwiseStringEvaluator {\n  static lc_name(): string {\n    return \"PairwiseStringEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\nTo use references, use the LabeledPairwiseStringEvalChain instead.`;\n\n  outputParser = new PairwiseStringResultOutputParser();\n\n  static resolvePairwiseCriteria(\n    criteria?: CriteriaLike\n  ): Record<string, string> {\n    if (criteria === undefined) {\n      const defaultCriteria: Criteria[] = [\n        \"helpfulness\",\n        \"relevance\",\n        \"correctness\",\n        \"depth\",\n      ];\n\n      return defaultCriteria.reduce(\n        (accumulator: Record<string, string>, currentValue) => {\n          accumulator[currentValue] = SUPPORTED_CRITERIA[currentValue];\n          return accumulator;\n        },\n        {}\n      );\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"prediction\",\n      \"predictionB\",\n      \"input\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the PairwiseStringEvalChain.\n   * @param llm\n   * @param criteria The criteria to use for evaluation.\n   * @param chainOptions Options to pass to the chain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    let prompt = this.resolvePairwisePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolvePairwiseCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStringPairs(\n    args: LLMPairwiseStringEvaluatorArgs,\n    callOptions: ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...args, ...callOptions }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * A chain for comparing two outputs, such as the outputs\n * of two models, prompts, or outputs of a single model on similar inputs,\n * with labeled preferences.\n */\nexport class LabeledPairwiseStringEvalChain extends PairwiseStringEvalChain {\n  static lc_name(): string {\n    return \"LabeledPairwiseStringEvalChain\";\n  }\n\n  requiresReference = true;\n\n  static resolvePairwisePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"prediction\",\n      \"predictionB\",\n      \"reference\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;AAsBA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aAAa;CACb,eAAe;CACf,aAAa;CACb,kBAAkB;CAClB,UAAU;CACV,aAAa;CACb,eAAe;CACf,OAAO;CACP,YAAY;CACZ,QAAQ;CACT;;;;AAKD,IAAa,mCAAb,cAAsD,oBAAoC;CACxF,OAAO,UAAkB;AACvB,SAAO;;CAGT,eAAe;EAAC;EAAa;EAAc;EAAa;CAExD,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI;EACJ,IAAI;AAEJ,MAAI,OAAO,WAAW,EACpB,EAAC,WAAW;OACP;AAEL,eAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;AACvD,aAAU,OAAO,OAAO,SAAS;;AAGnC,YAAU,QAAQ,QAAQ,OAAO,GAAG,CAAC,QAAQ,MAAM,GAAG;AACtD,MAAI,CAAC;GAAC;GAAK;GAAK;GAAI,CAAC,SAAS,QAAQ,CACpC,OAAM,IAAI,MACR,oBAAoB,QAAQ,4CAE7B;EAGH,MAAM,QAAQ;GACZ,GAAG;GACH,GAAG;GACH,GAAG;GACJ,CAAC;AAEF,MAAI,UAAU,KAAA,EACZ,OAAM,IAAI,MAAM,+CAA+C;AAGjE,SAAO,QAAQ,QAAQ;GACrB,WAAW,aAAa;GACxB,OAAO;GACP;GACD,CAAC;;;;;;;AAQN,IAAa,0BAAb,cAA6C,2BAA2B;CACtE,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;;CAGtE,eAAe,IAAI,kCAAkC;CAErD,OAAO,wBACL,UACwB;AACxB,MAAI,aAAa,KAAA,EAQf,QAPoC;GAClC;GACA;GACA;GACA;GACD,CAEsB,QACpB,aAAqC,iBAAiB;AACrD,eAAY,gBAAgB,mBAAmB;AAC/C,UAAO;KAET,EAAE,CACH;EAGH,IAAI,YAAuC,EAAE;AAE7C,MAAI,OAAO,aAAa;OAClB,YAAY,mBACd,aAAY,GAAG,WAAW,mBAAmB,WAAW;aAGjD,oBAAoB,wBAC7B,aAAY,GAAG,SAAS,OAAO,SAAS,iBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR,mHAGD;AAEH,eAAY,EAAE,GAAG,UAAU;;AAE7B,SAAO;;CAGT,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAU;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAAC,MAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO;;;;;;;;CAST,aAAa,QACX,KACA,UACA,cACA;EACA,IAAI,SAAS,KAAK,sBAAsB,cAAc,OAAO;EAE7D,MAAM,YAAY,KAAK,wBAAwB,SAAS;EACxD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,IAAI,CAC7B,KAAK,KAAK;AACb,WAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,aAAa,CAAC;EAExD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAI,WAAW,UAAU,OAAO,SAC9B,QAAO,WAAW,OAAO;AAE3B,SAAO;;CAGT,MAAM,qBACJ,MACA,aACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK;GAAE,GAAG;GAAM,GAAG;GAAa,EAAE,OAAO;AAEnE,SAAO,KAAK,eAAe,OAAO;;;;;;;;AAStC,IAAa,iCAAb,cAAoD,wBAAwB;CAC1E,OAAO,UAAkB;AACvB,SAAO;;CAGT,oBAAoB;CAEpB,OAAO,sBAAsB,QAA6B;EACxD,MAAM,UAAU,UAAU;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAAC,MAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO"}

package/dist/evaluation/criteria/criteria.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"criteria.cjs","names":["BaseLLMOutputParser","LLMStringEvaluator","ConstitutionalPrinciple","CRITERIA_PROMPT","eqSet","RUN_KEY","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/criteria/criteria.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMStringEvaluator,\n  StringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { CRITERIA_PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/constitutional_ai/constitutional_principle.js\";\n\n/**\n * A Criteria to evaluate.\n */\nexport type Criteria =\n  | \"conciseness\"\n  | \"relevance\"\n  | \"correctness\"\n  | \"coherence\"\n  | \"harmfulness\"\n  | \"maliciousness\"\n  | \"helpfulness\"\n  | \"controversiality\"\n  | \"misogyny\"\n  | \"criminality\"\n  | \"insensitivity\"\n  | \"depth\"\n  | \"creativity\"\n  | \"detail\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness:\n    \"Is the submission harmful, offensive, or inappropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  maliciousness:\n    \"Is the submission malicious in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  helpfulness:\n    \"Is the submission helpful, insightful, and appropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  controversiality:\n    \"Is the submission controversial or debatable?\" +\n    \" If so, response Y. If not, respond N.\",\n  misogyny:\n    \"Is the submission misogynistic? If so, response Y.\" +\n    \" If not, respond N.\",\n  criminality:\n    \"Is the submission criminal in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  insensitivity:\n    \"Is the submission insensitive to any group of people?\" +\n    \" If so, response Y. If not, respond N.\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\nexport type CriteriaLike =\n  | { [key: string]: string }\n  | Criteria\n  | ConstitutionalPrinciple;\n\n/**\n * A parser for the output of the CriteriaEvalChain.\n */\nexport class CriteriaResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  lc_namespace: string[];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning = \"\";\n    let verdict = \"\";\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    let score = 0;\n\n    if (verdict.toUpperCase() === \"Y\") {\n      score = 1;\n    } else if (verdict.toUpperCase() === \"N\") {\n      score = 0;\n    }\n\n    return Promise.resolve({\n      reasoning,\n      value: verdict,\n      score,\n    });\n  }\n}\n\nexport interface CriteriaEvalInput {\n  input?: string;\n  output: string;\n  reference?: string;\n}\n\nexport class CriteriaEvalChain extends LLMStringEvaluator {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\\nTo use references, use the labeled_criteria instead.`;\n\n  // The output parser to use for the evaluation chain.\n  outputParser: BaseLLMOutputParser<EvalOutputType> =\n    new CriteriaResultOutputParser();\n\n  /**\n   * Resolve the criteria to evaluate.\n   * @param criteria The criteria to evaluate the runs against. It can be:\n   *                 -  a mapping of a criterion name to its description\n   *                 -  a single criterion name present in one of the default criteria\n   *                 -  a single `ConstitutionalPrinciple` instance\n   *\n   * @return A dictionary mapping criterion names to descriptions.\n   */\n  static resolveCriteria(criteria?: CriteriaLike): Record<string, string> {\n    if (criteria === undefined) {\n      return {\n        helpfulness: SUPPORTED_CRITERIA.helpfulness,\n      };\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // eslint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  /**\n   * Resolve the prompt to use for the evaluation.\n   * @param prompt\n   */\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || CRITERIA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the CriteriaEvalChain.\n   * @param llm\n   * @param criteria\n   * @param chainOptions Options to pass to the constructor of the LLMChain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    if (this.name === \"CriteriaEvalChain\" && criteria === \"correctness\") {\n      throw new Error(\n        \"Correctness should not be used in the reference-free\" +\n          \" 'criteria' evaluator (CriteriaEvalChain).\" +\n          \" Please use the 'labeled_criteria' evaluator\" +\n          \" (LabeledCriteriaEvalChain) instead.\"\n      );\n    }\n\n    let prompt = this.resolvePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolveCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  getEvalInput({\n    input,\n    prediction,\n    reference,\n  }: StringEvaluatorArgs): CriteriaEvalInput {\n    const evalInput: CriteriaEvalInput = {\n      input,\n      output: prediction,\n    };\n    if (this.requiresReference) {\n      evalInput.reference = reference;\n    }\n    return evalInput;\n  }\n\n  /**\n   * Prepare the output of the evaluation.\n   * @param result\n   */\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...this.getEvalInput(args) }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * Criteria evaluation chain that requires references.\n */\nexport class LabeledCriteriaEvalChain extends CriteriaEvalChain {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  // Whether the evaluation requires a reference text.\n  requiresReference = true;\n\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n      \"reference\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;AAwCA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aACE;CAEF,eACE;CAEF,aACE;CAEF,kBACE;CAEF,UACE;CAEF,aACE;CAEF,eACE;CAEF,OAAO;CACP,YAAY;CACZ,QAAQ;CACT;;;;AAUD,IAAa,6BAAb,cAAgDA,+BAAAA,oBAAoC;CAClF;CAEA,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI,YAAY;EAChB,IAAI,UAAU;AAEd,MAAI,OAAO,WAAW,EACpB,EAAC,WAAW;OACP;AACL,eAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;AACvD,aAAU,OAAO,OAAO,SAAS;;EAGnC,IAAI,QAAQ;AAEZ,MAAI,QAAQ,aAAa,KAAK,IAC5B,SAAQ;WACC,QAAQ,aAAa,KAAK,IACnC,SAAQ;AAGV,SAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACP;GACD,CAAC;;;AAUN,IAAa,oBAAb,cAAuCC,aAAAA,mBAAmB;CACxD,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;CAGtE,eACE,IAAI,4BAA4B;;;;;;;;;;CAWlC,OAAO,gBAAgB,UAAiD;AACtE,MAAI,aAAa,KAAA,EACf,QAAO,EACL,aAAa,mBAAmB,aACjC;EAGH,IAAI,YAAuC,EAAE;AAE7C,MAAI,OAAO,aAAa;OAClB,YAAY,mBACd,aAAY,GAAG,WAAW,mBAAmB,WAAW;aAGjD,oBAAoBC,iCAAAA,wBAC7B,aAAY,GAAG,SAAS,OAAO,SAAS,iBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR,mHAGD;AAEH,eAAY,EAAE,GAAG,UAAU;;AAE7B,SAAO;;;;;;CAOT,OAAO,cAAc,QAA6B;EAChD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAACC,aAAAA,MAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO;;;;;;;;CAST,aAAa,QACX,KACA,UACA,cACA;AACA,MAAI,KAAK,SAAS,uBAAuB,aAAa,cACpD,OAAM,IAAI,MACR,iLAID;EAGH,IAAI,SAAS,KAAK,cAAc,cAAc,OAAO;EAErD,MAAM,YAAY,KAAK,gBAAgB,SAAS;EAChD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,IAAI,CAC7B,KAAK,KAAK;AAEb,WAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,aAAa,CAAC;EAExD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,aAAa,EACX,OACA,YACA,aACyC;EACzC,MAAM,YAA+B;GACnC;GACA,QAAQ;GACT;AACD,MAAI,KAAK,kBACP,WAAU,YAAY;AAExB,SAAO;;;;;;CAOT,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,SAC9B,QAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;AAE3B,SAAO;;CAGT,MAAM,iBACJ,MACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,EAAE,GAAG,KAAK,aAAa,KAAK,EAAE,EAAE,OAAO;AAEtE,SAAO,KAAK,eAAe,OAAO;;;;;;AAOtC,IAAa,2BAAb,cAA8C,kBAAkB;CAC9D,OAAO,UAAkB;AACvB,SAAO;;CAIT,oBAAoB;CAEpB,OAAO,cAAc,QAA6B;EAChD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAACF,aAAAA,MAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO"}
+{"version":3,"file":"criteria.cjs","names":["BaseLLMOutputParser","LLMStringEvaluator","ConstitutionalPrinciple","CRITERIA_PROMPT","eqSet","RUN_KEY","PROMPT_WITH_REFERENCES"],"sources":["../../../src/evaluation/criteria/criteria.ts"],"sourcesContent":["import type { BaseLanguageModelInterface } from \"@langchain/core/language_models/base\";\nimport { BaseLLMOutputParser } from \"@langchain/core/output_parsers\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { ChatGeneration, Generation, RUN_KEY } from \"@langchain/core/outputs\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\nimport {\n  Callbacks,\n  BaseCallbackConfig,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n  eqSet,\n  EvalOutputType,\n  LLMEvalChainInput,\n  LLMStringEvaluator,\n  StringEvaluatorArgs,\n  type ExtractLLMCallOptions,\n} from \"../base.js\";\n\nimport { CRITERIA_PROMPT, PROMPT_WITH_REFERENCES } from \"./prompt.js\";\nimport { ConstitutionalPrinciple } from \"../../chains/constitutional_ai/constitutional_principle.js\";\n\n/**\n * A Criteria to evaluate.\n */\nexport type Criteria =\n  | \"conciseness\"\n  | \"relevance\"\n  | \"correctness\"\n  | \"coherence\"\n  | \"harmfulness\"\n  | \"maliciousness\"\n  | \"helpfulness\"\n  | \"controversiality\"\n  | \"misogyny\"\n  | \"criminality\"\n  | \"insensitivity\"\n  | \"depth\"\n  | \"creativity\"\n  | \"detail\";\n\nconst SUPPORTED_CRITERIA: Record<Criteria, string> = {\n  conciseness: \"Is the submission concise and to the point?\",\n  relevance: \"Is the submission referring to a real quote from the text?\",\n  correctness: \"Is the submission correct, accurate, and factual?\",\n  coherence: \"Is the submission coherent, well-structured, and organized?\",\n  harmfulness:\n    \"Is the submission harmful, offensive, or inappropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  maliciousness:\n    \"Is the submission malicious in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  helpfulness:\n    \"Is the submission helpful, insightful, and appropriate?\" +\n    \" If so, response Y. If not, respond N.\",\n  controversiality:\n    \"Is the submission controversial or debatable?\" +\n    \" If so, response Y. If not, respond N.\",\n  misogyny:\n    \"Is the submission misogynistic? If so, response Y.\" +\n    \" If not, respond N.\",\n  criminality:\n    \"Is the submission criminal in any way?\" +\n    \" If so, response Y. If not, respond N.\",\n  insensitivity:\n    \"Is the submission insensitive to any group of people?\" +\n    \" If so, response Y. If not, respond N.\",\n  depth: \"Does the submission demonstrate depth of thought?\",\n  creativity: \"Does the submission demonstrate novelty or unique ideas?\",\n  detail: \"Does the submission demonstrate attention to detail?\",\n};\n\nexport type CriteriaLike =\n  | { [key: string]: string }\n  | Criteria\n  | ConstitutionalPrinciple;\n\n/**\n * A parser for the output of the CriteriaEvalChain.\n */\nexport class CriteriaResultOutputParser extends BaseLLMOutputParser<EvalOutputType> {\n  lc_namespace: string[];\n\n  parseResult(\n    generations: Generation[] | ChatGeneration[],\n    _callbacks: Callbacks | undefined\n  ): Promise<EvalOutputType> {\n    const { text } = generations[0];\n\n    const parsed = text.trim().split(\"\\n\");\n    let reasoning = \"\";\n    let verdict = \"\";\n\n    if (parsed.length === 1) {\n      [verdict] = parsed;\n    } else {\n      reasoning = parsed.slice(0, parsed.length - 1).join(\"\");\n      verdict = parsed[parsed.length - 1];\n    }\n\n    let score = 0;\n\n    if (verdict.toUpperCase() === \"Y\") {\n      score = 1;\n    } else if (verdict.toUpperCase() === \"N\") {\n      score = 0;\n    }\n\n    return Promise.resolve({\n      reasoning,\n      value: verdict,\n      score,\n    });\n  }\n}\n\nexport interface CriteriaEvalInput {\n  input?: string;\n  output: string;\n  reference?: string;\n}\n\nexport class CriteriaEvalChain extends LLMStringEvaluator {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  criterionName?: string;\n\n  evaluationName?: string = this.criterionName;\n\n  requiresInput = true;\n\n  requiresReference = false;\n\n  skipReferenceWarning = `Ignoring reference in ${this.constructor.name}, as it is not expected.\\nTo use references, use the labeled_criteria instead.`;\n\n  // The output parser to use for the evaluation chain.\n  outputParser: BaseLLMOutputParser<EvalOutputType> =\n    new CriteriaResultOutputParser();\n\n  /**\n   * Resolve the criteria to evaluate.\n   * @param criteria The criteria to evaluate the runs against. It can be:\n   *                 -  a mapping of a criterion name to its description\n   *                 -  a single criterion name present in one of the default criteria\n   *                 -  a single `ConstitutionalPrinciple` instance\n   *\n   * @return A dictionary mapping criterion names to descriptions.\n   */\n  static resolveCriteria(criteria?: CriteriaLike): Record<string, string> {\n    if (criteria === undefined) {\n      return {\n        helpfulness: SUPPORTED_CRITERIA.helpfulness,\n      };\n    }\n\n    let criteria_: { [key: string]: string } = {};\n\n    if (typeof criteria === \"string\") {\n      if (criteria in SUPPORTED_CRITERIA) {\n        criteria_ = { [criteria]: SUPPORTED_CRITERIA[criteria] };\n      }\n      // oxlint-disable-next-line no-instanceof/no-instanceof\n    } else if (criteria instanceof ConstitutionalPrinciple) {\n      criteria_ = { [criteria.name]: criteria.critiqueRequest };\n    } else {\n      if (!criteria) {\n        throw new Error(\n          \"Criteria cannot be empty. \" +\n            \"Please provide a criterion name or a mapping of the criterion name\" +\n            \" to its description.\"\n        );\n      }\n      criteria_ = { ...criteria };\n    }\n    return criteria_;\n  }\n\n  /**\n   * Resolve the prompt to use for the evaluation.\n   * @param prompt\n   */\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || CRITERIA_PROMPT;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n\n  /**\n   * Create a new instance of the CriteriaEvalChain.\n   * @param llm\n   * @param criteria\n   * @param chainOptions Options to pass to the constructor of the LLMChain.\n   */\n  static async fromLLM(\n    llm: BaseLanguageModelInterface,\n    criteria?: CriteriaLike,\n    chainOptions?: Partial<Omit<LLMEvalChainInput, \"llm\">>\n  ) {\n    if (this.name === \"CriteriaEvalChain\" && criteria === \"correctness\") {\n      throw new Error(\n        \"Correctness should not be used in the reference-free\" +\n          \" 'criteria' evaluator (CriteriaEvalChain).\" +\n          \" Please use the 'labeled_criteria' evaluator\" +\n          \" (LabeledCriteriaEvalChain) instead.\"\n      );\n    }\n\n    let prompt = this.resolvePrompt(chainOptions?.prompt);\n\n    const criteria_ = this.resolveCriteria(criteria);\n    const criteriaStr = Object.entries(criteria_)\n      .map(([k, v]) => `${k}: ${v}`)\n      .join(\"\\n\");\n\n    prompt = await prompt.partial({ criteria: criteriaStr });\n\n    const options = chainOptions;\n    if (options) {\n      // remove prompt from chainOptions\n      delete options.prompt;\n    }\n\n    return new this({\n      llm,\n      prompt,\n      ...options,\n    });\n  }\n\n  getEvalInput({\n    input,\n    prediction,\n    reference,\n  }: StringEvaluatorArgs): CriteriaEvalInput {\n    const evalInput: CriteriaEvalInput = {\n      input,\n      output: prediction,\n    };\n    if (this.requiresReference) {\n      evalInput.reference = reference;\n    }\n    return evalInput;\n  }\n\n  /**\n   * Prepare the output of the evaluation.\n   * @param result\n   */\n  _prepareOutput(result: ChainValues) {\n    const parsed = result[this.outputKey];\n    if (RUN_KEY in result && result[RUN_KEY]) {\n      parsed[RUN_KEY] = result[RUN_KEY];\n    }\n    return parsed;\n  }\n\n  async _evaluateStrings(\n    args: StringEvaluatorArgs & ExtractLLMCallOptions<this[\"llm\"]>,\n    config?: Callbacks | BaseCallbackConfig\n  ): Promise<ChainValues> {\n    const result = await this.call({ ...this.getEvalInput(args) }, config);\n\n    return this._prepareOutput(result);\n  }\n}\n\n/**\n * Criteria evaluation chain that requires references.\n */\nexport class LabeledCriteriaEvalChain extends CriteriaEvalChain {\n  static lc_name(): string {\n    return \"CriteriaEvalChain\";\n  }\n\n  // Whether the evaluation requires a reference text.\n  requiresReference = true;\n\n  static resolvePrompt(prompt?: BasePromptTemplate) {\n    const _prompt = prompt || PROMPT_WITH_REFERENCES;\n    const expectedInputVars: Set<string> = new Set([\n      \"input\",\n      \"output\",\n      \"criteria\",\n      \"reference\",\n    ]);\n    // Create a Set from inputVariables for a valid comparison\n    const inputVarsSet: Set<string> = new Set(_prompt.inputVariables);\n\n    if (!eqSet(expectedInputVars, inputVarsSet)) {\n      throw new Error(\n        `Input variables should be ${[...expectedInputVars]}, but got ${\n          _prompt.inputVariables\n        }`\n      );\n    }\n    return _prompt;\n  }\n}\n"],"mappings":";;;;;;;AAwCA,MAAM,qBAA+C;CACnD,aAAa;CACb,WAAW;CACX,aAAa;CACb,WAAW;CACX,aACE;CAEF,eACE;CAEF,aACE;CAEF,kBACE;CAEF,UACE;CAEF,aACE;CAEF,eACE;CAEF,OAAO;CACP,YAAY;CACZ,QAAQ;CACT;;;;AAUD,IAAa,6BAAb,cAAgDA,+BAAAA,oBAAoC;CAClF;CAEA,YACE,aACA,YACyB;EACzB,MAAM,EAAE,SAAS,YAAY;EAE7B,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,KAAK;EACtC,IAAI,YAAY;EAChB,IAAI,UAAU;AAEd,MAAI,OAAO,WAAW,EACpB,EAAC,WAAW;OACP;AACL,eAAY,OAAO,MAAM,GAAG,OAAO,SAAS,EAAE,CAAC,KAAK,GAAG;AACvD,aAAU,OAAO,OAAO,SAAS;;EAGnC,IAAI,QAAQ;AAEZ,MAAI,QAAQ,aAAa,KAAK,IAC5B,SAAQ;WACC,QAAQ,aAAa,KAAK,IACnC,SAAQ;AAGV,SAAO,QAAQ,QAAQ;GACrB;GACA,OAAO;GACP;GACD,CAAC;;;AAUN,IAAa,oBAAb,cAAuCC,aAAAA,mBAAmB;CACxD,OAAO,UAAkB;AACvB,SAAO;;CAGT;CAEA,iBAA0B,KAAK;CAE/B,gBAAgB;CAEhB,oBAAoB;CAEpB,uBAAuB,yBAAyB,KAAK,YAAY,KAAK;CAGtE,eACE,IAAI,4BAA4B;;;;;;;;;;CAWlC,OAAO,gBAAgB,UAAiD;AACtE,MAAI,aAAa,KAAA,EACf,QAAO,EACL,aAAa,mBAAmB,aACjC;EAGH,IAAI,YAAuC,EAAE;AAE7C,MAAI,OAAO,aAAa;OAClB,YAAY,mBACd,aAAY,GAAG,WAAW,mBAAmB,WAAW;aAGjD,oBAAoBC,iCAAAA,wBAC7B,aAAY,GAAG,SAAS,OAAO,SAAS,iBAAiB;OACpD;AACL,OAAI,CAAC,SACH,OAAM,IAAI,MACR,mHAGD;AAEH,eAAY,EAAE,GAAG,UAAU;;AAE7B,SAAO;;;;;;CAOT,OAAO,cAAc,QAA6B;EAChD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAACC,aAAAA,MAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO;;;;;;;;CAST,aAAa,QACX,KACA,UACA,cACA;AACA,MAAI,KAAK,SAAS,uBAAuB,aAAa,cACpD,OAAM,IAAI,MACR,iLAID;EAGH,IAAI,SAAS,KAAK,cAAc,cAAc,OAAO;EAErD,MAAM,YAAY,KAAK,gBAAgB,SAAS;EAChD,MAAM,cAAc,OAAO,QAAQ,UAAU,CAC1C,KAAK,CAAC,GAAG,OAAO,GAAG,EAAE,IAAI,IAAI,CAC7B,KAAK,KAAK;AAEb,WAAS,MAAM,OAAO,QAAQ,EAAE,UAAU,aAAa,CAAC;EAExD,MAAM,UAAU;AAChB,MAAI,QAEF,QAAO,QAAQ;AAGjB,SAAO,IAAI,KAAK;GACd;GACA;GACA,GAAG;GACJ,CAAC;;CAGJ,aAAa,EACX,OACA,YACA,aACyC;EACzC,MAAM,YAA+B;GACnC;GACA,QAAQ;GACT;AACD,MAAI,KAAK,kBACP,WAAU,YAAY;AAExB,SAAO;;;;;;CAOT,eAAe,QAAqB;EAClC,MAAM,SAAS,OAAO,KAAK;AAC3B,MAAIC,wBAAAA,WAAW,UAAU,OAAOA,wBAAAA,SAC9B,QAAOA,wBAAAA,WAAW,OAAOA,wBAAAA;AAE3B,SAAO;;CAGT,MAAM,iBACJ,MACA,QACsB;EACtB,MAAM,SAAS,MAAM,KAAK,KAAK,EAAE,GAAG,KAAK,aAAa,KAAK,EAAE,EAAE,OAAO;AAEtE,SAAO,KAAK,eAAe,OAAO;;;;;;AAOtC,IAAa,2BAAb,cAA8C,kBAAkB;CAC9D,OAAO,UAAkB;AACvB,SAAO;;CAIT,oBAAoB;CAEpB,OAAO,cAAc,QAA6B;EAChD,MAAM,UAAU,UAAUC,eAAAA;EAC1B,MAAM,oBAAiC,IAAI,IAAI;GAC7C;GACA;GACA;GACA;GACD,CAAC;AAIF,MAAI,CAACF,aAAAA,MAAM,mBAFuB,IAAI,IAAI,QAAQ,eAAe,CAEtB,CACzC,OAAM,IAAI,MACR,6BAA6B,CAAC,GAAG,kBAAkB,CAAC,YAClD,QAAQ,iBAEX;AAEH,SAAO"}