npm - @langchain/classic - Versions diffs - 1.0.5 → 1.0.6-dev-1765431816670 - Mend

@langchain/classic 1.0.5 → 1.0.6-dev-1765431816670

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (1277) hide show

package/dist/schema/prompt_template.d.cts CHANGED Viewed

@@ -1,5 +1,5 @@
-import { BasePromptValueInterface } from "../langchain-core/dist/prompt_values.cjs";
 import { BasePromptTemplate } from "@langchain/core/prompts";
+import * as _langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 import { Document } from "@langchain/core/documents";
 //#region src/schema/prompt_template.d.ts
@@ -13,7 +13,7 @@ import { Document } from "@langchain/core/documents";
  * @returns {Promise<string>} A Promise that resolves to the formatted document as a string.
  * @throws {Error} If the document is missing required metadata variables specified in the prompt template.
  */
-declare const formatDocument: (document: Document<Record<string, any>>, prompt: BasePromptTemplate<any, BasePromptValueInterface, any>) => Promise<string>;
+declare const formatDocument: (document: Document<Record<string, any>>, prompt: BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>) => Promise<string>;
 //#endregion
 export { formatDocument };
 //# sourceMappingURL=prompt_template.d.cts.map

package/dist/schema/prompt_template.d.cts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"prompt_template.d.cts","names":["Document","BasePromptTemplate","formatDocument","Record","_langchain_core_prompt_values0","BasePromptValueInterface","Promise"],"sources":["../../src/schema/prompt_template.d.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\n/**\n * Formats a document using a given prompt template.\n *\n * @async\n * @param {Document} document - The document to format.\n * @param {BasePromptTemplate} prompt - The prompt template to use for formatting.\n * @returns {Promise<string>} A Promise that resolves to the formatted document as a string.\n * @throws {Error} If the document is missing required metadata variables specified in the prompt template.\n */\nexport declare const formatDocument: (document: Document<Record<string, any>>, prompt: BasePromptTemplate<any, import(\"@langchain/core/prompt_values\").BasePromptValueInterface, any>) => Promise<string>;\n//# sourceMappingURL=prompt_template.d.ts.map"],"mappings":";;;;;;;;;AAWA;;;;;;AAAiM,cAA5KE,cAA4K,EAAA,CAAA,QAAA,EAAjJF,QAAiJ,CAAxIG,MAAwI,CAAA,MAAA,EAAA,GAAA,CAAA,CAAA,EAAA,MAAA,EAA1GF,kBAA0G,CAAA,GAAA,EAAzIG,8BAAAA,CAA+FC,wBAAAA,EAA0C,GAAA,CAAA,EAAA,GAAPC,OAAO,CAAA,MAAA,CAAA"}

package/dist/schema/prompt_template.d.ts CHANGED Viewed

@@ -1,6 +1,6 @@
-import { BasePromptValueInterface } from "../libs/langchain-core/dist/prompt_values.js";
 import { Document } from "@langchain/core/documents";
 import { BasePromptTemplate } from "@langchain/core/prompts";
+import * as _langchain_core_prompt_values0 from "@langchain/core/prompt_values";
 //#region src/schema/prompt_template.d.ts
@@ -13,7 +13,7 @@ import { BasePromptTemplate } from "@langchain/core/prompts";
  * @returns {Promise<string>} A Promise that resolves to the formatted document as a string.
  * @throws {Error} If the document is missing required metadata variables specified in the prompt template.
  */
-declare const formatDocument: (document: Document<Record<string, any>>, prompt: BasePromptTemplate<any, BasePromptValueInterface, any>) => Promise<string>;
+declare const formatDocument: (document: Document<Record<string, any>>, prompt: BasePromptTemplate<any, _langchain_core_prompt_values0.BasePromptValueInterface, any>) => Promise<string>;
 //#endregion
 export { formatDocument };
 //# sourceMappingURL=prompt_template.d.ts.map

package/dist/schema/prompt_template.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"prompt_template.d.ts","names":["Document","BasePromptTemplate","formatDocument","Record","_langchain_core_prompt_values0","BasePromptValueInterface","Promise"],"sources":["../../src/schema/prompt_template.d.ts"],"sourcesContent":["import { Document } from \"@langchain/core/documents\";\nimport { BasePromptTemplate } from \"@langchain/core/prompts\";\n/**\n * Formats a document using a given prompt template.\n *\n * @async\n * @param {Document} document - The document to format.\n * @param {BasePromptTemplate} prompt - The prompt template to use for formatting.\n * @returns {Promise<string>} A Promise that resolves to the formatted document as a string.\n * @throws {Error} If the document is missing required metadata variables specified in the prompt template.\n */\nexport declare const formatDocument: (document: Document<Record<string, any>>, prompt: BasePromptTemplate<any, import(\"@langchain/core/prompt_values\").BasePromptValueInterface, any>) => Promise<string>;\n//# sourceMappingURL=prompt_template.d.ts.map"],"mappings":";;;;;;;;;AAWA;;;;;;AAAiM,cAA5KE,cAA4K,EAAA,CAAA,QAAA,EAAjJF,QAAiJ,CAAxIG,MAAwI,CAAA,MAAA,EAAA,GAAA,CAAA,CAAA,EAAA,MAAA,EAA1GF,kBAA0G,CAAA,GAAA,EAAzIG,8BAAAA,CAA+FC,wBAAAA,EAA0C,GAAA,CAAA,EAAA,GAAPC,OAAO,CAAA,MAAA,CAAA"}

package/dist/schema/query_constructor.d.cts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"query_constructor.d.cts","names":["AttributeInfo"],"sources":["../../src/schema/query_constructor.d.ts"],"sourcesContent":["/**\n * A simple data structure that holds information about an attribute. It\n * is typically used to provide metadata about attributes in other classes\n * or data structures within the LangChain framework.\n */\nexport declare class AttributeInfo {\n name: string;\n type: string;\n description: string;\n constructor(name: string, type: string, description: string);\n}\n//# sourceMappingURL=query_constructor.d.ts.map"],"mappings":";;AAKA;;;;cAAqBA,aAAAA"}

package/dist/schema/query_constructor.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"query_constructor.d.ts","names":["AttributeInfo"],"sources":["../../src/schema/query_constructor.d.ts"],"sourcesContent":["/**\n * A simple data structure that holds information about an attribute. It\n * is typically used to provide metadata about attributes in other classes\n * or data structures within the LangChain framework.\n */\nexport declare class AttributeInfo {\n name: string;\n type: string;\n description: string;\n constructor(name: string, type: string, description: string);\n}\n//# sourceMappingURL=query_constructor.d.ts.map"],"mappings":";;AAKA;;;;cAAqBA,aAAAA"}

package/dist/smith/config.d.cts CHANGED Viewed

@@ -19,21 +19,12 @@ type EvaluatorInputFormatter = ({
   rawReferenceOutput,
   run
 }: {
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   rawInput: any;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   rawPrediction: any;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   rawReferenceOutput?: any;
   run: Run;
 }) => EvaluatorInputs;
-type DynamicRunEvaluatorParams<
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-Input extends Record<string, any> = Record<string, unknown>,
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-Prediction extends Record<string, any> = Record<string, unknown>,
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-Reference extends Record<string, any> = Record<string, unknown>> = {
+type DynamicRunEvaluatorParams<Input extends Record<string, any> = Record<string, unknown>, Prediction extends Record<string, any> = Record<string, unknown>, Reference extends Record<string, any> = Record<string, unknown>> = {
   input: Input;
   prediction?: Prediction;
   reference?: Reference;
@@ -191,7 +182,6 @@ type Criteria$1 = EvalConfig & {
    */
   llm?: BaseLanguageModel;
 };
-// for compatibility reasons
 type CriteriaEvalChainConfig = Criteria$1;
 declare function Criteria$1(criteria: Criteria | Record<string, string>, config?: Pick<Partial<LabeledCriteria>, "formatEvaluatorInputs" | "llm" | "feedbackKey">): EvalConfig;
 /**

package/dist/smith/config.d.cts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"config.d.cts","names":["BaseLanguageModel","RunnableConfig","Example","Run","EvaluationResult","RunEvaluator","Criteria","CriteriaType","EmbeddingDistanceEvalChainInput","LoadEvaluatorOptions","EvaluatorType","EvaluatorInputs","EvaluatorInputFormatter","rawInput","rawPrediction","rawReferenceOutput","run","DynamicRunEvaluatorParams","Input","Prediction","Reference","Record","RunEvaluatorLike","Promise","isOffTheShelfEvaluator","T","U","EvalConfig","isCustomEvaluator","RunEvalType","RunEvalConfig","CriteriaEvalChainConfig","LabeledCriteria","Partial","Pick","EmbeddingDistance"],"sources":["../../src/smith/config.d.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { RunnableConfig } from \"@langchain/core/runnables\";\nimport { Example, Run } from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { Criteria as CriteriaType, type EmbeddingDistanceEvalChainInput } from \"../evaluation/index.js\";\nimport { LoadEvaluatorOptions } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nexport type EvaluatorInputs = {\n input?: string | unknown;\n prediction: string | unknown;\n reference?: string | unknown;\n};\nexport type EvaluatorInputFormatter = ({ rawInput, rawPrediction, rawReferenceOutput, run }: {\n rawInput: any;\n rawPrediction: any;\n rawReferenceOutput?: any;\n run: Run;\n}) => EvaluatorInputs;\nexport type DynamicRunEvaluatorParams<Input extends Record<string, any> = Record<string, unknown>, Prediction extends Record<string, any> = Record<string, unknown>, Reference extends Record<string, any> = Record<string, unknown>> = {\n input: Input;\n prediction?: Prediction;\n reference?: Reference;\n run: Run;\n example?: Example;\n};\n/**\n * Type of a function that can be coerced into a RunEvaluator function.\n * While we have the class-based RunEvaluator, it's often more convenient to directly\n * pass a function to the runner. This type allows us to do that.\n */\nexport type RunEvaluatorLike = ((props: DynamicRunEvaluatorParams, options: RunnableConfig) => Promise<EvaluationResult>) | ((props: DynamicRunEvaluatorParams, options: RunnableConfig) => EvaluationResult);\nexport declare function isOffTheShelfEvaluator<T extends keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike>(evaluator: T | EvalConfig | U): evaluator is T | EvalConfig;\nexport declare function isCustomEvaluator<T extends keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike>(evaluator: T | EvalConfig | U): evaluator is U;\nexport type RunEvalType<T extends keyof EvaluatorType = \"criteria\" | \"labeled_criteria\" | \"embedding_distance\", U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = T | EvalConfig | U;\n/**\n * Configuration class for running evaluations on datasets.\n *\n * @remarks\n * RunEvalConfig in LangSmith is a configuration class for running evaluations on datasets. Its primary purpose is to define the parameters and evaluators that will be applied during the evaluation of a dataset. This configuration can include various evaluators, custom evaluators, and different keys for inputs, predictions, and references.\n *\n * @typeparam T - The type of evaluators.\n * @typeparam U - The type of custom evaluators.\n */\nexport type RunEvalConfig<T extends keyof EvaluatorType = \"criteria\" | \"labeled_criteria\" | \"embedding_distance\", U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = {\n /**\n * Evaluators to apply to a dataset run.\n * You can optionally specify these by name, or by\n * configuring them with an EvalConfig object.\n */\n evaluators?: RunEvalType<T, U>[];\n /**\n * Convert the evaluation data into formats that can be used by the evaluator.\n * This should most commonly be a string.\n * Parameters are the raw input from the run, the raw output, raw reference output, and the raw run.\n * @example\n * ```ts\n * // Chain input: { input: \"some string\" }\n * // Chain output: { output: \"some output\" }\n * // Reference example output format: { output: \"some reference output\" }\n * const formatEvaluatorInputs = ({\n * rawInput,\n * rawPrediction,\n * rawReferenceOutput,\n * }) => {\n * return {\n * input: rawInput.input,\n * prediction: rawPrediction.output,\n * reference: rawReferenceOutput.output,\n * };\n * };\n * ```\n * @returns The prepared data.\n */\n formatEvaluatorInputs?: EvaluatorInputFormatter;\n /**\n * Custom evaluators to apply to a dataset run.\n * Each evaluator is provided with a run trace containing the model\n * outputs, as well as an \"example\" object representing a record\n * in the dataset.\n *\n * @deprecated Use `evaluators` instead.\n */\n customEvaluators?: U[];\n};\nexport interface EvalConfig extends LoadEvaluatorOptions {\n /**\n * The name of the evaluator to use.\n * Example: labeled_criteria, criteria, etc.\n */\n evaluatorType: keyof EvaluatorType;\n /**\n * The feedback (or metric) name to use for the logged\n * evaluation results. If none provided, we default to\n * the evaluationName.\n */\n feedbackKey?: string;\n /**\n * Convert the evaluation data into formats that can be used by the evaluator.\n * This should most commonly be a string.\n * Parameters are the raw input from the run, the raw output, raw reference output, and the raw run.\n * @example\n * ```ts\n * // Chain input: { input: \"some string\" }\n * // Chain output: { output: \"some output\" }\n * // Reference example output format: { output: \"some reference output\" }\n * const formatEvaluatorInputs = ({\n * rawInput,\n * rawPrediction,\n * rawReferenceOutput,\n * }) => {\n * return {\n * input: rawInput.input,\n * prediction: rawPrediction.output,\n * reference: rawReferenceOutput.output,\n * };\n * };\n * ```\n * @returns The prepared data.\n */\n formatEvaluatorInputs: EvaluatorInputFormatter;\n}\n/**\n * Configuration to load a \"CriteriaEvalChain\" evaluator,\n * which prompts an LLM to determine whether the model's\n * prediction complies with the provided criteria.\n * @param criteria - The criteria to use for the evaluator.\n * @param llm - The language model to use for the evaluator.\n * @returns The configuration for the evaluator.\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [Criteria(\"helpfulness\")],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [\n * Criteria({\n * \"isCompliant\": \"Does the submission comply with the requirements of XYZ\"\n * })\n * ],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [{\n * evaluatorType: \"criteria\",\n * criteria: \"helpfulness\"\n * formatEvaluatorInputs: ...\n * }]\n * };\n * ```\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [{\n * evaluatorType: \"criteria\",\n * criteria: { \"isCompliant\": \"Does the submission comply with the requirements of XYZ\" },\n * formatEvaluatorInputs: ...\n * }]\n * };\n */\nexport type Criteria = EvalConfig & {\n evaluatorType: \"criteria\";\n /**\n * The \"criteria\" to insert into the prompt template\n * used for evaluation. See the prompt at\n * https://smith.langchain.com/hub/langchain-ai/criteria-evaluator\n * for more information.\n */\n criteria?: CriteriaType | Record<string, string>;\n /**\n * The language model to use as the evaluator, defaults to GPT-4\n */\n llm?: BaseLanguageModel;\n};\nexport type CriteriaEvalChainConfig = Criteria;\nexport declare function Criteria(criteria: CriteriaType | Record<string, string>, config?: Pick<Partial<LabeledCriteria>, \"formatEvaluatorInputs\" | \"llm\" | \"feedbackKey\">): EvalConfig;\n/**\n * Configuration to load a \"LabeledCriteriaEvalChain\" evaluator,\n * which prompts an LLM to determine whether the model's\n * prediction complies with the provided criteria and also\n * provides a \"ground truth\" label for the evaluator to incorporate\n * in its evaluation.\n * @param criteria - The criteria to use for the evaluator.\n * @param llm - The language model to use for the evaluator.\n * @returns The configuration for the evaluator.\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [LabeledCriteria(\"correctness\")],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [\n * LabeledCriteria({\n * \"mentionsAllFacts\": \"Does the include all facts provided in the reference?\"\n * })\n * ],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [{\n * evaluatorType: \"labeled_criteria\",\n * criteria: \"correctness\",\n * formatEvaluatorInputs: ...\n * }],\n * };\n * ```\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [{\n * evaluatorType: \"labeled_criteria\",\n * criteria: { \"mentionsAllFacts\": \"Does the include all facts provided in the reference?\" },\n * formatEvaluatorInputs: ...\n * }],\n * };\n */\nexport type LabeledCriteria = EvalConfig & {\n evaluatorType: \"labeled_criteria\";\n /**\n * The \"criteria\" to insert into the prompt template\n * used for evaluation. See the prompt at\n * https://smith.langchain.com/hub/langchain-ai/labeled-criteria\n * for more information.\n */\n criteria?: CriteriaType | Record<string, string>;\n /**\n * The language model to use as the evaluator, defaults to GPT-4\n */\n llm?: BaseLanguageModel;\n};\nexport declare function LabeledCriteria(criteria: CriteriaType | Record<string, string>, config?: Pick<Partial<LabeledCriteria>, \"formatEvaluatorInputs\" | \"llm\" | \"feedbackKey\">): LabeledCriteria;\n/**\n * Configuration to load a \"EmbeddingDistanceEvalChain\" evaluator,\n * which embeds distances to score semantic difference between\n * a prediction and reference.\n */\nexport type EmbeddingDistance = EvalConfig & EmbeddingDistanceEvalChainInput & {\n evaluatorType: \"embedding_distance\";\n};\nexport declare function EmbeddingDistance(distanceMetric: EmbeddingDistanceEvalChainInput[\"distanceMetric\"], config?: Pick<Partial<LabeledCriteria>, \"formatEvaluatorInputs\" | \"embedding\" | \"feedbackKey\">): EmbeddingDistance;\n//# sourceMappingURL=config.d.ts.map"],"mappings":";;;;;;;;;;KAOYW,eAAAA;;;EAAAA,SAAAA,CAAAA,EAAAA,MAAe,GAAA,OAAA;AAK3B,CAAA;AAAyCE,KAA7BD,uBAAAA,GAA6BC,CAAAA;EAAAA,QAAAA;EAAAA,aAAAA;EAAAA,kBAAAA;EAAAA;CAAAA,EAAAA;EAAUC,QAAAA,EAAAA,GAAAA;EAAeC,aAAAA,EAAAA,GAAAA;EAAoBC,kBAAAA,CAAAA,EAAAA,GAAAA;EAI7Eb,GAAAA,EAAAA,GAAAA;CACHQ,EAAAA,GAAAA,eAAAA;AAAe,KACTM,yBADS,CAAA,cAC+BI,MAD/B,CAAA,MAAA,EAAA,GAAA,CAAA,GACqDA,MADrD,CAAA,MAAA,EAAA,OAAA,CAAA,EAAA,mBACiGA,MADjG,CAAA,MAAA,EAAA,GAAA,CAAA,GACuHA,MADvH,CAAA,MAAA,EAAA,OAAA,CAAA,EAAA,kBACkKA,MADlK,CAAA,MAAA,EAAA,GAAA,CAAA,GACwLA,MADxL,CAAA,MAAA,EAAA,OAAA,CAAA,CAAA,GAAA;EACTJ,KAAAA,EACDC,KADCD;EAAwCI,UAAAA,CAAAA,EAEnCF,UAFmCE;EAAsBA,SAAAA,CAAAA,EAG1DD,SAH0DC;EAA4CA,GAAAA,EAI7GlB,GAJ6GkB;EAAsBA,OAAAA,CAAAA,EAK9HnB,OAL8HmB;CAA2CA;;;;;;AAKzKnB,KAOFoB,gBAAAA,GAPEpB,CAAAA,CAAAA,KAAAA,EAO0Be,yBAP1Bf,EAAAA,OAAAA,EAO8DD,cAP9DC,EAAAA,GAOiFqB,OAPjFrB,CAOyFE,gBAPzFF,CAAAA,CAAAA,GAAAA,CAAAA,CAAAA,KAAAA,EAOuHe,yBAPvHf,EAAAA,OAAAA,EAO2JD,cAP3JC,EAAAA,GAO8KE,gBAP9KF,CAAAA;AAAO,iBAQGsB,sBARH,CAAA,UAAA,MAQ0Cd,aAR1C,EAAA,UAQmEL,YARnE,GAQkFiB,gBARlF,GAQqGjB,YARrG,GAQoHiB,gBARpH,CAAA,CAAA,SAAA,EAQiJG,CARjJ,GAQqJE,UARrJ,GAQkKD,CARlK,CAAA,EAAA,SAAA,IAQmLD,CARnL,GAQuLE,UARvL;AAOTL,iBAEYM,iBAFI,CAAA,UAAA,MAE8BlB,aAF9B,EAAA,UAEuDL,YAFvD,GAEsEiB,gBAFtE,GAEyFjB,YAFzF,GAEwGiB,gBAFxG,CAAA,CAAA,SAAA,EAEqIG,CAFrI,GAEyIE,UAFzI,GAEsJD,CAFtJ,CAAA,EAAA,SAAA,IAEuKA,CAFvK;AAAYT,KAG5BY,WAH4BZ,CAAAA,UAAAA,MAGAP,aAHAO,GAAAA,UAAAA,GAAAA,kBAAAA,GAAAA,oBAAAA,EAAAA,UAGkFZ,YAHlFY,GAGiGK,gBAHjGL,GAGoHZ,YAHpHY,GAGmIK,gBAHnIL,CAAAA,GAGuJQ,CAHvJR,GAG2JU,UAH3JV,GAGwKS,CAHxKT;;;;;;;AAAoK;AAC5M;;AAAwFZ,KAY5EyB,aAZ4EzB,CAAAA,UAAAA,MAY9CK,aAZ8CL,GAAAA,UAAAA,GAAAA,kBAAAA,GAAAA,oBAAAA,EAAAA,UAYoCA,YAZpCA,GAYmDiB,gBAZnDjB,GAYsEA,YAZtEA,GAYqFiB,gBAZrFjB,CAAAA,GAAAA;EAAeiB;;;;;EAAgFI,UAAAA,CAAAA,EAkBtKG,WAlBsKH,CAkB1JD,CAlB0JC,EAkBvJA,CAlBuJA,CAAAA,EAAAA;EAAiBD;;AAAc;AACtN;;;;;;;;;;AAAoM;AACpM;;;;;;;;;EAAiN,qBAAA,CAAA,EAwCrLb,uBAxCqL;EAUrMkB;;;;;;;;EAMKD,gBAAAA,CAAAA,EAiCMH,CAjCNG,EAAAA;CAwBWjB;AASLc,UAENC,UAAAA,SAAmBlB,oBAFbiB,CAAAA;EAAC;AAExB;;;EAAoCjB,aAAAA,EAAAA,MAKXC,aALWD;EAAoB;AA8ExD;;;;EAYUT,WAAAA,CAAAA,EAAAA,MAAAA;EAAiB;AAE3B;AACA;;;;;;;AAAuL;AA4CvL;;;;;AAY2B;AAE3B;;;;;;;EAAmM,qBAAA,EApHxKY,uBAoHwK;AAMnM;AAGA;;;;;;AAA+N;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAlFnNN,UAAAA,GAAWqB;;;;;;;;aAQRpB,WAAec;;;;QAIpBrB;;KAEE+B,uBAAAA,GAA0BzB;iBACdA,UAAAA,WAAmBC,WAAec,iCAAiCa,KAAKD,QAAQD,qEAAqEL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA4CjKK,eAAAA,GAAkBL;;;;;;;;aAQfpB,WAAec;;;;QAIpBrB;;iBAEcgC,eAAAA,WAA0BzB,WAAec,iCAAiCa,KAAKD,QAAQD,qEAAqEA;;;;;;KAMxKG,iBAAAA,GAAoBR,aAAanB;;;iBAGrB2B,iBAAAA,iBAAkC3B,4DAA4D0B,KAAKD,QAAQD,2EAA2EG"}

package/dist/smith/config.d.ts CHANGED Viewed

@@ -20,21 +20,12 @@ type EvaluatorInputFormatter = ({
   rawReferenceOutput,
   run
 }: {
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   rawInput: any;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   rawPrediction: any;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   rawReferenceOutput?: any;
   run: Run;
 }) => EvaluatorInputs;
-type DynamicRunEvaluatorParams<
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-Input extends Record<string, any> = Record<string, unknown>,
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-Prediction extends Record<string, any> = Record<string, unknown>,
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-Reference extends Record<string, any> = Record<string, unknown>> = {
+type DynamicRunEvaluatorParams<Input extends Record<string, any> = Record<string, unknown>, Prediction extends Record<string, any> = Record<string, unknown>, Reference extends Record<string, any> = Record<string, unknown>> = {
   input: Input;
   prediction?: Prediction;
   reference?: Reference;
@@ -192,7 +183,6 @@ type Criteria$1 = EvalConfig & {
    */
   llm?: BaseLanguageModel;
 };
-// for compatibility reasons
 type CriteriaEvalChainConfig = Criteria$1;
 declare function Criteria$1(criteria: Criteria | Record<string, string>, config?: Pick<Partial<LabeledCriteria>, "formatEvaluatorInputs" | "llm" | "feedbackKey">): EvalConfig;
 /**

package/dist/smith/config.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"config.d.ts","names":["BaseLanguageModel","RunnableConfig","Example","Run","EvaluationResult","RunEvaluator","Criteria","CriteriaType","EmbeddingDistanceEvalChainInput","LoadEvaluatorOptions","EvaluatorType","EvaluatorInputs","EvaluatorInputFormatter","rawInput","rawPrediction","rawReferenceOutput","run","DynamicRunEvaluatorParams","Input","Prediction","Reference","Record","RunEvaluatorLike","Promise","isOffTheShelfEvaluator","T","U","EvalConfig","isCustomEvaluator","RunEvalType","RunEvalConfig","CriteriaEvalChainConfig","LabeledCriteria","Partial","Pick","EmbeddingDistance"],"sources":["../../src/smith/config.d.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { RunnableConfig } from \"@langchain/core/runnables\";\nimport { Example, Run } from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { Criteria as CriteriaType, type EmbeddingDistanceEvalChainInput } from \"../evaluation/index.js\";\nimport { LoadEvaluatorOptions } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nexport type EvaluatorInputs = {\n input?: string | unknown;\n prediction: string | unknown;\n reference?: string | unknown;\n};\nexport type EvaluatorInputFormatter = ({ rawInput, rawPrediction, rawReferenceOutput, run }: {\n rawInput: any;\n rawPrediction: any;\n rawReferenceOutput?: any;\n run: Run;\n}) => EvaluatorInputs;\nexport type DynamicRunEvaluatorParams<Input extends Record<string, any> = Record<string, unknown>, Prediction extends Record<string, any> = Record<string, unknown>, Reference extends Record<string, any> = Record<string, unknown>> = {\n input: Input;\n prediction?: Prediction;\n reference?: Reference;\n run: Run;\n example?: Example;\n};\n/**\n * Type of a function that can be coerced into a RunEvaluator function.\n * While we have the class-based RunEvaluator, it's often more convenient to directly\n * pass a function to the runner. This type allows us to do that.\n */\nexport type RunEvaluatorLike = ((props: DynamicRunEvaluatorParams, options: RunnableConfig) => Promise<EvaluationResult>) | ((props: DynamicRunEvaluatorParams, options: RunnableConfig) => EvaluationResult);\nexport declare function isOffTheShelfEvaluator<T extends keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike>(evaluator: T | EvalConfig | U): evaluator is T | EvalConfig;\nexport declare function isCustomEvaluator<T extends keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike>(evaluator: T | EvalConfig | U): evaluator is U;\nexport type RunEvalType<T extends keyof EvaluatorType = \"criteria\" | \"labeled_criteria\" | \"embedding_distance\", U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = T | EvalConfig | U;\n/**\n * Configuration class for running evaluations on datasets.\n *\n * @remarks\n * RunEvalConfig in LangSmith is a configuration class for running evaluations on datasets. Its primary purpose is to define the parameters and evaluators that will be applied during the evaluation of a dataset. This configuration can include various evaluators, custom evaluators, and different keys for inputs, predictions, and references.\n *\n * @typeparam T - The type of evaluators.\n * @typeparam U - The type of custom evaluators.\n */\nexport type RunEvalConfig<T extends keyof EvaluatorType = \"criteria\" | \"labeled_criteria\" | \"embedding_distance\", U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = {\n /**\n * Evaluators to apply to a dataset run.\n * You can optionally specify these by name, or by\n * configuring them with an EvalConfig object.\n */\n evaluators?: RunEvalType<T, U>[];\n /**\n * Convert the evaluation data into formats that can be used by the evaluator.\n * This should most commonly be a string.\n * Parameters are the raw input from the run, the raw output, raw reference output, and the raw run.\n * @example\n * ```ts\n * // Chain input: { input: \"some string\" }\n * // Chain output: { output: \"some output\" }\n * // Reference example output format: { output: \"some reference output\" }\n * const formatEvaluatorInputs = ({\n * rawInput,\n * rawPrediction,\n * rawReferenceOutput,\n * }) => {\n * return {\n * input: rawInput.input,\n * prediction: rawPrediction.output,\n * reference: rawReferenceOutput.output,\n * };\n * };\n * ```\n * @returns The prepared data.\n */\n formatEvaluatorInputs?: EvaluatorInputFormatter;\n /**\n * Custom evaluators to apply to a dataset run.\n * Each evaluator is provided with a run trace containing the model\n * outputs, as well as an \"example\" object representing a record\n * in the dataset.\n *\n * @deprecated Use `evaluators` instead.\n */\n customEvaluators?: U[];\n};\nexport interface EvalConfig extends LoadEvaluatorOptions {\n /**\n * The name of the evaluator to use.\n * Example: labeled_criteria, criteria, etc.\n */\n evaluatorType: keyof EvaluatorType;\n /**\n * The feedback (or metric) name to use for the logged\n * evaluation results. If none provided, we default to\n * the evaluationName.\n */\n feedbackKey?: string;\n /**\n * Convert the evaluation data into formats that can be used by the evaluator.\n * This should most commonly be a string.\n * Parameters are the raw input from the run, the raw output, raw reference output, and the raw run.\n * @example\n * ```ts\n * // Chain input: { input: \"some string\" }\n * // Chain output: { output: \"some output\" }\n * // Reference example output format: { output: \"some reference output\" }\n * const formatEvaluatorInputs = ({\n * rawInput,\n * rawPrediction,\n * rawReferenceOutput,\n * }) => {\n * return {\n * input: rawInput.input,\n * prediction: rawPrediction.output,\n * reference: rawReferenceOutput.output,\n * };\n * };\n * ```\n * @returns The prepared data.\n */\n formatEvaluatorInputs: EvaluatorInputFormatter;\n}\n/**\n * Configuration to load a \"CriteriaEvalChain\" evaluator,\n * which prompts an LLM to determine whether the model's\n * prediction complies with the provided criteria.\n * @param criteria - The criteria to use for the evaluator.\n * @param llm - The language model to use for the evaluator.\n * @returns The configuration for the evaluator.\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [Criteria(\"helpfulness\")],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [\n * Criteria({\n * \"isCompliant\": \"Does the submission comply with the requirements of XYZ\"\n * })\n * ],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [{\n * evaluatorType: \"criteria\",\n * criteria: \"helpfulness\"\n * formatEvaluatorInputs: ...\n * }]\n * };\n * ```\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [{\n * evaluatorType: \"criteria\",\n * criteria: { \"isCompliant\": \"Does the submission comply with the requirements of XYZ\" },\n * formatEvaluatorInputs: ...\n * }]\n * };\n */\nexport type Criteria = EvalConfig & {\n evaluatorType: \"criteria\";\n /**\n * The \"criteria\" to insert into the prompt template\n * used for evaluation. See the prompt at\n * https://smith.langchain.com/hub/langchain-ai/criteria-evaluator\n * for more information.\n */\n criteria?: CriteriaType | Record<string, string>;\n /**\n * The language model to use as the evaluator, defaults to GPT-4\n */\n llm?: BaseLanguageModel;\n};\nexport type CriteriaEvalChainConfig = Criteria;\nexport declare function Criteria(criteria: CriteriaType | Record<string, string>, config?: Pick<Partial<LabeledCriteria>, \"formatEvaluatorInputs\" | \"llm\" | \"feedbackKey\">): EvalConfig;\n/**\n * Configuration to load a \"LabeledCriteriaEvalChain\" evaluator,\n * which prompts an LLM to determine whether the model's\n * prediction complies with the provided criteria and also\n * provides a \"ground truth\" label for the evaluator to incorporate\n * in its evaluation.\n * @param criteria - The criteria to use for the evaluator.\n * @param llm - The language model to use for the evaluator.\n * @returns The configuration for the evaluator.\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [LabeledCriteria(\"correctness\")],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [\n * LabeledCriteria({\n * \"mentionsAllFacts\": \"Does the include all facts provided in the reference?\"\n * })\n * ],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [{\n * evaluatorType: \"labeled_criteria\",\n * criteria: \"correctness\",\n * formatEvaluatorInputs: ...\n * }],\n * };\n * ```\n * @example\n * ```ts\n * const evalConfig = {\n * evaluators: [{\n * evaluatorType: \"labeled_criteria\",\n * criteria: { \"mentionsAllFacts\": \"Does the include all facts provided in the reference?\" },\n * formatEvaluatorInputs: ...\n * }],\n * };\n */\nexport type LabeledCriteria = EvalConfig & {\n evaluatorType: \"labeled_criteria\";\n /**\n * The \"criteria\" to insert into the prompt template\n * used for evaluation. See the prompt at\n * https://smith.langchain.com/hub/langchain-ai/labeled-criteria\n * for more information.\n */\n criteria?: CriteriaType | Record<string, string>;\n /**\n * The language model to use as the evaluator, defaults to GPT-4\n */\n llm?: BaseLanguageModel;\n};\nexport declare function LabeledCriteria(criteria: CriteriaType | Record<string, string>, config?: Pick<Partial<LabeledCriteria>, \"formatEvaluatorInputs\" | \"llm\" | \"feedbackKey\">): LabeledCriteria;\n/**\n * Configuration to load a \"EmbeddingDistanceEvalChain\" evaluator,\n * which embeds distances to score semantic difference between\n * a prediction and reference.\n */\nexport type EmbeddingDistance = EvalConfig & EmbeddingDistanceEvalChainInput & {\n evaluatorType: \"embedding_distance\";\n};\nexport declare function EmbeddingDistance(distanceMetric: EmbeddingDistanceEvalChainInput[\"distanceMetric\"], config?: Pick<Partial<LabeledCriteria>, \"formatEvaluatorInputs\" | \"embedding\" | \"feedbackKey\">): EmbeddingDistance;\n//# sourceMappingURL=config.d.ts.map"],"mappings":";;;;;;;;;;;KAOYW,eAAAA;;;;AAAZ,CAAA;AAKYC,KAAAA,uBAAAA,GAAuB,CAAA;EAAA,QAAA;EAAA,aAAA;EAAA,kBAAA;EAAA;CAAA,EAAA;EAAMC,QAAAA,EAAAA,GAAAA;EAAUC,aAAAA,EAAAA,GAAAA;EAAeC,kBAAAA,CAAAA,EAAAA,GAAAA;EAAoBC,GAAAA,EAI7Eb,GAJ6Ea;CAI7Eb,EAAAA,GACHQ,eADGR;AACHQ,KACMM,yBADNN,CAAAA,cAC8CU,MAD9CV,CAAAA,MAAAA,EAAAA,GAAAA,CAAAA,GACoEU,MADpEV,CAAAA,MAAAA,EAAAA,OAAAA,CAAAA,EAAAA,mBACgHU,MADhHV,CAAAA,MAAAA,EAAAA,GAAAA,CAAAA,GACsIU,MADtIV,CAAAA,MAAAA,EAAAA,OAAAA,CAAAA,EAAAA,kBACiLU,MADjLV,CAAAA,MAAAA,EAAAA,GAAAA,CAAAA,GACuMU,MADvMV,CAAAA,MAAAA,EAAAA,OAAAA,CAAAA,CAAAA,GAAAA;EAAe,KAAA,EAEVO,KAFU;EACTD,UAAAA,CAAAA,EAEKE,UAFLF;EAAwCI,SAAAA,CAAAA,EAGpCD,SAHoCC;EAAsBA,GAAAA,EAIjElB,GAJiEkB;EAA4CA,OAAAA,CAAAA,EAKxGnB,OALwGmB;CAAsBA;;;;;;AAInIlB,KAQGmB,gBAAAA,GARHnB,CAAAA,CAAAA,KAAAA,EAQ+Bc,yBAR/Bd,EAAAA,OAAAA,EAQmEF,cARnEE,EAAAA,GAQsFoB,OARtFpB,CAQ8FC,gBAR9FD,CAAAA,CAAAA,GAAAA,CAAAA,CAAAA,KAAAA,EAQ4Hc,yBAR5Hd,EAAAA,OAAAA,EAQgKF,cARhKE,EAAAA,GAQmLC,gBARnLD,CAAAA;AACKD,iBAQUsB,sBARVtB,CAAAA,UAAAA,MAQiDQ,aARjDR,EAAAA,UAQ0EG,YAR1EH,GAQyFoB,gBARzFpB,GAQ4GG,YAR5GH,GAQ2HoB,gBAR3HpB,CAAAA,CAAAA,SAAAA,EAQwJuB,CARxJvB,GAQ4JyB,UAR5JzB,GAQyKwB,CARzKxB,CAAAA,EAAAA,SAAAA,IAQ0LuB,CAR1LvB,GAQ8LyB,UAR9LzB;AAAO,iBASG0B,iBATH,CAAA,UAAA,MASqClB,aATrC,EAAA,UAS8DL,YAT9D,GAS6EiB,gBAT7E,GASgGjB,YAThG,GAS+GiB,gBAT/G,CAAA,CAAA,SAAA,EAS4IG,CAT5I,GASgJE,UAThJ,GAS6JD,CAT7J,CAAA,EAAA,SAAA,IAS8KA,CAT9K;AAOTJ,KAGAO,WAHgB,CAAA,UAAA,MAGYnB,aAHZ,GAAA,UAAA,GAAA,kBAAA,GAAA,oBAAA,EAAA,UAG8FL,YAH9F,GAG6GiB,gBAH7G,GAGgIjB,YAHhI,GAG+IiB,gBAH/I,CAAA,GAGmKG,CAHnK,GAGuKE,UAHvK,GAGoLD,CAHpL;;;;;;;;AAAgL;AAC5M;AAA+DhB,KAYnDoB,aAZmDpB,CAAAA,UAAAA,MAYrBA,aAZqBA,GAAAA,UAAAA,GAAAA,kBAAAA,GAAAA,oBAAAA,EAAAA,UAY6DL,YAZ7DK,GAY4EY,gBAZ5EZ,GAY+FL,YAZ/FK,GAY8GY,gBAZ9GZ,CAAAA,GAAAA;EAAyBL;;;;;EAAkFsB,UAAAA,CAAAA,EAkBzJE,WAlByJF,CAkB7IF,CAlB6IE,EAkB1ID,CAlB0IC,CAAAA,EAAAA;EAAaD;;;AAA+B;AACtN;;;;;;;;;;AAAoM;AACpM;;;;;;;;EAAgNA,qBAAAA,CAAAA,EAwCpLd,uBAxCoLc;EAAC;AAUjN;;;;;;;EAMgCA,gBAAAA,CAAAA,EAiCTA,CAjCSA,EAAAA;CAAfG;AAwBWjB,UAWXe,UAAAA,SAAmBlB,oBAXRG,CAAAA;EASLc;AAAC;AAExB;;EAmC2Bd,aAAAA,EAAAA,MA9BFF,aA8BEE;EAnCSH;AAAoB;AA8ExD;;;EAQ8BY,WAAAA,CAAAA,EAAAA,MAAAA;EAIpBrB;AAAiB;AAE3B;AACA;;;;;;;AAAuL;AA4CvL;;;;;AAY2B;AAE3B;;;;;;EAAoLgC,qBAAAA,EApHzJpB,uBAoHyJoB;AAAe;AAMnM;AAGA;;;;;;AAA+N;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAlFnN1B,UAAAA,GAAWqB;;;;;;;;aAQRpB,WAAec;;;;QAIpBrB;;KAEE+B,uBAAAA,GAA0BzB;iBACdA,UAAAA,WAAmBC,WAAec,iCAAiCa,KAAKD,QAAQD,qEAAqEL;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA4CjKK,eAAAA,GAAkBL;;;;;;;;aAQfpB,WAAec;;;;QAIpBrB;;iBAEcgC,eAAAA,WAA0BzB,WAAec,iCAAiCa,KAAKD,QAAQD,qEAAqEA;;;;;;KAMxKG,iBAAAA,GAAoBR,aAAanB;;;iBAGrB2B,iBAAAA,iBAAkC3B,4DAA4D0B,KAAKD,QAAQD,2EAA2EG"}

package/dist/smith/runner_utils.cjs CHANGED Viewed

@@ -1,8 +1,5 @@
 const require_rolldown_runtime = require('../_virtual/rolldown_runtime.cjs');
 const require_loader = require('../evaluation/loader.cjs');
-const require_client = require('../node_modules/.pnpm/langsmith@0.3.74_@opentelemetry_api@1.9.0_openai@5.12.2_ws@8.18.3_bufferutil@4.0.9_utf-8-validate@6.0.5__zod@3.25.76_/node_modules/langsmith/dist/client.cjs');
-const require_run_trees = require('../node_modules/.pnpm/langsmith@0.3.74_@opentelemetry_api@1.9.0_openai@5.12.2_ws@8.18.3_bufferutil@4.0.9_utf-8-validate@6.0.5__zod@3.25.76_/node_modules/langsmith/dist/run_trees.cjs');
-require('../node_modules/.pnpm/langsmith@0.3.74_@opentelemetry_api@1.9.0_openai@5.12.2_ws@8.18.3_bufferutil@4.0.9_utf-8-validate@6.0.5__zod@3.25.76_/node_modules/langsmith/index.cjs');
 const require_config = require('./config.cjs');
 const require_name_generation = require('./name_generation.cjs');
 const require_progress = require('./progress.cjs');
@@ -11,6 +8,7 @@ const __langchain_core_messages = require_rolldown_runtime.__toESM(require("@lan
 const __langchain_core_tracers_base = require_rolldown_runtime.__toESM(require("@langchain/core/tracers/base"));
 const __langchain_core_tracers_tracer_langchain = require_rolldown_runtime.__toESM(require("@langchain/core/tracers/tracer_langchain"));
 const __langchain_core_utils_async_caller = require_rolldown_runtime.__toESM(require("@langchain/core/utils/async_caller"));
+const langsmith = require_rolldown_runtime.__toESM(require("langsmith"));
 //#region src/smith/runner_utils.ts
 var SingleRunIdExtractor = class {
@@ -87,7 +85,7 @@ function isLLMStringEvaluator(evaluator) {
 * The goal of this class is to ensure seamless interop when intergrated
 * with other Runnables.
 */
-var CallbackManagerRunTree = class CallbackManagerRunTree extends require_run_trees.RunTree {
+var CallbackManagerRunTree = class CallbackManagerRunTree extends langsmith.RunTree {
 	callbackManager;
 	activeCallbackManager = void 0;
 	constructor(config, callbackManager) {
@@ -136,7 +134,7 @@ var RunnableTraceable = class extends __langchain_core_runnables.Runnable {
 		if (!callbackManager) throw new Error("CallbackManager not found");
 		const runTree = new CallbackManagerRunTree({
 			...partialConfig,
-			parent_run: callbackManager?._parentRunId ? new require_run_trees.RunTree({
+			parent_run: callbackManager?._parentRunId ? new langsmith.RunTree({
 				name: "<parent>",
 				id: callbackManager?._parentRunId
 			}) : void 0
@@ -356,7 +354,7 @@ async function runOnDataset(chainOrFactory, datasetName, options) {
 		formatEvaluatorInputs: options.formatEvaluatorInputs
 	} : void 0);
 	const wrappedModel = await createWrappedModel(chainOrFactory);
-	const testClient = client ?? new require_client.Client();
+	const testClient = client ?? new langsmith.Client();
 	const testProjectName = projectName ?? require_name_generation.randomName();
 	const dataset = await testClient.readDataset({ datasetName });
 	const datasetId = dataset.id;

package/dist/smith/runner_utils.cjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"runner_utils.cjs","names":["_chain: Serialized","_inputs: ChainValues","runId: string","BaseTracer","run: Run","evaluator: RunEvaluatorLike","RunnableLambda","example?: Example","LangChainTracer","evaluator: any","RunTree","config: RunTreeConfig","callbackManager: CallbackManager","Runnable","fields: { func: AnyTraceableFunction }","input: RunInput","options?: Partial<RunnableConfig>","evaluator: LLMStringEvaluator","evaluationName: string","formatEvaluatorInputs: EvaluatorInputFormatter","config: EvalConfig \| keyof EvaluatorType","loadEvaluator","evaluators: (RunEvaluator \| DynamicRunEvaluator)[]","config: RunEvalConfig<keyof EvaluatorType>","isCustomEvaluator","isOffTheShelfEvaluator","modelOrFactory: ChainOrFactory","configs: RunnableConfig[]","ProgressBar","AsyncCaller","examples: Example[]","chainOrFactory: ChainOrFactory","dataType?: DataType","datasetName: string","options?: RunOnDatasetParams","evaluationConfig: RunEvalConfig \| undefined","Client","randomName","wrappedRunnable: Runnable","runs: Run[]","evalResults: Record<\n string,\n { run_id: string; execution_time?: number; feedback: Feedback[] }\n >","results: EvalResults","x: unknown"],"sources":["../../src/smith/runner_utils.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { Serialized } from \"@langchain/core/load/serializable\";\nimport { mapStoredMessagesToChatMessages } from \"@langchain/core/messages\";\nimport {\n Runnable,\n RunnableConfig,\n RunnableLambda,\n getCallbackManagerForConfig,\n} from \"@langchain/core/runnables\";\nimport { LangChainTracer } from \"@langchain/core/tracers/tracer_langchain\";\nimport { BaseTracer } from \"@langchain/core/tracers/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport type {\n CallbackManager,\n CallbackManagerForChainRun,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n Client,\n Example,\n Feedback,\n Run,\n RunTree,\n RunTreeConfig,\n} from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { DataType } from \"langsmith/schemas\";\nimport type { TraceableFunction } from \"langsmith/singletons/traceable\";\nimport { LLMStringEvaluator } from \"../evaluation/base.js\";\nimport { loadEvaluator } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nimport {\n isOffTheShelfEvaluator,\n type DynamicRunEvaluatorParams,\n type EvalConfig,\n type EvaluatorInputFormatter,\n type RunEvalConfig,\n type RunEvaluatorLike,\n isCustomEvaluator,\n} from \"./config.js\";\nimport { randomName } from \"./name_generation.js\";\nimport { ProgressBar } from \"./progress.js\";\n\nexport type ChainOrFactory =\n \| Runnable\n \| (() => Runnable)\n \| AnyTraceableFunction\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n \| ((obj: any) => any)\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n \| ((obj: any) => Promise<any>)\n \| (() => (obj: unknown) => unknown)\n \| (() => (obj: unknown) => Promise<unknown>);\n\nclass SingleRunIdExtractor {\n runIdPromiseResolver: (runId: string) => void;\n\n runIdPromise: Promise<string>;\n\n constructor() {\n this.runIdPromise = new Promise<string>((extract) => {\n this.runIdPromiseResolver = extract;\n });\n }\n\n handleChainStart = (\n _chain: Serialized,\n _inputs: ChainValues,\n runId: string\n ) => {\n this.runIdPromiseResolver(runId);\n };\n\n async extract(): Promise<string> {\n return this.runIdPromise;\n }\n}\n\nclass SingleRunExtractor extends BaseTracer {\n runPromiseResolver: (run: Run) => void;\n\n runPromise: Promise<Run>;\n\n /** The name of the callback handler. /\n name = \"single_run_extractor\";\n\n constructor() {\n super();\n this.runPromise = new Promise<Run>((extract) => {\n this.runPromiseResolver = extract;\n });\n }\n\n async persistRun(run: Run) {\n this.runPromiseResolver(run);\n }\n\n async extract(): Promise<Run> {\n return this.runPromise;\n }\n}\n\n/\n Wraps an evaluator function + implements the RunEvaluator interface.\n /\nclass DynamicRunEvaluator implements RunEvaluator {\n evaluator: RunnableLambda<DynamicRunEvaluatorParams, EvaluationResult>;\n\n constructor(evaluator: RunEvaluatorLike) {\n this.evaluator = new RunnableLambda({ func: evaluator });\n }\n\n /\n Evaluates a run with an optional example and returns the evaluation result.\n * @param run The run to evaluate.\n * @param example The optional example to use for evaluation.\n * @returns A promise that extracts to the evaluation result.\n /\n async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n const extractor = new SingleRunIdExtractor();\n const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n const result = await this.evaluator.invoke(\n {\n run,\n example,\n input: run.inputs,\n prediction: run.outputs,\n reference: example?.outputs,\n },\n {\n callbacks: [extractor, tracer],\n }\n );\n const runId = await extractor.extract();\n return {\n sourceRunId: runId,\n ...result,\n };\n }\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction isLLMStringEvaluator(evaluator: any): evaluator is LLMStringEvaluator {\n return evaluator && typeof evaluator.evaluateStrings === \"function\";\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;\n\n/\n Internal implementation of RunTree, which uses the\n * provided callback manager instead of the internal LangSmith client.\n \n The goal of this class is to ensure seamless interop when intergrated\n * with other Runnables.\n /\nclass CallbackManagerRunTree extends RunTree {\n callbackManager: CallbackManager;\n\n activeCallbackManager: CallbackManagerForChainRun \| undefined = undefined;\n\n constructor(config: RunTreeConfig, callbackManager: CallbackManager) {\n super(config);\n\n this.callbackManager = callbackManager;\n }\n\n createChild(config: RunTreeConfig): CallbackManagerRunTree {\n const child = new CallbackManagerRunTree(\n {\n ...config,\n parent_run: this,\n project_name: this.project_name,\n client: this.client,\n },\n this.activeCallbackManager?.getChild() ?? this.callbackManager\n );\n this.child_runs.push(child);\n return child;\n }\n\n async postRun(): Promise<void> {\n // how it is translated in comparison to basic RunTree?\n this.activeCallbackManager = await this.callbackManager.handleChainStart(\n typeof this.serialized !== \"object\" &&\n this.serialized != null &&\n \"lc\" in this.serialized\n ? this.serialized\n : {\n id: [\"langchain\", \"smith\", \"CallbackManagerRunTree\"],\n lc: 1,\n type: \"not_implemented\",\n },\n this.inputs,\n this.id,\n this.run_type,\n undefined,\n undefined,\n this.name\n );\n }\n\n async patchRun(): Promise<void> {\n if (this.error) {\n await this.activeCallbackManager?.handleChainError(\n this.error,\n this.id,\n this.parent_run?.id,\n undefined,\n undefined\n );\n } else {\n await this.activeCallbackManager?.handleChainEnd(\n this.outputs ?? {},\n this.id,\n this.parent_run?.id,\n undefined,\n undefined\n );\n }\n }\n}\n\nclass RunnableTraceable<RunInput, RunOutput> extends Runnable<\n RunInput,\n RunOutput\n> {\n lc_serializable = false;\n\n lc_namespace = [\"langchain_core\", \"runnables\"];\n\n protected func: AnyTraceableFunction;\n\n constructor(fields: { func: AnyTraceableFunction }) {\n super(fields);\n\n if (!isLangsmithTraceableFunction(fields.func)) {\n throw new Error(\n \"RunnableTraceable requires a function that is wrapped in traceable higher-order function\"\n );\n }\n\n this.func = fields.func;\n }\n\n async invoke(input: RunInput, options?: Partial<RunnableConfig>) {\n const [config] = this._getOptionsList(options ?? {}, 1);\n const callbackManager = await getCallbackManagerForConfig(config);\n\n const partialConfig =\n \"langsmith:traceable\" in this.func\n ? (this.func[\"langsmith:traceable\"] as RunTreeConfig)\n : { name: \"<lambda>\" };\n\n if (!callbackManager) throw new Error(\"CallbackManager not found\");\n const runTree = new CallbackManagerRunTree(\n {\n ...partialConfig,\n parent_run: callbackManager?._parentRunId\n ? new RunTree({ name: \"<parent>\", id: callbackManager?._parentRunId })\n : undefined,\n },\n callbackManager\n );\n\n if (\n typeof input === \"object\" &&\n input != null &&\n Object.keys(input).length === 1\n ) {\n if (\"args\" in input && Array.isArray(input)) {\n return (await this.func(runTree, ...input)) as RunOutput;\n }\n\n if (\n \"input\" in input &&\n !(\n typeof input === \"object\" &&\n input != null &&\n !Array.isArray(input) &&\n // eslint-disable-next-line no-instanceof/no-instanceof\n !(input instanceof Date)\n )\n ) {\n try {\n return (await this.func(runTree, input.input)) as RunOutput;\n } catch {\n return (await this.func(runTree, input)) as RunOutput;\n }\n }\n }\n\n return (await this.func(runTree, input)) as RunOutput;\n }\n}\n\n/\n Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])\n * and composes with a prepareData function so the user can prepare the trace and\n * dataset data for the evaluator.\n /\nclass PreparedRunEvaluator implements RunEvaluator {\n evaluator: LLMStringEvaluator;\n\n formatEvaluatorInputs: EvaluatorInputFormatter;\n\n isStringEvaluator: boolean;\n\n evaluationName: string;\n\n constructor(\n evaluator: LLMStringEvaluator,\n evaluationName: string,\n formatEvaluatorInputs: EvaluatorInputFormatter\n ) {\n this.evaluator = evaluator;\n this.isStringEvaluator = typeof evaluator?.evaluateStrings === \"function\";\n this.evaluationName = evaluationName;\n this.formatEvaluatorInputs = formatEvaluatorInputs;\n }\n\n static async fromEvalConfig(\n config: EvalConfig \| keyof EvaluatorType\n ): Promise<PreparedRunEvaluator> {\n const evaluatorType =\n typeof config === \"string\" ? config : config.evaluatorType;\n const evalConfig = typeof config === \"string\" ? ({} as EvalConfig) : config;\n const evaluator = await loadEvaluator(evaluatorType, evalConfig);\n const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;\n if (!isLLMStringEvaluator(evaluator)) {\n throw new Error(\n `Evaluator of type ${evaluatorType} not yet supported. ` +\n \"Please use a string evaluator, or implement your \" +\n \"evaluation logic as a custom evaluator.\"\n );\n }\n if (!feedbackKey) {\n throw new Error(\n `Evaluator of type ${evaluatorType} must have an evaluationName` +\n ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`\n );\n }\n return new PreparedRunEvaluator(\n evaluator as LLMStringEvaluator,\n feedbackKey,\n evalConfig?.formatEvaluatorInputs\n );\n }\n\n /\n Evaluates a run with an optional example and returns the evaluation result.\n * @param run The run to evaluate.\n * @param example The optional example to use for evaluation.\n * @returns A promise that extracts to the evaluation result.\n /\n async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n const { prediction, input, reference } = this.formatEvaluatorInputs({\n rawInput: run.inputs,\n rawPrediction: run.outputs,\n rawReferenceOutput: example?.outputs,\n run,\n });\n const extractor = new SingleRunIdExtractor();\n const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n if (this.isStringEvaluator) {\n const evalResult = await this.evaluator.evaluateStrings(\n {\n prediction: prediction as string,\n reference: reference as string,\n input: input as string,\n },\n {\n callbacks: [extractor, tracer],\n }\n );\n const runId = await extractor.extract();\n return {\n key: this.evaluationName,\n comment: evalResult?.reasoning,\n sourceRunId: runId,\n ...evalResult,\n };\n }\n throw new Error(\n \"Evaluator not yet supported. \" +\n \"Please use a string evaluator, or implement your \" +\n \"evaluation logic as a custom evaluator.\"\n );\n }\n}\n\nclass LoadedEvalConfig {\n constructor(public evaluators: (RunEvaluator \| DynamicRunEvaluator)[]) {}\n\n static async fromRunEvalConfig(\n config: RunEvalConfig<keyof EvaluatorType>\n ): Promise<LoadedEvalConfig> {\n // Custom evaluators are applied \"as-is\"\n const customEvaluators = (\n config?.customEvaluators ?? config.evaluators?.filter(isCustomEvaluator)\n )?.map((evaluator) => {\n if (typeof evaluator === \"function\") {\n return new DynamicRunEvaluator(evaluator);\n } else {\n return evaluator;\n }\n });\n\n const offTheShelfEvaluators = await Promise.all(\n config?.evaluators\n ?.filter(isOffTheShelfEvaluator)\n ?.map(\n async (evaluator) =>\n await PreparedRunEvaluator.fromEvalConfig(evaluator)\n ) ?? []\n );\n return new LoadedEvalConfig(\n (customEvaluators ?? []).concat(offTheShelfEvaluators ?? [])\n );\n }\n}\n\nexport interface RunOnDatasetParams\n extends Omit<RunEvalConfig, \"customEvaluators\"> {\n /\n Name of the project for logging and tracking.\n /\n projectName?: string;\n\n /\n Additional metadata for the project.\n /\n projectMetadata?: Record<string, unknown>;\n\n /\n Client instance for LangSmith service interaction.\n /\n client?: Client;\n\n /\n Maximum concurrency level for dataset processing.\n /\n maxConcurrency?: number;\n\n /\n @deprecated Pass keys directly to the RunOnDatasetParams instead\n /\n evaluationConfig?: RunEvalConfig;\n}\n\n/\n Internals expect a constructor () -> Runnable. This function wraps/coerces\n * the provided LangChain object, custom function, or factory function into\n * a constructor of a runnable.\n * @param modelOrFactory The model or factory to create a wrapped model from.\n * @returns A function that returns the wrapped model.\n * @throws Error if the modelOrFactory is invalid.\n /\nconst createWrappedModel = async (modelOrFactory: ChainOrFactory) => {\n if (Runnable.isRunnable(modelOrFactory)) {\n return () => modelOrFactory;\n }\n if (typeof modelOrFactory === \"function\") {\n if (isLangsmithTraceableFunction(modelOrFactory)) {\n const wrappedModel = new RunnableTraceable({ func: modelOrFactory });\n return () => wrappedModel;\n }\n\n try {\n // If it works with no arguments, assume it's a factory\n let res = (modelOrFactory as () => Runnable)();\n if (\n res &&\n typeof (res as unknown as Promise<Runnable>).then === \"function\"\n ) {\n res = await res;\n }\n return modelOrFactory as () => Runnable;\n } catch {\n // Otherwise, it's a custom UDF, and we'll wrap\n // the function in a lambda\n const wrappedModel = new RunnableLambda({ func: modelOrFactory });\n return () => wrappedModel;\n }\n }\n throw new Error(\"Invalid modelOrFactory\");\n};\n\nconst loadExamples = async ({\n datasetName,\n client,\n projectName,\n}: {\n datasetName: string;\n client: Client;\n projectName: string;\n maxConcurrency: number;\n}) => {\n const exampleIterator = client.listExamples({ datasetName });\n const configs: RunnableConfig[] = [];\n const runExtractors = [];\n const examples = [];\n for await (const example of exampleIterator) {\n const runExtractor = new SingleRunExtractor();\n configs.push({\n callbacks: [\n new LangChainTracer({ exampleId: example.id, projectName }),\n runExtractor,\n ],\n });\n examples.push(example);\n runExtractors.push(runExtractor);\n }\n return {\n configs,\n examples,\n runExtractors,\n };\n};\n\nconst applyEvaluators = async ({\n evaluation,\n runs,\n examples,\n client,\n maxConcurrency,\n}: {\n evaluation: LoadedEvalConfig;\n runs: Run[];\n examples: Example[];\n client: Client;\n maxConcurrency: number;\n}): Promise<{\n [key: string]: {\n execution_time?: number;\n run_id: string;\n feedback: Feedback[];\n };\n}> => {\n // TODO: Parallelize and/or put in callbacks to speed up evals.\n const { evaluators } = evaluation;\n const progress = new ProgressBar({\n total: examples.length,\n format: \"Running Evaluators: {bar} {percentage}% \| {value}/{total}\\n\",\n });\n const caller = new AsyncCaller({\n maxConcurrency,\n });\n const requests = runs.map(\n async (\n run,\n i\n ): Promise<{\n run_id: string;\n execution_time?: number;\n feedback: Feedback[];\n }> =>\n caller.call(async () => {\n const evaluatorResults = await Promise.allSettled(\n evaluators.map((evaluator) =>\n client.evaluateRun(run, evaluator, {\n referenceExample: examples[i],\n loadChildRuns: false,\n })\n )\n );\n progress.increment();\n return {\n execution_time:\n run?.end_time && run.start_time\n ? new Date(run.end_time).getTime() -\n new Date(run.start_time).getTime()\n : undefined,\n feedback: evaluatorResults.map((evalResult) =>\n evalResult.status === \"fulfilled\"\n ? evalResult.value\n : evalResult.reason\n ),\n run_id: run.id,\n };\n })\n );\n const results = await Promise.all(requests);\n\n return results.reduce(\n (acc, result, i) => ({\n ...acc,\n [examples[i].id]: result,\n }),\n {}\n );\n};\n\nexport type EvalResults = {\n projectName: string;\n results: {\n [key: string]: {\n execution_time?: number;\n run_id: string;\n feedback: Feedback[];\n };\n };\n};\n\nconst getExamplesInputs = (\n examples: Example[],\n chainOrFactory: ChainOrFactory,\n dataType?: DataType\n) => {\n if (dataType === \"chat\") {\n // For some batty reason, we store the chat dataset differently.\n // { type: \"system\", data: { content: inputs.input } },\n // But we need to create AIMesage, SystemMessage, etc.\n return examples.map(({ inputs }) =>\n mapStoredMessagesToChatMessages(inputs.input)\n );\n }\n // If it's a language model and ALL example inputs have a single value,\n // then we can be friendly and flatten the inputs to a list of strings.\n const isLanguageModel =\n typeof chainOrFactory === \"object\" &&\n typeof (chainOrFactory as BaseLanguageModel)._llmType === \"function\";\n if (\n isLanguageModel &&\n examples.every(({ inputs }) => Object.keys(inputs).length === 1)\n ) {\n return examples.map(({ inputs }) => Object.values(inputs)[0]);\n }\n return examples.map(({ inputs }) => inputs);\n};\n\n/\n Evaluates a given model or chain against a specified LangSmith dataset.\n \n This function fetches example records from the specified dataset,\n * runs the model or chain against each example, and returns the evaluation\n * results.\n \n @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a\n * Runnable instance, a factory function that returns a Runnable, or a user-defined\n * function or factory.\n \n @param datasetName - The name of the dataset against which the evaluation will be\n * performed. This dataset should already be defined and contain the relevant data\n * for evaluation.\n \n @param options - (Optional) Additional parameters for the evaluation process:\n * - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.\n * - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.\n * - `projectName` (string): Name of the project for logging and tracking.\n * - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.\n * - `client` (Client): Client instance for LangSmith service interaction.\n * - `maxConcurrency` (number): Maximum concurrency level for dataset processing.\n \n @returns A promise that resolves to an `EvalResults` object. This object includes\n * detailed results of the evaluation, such as execution time, run IDs, and feedback\n * for each entry in the dataset.\n \n @example\n * ```typescript\n * // Example usage for evaluating a model on a dataset\n * async function evaluateModel() {\n * const chain = /* ...create your model or chain...\\//\n const datasetName = 'example-dataset';\n * const client = new Client(/* ...config... \\//);\n \n * const results = await runOnDataset(chain, datasetName, {\n * evaluators: [/* ...evaluators... \\//],\n client,\n * });\n \n console.log('Evaluation Results:', results);\n * }\n \n evaluateModel();\n * ```\n * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against\n * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams[\"evaluators\"]`, which can\n * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.\n * The function returns the evaluation results, which can be logged or further processed as needed.\n */\n\nexport async function runOnDataset(\n chainOrFactory: ChainOrFactory,\n datasetName: string,\n options?: RunOnDatasetParams\n) {\n const {\n projectName,\n projectMetadata,\n client,\n maxConcurrency,\n }: RunOnDatasetParams = options ?? {};\n\n const evaluationConfig: RunEvalConfig \| undefined =\n options?.evaluationConfig ??\n (options?.evaluators != null\n ? {\n evaluators: options.evaluators,\n formatEvaluatorInputs: options.formatEvaluatorInputs,\n }\n : undefined);\n\n const wrappedModel = await createWrappedModel(chainOrFactory);\n const testClient = client ?? new Client();\n const testProjectName = projectName ?? randomName();\n const dataset = await testClient.readDataset({ datasetName });\n const datasetId = dataset.id;\n const testConcurrency = maxConcurrency ?? 5;\n const { configs, examples, runExtractors } = await loadExamples({\n datasetName,\n client: testClient,\n projectName: testProjectName,\n maxConcurrency: testConcurrency,\n });\n\n await testClient.createProject({\n projectName: testProjectName,\n referenceDatasetId: datasetId,\n projectExtra: { metadata: { ...projectMetadata } },\n });\n const wrappedRunnable: Runnable = new RunnableLambda({\n func: wrappedModel,\n }).withConfig({ runName: \"evaluationRun\" });\n const runInputs = getExamplesInputs(\n examples,\n chainOrFactory,\n dataset.data_type\n );\n const progress = new ProgressBar({\n total: runInputs.length,\n format: \"Predicting: {bar} {percentage}% \| {value}/{total}\",\n });\n // TODO: Collect the runs as well.\n await wrappedRunnable\n .withListeners({\n onEnd: () => progress.increment(),\n })\n // TODO: Insert evaluation inline for immediate feedback.\n .batch(runInputs, configs, {\n maxConcurrency,\n returnExceptions: true,\n });\n\n progress.complete();\n const runs: Run[] = [];\n for (let i = 0; i < examples.length; i += 1) {\n runs.push(await runExtractors[i].extract());\n }\n let evalResults: Record<\n string,\n { run_id: string; execution_time?: number; feedback: Feedback[] }\n > = {};\n if (evaluationConfig) {\n const loadedEvalConfig = await LoadedEvalConfig.fromRunEvalConfig(\n evaluationConfig\n );\n evalResults = await applyEvaluators({\n evaluation: loadedEvalConfig,\n runs,\n examples,\n client: testClient,\n maxConcurrency: testConcurrency,\n });\n }\n const results: EvalResults = {\n projectName: testProjectName,\n results: evalResults ?? {},\n };\n return results;\n}\n\nfunction isLangsmithTraceableFunction(x: unknown): x is AnyTraceableFunction {\n return typeof x === \"function\" && \"langsmith:traceable\" in x;\n}\n"],"mappings":";;;;;;;;;;;;;;;AAsDA,IAAM,uBAAN,MAA2B;CACzB;CAEA;CAEA,cAAc;EACZ,KAAK,eAAe,IAAI,QAAgB,CAAC,YAAY;GACnD,KAAK,uBAAuB;EAC7B;CACF;CAED,mBAAmB,CACjBA,QACAC,SACAC,UACG;EACH,KAAK,qBAAqB,MAAM;CACjC;CAED,MAAM,UAA2B;AAC/B,SAAO,KAAK;CACb;AACF;AAED,IAAM,qBAAN,cAAiCC,yCAAW;CAC1C;CAEA;;CAGA,OAAO;CAEP,cAAc;EACZ,OAAO;EACP,KAAK,aAAa,IAAI,QAAa,CAAC,YAAY;GAC9C,KAAK,qBAAqB;EAC3B;CACF;CAED,MAAM,WAAWC,KAAU;EACzB,KAAK,mBAAmB,IAAI;CAC7B;CAED,MAAM,UAAwB;AAC5B,SAAO,KAAK;CACb;AACF;;;;AAKD,IAAM,sBAAN,MAAkD;CAChD;CAEA,YAAYC,WAA6B;EACvC,KAAK,YAAY,IAAIC,0CAAe,EAAE,MAAM,UAAW;CACxD;;;;;;;CAQD,MAAM,YAAYF,KAAUG,SAA8C;EACxE,MAAM,YAAY,IAAI;EACtB,MAAM,SAAS,IAAIC,0DAAgB,EAAE,aAAa,aAAc;EAChE,MAAM,SAAS,MAAM,KAAK,UAAU,OAClC;GACE;GACA;GACA,OAAO,IAAI;GACX,YAAY,IAAI;GAChB,WAAW,SAAS;EACrB,GACD,EACE,WAAW,CAAC,WAAW,MAAO,EAC/B,EACF;EACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,SAAO;GACL,aAAa;GACb,GAAG;EACJ;CACF;AACF;AAGD,SAAS,qBAAqBC,WAAiD;AAC7E,QAAO,aAAa,OAAO,UAAU,oBAAoB;AAC1D;;;;;;;;AAYD,IAAM,yBAAN,MAAM,+BAA+BC,0BAAQ;CAC3C;CAEA,wBAAgE;CAEhE,YAAYC,QAAuBC,iBAAkC;EACnE,MAAM,OAAO;EAEb,KAAK,kBAAkB;CACxB;CAED,YAAYD,QAA+C;EACzD,MAAM,QAAQ,IAAI,uBAChB;GACE,GAAG;GACH,YAAY;GACZ,cAAc,KAAK;GACnB,QAAQ,KAAK;EACd,GACD,KAAK,uBAAuB,UAAU,IAAI,KAAK;EAEjD,KAAK,WAAW,KAAK,MAAM;AAC3B,SAAO;CACR;CAED,MAAM,UAAyB;EAE7B,KAAK,wBAAwB,MAAM,KAAK,gBAAgB,iBACtD,OAAO,KAAK,eAAe,YACzB,KAAK,cAAc,QACnB,QAAQ,KAAK,aACX,KAAK,aACL;GACE,IAAI;IAAC;IAAa;IAAS;GAAyB;GACpD,IAAI;GACJ,MAAM;EACP,GACL,KAAK,QACL,KAAK,IACL,KAAK,UACL,QACA,QACA,KAAK,KACN;CACF;CAED,MAAM,WAA0B;AAC9B,MAAI,KAAK,OACP,MAAM,KAAK,uBAAuB,iBAChC,KAAK,OACL,KAAK,IACL,KAAK,YAAY,IACjB,QACA,OACD;OAED,MAAM,KAAK,uBAAuB,eAChC,KAAK,WAAW,CAAE,GAClB,KAAK,IACL,KAAK,YAAY,IACjB,QACA,OACD;CAEJ;AACF;AAED,IAAM,oBAAN,cAAqDE,oCAGnD;CACA,kBAAkB;CAElB,eAAe,CAAC,kBAAkB,WAAY;CAE9C,AAAU;CAEV,YAAYC,QAAwC;EAClD,MAAM,OAAO;AAEb,MAAI,CAAC,6BAA6B,OAAO,KAAK,CAC5C,OAAM,IAAI,MACR;EAIJ,KAAK,OAAO,OAAO;CACpB;CAED,MAAM,OAAOC,OAAiBC,SAAmC;EAC/D,MAAM,CAAC,OAAO,GAAG,KAAK,gBAAgB,WAAW,CAAE,GAAE,EAAE;EACvD,MAAM,kBAAkB,kEAAkC,OAAO;EAEjE,MAAM,gBACJ,yBAAyB,KAAK,OACzB,KAAK,KAAK,yBACX,EAAE,MAAM,WAAY;AAE1B,MAAI,CAAC,gBAAiB,OAAM,IAAI,MAAM;EACtC,MAAM,UAAU,IAAI,uBAClB;GACE,GAAG;GACH,YAAY,iBAAiB,eACzB,IAAIN,0BAAQ;IAAE,MAAM;IAAY,IAAI,iBAAiB;GAAc,KACnE;EACL,GACD;AAGF,MACE,OAAO,UAAU,YACjB,SAAS,QACT,OAAO,KAAK,MAAM,CAAC,WAAW,GAC9B;AACA,OAAI,UAAU,SAAS,MAAM,QAAQ,MAAM,CACzC,QAAQ,MAAM,KAAK,KAAK,SAAS,GAAG,MAAM;AAG5C,OACE,WAAW,SACX,EACE,OAAO,UAAU,YACjB,SAAS,QACT,CAAC,MAAM,QAAQ,MAAM,IAErB,EAAE,iBAAiB,OAGrB,KAAI;AACF,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM,MAAM;GAC9C,QAAO;AACN,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;GACxC;EAEJ;AAED,SAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;CACxC;AACF;;;;;;AAOD,IAAM,uBAAN,MAAM,qBAA6C;CACjD;CAEA;CAEA;CAEA;CAEA,YACEO,WACAC,gBACAC,uBACA;EACA,KAAK,YAAY;EACjB,KAAK,oBAAoB,OAAO,WAAW,oBAAoB;EAC/D,KAAK,iBAAiB;EACtB,KAAK,wBAAwB;CAC9B;CAED,aAAa,eACXC,QAC+B;EAC/B,MAAM,gBACJ,OAAO,WAAW,WAAW,SAAS,OAAO;EAC/C,MAAM,aAAa,OAAO,WAAW,WAAY,CAAE,IAAkB;EACrE,MAAM,YAAY,MAAMC,6BAAc,eAAe,WAAW;EAChE,MAAM,cAAc,YAAY,eAAe,WAAW;AAC1D,MAAI,CAAC,qBAAqB,UAAU,CAClC,OAAM,IAAI,MACR,CAAC,kBAAkB,EAAE,cAAc,4GAAoB,CAEZ;AAG/C,MAAI,CAAC,YACH,OAAM,IAAI,MACR,CAAC,kBAAkB,EAAE,cAAc,qGAA4B,CACc;AAGjF,SAAO,IAAI,qBACT,WACA,aACA,YAAY;CAEf;;;;;;;CAQD,MAAM,YAAYjB,KAAUG,SAA8C;EACxE,MAAM,EAAE,YAAY,OAAO,WAAW,GAAG,KAAK,sBAAsB;GAClE,UAAU,IAAI;GACd,eAAe,IAAI;GACnB,oBAAoB,SAAS;GAC7B;EACD,EAAC;EACF,MAAM,YAAY,IAAI;EACtB,MAAM,SAAS,IAAIC,0DAAgB,EAAE,aAAa,aAAc;AAChE,MAAI,KAAK,mBAAmB;GAC1B,MAAM,aAAa,MAAM,KAAK,UAAU,gBACtC;IACc;IACD;IACJ;GACR,GACD,EACE,WAAW,CAAC,WAAW,MAAO,EAC/B,EACF;GACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,UAAO;IACL,KAAK,KAAK;IACV,SAAS,YAAY;IACrB,aAAa;IACb,GAAG;GACJ;EACF;AACD,QAAM,IAAI,MACR;CAIH;AACF;AAED,IAAM,mBAAN,MAAM,iBAAiB;CACrB,YAAmBc,YAAoD;EAApD;CAAsD;CAEzE,aAAa,kBACXC,QAC2B;EAE3B,MAAM,oBACJ,QAAQ,oBAAoB,OAAO,YAAY,OAAOC,iCAAkB,GACvE,IAAI,CAAC,cAAc;AACpB,OAAI,OAAO,cAAc,WACvB,QAAO,IAAI,oBAAoB;OAE/B,QAAO;EAEV,EAAC;EAEF,MAAM,wBAAwB,MAAM,QAAQ,IAC1C,QAAQ,YACJ,OAAOC,sCAAuB,EAC9B,IACA,OAAO,cACL,MAAM,qBAAqB,eAAe,UAAU,CACvD,IAAI,CAAE,EACV;AACD,SAAO,IAAI,kBACR,oBAAoB,CAAE,GAAE,OAAO,yBAAyB,CAAE,EAAC;CAE/D;AACF;;;;;;;;;AAsCD,MAAM,qBAAqB,OAAOC,mBAAmC;AACnE,KAAIb,oCAAS,WAAW,eAAe,CACrC,QAAO,MAAM;AAEf,KAAI,OAAO,mBAAmB,YAAY;AACxC,MAAI,6BAA6B,eAAe,EAAE;GAChD,MAAM,eAAe,IAAI,kBAAkB,EAAE,MAAM,eAAgB;AACnE,UAAO,MAAM;EACd;AAED,MAAI;GAEF,IAAI,MAAO,gBAAmC;AAC9C,OACE,OACA,OAAQ,IAAqC,SAAS,YAEtD,MAAM,MAAM;AAEd,UAAO;EACR,QAAO;GAGN,MAAM,eAAe,IAAIP,0CAAe,EAAE,MAAM,eAAgB;AAChE,UAAO,MAAM;EACd;CACF;AACD,OAAM,IAAI,MAAM;AACjB;AAED,MAAM,eAAe,OAAO,EAC1B,aACA,QACA,aAMD,KAAK;CACJ,MAAM,kBAAkB,OAAO,aAAa,EAAE,YAAa,EAAC;CAC5D,MAAMqB,UAA4B,CAAE;CACpC,MAAM,gBAAgB,CAAE;CACxB,MAAM,WAAW,CAAE;AACnB,YAAW,MAAM,WAAW,iBAAiB;EAC3C,MAAM,eAAe,IAAI;EACzB,QAAQ,KAAK,EACX,WAAW,CACT,IAAInB,0DAAgB;GAAE,WAAW,QAAQ;GAAI;EAAa,IAC1D,YACD,EACF,EAAC;EACF,SAAS,KAAK,QAAQ;EACtB,cAAc,KAAK,aAAa;CACjC;AACD,QAAO;EACL;EACA;EACA;CACD;AACF;AAED,MAAM,kBAAkB,OAAO,EAC7B,YACA,MACA,UACA,QACA,gBAOD,KAMK;CAEJ,MAAM,EAAE,YAAY,GAAG;CACvB,MAAM,WAAW,IAAIoB,6BAAY;EAC/B,OAAO,SAAS;EAChB,QAAQ;CACT;CACD,MAAM,SAAS,IAAIC,gDAAY,EAC7B,eACD;CACD,MAAM,WAAW,KAAK,IACpB,OACE,KACA,MAMA,OAAO,KAAK,YAAY;EACtB,MAAM,mBAAmB,MAAM,QAAQ,WACrC,WAAW,IAAI,CAAC,cACd,OAAO,YAAY,KAAK,WAAW;GACjC,kBAAkB,SAAS;GAC3B,eAAe;EAChB,EAAC,CACH,CACF;EACD,SAAS,WAAW;AACpB,SAAO;GACL,gBACE,KAAK,YAAY,IAAI,aACjB,IAAI,KAAK,IAAI,UAAU,SAAS,GAChC,IAAI,KAAK,IAAI,YAAY,SAAS,GAClC;GACN,UAAU,iBAAiB,IAAI,CAAC,eAC9B,WAAW,WAAW,cAClB,WAAW,QACX,WAAW,OAChB;GACD,QAAQ,IAAI;EACb;CACF,EAAC,CACL;CACD,MAAM,UAAU,MAAM,QAAQ,IAAI,SAAS;AAE3C,QAAO,QAAQ,OACb,CAAC,KAAK,QAAQ,OAAO;EACnB,GAAG;GACF,SAAS,GAAG,KAAK;CACnB,IACD,CAAE,EACH;AACF;AAaD,MAAM,oBAAoB,CACxBC,UACAC,gBACAC,aACG;AACH,KAAI,aAAa,OAIf,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,oEACG,OAAO,MAAM,CAC9C;CAIH,MAAM,kBACJ,OAAO,mBAAmB,YAC1B,OAAQ,eAAqC,aAAa;AAC5D,KACE,mBACA,SAAS,MAAM,CAAC,EAAE,QAAQ,KAAK,OAAO,KAAK,OAAO,CAAC,WAAW,EAAE,CAEhE,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAAK,OAAO,OAAO,OAAO,CAAC,GAAG;AAE/D,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAAK,OAAO;AAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDD,eAAsB,aACpBD,gBACAE,aACAC,SACA;CACA,MAAM,EACJ,aACA,iBACA,QACA,gBACmB,GAAG,WAAW,CAAE;CAErC,MAAMC,mBACJ,SAAS,qBACR,SAAS,cAAc,OACpB;EACE,YAAY,QAAQ;EACpB,uBAAuB,QAAQ;CAChC,IACD;CAEN,MAAM,eAAe,MAAM,mBAAmB,eAAe;CAC7D,MAAM,aAAa,UAAU,IAAIC;CACjC,MAAM,kBAAkB,eAAeC,oCAAY;CACnD,MAAM,UAAU,MAAM,WAAW,YAAY,EAAE,YAAa,EAAC;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,kBAAkB,kBAAkB;CAC1C,MAAM,EAAE,SAAS,UAAU,eAAe,GAAG,MAAM,aAAa;EAC9D;EACA,QAAQ;EACR,aAAa;EACb,gBAAgB;CACjB,EAAC;CAEF,MAAM,WAAW,cAAc;EAC7B,aAAa;EACb,oBAAoB;EACpB,cAAc,EAAE,UAAU,EAAE,GAAG,gBAAiB,EAAE;CACnD,EAAC;CACF,MAAMC,kBAA4B,IAAIhC,0CAAe,EACnD,MAAM,aACP,GAAE,WAAW,EAAE,SAAS,gBAAiB,EAAC;CAC3C,MAAM,YAAY,kBAChB,UACA,gBACA,QAAQ,UACT;CACD,MAAM,WAAW,IAAIsB,6BAAY;EAC/B,OAAO,UAAU;EACjB,QAAQ;CACT;CAED,MAAM,gBACH,cAAc,EACb,OAAO,MAAM,SAAS,WAAW,CAClC,EAAC,CAED,MAAM,WAAW,SAAS;EACzB;EACA,kBAAkB;CACnB,EAAC;CAEJ,SAAS,UAAU;CACnB,MAAMW,OAAc,CAAE;AACtB,MAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,GACxC,KAAK,KAAK,MAAM,cAAc,GAAG,SAAS,CAAC;CAE7C,IAAIC,cAGA,CAAE;AACN,KAAI,kBAAkB;EACpB,MAAM,mBAAmB,MAAM,iBAAiB,kBAC9C,iBACD;EACD,cAAc,MAAM,gBAAgB;GAClC,YAAY;GACZ;GACA;GACA,QAAQ;GACR,gBAAgB;EACjB,EAAC;CACH;CACD,MAAMC,UAAuB;EAC3B,aAAa;EACb,SAAS,eAAe,CAAE;CAC3B;AACD,QAAO;AACR;AAED,SAAS,6BAA6BC,GAAuC;AAC3E,QAAO,OAAO,MAAM,cAAc,yBAAyB;AAC5D"}
1	+ {"version":3,"file":"runner_utils.cjs","names":["_chain: Serialized","_inputs: ChainValues","runId: string","BaseTracer","run: Run","evaluator: RunEvaluatorLike","RunnableLambda","example?: Example","LangChainTracer","evaluator: any","RunTree","config: RunTreeConfig","callbackManager: CallbackManager","Runnable","fields: { func: AnyTraceableFunction }","input: RunInput","options?: Partial<RunnableConfig>","evaluator: LLMStringEvaluator","evaluationName: string","formatEvaluatorInputs: EvaluatorInputFormatter","config: EvalConfig \| keyof EvaluatorType","loadEvaluator","evaluators: (RunEvaluator \| DynamicRunEvaluator)[]","config: RunEvalConfig<keyof EvaluatorType>","isCustomEvaluator","isOffTheShelfEvaluator","modelOrFactory: ChainOrFactory","configs: RunnableConfig[]","ProgressBar","AsyncCaller","examples: Example[]","chainOrFactory: ChainOrFactory","dataType?: DataType","datasetName: string","options?: RunOnDatasetParams","evaluationConfig: RunEvalConfig \| undefined","Client","randomName","wrappedRunnable: Runnable","runs: Run[]","evalResults: Record<\n string,\n { run_id: string; execution_time?: number; feedback: Feedback[] }\n >","results: EvalResults","x: unknown"],"sources":["../../src/smith/runner_utils.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { Serialized } from \"@langchain/core/load/serializable\";\nimport { mapStoredMessagesToChatMessages } from \"@langchain/core/messages\";\nimport {\n Runnable,\n RunnableConfig,\n RunnableLambda,\n getCallbackManagerForConfig,\n} from \"@langchain/core/runnables\";\nimport { LangChainTracer } from \"@langchain/core/tracers/tracer_langchain\";\nimport { BaseTracer } from \"@langchain/core/tracers/base\";\nimport { ChainValues } from \"@langchain/core/utils/types\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport type {\n CallbackManager,\n CallbackManagerForChainRun,\n} from \"@langchain/core/callbacks/manager\";\nimport {\n Client,\n Example,\n Feedback,\n Run,\n RunTree,\n RunTreeConfig,\n} from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { DataType } from \"langsmith/schemas\";\nimport type { TraceableFunction } from \"langsmith/singletons/traceable\";\nimport { LLMStringEvaluator } from \"../evaluation/base.js\";\nimport { loadEvaluator } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nimport {\n isOffTheShelfEvaluator,\n type DynamicRunEvaluatorParams,\n type EvalConfig,\n type EvaluatorInputFormatter,\n type RunEvalConfig,\n type RunEvaluatorLike,\n isCustomEvaluator,\n} from \"./config.js\";\nimport { randomName } from \"./name_generation.js\";\nimport { ProgressBar } from \"./progress.js\";\n\nexport type ChainOrFactory =\n \| Runnable\n \| (() => Runnable)\n \| AnyTraceableFunction\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n \| ((obj: any) => any)\n // eslint-disable-next-line @typescript-eslint/no-explicit-any\n \| ((obj: any) => Promise<any>)\n \| (() => (obj: unknown) => unknown)\n \| (() => (obj: unknown) => Promise<unknown>);\n\nclass SingleRunIdExtractor {\n runIdPromiseResolver: (runId: string) => void;\n\n runIdPromise: Promise<string>;\n\n constructor() {\n this.runIdPromise = new Promise<string>((extract) => {\n this.runIdPromiseResolver = extract;\n });\n }\n\n handleChainStart = (\n _chain: Serialized,\n _inputs: ChainValues,\n runId: string\n ) => {\n this.runIdPromiseResolver(runId);\n };\n\n async extract(): Promise<string> {\n return this.runIdPromise;\n }\n}\n\nclass SingleRunExtractor extends BaseTracer {\n runPromiseResolver: (run: Run) => void;\n\n runPromise: Promise<Run>;\n\n /** The name of the callback handler. /\n name = \"single_run_extractor\";\n\n constructor() {\n super();\n this.runPromise = new Promise<Run>((extract) => {\n this.runPromiseResolver = extract;\n });\n }\n\n async persistRun(run: Run) {\n this.runPromiseResolver(run);\n }\n\n async extract(): Promise<Run> {\n return this.runPromise;\n }\n}\n\n/\n Wraps an evaluator function + implements the RunEvaluator interface.\n /\nclass DynamicRunEvaluator implements RunEvaluator {\n evaluator: RunnableLambda<DynamicRunEvaluatorParams, EvaluationResult>;\n\n constructor(evaluator: RunEvaluatorLike) {\n this.evaluator = new RunnableLambda({ func: evaluator });\n }\n\n /\n Evaluates a run with an optional example and returns the evaluation result.\n * @param run The run to evaluate.\n * @param example The optional example to use for evaluation.\n * @returns A promise that extracts to the evaluation result.\n /\n async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n const extractor = new SingleRunIdExtractor();\n const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n const result = await this.evaluator.invoke(\n {\n run,\n example,\n input: run.inputs,\n prediction: run.outputs,\n reference: example?.outputs,\n },\n {\n callbacks: [extractor, tracer],\n }\n );\n const runId = await extractor.extract();\n return {\n sourceRunId: runId,\n ...result,\n };\n }\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nfunction isLLMStringEvaluator(evaluator: any): evaluator is LLMStringEvaluator {\n return evaluator && typeof evaluator.evaluateStrings === \"function\";\n}\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\ntype AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;\n\n/\n Internal implementation of RunTree, which uses the\n * provided callback manager instead of the internal LangSmith client.\n \n The goal of this class is to ensure seamless interop when intergrated\n * with other Runnables.\n /\nclass CallbackManagerRunTree extends RunTree {\n callbackManager: CallbackManager;\n\n activeCallbackManager: CallbackManagerForChainRun \| undefined = undefined;\n\n constructor(config: RunTreeConfig, callbackManager: CallbackManager) {\n super(config);\n\n this.callbackManager = callbackManager;\n }\n\n createChild(config: RunTreeConfig): CallbackManagerRunTree {\n const child = new CallbackManagerRunTree(\n {\n ...config,\n parent_run: this,\n project_name: this.project_name,\n client: this.client,\n },\n this.activeCallbackManager?.getChild() ?? this.callbackManager\n );\n this.child_runs.push(child);\n return child;\n }\n\n async postRun(): Promise<void> {\n // how it is translated in comparison to basic RunTree?\n this.activeCallbackManager = await this.callbackManager.handleChainStart(\n typeof this.serialized !== \"object\" &&\n this.serialized != null &&\n \"lc\" in this.serialized\n ? this.serialized\n : {\n id: [\"langchain\", \"smith\", \"CallbackManagerRunTree\"],\n lc: 1,\n type: \"not_implemented\",\n },\n this.inputs,\n this.id,\n this.run_type,\n undefined,\n undefined,\n this.name\n );\n }\n\n async patchRun(): Promise<void> {\n if (this.error) {\n await this.activeCallbackManager?.handleChainError(\n this.error,\n this.id,\n this.parent_run?.id,\n undefined,\n undefined\n );\n } else {\n await this.activeCallbackManager?.handleChainEnd(\n this.outputs ?? {},\n this.id,\n this.parent_run?.id,\n undefined,\n undefined\n );\n }\n }\n}\n\nclass RunnableTraceable<RunInput, RunOutput> extends Runnable<\n RunInput,\n RunOutput\n> {\n lc_serializable = false;\n\n lc_namespace = [\"langchain_core\", \"runnables\"];\n\n protected func: AnyTraceableFunction;\n\n constructor(fields: { func: AnyTraceableFunction }) {\n super(fields);\n\n if (!isLangsmithTraceableFunction(fields.func)) {\n throw new Error(\n \"RunnableTraceable requires a function that is wrapped in traceable higher-order function\"\n );\n }\n\n this.func = fields.func;\n }\n\n async invoke(input: RunInput, options?: Partial<RunnableConfig>) {\n const [config] = this._getOptionsList(options ?? {}, 1);\n const callbackManager = await getCallbackManagerForConfig(config);\n\n const partialConfig =\n \"langsmith:traceable\" in this.func\n ? (this.func[\"langsmith:traceable\"] as RunTreeConfig)\n : { name: \"<lambda>\" };\n\n if (!callbackManager) throw new Error(\"CallbackManager not found\");\n const runTree = new CallbackManagerRunTree(\n {\n ...partialConfig,\n parent_run: callbackManager?._parentRunId\n ? new RunTree({ name: \"<parent>\", id: callbackManager?._parentRunId })\n : undefined,\n },\n callbackManager\n );\n\n if (\n typeof input === \"object\" &&\n input != null &&\n Object.keys(input).length === 1\n ) {\n if (\"args\" in input && Array.isArray(input)) {\n return (await this.func(runTree, ...input)) as RunOutput;\n }\n\n if (\n \"input\" in input &&\n !(\n typeof input === \"object\" &&\n input != null &&\n !Array.isArray(input) &&\n // eslint-disable-next-line no-instanceof/no-instanceof\n !(input instanceof Date)\n )\n ) {\n try {\n return (await this.func(runTree, input.input)) as RunOutput;\n } catch {\n return (await this.func(runTree, input)) as RunOutput;\n }\n }\n }\n\n return (await this.func(runTree, input)) as RunOutput;\n }\n}\n\n/\n Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])\n * and composes with a prepareData function so the user can prepare the trace and\n * dataset data for the evaluator.\n /\nclass PreparedRunEvaluator implements RunEvaluator {\n evaluator: LLMStringEvaluator;\n\n formatEvaluatorInputs: EvaluatorInputFormatter;\n\n isStringEvaluator: boolean;\n\n evaluationName: string;\n\n constructor(\n evaluator: LLMStringEvaluator,\n evaluationName: string,\n formatEvaluatorInputs: EvaluatorInputFormatter\n ) {\n this.evaluator = evaluator;\n this.isStringEvaluator = typeof evaluator?.evaluateStrings === \"function\";\n this.evaluationName = evaluationName;\n this.formatEvaluatorInputs = formatEvaluatorInputs;\n }\n\n static async fromEvalConfig(\n config: EvalConfig \| keyof EvaluatorType\n ): Promise<PreparedRunEvaluator> {\n const evaluatorType =\n typeof config === \"string\" ? config : config.evaluatorType;\n const evalConfig = typeof config === \"string\" ? ({} as EvalConfig) : config;\n const evaluator = await loadEvaluator(evaluatorType, evalConfig);\n const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;\n if (!isLLMStringEvaluator(evaluator)) {\n throw new Error(\n `Evaluator of type ${evaluatorType} not yet supported. ` +\n \"Please use a string evaluator, or implement your \" +\n \"evaluation logic as a custom evaluator.\"\n );\n }\n if (!feedbackKey) {\n throw new Error(\n `Evaluator of type ${evaluatorType} must have an evaluationName` +\n ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`\n );\n }\n return new PreparedRunEvaluator(\n evaluator as LLMStringEvaluator,\n feedbackKey,\n evalConfig?.formatEvaluatorInputs\n );\n }\n\n /\n Evaluates a run with an optional example and returns the evaluation result.\n * @param run The run to evaluate.\n * @param example The optional example to use for evaluation.\n * @returns A promise that extracts to the evaluation result.\n /\n async evaluateRun(run: Run, example?: Example): Promise<EvaluationResult> {\n const { prediction, input, reference } = this.formatEvaluatorInputs({\n rawInput: run.inputs,\n rawPrediction: run.outputs,\n rawReferenceOutput: example?.outputs,\n run,\n });\n const extractor = new SingleRunIdExtractor();\n const tracer = new LangChainTracer({ projectName: \"evaluators\" });\n if (this.isStringEvaluator) {\n const evalResult = await this.evaluator.evaluateStrings(\n {\n prediction: prediction as string,\n reference: reference as string,\n input: input as string,\n },\n {\n callbacks: [extractor, tracer],\n }\n );\n const runId = await extractor.extract();\n return {\n key: this.evaluationName,\n comment: evalResult?.reasoning,\n sourceRunId: runId,\n ...evalResult,\n };\n }\n throw new Error(\n \"Evaluator not yet supported. \" +\n \"Please use a string evaluator, or implement your \" +\n \"evaluation logic as a custom evaluator.\"\n );\n }\n}\n\nclass LoadedEvalConfig {\n constructor(public evaluators: (RunEvaluator \| DynamicRunEvaluator)[]) {}\n\n static async fromRunEvalConfig(\n config: RunEvalConfig<keyof EvaluatorType>\n ): Promise<LoadedEvalConfig> {\n // Custom evaluators are applied \"as-is\"\n const customEvaluators = (\n config?.customEvaluators ?? config.evaluators?.filter(isCustomEvaluator)\n )?.map((evaluator) => {\n if (typeof evaluator === \"function\") {\n return new DynamicRunEvaluator(evaluator);\n } else {\n return evaluator;\n }\n });\n\n const offTheShelfEvaluators = await Promise.all(\n config?.evaluators\n ?.filter(isOffTheShelfEvaluator)\n ?.map(\n async (evaluator) =>\n await PreparedRunEvaluator.fromEvalConfig(evaluator)\n ) ?? []\n );\n return new LoadedEvalConfig(\n (customEvaluators ?? []).concat(offTheShelfEvaluators ?? [])\n );\n }\n}\n\nexport interface RunOnDatasetParams\n extends Omit<RunEvalConfig, \"customEvaluators\"> {\n /\n Name of the project for logging and tracking.\n /\n projectName?: string;\n\n /\n Additional metadata for the project.\n /\n projectMetadata?: Record<string, unknown>;\n\n /\n Client instance for LangSmith service interaction.\n /\n client?: Client;\n\n /\n Maximum concurrency level for dataset processing.\n /\n maxConcurrency?: number;\n\n /\n @deprecated Pass keys directly to the RunOnDatasetParams instead\n /\n evaluationConfig?: RunEvalConfig;\n}\n\n/\n Internals expect a constructor () -> Runnable. This function wraps/coerces\n * the provided LangChain object, custom function, or factory function into\n * a constructor of a runnable.\n * @param modelOrFactory The model or factory to create a wrapped model from.\n * @returns A function that returns the wrapped model.\n * @throws Error if the modelOrFactory is invalid.\n /\nconst createWrappedModel = async (modelOrFactory: ChainOrFactory) => {\n if (Runnable.isRunnable(modelOrFactory)) {\n return () => modelOrFactory;\n }\n if (typeof modelOrFactory === \"function\") {\n if (isLangsmithTraceableFunction(modelOrFactory)) {\n const wrappedModel = new RunnableTraceable({ func: modelOrFactory });\n return () => wrappedModel;\n }\n\n try {\n // If it works with no arguments, assume it's a factory\n let res = (modelOrFactory as () => Runnable)();\n if (\n res &&\n typeof (res as unknown as Promise<Runnable>).then === \"function\"\n ) {\n res = await res;\n }\n return modelOrFactory as () => Runnable;\n } catch {\n // Otherwise, it's a custom UDF, and we'll wrap\n // the function in a lambda\n const wrappedModel = new RunnableLambda({ func: modelOrFactory });\n return () => wrappedModel;\n }\n }\n throw new Error(\"Invalid modelOrFactory\");\n};\n\nconst loadExamples = async ({\n datasetName,\n client,\n projectName,\n}: {\n datasetName: string;\n client: Client;\n projectName: string;\n maxConcurrency: number;\n}) => {\n const exampleIterator = client.listExamples({ datasetName });\n const configs: RunnableConfig[] = [];\n const runExtractors = [];\n const examples = [];\n for await (const example of exampleIterator) {\n const runExtractor = new SingleRunExtractor();\n configs.push({\n callbacks: [\n new LangChainTracer({ exampleId: example.id, projectName }),\n runExtractor,\n ],\n });\n examples.push(example);\n runExtractors.push(runExtractor);\n }\n return {\n configs,\n examples,\n runExtractors,\n };\n};\n\nconst applyEvaluators = async ({\n evaluation,\n runs,\n examples,\n client,\n maxConcurrency,\n}: {\n evaluation: LoadedEvalConfig;\n runs: Run[];\n examples: Example[];\n client: Client;\n maxConcurrency: number;\n}): Promise<{\n [key: string]: {\n execution_time?: number;\n run_id: string;\n feedback: Feedback[];\n };\n}> => {\n // TODO: Parallelize and/or put in callbacks to speed up evals.\n const { evaluators } = evaluation;\n const progress = new ProgressBar({\n total: examples.length,\n format: \"Running Evaluators: {bar} {percentage}% \| {value}/{total}\\n\",\n });\n const caller = new AsyncCaller({\n maxConcurrency,\n });\n const requests = runs.map(\n async (\n run,\n i\n ): Promise<{\n run_id: string;\n execution_time?: number;\n feedback: Feedback[];\n }> =>\n caller.call(async () => {\n const evaluatorResults = await Promise.allSettled(\n evaluators.map((evaluator) =>\n client.evaluateRun(run, evaluator, {\n referenceExample: examples[i],\n loadChildRuns: false,\n })\n )\n );\n progress.increment();\n return {\n execution_time:\n run?.end_time && run.start_time\n ? new Date(run.end_time).getTime() -\n new Date(run.start_time).getTime()\n : undefined,\n feedback: evaluatorResults.map((evalResult) =>\n evalResult.status === \"fulfilled\"\n ? evalResult.value\n : evalResult.reason\n ),\n run_id: run.id,\n };\n })\n );\n const results = await Promise.all(requests);\n\n return results.reduce(\n (acc, result, i) => ({\n ...acc,\n [examples[i].id]: result,\n }),\n {}\n );\n};\n\nexport type EvalResults = {\n projectName: string;\n results: {\n [key: string]: {\n execution_time?: number;\n run_id: string;\n feedback: Feedback[];\n };\n };\n};\n\nconst getExamplesInputs = (\n examples: Example[],\n chainOrFactory: ChainOrFactory,\n dataType?: DataType\n) => {\n if (dataType === \"chat\") {\n // For some batty reason, we store the chat dataset differently.\n // { type: \"system\", data: { content: inputs.input } },\n // But we need to create AIMesage, SystemMessage, etc.\n return examples.map(({ inputs }) =>\n mapStoredMessagesToChatMessages(inputs.input)\n );\n }\n // If it's a language model and ALL example inputs have a single value,\n // then we can be friendly and flatten the inputs to a list of strings.\n const isLanguageModel =\n typeof chainOrFactory === \"object\" &&\n typeof (chainOrFactory as BaseLanguageModel)._llmType === \"function\";\n if (\n isLanguageModel &&\n examples.every(({ inputs }) => Object.keys(inputs).length === 1)\n ) {\n return examples.map(({ inputs }) => Object.values(inputs)[0]);\n }\n return examples.map(({ inputs }) => inputs);\n};\n\n/\n Evaluates a given model or chain against a specified LangSmith dataset.\n \n This function fetches example records from the specified dataset,\n * runs the model or chain against each example, and returns the evaluation\n * results.\n \n @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a\n * Runnable instance, a factory function that returns a Runnable, or a user-defined\n * function or factory.\n \n @param datasetName - The name of the dataset against which the evaluation will be\n * performed. This dataset should already be defined and contain the relevant data\n * for evaluation.\n \n @param options - (Optional) Additional parameters for the evaluation process:\n * - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.\n * - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.\n * - `projectName` (string): Name of the project for logging and tracking.\n * - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.\n * - `client` (Client): Client instance for LangSmith service interaction.\n * - `maxConcurrency` (number): Maximum concurrency level for dataset processing.\n \n @returns A promise that resolves to an `EvalResults` object. This object includes\n * detailed results of the evaluation, such as execution time, run IDs, and feedback\n * for each entry in the dataset.\n \n @example\n * ```typescript\n * // Example usage for evaluating a model on a dataset\n * async function evaluateModel() {\n * const chain = /* ...create your model or chain...\\//\n const datasetName = 'example-dataset';\n * const client = new Client(/* ...config... \\//);\n \n * const results = await runOnDataset(chain, datasetName, {\n * evaluators: [/* ...evaluators... \\//],\n client,\n * });\n \n console.log('Evaluation Results:', results);\n * }\n \n evaluateModel();\n * ```\n * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against\n * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams[\"evaluators\"]`, which can\n * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.\n * The function returns the evaluation results, which can be logged or further processed as needed.\n */\n\nexport async function runOnDataset(\n chainOrFactory: ChainOrFactory,\n datasetName: string,\n options?: RunOnDatasetParams\n) {\n const {\n projectName,\n projectMetadata,\n client,\n maxConcurrency,\n }: RunOnDatasetParams = options ?? {};\n\n const evaluationConfig: RunEvalConfig \| undefined =\n options?.evaluationConfig ??\n (options?.evaluators != null\n ? {\n evaluators: options.evaluators,\n formatEvaluatorInputs: options.formatEvaluatorInputs,\n }\n : undefined);\n\n const wrappedModel = await createWrappedModel(chainOrFactory);\n const testClient = client ?? new Client();\n const testProjectName = projectName ?? randomName();\n const dataset = await testClient.readDataset({ datasetName });\n const datasetId = dataset.id;\n const testConcurrency = maxConcurrency ?? 5;\n const { configs, examples, runExtractors } = await loadExamples({\n datasetName,\n client: testClient,\n projectName: testProjectName,\n maxConcurrency: testConcurrency,\n });\n\n await testClient.createProject({\n projectName: testProjectName,\n referenceDatasetId: datasetId,\n projectExtra: { metadata: { ...projectMetadata } },\n });\n const wrappedRunnable: Runnable = new RunnableLambda({\n func: wrappedModel,\n }).withConfig({ runName: \"evaluationRun\" });\n const runInputs = getExamplesInputs(\n examples,\n chainOrFactory,\n dataset.data_type\n );\n const progress = new ProgressBar({\n total: runInputs.length,\n format: \"Predicting: {bar} {percentage}% \| {value}/{total}\",\n });\n // TODO: Collect the runs as well.\n await wrappedRunnable\n .withListeners({\n onEnd: () => progress.increment(),\n })\n // TODO: Insert evaluation inline for immediate feedback.\n .batch(runInputs, configs, {\n maxConcurrency,\n returnExceptions: true,\n });\n\n progress.complete();\n const runs: Run[] = [];\n for (let i = 0; i < examples.length; i += 1) {\n runs.push(await runExtractors[i].extract());\n }\n let evalResults: Record<\n string,\n { run_id: string; execution_time?: number; feedback: Feedback[] }\n > = {};\n if (evaluationConfig) {\n const loadedEvalConfig = await LoadedEvalConfig.fromRunEvalConfig(\n evaluationConfig\n );\n evalResults = await applyEvaluators({\n evaluation: loadedEvalConfig,\n runs,\n examples,\n client: testClient,\n maxConcurrency: testConcurrency,\n });\n }\n const results: EvalResults = {\n projectName: testProjectName,\n results: evalResults ?? {},\n };\n return results;\n}\n\nfunction isLangsmithTraceableFunction(x: unknown): x is AnyTraceableFunction {\n return typeof x === \"function\" && \"langsmith:traceable\" in x;\n}\n"],"mappings":";;;;;;;;;;;;;AAsDA,IAAM,uBAAN,MAA2B;CACzB;CAEA;CAEA,cAAc;EACZ,KAAK,eAAe,IAAI,QAAgB,CAAC,YAAY;GACnD,KAAK,uBAAuB;EAC7B;CACF;CAED,mBAAmB,CACjBA,QACAC,SACAC,UACG;EACH,KAAK,qBAAqB,MAAM;CACjC;CAED,MAAM,UAA2B;AAC/B,SAAO,KAAK;CACb;AACF;AAED,IAAM,qBAAN,cAAiCC,yCAAW;CAC1C;CAEA;;CAGA,OAAO;CAEP,cAAc;EACZ,OAAO;EACP,KAAK,aAAa,IAAI,QAAa,CAAC,YAAY;GAC9C,KAAK,qBAAqB;EAC3B;CACF;CAED,MAAM,WAAWC,KAAU;EACzB,KAAK,mBAAmB,IAAI;CAC7B;CAED,MAAM,UAAwB;AAC5B,SAAO,KAAK;CACb;AACF;;;;AAKD,IAAM,sBAAN,MAAkD;CAChD;CAEA,YAAYC,WAA6B;EACvC,KAAK,YAAY,IAAIC,0CAAe,EAAE,MAAM,UAAW;CACxD;;;;;;;CAQD,MAAM,YAAYF,KAAUG,SAA8C;EACxE,MAAM,YAAY,IAAI;EACtB,MAAM,SAAS,IAAIC,0DAAgB,EAAE,aAAa,aAAc;EAChE,MAAM,SAAS,MAAM,KAAK,UAAU,OAClC;GACE;GACA;GACA,OAAO,IAAI;GACX,YAAY,IAAI;GAChB,WAAW,SAAS;EACrB,GACD,EACE,WAAW,CAAC,WAAW,MAAO,EAC/B,EACF;EACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,SAAO;GACL,aAAa;GACb,GAAG;EACJ;CACF;AACF;AAGD,SAAS,qBAAqBC,WAAiD;AAC7E,QAAO,aAAa,OAAO,UAAU,oBAAoB;AAC1D;;;;;;;;AAYD,IAAM,yBAAN,MAAM,+BAA+BC,kBAAQ;CAC3C;CAEA,wBAAgE;CAEhE,YAAYC,QAAuBC,iBAAkC;EACnE,MAAM,OAAO;EAEb,KAAK,kBAAkB;CACxB;CAED,YAAYD,QAA+C;EACzD,MAAM,QAAQ,IAAI,uBAChB;GACE,GAAG;GACH,YAAY;GACZ,cAAc,KAAK;GACnB,QAAQ,KAAK;EACd,GACD,KAAK,uBAAuB,UAAU,IAAI,KAAK;EAEjD,KAAK,WAAW,KAAK,MAAM;AAC3B,SAAO;CACR;CAED,MAAM,UAAyB;EAE7B,KAAK,wBAAwB,MAAM,KAAK,gBAAgB,iBACtD,OAAO,KAAK,eAAe,YACzB,KAAK,cAAc,QACnB,QAAQ,KAAK,aACX,KAAK,aACL;GACE,IAAI;IAAC;IAAa;IAAS;GAAyB;GACpD,IAAI;GACJ,MAAM;EACP,GACL,KAAK,QACL,KAAK,IACL,KAAK,UACL,QACA,QACA,KAAK,KACN;CACF;CAED,MAAM,WAA0B;AAC9B,MAAI,KAAK,OACP,MAAM,KAAK,uBAAuB,iBAChC,KAAK,OACL,KAAK,IACL,KAAK,YAAY,IACjB,QACA,OACD;OAED,MAAM,KAAK,uBAAuB,eAChC,KAAK,WAAW,CAAE,GAClB,KAAK,IACL,KAAK,YAAY,IACjB,QACA,OACD;CAEJ;AACF;AAED,IAAM,oBAAN,cAAqDE,oCAGnD;CACA,kBAAkB;CAElB,eAAe,CAAC,kBAAkB,WAAY;CAE9C,AAAU;CAEV,YAAYC,QAAwC;EAClD,MAAM,OAAO;AAEb,MAAI,CAAC,6BAA6B,OAAO,KAAK,CAC5C,OAAM,IAAI,MACR;EAIJ,KAAK,OAAO,OAAO;CACpB;CAED,MAAM,OAAOC,OAAiBC,SAAmC;EAC/D,MAAM,CAAC,OAAO,GAAG,KAAK,gBAAgB,WAAW,CAAE,GAAE,EAAE;EACvD,MAAM,kBAAkB,kEAAkC,OAAO;EAEjE,MAAM,gBACJ,yBAAyB,KAAK,OACzB,KAAK,KAAK,yBACX,EAAE,MAAM,WAAY;AAE1B,MAAI,CAAC,gBAAiB,OAAM,IAAI,MAAM;EACtC,MAAM,UAAU,IAAI,uBAClB;GACE,GAAG;GACH,YAAY,iBAAiB,eACzB,IAAIN,kBAAQ;IAAE,MAAM;IAAY,IAAI,iBAAiB;GAAc,KACnE;EACL,GACD;AAGF,MACE,OAAO,UAAU,YACjB,SAAS,QACT,OAAO,KAAK,MAAM,CAAC,WAAW,GAC9B;AACA,OAAI,UAAU,SAAS,MAAM,QAAQ,MAAM,CACzC,QAAQ,MAAM,KAAK,KAAK,SAAS,GAAG,MAAM;AAG5C,OACE,WAAW,SACX,EACE,OAAO,UAAU,YACjB,SAAS,QACT,CAAC,MAAM,QAAQ,MAAM,IAErB,EAAE,iBAAiB,OAGrB,KAAI;AACF,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM,MAAM;GAC9C,QAAO;AACN,WAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;GACxC;EAEJ;AAED,SAAQ,MAAM,KAAK,KAAK,SAAS,MAAM;CACxC;AACF;;;;;;AAOD,IAAM,uBAAN,MAAM,qBAA6C;CACjD;CAEA;CAEA;CAEA;CAEA,YACEO,WACAC,gBACAC,uBACA;EACA,KAAK,YAAY;EACjB,KAAK,oBAAoB,OAAO,WAAW,oBAAoB;EAC/D,KAAK,iBAAiB;EACtB,KAAK,wBAAwB;CAC9B;CAED,aAAa,eACXC,QAC+B;EAC/B,MAAM,gBACJ,OAAO,WAAW,WAAW,SAAS,OAAO;EAC/C,MAAM,aAAa,OAAO,WAAW,WAAY,CAAE,IAAkB;EACrE,MAAM,YAAY,MAAMC,6BAAc,eAAe,WAAW;EAChE,MAAM,cAAc,YAAY,eAAe,WAAW;AAC1D,MAAI,CAAC,qBAAqB,UAAU,CAClC,OAAM,IAAI,MACR,CAAC,kBAAkB,EAAE,cAAc,4GAAoB,CAEZ;AAG/C,MAAI,CAAC,YACH,OAAM,IAAI,MACR,CAAC,kBAAkB,EAAE,cAAc,qGAA4B,CACc;AAGjF,SAAO,IAAI,qBACT,WACA,aACA,YAAY;CAEf;;;;;;;CAQD,MAAM,YAAYjB,KAAUG,SAA8C;EACxE,MAAM,EAAE,YAAY,OAAO,WAAW,GAAG,KAAK,sBAAsB;GAClE,UAAU,IAAI;GACd,eAAe,IAAI;GACnB,oBAAoB,SAAS;GAC7B;EACD,EAAC;EACF,MAAM,YAAY,IAAI;EACtB,MAAM,SAAS,IAAIC,0DAAgB,EAAE,aAAa,aAAc;AAChE,MAAI,KAAK,mBAAmB;GAC1B,MAAM,aAAa,MAAM,KAAK,UAAU,gBACtC;IACc;IACD;IACJ;GACR,GACD,EACE,WAAW,CAAC,WAAW,MAAO,EAC/B,EACF;GACD,MAAM,QAAQ,MAAM,UAAU,SAAS;AACvC,UAAO;IACL,KAAK,KAAK;IACV,SAAS,YAAY;IACrB,aAAa;IACb,GAAG;GACJ;EACF;AACD,QAAM,IAAI,MACR;CAIH;AACF;AAED,IAAM,mBAAN,MAAM,iBAAiB;CACrB,YAAmBc,YAAoD;EAApD;CAAsD;CAEzE,aAAa,kBACXC,QAC2B;EAE3B,MAAM,oBACJ,QAAQ,oBAAoB,OAAO,YAAY,OAAOC,iCAAkB,GACvE,IAAI,CAAC,cAAc;AACpB,OAAI,OAAO,cAAc,WACvB,QAAO,IAAI,oBAAoB;OAE/B,QAAO;EAEV,EAAC;EAEF,MAAM,wBAAwB,MAAM,QAAQ,IAC1C,QAAQ,YACJ,OAAOC,sCAAuB,EAC9B,IACA,OAAO,cACL,MAAM,qBAAqB,eAAe,UAAU,CACvD,IAAI,CAAE,EACV;AACD,SAAO,IAAI,kBACR,oBAAoB,CAAE,GAAE,OAAO,yBAAyB,CAAE,EAAC;CAE/D;AACF;;;;;;;;;AAsCD,MAAM,qBAAqB,OAAOC,mBAAmC;AACnE,KAAIb,oCAAS,WAAW,eAAe,CACrC,QAAO,MAAM;AAEf,KAAI,OAAO,mBAAmB,YAAY;AACxC,MAAI,6BAA6B,eAAe,EAAE;GAChD,MAAM,eAAe,IAAI,kBAAkB,EAAE,MAAM,eAAgB;AACnE,UAAO,MAAM;EACd;AAED,MAAI;GAEF,IAAI,MAAO,gBAAmC;AAC9C,OACE,OACA,OAAQ,IAAqC,SAAS,YAEtD,MAAM,MAAM;AAEd,UAAO;EACR,QAAO;GAGN,MAAM,eAAe,IAAIP,0CAAe,EAAE,MAAM,eAAgB;AAChE,UAAO,MAAM;EACd;CACF;AACD,OAAM,IAAI,MAAM;AACjB;AAED,MAAM,eAAe,OAAO,EAC1B,aACA,QACA,aAMD,KAAK;CACJ,MAAM,kBAAkB,OAAO,aAAa,EAAE,YAAa,EAAC;CAC5D,MAAMqB,UAA4B,CAAE;CACpC,MAAM,gBAAgB,CAAE;CACxB,MAAM,WAAW,CAAE;AACnB,YAAW,MAAM,WAAW,iBAAiB;EAC3C,MAAM,eAAe,IAAI;EACzB,QAAQ,KAAK,EACX,WAAW,CACT,IAAInB,0DAAgB;GAAE,WAAW,QAAQ;GAAI;EAAa,IAC1D,YACD,EACF,EAAC;EACF,SAAS,KAAK,QAAQ;EACtB,cAAc,KAAK,aAAa;CACjC;AACD,QAAO;EACL;EACA;EACA;CACD;AACF;AAED,MAAM,kBAAkB,OAAO,EAC7B,YACA,MACA,UACA,QACA,gBAOD,KAMK;CAEJ,MAAM,EAAE,YAAY,GAAG;CACvB,MAAM,WAAW,IAAIoB,6BAAY;EAC/B,OAAO,SAAS;EAChB,QAAQ;CACT;CACD,MAAM,SAAS,IAAIC,gDAAY,EAC7B,eACD;CACD,MAAM,WAAW,KAAK,IACpB,OACE,KACA,MAMA,OAAO,KAAK,YAAY;EACtB,MAAM,mBAAmB,MAAM,QAAQ,WACrC,WAAW,IAAI,CAAC,cACd,OAAO,YAAY,KAAK,WAAW;GACjC,kBAAkB,SAAS;GAC3B,eAAe;EAChB,EAAC,CACH,CACF;EACD,SAAS,WAAW;AACpB,SAAO;GACL,gBACE,KAAK,YAAY,IAAI,aACjB,IAAI,KAAK,IAAI,UAAU,SAAS,GAChC,IAAI,KAAK,IAAI,YAAY,SAAS,GAClC;GACN,UAAU,iBAAiB,IAAI,CAAC,eAC9B,WAAW,WAAW,cAClB,WAAW,QACX,WAAW,OAChB;GACD,QAAQ,IAAI;EACb;CACF,EAAC,CACL;CACD,MAAM,UAAU,MAAM,QAAQ,IAAI,SAAS;AAE3C,QAAO,QAAQ,OACb,CAAC,KAAK,QAAQ,OAAO;EACnB,GAAG;GACF,SAAS,GAAG,KAAK;CACnB,IACD,CAAE,EACH;AACF;AAaD,MAAM,oBAAoB,CACxBC,UACAC,gBACAC,aACG;AACH,KAAI,aAAa,OAIf,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,oEACG,OAAO,MAAM,CAC9C;CAIH,MAAM,kBACJ,OAAO,mBAAmB,YAC1B,OAAQ,eAAqC,aAAa;AAC5D,KACE,mBACA,SAAS,MAAM,CAAC,EAAE,QAAQ,KAAK,OAAO,KAAK,OAAO,CAAC,WAAW,EAAE,CAEhE,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAAK,OAAO,OAAO,OAAO,CAAC,GAAG;AAE/D,QAAO,SAAS,IAAI,CAAC,EAAE,QAAQ,KAAK,OAAO;AAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqDD,eAAsB,aACpBD,gBACAE,aACAC,SACA;CACA,MAAM,EACJ,aACA,iBACA,QACA,gBACmB,GAAG,WAAW,CAAE;CAErC,MAAMC,mBACJ,SAAS,qBACR,SAAS,cAAc,OACpB;EACE,YAAY,QAAQ;EACpB,uBAAuB,QAAQ;CAChC,IACD;CAEN,MAAM,eAAe,MAAM,mBAAmB,eAAe;CAC7D,MAAM,aAAa,UAAU,IAAIC;CACjC,MAAM,kBAAkB,eAAeC,oCAAY;CACnD,MAAM,UAAU,MAAM,WAAW,YAAY,EAAE,YAAa,EAAC;CAC7D,MAAM,YAAY,QAAQ;CAC1B,MAAM,kBAAkB,kBAAkB;CAC1C,MAAM,EAAE,SAAS,UAAU,eAAe,GAAG,MAAM,aAAa;EAC9D;EACA,QAAQ;EACR,aAAa;EACb,gBAAgB;CACjB,EAAC;CAEF,MAAM,WAAW,cAAc;EAC7B,aAAa;EACb,oBAAoB;EACpB,cAAc,EAAE,UAAU,EAAE,GAAG,gBAAiB,EAAE;CACnD,EAAC;CACF,MAAMC,kBAA4B,IAAIhC,0CAAe,EACnD,MAAM,aACP,GAAE,WAAW,EAAE,SAAS,gBAAiB,EAAC;CAC3C,MAAM,YAAY,kBAChB,UACA,gBACA,QAAQ,UACT;CACD,MAAM,WAAW,IAAIsB,6BAAY;EAC/B,OAAO,UAAU;EACjB,QAAQ;CACT;CAED,MAAM,gBACH,cAAc,EACb,OAAO,MAAM,SAAS,WAAW,CAClC,EAAC,CAED,MAAM,WAAW,SAAS;EACzB;EACA,kBAAkB;CACnB,EAAC;CAEJ,SAAS,UAAU;CACnB,MAAMW,OAAc,CAAE;AACtB,MAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,GACxC,KAAK,KAAK,MAAM,cAAc,GAAG,SAAS,CAAC;CAE7C,IAAIC,cAGA,CAAE;AACN,KAAI,kBAAkB;EACpB,MAAM,mBAAmB,MAAM,iBAAiB,kBAC9C,iBACD;EACD,cAAc,MAAM,gBAAgB;GAClC,YAAY;GACZ;GACA;GACA,QAAQ;GACR,gBAAgB;EACjB,EAAC;CACH;CACD,MAAMC,UAAuB;EAC3B,aAAa;EACb,SAAS,eAAe,CAAE;CAC3B;AACD,QAAO;AACR;AAED,SAAS,6BAA6BC,GAAuC;AAC3E,QAAO,OAAO,MAAM,cAAc,yBAAyB;AAC5D"}

package/dist/smith/runner_utils.d.cts CHANGED Viewed

@@ -1,15 +1,10 @@
 import { RunEvalConfig } from "./config.cjs";
 import { Runnable } from "@langchain/core/runnables";
-import { TraceableFunction } from "langsmith/singletons/traceable";
 import { Client, Feedback } from "langsmith";
+import { TraceableFunction } from "langsmith/singletons/traceable";
 //#region src/smith/runner_utils.d.ts
-type ChainOrFactory = Runnable | (() => Runnable) | AnyTraceableFunction
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-| ((obj: any) => any)
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-| ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type ChainOrFactory = Runnable | (() => Runnable) | AnyTraceableFunction | ((obj: any) => any) | ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);
 type AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;
 interface RunOnDatasetParams extends Omit<RunEvalConfig, "customEvaluators"> {
   /**

package/dist/smith/runner_utils.d.cts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"runner_utils.d.cts","names":["Runnable","Client","Feedback","TraceableFunction","RunEvalConfig","ChainOrFactory","AnyTraceableFunction","Promise","RunOnDatasetParams","Record","Omit","EvalResults","runOnDataset"],"sources":["../../src/smith/runner_utils.d.ts"],"sourcesContent":["import { Runnable } from \"@langchain/core/runnables\";\nimport { Client, Feedback } from \"langsmith\";\nimport type { TraceableFunction } from \"langsmith/singletons/traceable\";\nimport { type RunEvalConfig } from \"./config.js\";\nexport type ChainOrFactory = Runnable | (() => Runnable) | AnyTraceableFunction | ((obj: any) => any) | ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);\ntype AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;\nexport interface RunOnDatasetParams extends Omit<RunEvalConfig, \"customEvaluators\"> {\n /**\n * Name of the project for logging and tracking.\n */\n projectName?: string;\n /**\n * Additional metadata for the project.\n */\n projectMetadata?: Record<string, unknown>;\n /**\n * Client instance for LangSmith service interaction.\n */\n client?: Client;\n /**\n * Maximum concurrency level for dataset processing.\n */\n maxConcurrency?: number;\n /**\n * @deprecated Pass keys directly to the RunOnDatasetParams instead\n */\n evaluationConfig?: RunEvalConfig;\n}\nexport type EvalResults = {\n projectName: string;\n results: {\n [key: string]: {\n execution_time?: number;\n run_id: string;\n feedback: Feedback[];\n };\n };\n};\n/**\n * Evaluates a given model or chain against a specified LangSmith dataset.\n *\n * This function fetches example records from the specified dataset,\n * runs the model or chain against each example, and returns the evaluation\n * results.\n *\n * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a\n * Runnable instance, a factory function that returns a Runnable, or a user-defined\n * function or factory.\n *\n * @param datasetName - The name of the dataset against which the evaluation will be\n * performed. This dataset should already be defined and contain the relevant data\n * for evaluation.\n *\n * @param options - (Optional) Additional parameters for the evaluation process:\n * - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.\n * - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.\n * - `projectName` (string): Name of the project for logging and tracking.\n * - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.\n * - `client` (Client): Client instance for LangSmith service interaction.\n * - `maxConcurrency` (number): Maximum concurrency level for dataset processing.\n *\n * @returns A promise that resolves to an `EvalResults` object. This object includes\n * detailed results of the evaluation, such as execution time, run IDs, and feedback\n * for each entry in the dataset.\n *\n * @example\n * ```typescript\n * // Example usage for evaluating a model on a dataset\n * async function evaluateModel() {\n * const chain = /* ...create your model or chain...*\\//\n * const datasetName = 'example-dataset';\n * const client = new Client(/* ...config... *\\//);\n *\n * const results = await runOnDataset(chain, datasetName, {\n * evaluators: [/* ...evaluators... *\\//],\n * client,\n * });\n *\n * console.log('Evaluation Results:', results);\n * }\n *\n * evaluateModel();\n * ```\n * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against\n * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams[\"evaluators\"]`, which can\n * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.\n * The function returns the evaluation results, which can be logged or further processed as needed.\n */\nexport declare function runOnDataset(chainOrFactory: ChainOrFactory, datasetName: string, options?: RunOnDatasetParams): Promise<EvalResults>;\nexport {};\n//# sourceMappingURL=runner_utils.d.ts.map"],"mappings":";;;;;;KAIYK,cAAAA,GAAiBL,kBAAkBA,YAAYM,4DAA4DC,6EAA6EA;KAC/LD,oBAAAA,GAAuBH;AADhBE,UAEKG,kBAAAA,SAA2BE,IAFlB,CAEuBN,aAFvB,EAAA,kBAAA,CAAA,CAAA;EAAGJ;;;EAA0FO,WAAAA,CAAAA,EAAAA,MAAAA;EAA6EA;AAAO;AAAW;EAErMC,eAAAA,CAAAA,EAQKC,MARa,CAAA,MAAA,EAAA,OAAA,CAAA;EAAcL;;;EAoB1BA,MAAAA,CAAAA,EARVH,MAQUG;EApBqBM;AAAI;AAsBhD;EA4DwBE,cAAAA,CAAY,EAAA,MAAA;EAAiBP;;;EAAoEE,gBAAAA,CAAAA,EA9DlGH,aA8DkGG;AAAO;KA5DpHI,WAAAA;;;;;;gBAMUT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAsDEU,YAAAA,iBAA6BP,+CAA+CG,qBAAqBD,QAAQI"}

package/dist/smith/runner_utils.d.ts CHANGED Viewed

@@ -1,15 +1,10 @@
 import { RunEvalConfig } from "./config.js";
 import { Runnable } from "@langchain/core/runnables";
-import { TraceableFunction } from "langsmith/singletons/traceable";
 import { Client, Feedback } from "langsmith";
+import { TraceableFunction } from "langsmith/singletons/traceable";
 //#region src/smith/runner_utils.d.ts
-type ChainOrFactory = Runnable | (() => Runnable) | AnyTraceableFunction
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-| ((obj: any) => any)
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-| ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type ChainOrFactory = Runnable | (() => Runnable) | AnyTraceableFunction | ((obj: any) => any) | ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);
 type AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;
 interface RunOnDatasetParams extends Omit<RunEvalConfig, "customEvaluators"> {
   /**

package/dist/smith/runner_utils.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"runner_utils.d.ts","names":["Runnable","Client","Feedback","TraceableFunction","RunEvalConfig","ChainOrFactory","AnyTraceableFunction","Promise","RunOnDatasetParams","Record","Omit","EvalResults","runOnDataset"],"sources":["../../src/smith/runner_utils.d.ts"],"sourcesContent":["import { Runnable } from \"@langchain/core/runnables\";\nimport { Client, Feedback } from \"langsmith\";\nimport type { TraceableFunction } from \"langsmith/singletons/traceable\";\nimport { type RunEvalConfig } from \"./config.js\";\nexport type ChainOrFactory = Runnable | (() => Runnable) | AnyTraceableFunction | ((obj: any) => any) | ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);\ntype AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;\nexport interface RunOnDatasetParams extends Omit<RunEvalConfig, \"customEvaluators\"> {\n /**\n * Name of the project for logging and tracking.\n */\n projectName?: string;\n /**\n * Additional metadata for the project.\n */\n projectMetadata?: Record<string, unknown>;\n /**\n * Client instance for LangSmith service interaction.\n */\n client?: Client;\n /**\n * Maximum concurrency level for dataset processing.\n */\n maxConcurrency?: number;\n /**\n * @deprecated Pass keys directly to the RunOnDatasetParams instead\n */\n evaluationConfig?: RunEvalConfig;\n}\nexport type EvalResults = {\n projectName: string;\n results: {\n [key: string]: {\n execution_time?: number;\n run_id: string;\n feedback: Feedback[];\n };\n };\n};\n/**\n * Evaluates a given model or chain against a specified LangSmith dataset.\n *\n * This function fetches example records from the specified dataset,\n * runs the model or chain against each example, and returns the evaluation\n * results.\n *\n * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a\n * Runnable instance, a factory function that returns a Runnable, or a user-defined\n * function or factory.\n *\n * @param datasetName - The name of the dataset against which the evaluation will be\n * performed. This dataset should already be defined and contain the relevant data\n * for evaluation.\n *\n * @param options - (Optional) Additional parameters for the evaluation process:\n * - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.\n * - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.\n * - `projectName` (string): Name of the project for logging and tracking.\n * - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.\n * - `client` (Client): Client instance for LangSmith service interaction.\n * - `maxConcurrency` (number): Maximum concurrency level for dataset processing.\n *\n * @returns A promise that resolves to an `EvalResults` object. This object includes\n * detailed results of the evaluation, such as execution time, run IDs, and feedback\n * for each entry in the dataset.\n *\n * @example\n * ```typescript\n * // Example usage for evaluating a model on a dataset\n * async function evaluateModel() {\n * const chain = /* ...create your model or chain...*\\//\n * const datasetName = 'example-dataset';\n * const client = new Client(/* ...config... *\\//);\n *\n * const results = await runOnDataset(chain, datasetName, {\n * evaluators: [/* ...evaluators... *\\//],\n * client,\n * });\n *\n * console.log('Evaluation Results:', results);\n * }\n *\n * evaluateModel();\n * ```\n * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against\n * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams[\"evaluators\"]`, which can\n * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.\n * The function returns the evaluation results, which can be logged or further processed as needed.\n */\nexport declare function runOnDataset(chainOrFactory: ChainOrFactory, datasetName: string, options?: RunOnDatasetParams): Promise<EvalResults>;\nexport {};\n//# sourceMappingURL=runner_utils.d.ts.map"],"mappings":";;;;;;KAIYK,cAAAA,GAAiBL,kBAAkBA,YAAYM,4DAA4DC,6EAA6EA;KAC/LD,oBAAAA,GAAuBH;AADhBE,UAEKG,kBAAAA,SAA2BE,IAFlB,CAEuBN,aAFvB,EAAA,kBAAA,CAAA,CAAA;EAAGJ;;;EAA0FO,WAAAA,CAAAA,EAAAA,MAAAA;EAA6EA;AAAO;AAAW;EAErMC,eAAAA,CAAAA,EAQKC,MARa,CAAA,MAAA,EAAA,OAAA,CAAA;EAAcL;;;EAoB1BA,MAAAA,CAAAA,EARVH,MAQUG;EApBqBM;AAAI;AAsBhD;EA4DwBE,cAAAA,CAAY,EAAA,MAAA;EAAiBP;;;EAAoEE,gBAAAA,CAAAA,EA9DlGH,aA8DkGG;AAAO;KA5DpHI,WAAAA;;;;;;gBAMUT;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAsDEU,YAAAA,iBAA6BP,+CAA+CG,qBAAqBD,QAAQI"}

package/dist/smith/runner_utils.js CHANGED Viewed

@@ -1,7 +1,4 @@
 import { loadEvaluator } from "../evaluation/loader.js";
-import { Client } from "../node_modules/.pnpm/langsmith@0.3.74_@opentelemetry_api@1.9.0_openai@5.12.2_ws@8.18.3_bufferutil@4.0.9_utf-8-validate@6.0.5__zod@3.25.76_/node_modules/langsmith/dist/client.js";
-import { RunTree } from "../node_modules/.pnpm/langsmith@0.3.74_@opentelemetry_api@1.9.0_openai@5.12.2_ws@8.18.3_bufferutil@4.0.9_utf-8-validate@6.0.5__zod@3.25.76_/node_modules/langsmith/dist/run_trees.js";
-import "../node_modules/.pnpm/langsmith@0.3.74_@opentelemetry_api@1.9.0_openai@5.12.2_ws@8.18.3_bufferutil@4.0.9_utf-8-validate@6.0.5__zod@3.25.76_/node_modules/langsmith/index.js";
 import { isCustomEvaluator, isOffTheShelfEvaluator } from "./config.js";
 import { randomName } from "./name_generation.js";
 import { ProgressBar } from "./progress.js";
@@ -10,6 +7,7 @@ import { mapStoredMessagesToChatMessages } from "@langchain/core/messages";
 import { BaseTracer } from "@langchain/core/tracers/base";
 import { LangChainTracer } from "@langchain/core/tracers/tracer_langchain";
 import { AsyncCaller } from "@langchain/core/utils/async_caller";
+import { Client, RunTree } from "langsmith";
 //#region src/smith/runner_utils.ts
 var SingleRunIdExtractor = class {