@langchain/classic 1.0.0-alpha.1

package/dist/smith/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","names":["BaseLanguageModel","RunnableConfig","Example","Run","EvaluationResult","RunEvaluator","Criteria","CriteriaType","EmbeddingDistanceEvalChainInput","LoadEvaluatorOptions","EvaluatorType","EvaluatorInputs","EvaluatorInputFormatter","rawInput","rawPrediction","rawReferenceOutput","run","DynamicRunEvaluatorParams","Record","Input","Prediction","Reference","RunEvaluatorLike","Promise","isOffTheShelfEvaluator","T","EvalConfig","U","isCustomEvaluator","RunEvalType","RunEvalConfig","CriteriaEvalChainConfig","LabeledCriteria","Partial","Pick","EmbeddingDistance"],"sources":["../../src/smith/config.d.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { RunnableConfig } from \"@langchain/core/runnables\";\nimport { Example, Run } from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport { Criteria as CriteriaType, type EmbeddingDistanceEvalChainInput } from \"../evaluation/index.js\";\nimport { LoadEvaluatorOptions } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\nexport type EvaluatorInputs = {\n    input?: string | unknown;\n    prediction: string | unknown;\n    reference?: string | unknown;\n};\nexport type EvaluatorInputFormatter = ({ rawInput, rawPrediction, rawReferenceOutput, run }: {\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    rawInput: any;\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    rawPrediction: any;\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    rawReferenceOutput?: any;\n    run: Run;\n}) => EvaluatorInputs;\nexport type DynamicRunEvaluatorParams<\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nInput extends Record<string, any> = Record<string, unknown>, \n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nPrediction extends Record<string, any> = Record<string, unknown>, \n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nReference extends Record<string, any> = Record<string, unknown>> = {\n    input: Input;\n    prediction?: Prediction;\n    reference?: Reference;\n    run: Run;\n    example?: Example;\n};\n/**\n * Type of a function that can be coerced into a RunEvaluator function.\n * While we have the class-based RunEvaluator, it's often more convenient to directly\n * pass a function to the runner. This type allows us to do that.\n */\nexport type RunEvaluatorLike = ((props: DynamicRunEvaluatorParams, options: RunnableConfig) => Promise<EvaluationResult>) | ((props: DynamicRunEvaluatorParams, options: RunnableConfig) => EvaluationResult);\nexport declare function isOffTheShelfEvaluator<T extends keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike>(evaluator: T | EvalConfig | U): evaluator is T | EvalConfig;\nexport declare function isCustomEvaluator<T extends keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike>(evaluator: T | EvalConfig | U): evaluator is U;\nexport type RunEvalType<T extends keyof EvaluatorType = \"criteria\" | \"labeled_criteria\" | \"embedding_distance\", U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = T | EvalConfig | U;\n/**\n * Configuration class for running evaluations on datasets.\n *\n * @remarks\n * RunEvalConfig in LangSmith is a configuration class for running evaluations on datasets. Its primary purpose is to define the parameters and evaluators that will be applied during the evaluation of a dataset. This configuration can include various evaluators, custom evaluators, and different keys for inputs, predictions, and references.\n *\n * @typeparam T - The type of evaluators.\n * @typeparam U - The type of custom evaluators.\n */\nexport type RunEvalConfig<T extends keyof EvaluatorType = \"criteria\" | \"labeled_criteria\" | \"embedding_distance\", U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = {\n    /**\n     * Evaluators to apply to a dataset run.\n     * You can optionally specify these by name, or by\n     * configuring them with an EvalConfig object.\n     */\n    evaluators?: RunEvalType<T, U>[];\n    /**\n     * Convert the evaluation data into formats that can be used by the evaluator.\n     * This should most commonly be a string.\n     * Parameters are the raw input from the run, the raw output, raw reference output, and the raw run.\n     * @example\n     * ```ts\n     * // Chain input: { input: \"some string\" }\n     * // Chain output: { output: \"some output\" }\n     * // Reference example output format: { output: \"some reference output\" }\n     * const formatEvaluatorInputs = ({\n     *   rawInput,\n     *   rawPrediction,\n     *   rawReferenceOutput,\n     * }) => {\n     *   return {\n     *     input: rawInput.input,\n     *     prediction: rawPrediction.output,\n     *     reference: rawReferenceOutput.output,\n     *   };\n     * };\n     * ```\n     * @returns The prepared data.\n     */\n    formatEvaluatorInputs?: EvaluatorInputFormatter;\n    /**\n     * Custom evaluators to apply to a dataset run.\n     * Each evaluator is provided with a run trace containing the model\n     * outputs, as well as an \"example\" object representing a record\n     * in the dataset.\n     *\n     * @deprecated Use `evaluators` instead.\n     */\n    customEvaluators?: U[];\n};\nexport interface EvalConfig extends LoadEvaluatorOptions {\n    /**\n     * The name of the evaluator to use.\n     * Example: labeled_criteria, criteria, etc.\n     */\n    evaluatorType: keyof EvaluatorType;\n    /**\n     * The feedback (or metric) name to use for the logged\n     * evaluation results. If none provided, we default to\n     * the evaluationName.\n     */\n    feedbackKey?: string;\n    /**\n     * Convert the evaluation data into formats that can be used by the evaluator.\n     * This should most commonly be a string.\n     * Parameters are the raw input from the run, the raw output, raw reference output, and the raw run.\n     * @example\n     * ```ts\n     * // Chain input: { input: \"some string\" }\n     * // Chain output: { output: \"some output\" }\n     * // Reference example output format: { output: \"some reference output\" }\n     * const formatEvaluatorInputs = ({\n     *   rawInput,\n     *   rawPrediction,\n     *   rawReferenceOutput,\n     * }) => {\n     *   return {\n     *     input: rawInput.input,\n     *     prediction: rawPrediction.output,\n     *     reference: rawReferenceOutput.output,\n     *   };\n     * };\n     * ```\n     * @returns The prepared data.\n     */\n    formatEvaluatorInputs: EvaluatorInputFormatter;\n}\n/**\n * Configuration to load a \"CriteriaEvalChain\" evaluator,\n * which prompts an LLM to determine whether the model's\n * prediction complies with the provided criteria.\n * @param criteria - The criteria to use for the evaluator.\n * @param llm - The language model to use for the evaluator.\n * @returns The configuration for the evaluator.\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [Criteria(\"helpfulness\")],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [\n *     Criteria({\n *       \"isCompliant\": \"Does the submission comply with the requirements of XYZ\"\n *     })\n *   ],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [{\n *     evaluatorType: \"criteria\",\n *     criteria: \"helpfulness\"\n *     formatEvaluatorInputs: ...\n *   }]\n * };\n * ```\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [{\n *     evaluatorType: \"criteria\",\n *     criteria: { \"isCompliant\": \"Does the submission comply with the requirements of XYZ\" },\n *     formatEvaluatorInputs: ...\n *   }]\n * };\n */\nexport type Criteria = EvalConfig & {\n    evaluatorType: \"criteria\";\n    /**\n     * The \"criteria\" to insert into the prompt template\n     * used for evaluation. See the prompt at\n     * https://smith.langchain.com/hub/langchain-ai/criteria-evaluator\n     * for more information.\n     */\n    criteria?: CriteriaType | Record<string, string>;\n    /**\n     * The language model to use as the evaluator, defaults to GPT-4\n     */\n    llm?: BaseLanguageModel;\n};\n// for compatibility reasons\nexport type CriteriaEvalChainConfig = Criteria;\nexport declare function Criteria(criteria: CriteriaType | Record<string, string>, config?: Pick<Partial<LabeledCriteria>, \"formatEvaluatorInputs\" | \"llm\" | \"feedbackKey\">): EvalConfig;\n/**\n * Configuration to load a \"LabeledCriteriaEvalChain\" evaluator,\n * which prompts an LLM to determine whether the model's\n * prediction complies with the provided criteria and also\n * provides a \"ground truth\" label for the evaluator to incorporate\n * in its evaluation.\n * @param criteria - The criteria to use for the evaluator.\n * @param llm - The language model to use for the evaluator.\n * @returns The configuration for the evaluator.\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [LabeledCriteria(\"correctness\")],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [\n *     LabeledCriteria({\n *       \"mentionsAllFacts\": \"Does the include all facts provided in the reference?\"\n *     })\n *   ],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [{\n *     evaluatorType: \"labeled_criteria\",\n *     criteria: \"correctness\",\n *     formatEvaluatorInputs: ...\n *   }],\n * };\n * ```\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [{\n *     evaluatorType: \"labeled_criteria\",\n *     criteria: { \"mentionsAllFacts\": \"Does the include all facts provided in the reference?\" },\n *     formatEvaluatorInputs: ...\n *   }],\n * };\n */\nexport type LabeledCriteria = EvalConfig & {\n    evaluatorType: \"labeled_criteria\";\n    /**\n     * The \"criteria\" to insert into the prompt template\n     * used for evaluation. See the prompt at\n     * https://smith.langchain.com/hub/langchain-ai/labeled-criteria\n     * for more information.\n     */\n    criteria?: CriteriaType | Record<string, string>;\n    /**\n     * The language model to use as the evaluator, defaults to GPT-4\n     */\n    llm?: BaseLanguageModel;\n};\nexport declare function LabeledCriteria(criteria: CriteriaType | Record<string, string>, config?: Pick<Partial<LabeledCriteria>, \"formatEvaluatorInputs\" | \"llm\" | \"feedbackKey\">): LabeledCriteria;\n/**\n * Configuration to load a \"EmbeddingDistanceEvalChain\" evaluator,\n * which embeds distances to score semantic difference between\n * a prediction and reference.\n */\nexport type EmbeddingDistance = EvalConfig & EmbeddingDistanceEvalChainInput & {\n    evaluatorType: \"embedding_distance\";\n};\nexport declare function EmbeddingDistance(distanceMetric: EmbeddingDistanceEvalChainInput[\"distanceMetric\"], config?: Pick<Partial<LabeledCriteria>, \"formatEvaluatorInputs\" | \"embedding\" | \"feedbackKey\">): EmbeddingDistance;\n"],"mappings":";;;;;;;;;;KAOYW,eAAAA;;;EAAAA,SAAAA,CAAAA,EAAAA,MAAe,GAAA,OAAA;AAK3B,CAAA;AAAmC,KAAvBC,uBAAAA,GAAuB,CAAA;EAAA,QAAA;EAAA,aAAA;EAAA,kBAAA;EAAA;AASnC,CATmC,EAAA;EAAA;EAAc,QAAEE,EAAAA,GAAAA;EAAa;EAAoB,aAAEE,EAAAA,GAAAA;EAAG;EAO7E,kBACNL,CAAAA,EAAAA,GAAAA;EAAe,GAAA,EADZR,GACY;AACrB,CAAA,EAAA,GADMQ,eACMM;AAAyB,KAAzBA,yBAAyB;;cAEvBC,MAAsBA,CAAAA,MAAAA,EAAAA,GAAAA,CAAAA,GAAAA,MAAAA,CAAAA,MAAAA,EAAAA,OAAAA,CAAAA;;mBAEjBA,MAAsBA,CAAAA,MAAAA,EAAAA,GAAAA,CAAAA,GAAAA,MAAAA,CAAAA,MAAAA,EAAAA,OAAAA,CAAAA;;kBAEvBA,MAAsBA,CAAAA,MAAAA,EAAAA,GAAAA,CAAAA,GAAAA,MAAAA,CAAAA,MAAAA,EAAAA,OAAAA,CAAAA,CAAAA,GAAAA;EAAM,KACnCC,EAAAA,KAAAA;EAAK,UACCC,CAAAA,EAAAA,UAAAA;EAAU,SACXC,CAAAA,EAAAA,SAAAA;EAAS,GAChBlB,EAAAA,GAAAA;EAAG,OACED,CAAAA,EAAAA,OAAAA;AAAO,CAAA;AAOrB;;;;;AAA+FqB,KAAnFD,gBAAAA,GAAmFC,CAAAA,CAAAA,KAAAA,EAAvDN,yBAAuDM,EAAAA,OAAAA,EAAnBtB,cAAmBsB,EAAAA,GAAAA,OAAAA,CAAQnB,gBAARmB,CAAAA,CAAAA,GAAAA,CAAAA,CAAAA,KAAAA,EAAsCN,yBAAtCM,EAAAA,OAAAA,EAA0EtB,cAA1EsB,EAAAA,GAA6FnB,gBAA7FmB,CAAAA;AAAsCN,iBAC7GO,sBAD6GP,CAAAA,UAAAA,MACtEP,aADsEO,EAAAA,UAC7CZ,YAD6CY,GAC9BK,gBAD8BL,GACXZ,YADWY,GACIK,gBADJL,CAAAA,CAAAA,SAAAA,EACiCQ,CADjCR,GACqCS,UADrCT,GACkDU,CADlDV,CAAAA,EAAAA,SAAAA,IACmEQ,CADnER,GACuES,UADvET;AAAoChB,iBAEjJ2B,iBAFiJ3B,CAAAA,UAAAA,MAE/GS,aAF+GT,EAAAA,UAEtFI,YAFsFJ,GAEvEqB,gBAFuErB,GAEpDI,YAFoDJ,GAErCqB,gBAFqCrB,CAAAA,CAAAA,SAAAA,EAERwB,CAFQxB,GAEJyB,UAFIzB,GAES0B,CAFT1B,CAAAA,EAAAA,SAAAA,IAE0B0B,CAF1B1B;AAAmBG,KAGhLyB,WAHgLzB,CAAAA,UAAAA,MAGpJM,aAHoJN,GAAAA,UAAAA,GAAAA,kBAAAA,GAAAA,oBAAAA,EAAAA,UAGlEC,YAHkED,GAGnDkB,gBAHmDlB,GAGhCC,YAHgCD,GAGjBkB,gBAHiBlB,CAAAA,GAGGqB,CAHHrB,GAGOsB,UAHPtB,GAGoBuB,CAHpBvB;AAAgB;AAC5M;;;;;;;;AAA0KsB,KAY9JI,aAZ8JJ,CAAAA,UAAAA,MAYhIhB,aAZgIgB,GAAAA,UAAAA,GAAAA,kBAAAA,GAAAA,oBAAAA,EAAAA,UAY9CrB,YAZ8CqB,GAY/BJ,gBAZ+BI,GAYZrB,YAZYqB,GAYGJ,gBAZHI,CAAAA,GAAAA;EAAU;;;AAAkC;AACtN;EAAyC,UAAA,CAAA,EAiBxBG,WAjBwB,CAiBZJ,CAjBY,EAiBTE,CAjBS,CAAA,EAAA;EAAA;;;;;;;;;AAA2J;AACpM;;;;;;;;;;AAAiN;AAUjN;;EAAyB,qBAAiBjB,CAAAA,EA8BdE,uBA9BcF;EAAa;;;;;;;;EA8BJ,gBAS5BiB,CAAAA,EAAAA,CAAAA,EAAAA;AAAC,CAAA;AAEPD,UAAAA,UAAAA,SAAmBjB,oBAAT,CAAA;EAAA;;;;EAA6B,aAAA,EAAA,MAK/BC,aAL+B;EA8E5CJ;;;;;EAQwB,WAI1BN,CAAAA,EAAAA,MAAAA;EAAiB;AAG3B;AACA;;;;;;;;AAAuL;AA4CvL;;;;;;AAY2B;AAE3B;;;;;EAA8H,qBAAvBiC,EArH5ErB,uBAqH4EqB;;;AAA4F;AAMnM;;;;AAA4E;AAG5E;;;;;;;AAA+N;;;;;;;;;;;;;;;;;;;;;;;;;;;KAnFnN3B,UAAAA,GAAWoB;;;;;;;;aAQRnB,WAAeW;;;;QAIpBlB;;;KAGE+B,uBAAAA,GAA0BzB;iBACdA,UAAAA,WAAmBC,WAAeW,iCAAiCgB,KAAKD,QAAQD,qEAAqEN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA4CjKM,eAAAA,GAAkBN;;;;;;;;aAQfnB,WAAeW;;;;QAIpBlB;;iBAEcgC,eAAAA,WAA0BzB,WAAeW,iCAAiCgB,KAAKD,QAAQD,qEAAqEA;;;;;;KAMxKG,iBAAAA,GAAoBT,aAAalB;;;iBAGrB2B,iBAAAA,iBAAkC3B,4DAA4D0B,KAAKD,QAAQD,2EAA2EG"}

package/dist/smith/config.js

@@ -0,0 +1,65 @@
+//#region src/smith/config.ts
+function isOffTheShelfEvaluator(evaluator) {
+	return typeof evaluator === "string" || "evaluatorType" in evaluator;
+}
+function isCustomEvaluator(evaluator) {
+	return !isOffTheShelfEvaluator(evaluator);
+}
+const isStringifiableValue = (value) => typeof value === "string" || typeof value === "number" || typeof value === "boolean" || typeof value === "bigint";
+const getSingleStringifiedValue = (value) => {
+	if (isStringifiableValue(value)) return `${value}`;
+	if (typeof value === "object" && value != null && !Array.isArray(value)) {
+		const entries = Object.entries(value);
+		if (entries.length === 1 && isStringifiableValue(entries[0][1])) return `${entries[0][1]}`;
+	}
+	console.warn("Non-stringifiable value found when coercing", value);
+	return `${value}`;
+};
+function Criteria(criteria, config) {
+	const formatEvaluatorInputs = config?.formatEvaluatorInputs ?? ((payload) => ({
+		prediction: getSingleStringifiedValue(payload.rawPrediction),
+		input: getSingleStringifiedValue(payload.rawInput)
+	}));
+	if (typeof criteria !== "string" && Object.keys(criteria).length !== 1) throw new Error("Only one criteria key is allowed when specifying custom criteria.");
+	const criteriaKey = typeof criteria === "string" ? criteria : Object.keys(criteria)[0];
+	return {
+		evaluatorType: "criteria",
+		criteria,
+		feedbackKey: config?.feedbackKey ?? criteriaKey,
+		llm: config?.llm,
+		formatEvaluatorInputs
+	};
+}
+function LabeledCriteria(criteria, config) {
+	const formatEvaluatorInputs = config?.formatEvaluatorInputs ?? ((payload) => ({
+		prediction: getSingleStringifiedValue(payload.rawPrediction),
+		input: getSingleStringifiedValue(payload.rawInput),
+		reference: getSingleStringifiedValue(payload.rawReferenceOutput)
+	}));
+	if (typeof criteria !== "string" && Object.keys(criteria).length !== 1) throw new Error("Only one labeled criteria key is allowed when specifying custom criteria.");
+	const criteriaKey = typeof criteria === "string" ? criteria : Object.keys(criteria)[0];
+	return {
+		evaluatorType: "labeled_criteria",
+		criteria,
+		feedbackKey: config?.feedbackKey ?? criteriaKey,
+		llm: config?.llm,
+		formatEvaluatorInputs
+	};
+}
+function EmbeddingDistance(distanceMetric, config) {
+	const formatEvaluatorInputs = config?.formatEvaluatorInputs ?? ((payload) => ({
+		prediction: getSingleStringifiedValue(payload.rawPrediction),
+		reference: getSingleStringifiedValue(payload.rawReferenceOutput)
+	}));
+	return {
+		evaluatorType: "embedding_distance",
+		embedding: config?.embedding,
+		distanceMetric,
+		feedbackKey: config?.feedbackKey ?? "embedding_distance",
+		formatEvaluatorInputs
+	};
+}
+
+//#endregion
+export { Criteria, EmbeddingDistance, LabeledCriteria, isCustomEvaluator, isOffTheShelfEvaluator };
+//# sourceMappingURL=config.js.map

package/dist/smith/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","names":["evaluator: T | EvalConfig | U","value: unknown","criteria: CriteriaType | Record<string, string>","config?: Pick<\n    Partial<LabeledCriteria>,\n    \"formatEvaluatorInputs\" | \"llm\" | \"feedbackKey\"\n  >","distanceMetric: EmbeddingDistanceEvalChainInput[\"distanceMetric\"]","config?: Pick<\n    Partial<LabeledCriteria>,\n    \"formatEvaluatorInputs\" | \"embedding\" | \"feedbackKey\"\n  >"],"sources":["../../src/smith/config.ts"],"sourcesContent":["import { BaseLanguageModel } from \"@langchain/core/language_models/base\";\nimport { RunnableConfig } from \"@langchain/core/runnables\";\nimport { Example, Run } from \"langsmith\";\nimport { EvaluationResult, RunEvaluator } from \"langsmith/evaluation\";\nimport {\n  Criteria as CriteriaType,\n  type EmbeddingDistanceEvalChainInput,\n} from \"../evaluation/index.js\";\nimport { LoadEvaluatorOptions } from \"../evaluation/loader.js\";\nimport { EvaluatorType } from \"../evaluation/types.js\";\n\nexport type EvaluatorInputs = {\n  input?: string | unknown;\n  prediction: string | unknown;\n  reference?: string | unknown;\n};\n\nexport type EvaluatorInputFormatter = ({\n  rawInput,\n  rawPrediction,\n  rawReferenceOutput,\n  run,\n}: {\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  rawInput: any;\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  rawPrediction: any;\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  rawReferenceOutput?: any;\n  run: Run;\n}) => EvaluatorInputs;\n\nexport type DynamicRunEvaluatorParams<\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  Input extends Record<string, any> = Record<string, unknown>,\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  Prediction extends Record<string, any> = Record<string, unknown>,\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  Reference extends Record<string, any> = Record<string, unknown>\n> = {\n  input: Input;\n  prediction?: Prediction;\n  reference?: Reference;\n  run: Run;\n  example?: Example;\n};\n\n/**\n * Type of a function that can be coerced into a RunEvaluator function.\n * While we have the class-based RunEvaluator, it's often more convenient to directly\n * pass a function to the runner. This type allows us to do that.\n */\nexport type RunEvaluatorLike =\n  | ((\n      props: DynamicRunEvaluatorParams,\n      options: RunnableConfig\n    ) => Promise<EvaluationResult>)\n  | ((\n      props: DynamicRunEvaluatorParams,\n      options: RunnableConfig\n    ) => EvaluationResult);\n\nexport function isOffTheShelfEvaluator<\n  T extends keyof EvaluatorType,\n  U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike\n>(evaluator: T | EvalConfig | U): evaluator is T | EvalConfig {\n  return typeof evaluator === \"string\" || \"evaluatorType\" in evaluator;\n}\n\nexport function isCustomEvaluator<\n  T extends keyof EvaluatorType,\n  U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike\n>(evaluator: T | EvalConfig | U): evaluator is U {\n  return !isOffTheShelfEvaluator(evaluator);\n}\n\nexport type RunEvalType<\n  T extends keyof EvaluatorType =\n    | \"criteria\"\n    | \"labeled_criteria\"\n    | \"embedding_distance\",\n  U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike\n> = T | EvalConfig | U;\n\n/**\n * Configuration class for running evaluations on datasets.\n *\n * @remarks\n * RunEvalConfig in LangSmith is a configuration class for running evaluations on datasets. Its primary purpose is to define the parameters and evaluators that will be applied during the evaluation of a dataset. This configuration can include various evaluators, custom evaluators, and different keys for inputs, predictions, and references.\n *\n * @typeparam T - The type of evaluators.\n * @typeparam U - The type of custom evaluators.\n */\nexport type RunEvalConfig<\n  T extends keyof EvaluatorType =\n    | \"criteria\"\n    | \"labeled_criteria\"\n    | \"embedding_distance\",\n  U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike\n> = {\n  /**\n   * Evaluators to apply to a dataset run.\n   * You can optionally specify these by name, or by\n   * configuring them with an EvalConfig object.\n   */\n  evaluators?: RunEvalType<T, U>[];\n\n  /**\n   * Convert the evaluation data into formats that can be used by the evaluator.\n   * This should most commonly be a string.\n   * Parameters are the raw input from the run, the raw output, raw reference output, and the raw run.\n   * @example\n   * ```ts\n   * // Chain input: { input: \"some string\" }\n   * // Chain output: { output: \"some output\" }\n   * // Reference example output format: { output: \"some reference output\" }\n   * const formatEvaluatorInputs = ({\n   *   rawInput,\n   *   rawPrediction,\n   *   rawReferenceOutput,\n   * }) => {\n   *   return {\n   *     input: rawInput.input,\n   *     prediction: rawPrediction.output,\n   *     reference: rawReferenceOutput.output,\n   *   };\n   * };\n   * ```\n   * @returns The prepared data.\n   */\n  formatEvaluatorInputs?: EvaluatorInputFormatter;\n\n  /**\n   * Custom evaluators to apply to a dataset run.\n   * Each evaluator is provided with a run trace containing the model\n   * outputs, as well as an \"example\" object representing a record\n   * in the dataset.\n   *\n   * @deprecated Use `evaluators` instead.\n   */\n  customEvaluators?: U[];\n};\n\nexport interface EvalConfig extends LoadEvaluatorOptions {\n  /**\n   * The name of the evaluator to use.\n   * Example: labeled_criteria, criteria, etc.\n   */\n  evaluatorType: keyof EvaluatorType;\n\n  /**\n   * The feedback (or metric) name to use for the logged\n   * evaluation results. If none provided, we default to\n   * the evaluationName.\n   */\n  feedbackKey?: string;\n\n  /**\n   * Convert the evaluation data into formats that can be used by the evaluator.\n   * This should most commonly be a string.\n   * Parameters are the raw input from the run, the raw output, raw reference output, and the raw run.\n   * @example\n   * ```ts\n   * // Chain input: { input: \"some string\" }\n   * // Chain output: { output: \"some output\" }\n   * // Reference example output format: { output: \"some reference output\" }\n   * const formatEvaluatorInputs = ({\n   *   rawInput,\n   *   rawPrediction,\n   *   rawReferenceOutput,\n   * }) => {\n   *   return {\n   *     input: rawInput.input,\n   *     prediction: rawPrediction.output,\n   *     reference: rawReferenceOutput.output,\n   *   };\n   * };\n   * ```\n   * @returns The prepared data.\n   */\n  formatEvaluatorInputs: EvaluatorInputFormatter;\n}\n\nconst isStringifiableValue = (\n  value: unknown\n): value is string | number | boolean | bigint =>\n  typeof value === \"string\" ||\n  typeof value === \"number\" ||\n  typeof value === \"boolean\" ||\n  typeof value === \"bigint\";\n\nconst getSingleStringifiedValue = (value: unknown) => {\n  if (isStringifiableValue(value)) {\n    return `${value}`;\n  }\n\n  if (typeof value === \"object\" && value != null && !Array.isArray(value)) {\n    const entries = Object.entries(value);\n\n    if (entries.length === 1 && isStringifiableValue(entries[0][1])) {\n      return `${entries[0][1]}`;\n    }\n  }\n\n  console.warn(\"Non-stringifiable value found when coercing\", value);\n  return `${value}`;\n};\n\n/**\n * Configuration to load a \"CriteriaEvalChain\" evaluator,\n * which prompts an LLM to determine whether the model's\n * prediction complies with the provided criteria.\n * @param criteria - The criteria to use for the evaluator.\n * @param llm - The language model to use for the evaluator.\n * @returns The configuration for the evaluator.\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [Criteria(\"helpfulness\")],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [\n *     Criteria({\n *       \"isCompliant\": \"Does the submission comply with the requirements of XYZ\"\n *     })\n *   ],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [{\n *     evaluatorType: \"criteria\",\n *     criteria: \"helpfulness\"\n *     formatEvaluatorInputs: ...\n *   }]\n * };\n * ```\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [{\n *     evaluatorType: \"criteria\",\n *     criteria: { \"isCompliant\": \"Does the submission comply with the requirements of XYZ\" },\n *     formatEvaluatorInputs: ...\n *   }]\n * };\n */\nexport type Criteria = EvalConfig & {\n  evaluatorType: \"criteria\";\n\n  /**\n   * The \"criteria\" to insert into the prompt template\n   * used for evaluation. See the prompt at\n   * https://smith.langchain.com/hub/langchain-ai/criteria-evaluator\n   * for more information.\n   */\n  criteria?: CriteriaType | Record<string, string>;\n\n  /**\n   * The language model to use as the evaluator, defaults to GPT-4\n   */\n  llm?: BaseLanguageModel;\n};\n\n// for compatibility reasons\nexport type CriteriaEvalChainConfig = Criteria;\n\nexport function Criteria(\n  criteria: CriteriaType | Record<string, string>,\n  config?: Pick<\n    Partial<LabeledCriteria>,\n    \"formatEvaluatorInputs\" | \"llm\" | \"feedbackKey\"\n  >\n): EvalConfig {\n  const formatEvaluatorInputs =\n    config?.formatEvaluatorInputs ??\n    ((payload) => ({\n      prediction: getSingleStringifiedValue(payload.rawPrediction),\n      input: getSingleStringifiedValue(payload.rawInput),\n    }));\n\n  if (typeof criteria !== \"string\" && Object.keys(criteria).length !== 1) {\n    throw new Error(\n      \"Only one criteria key is allowed when specifying custom criteria.\"\n    );\n  }\n\n  const criteriaKey =\n    typeof criteria === \"string\" ? criteria : Object.keys(criteria)[0];\n\n  return {\n    evaluatorType: \"criteria\",\n    criteria,\n    feedbackKey: config?.feedbackKey ?? criteriaKey,\n    llm: config?.llm,\n    formatEvaluatorInputs,\n  };\n}\n\n/**\n * Configuration to load a \"LabeledCriteriaEvalChain\" evaluator,\n * which prompts an LLM to determine whether the model's\n * prediction complies with the provided criteria and also\n * provides a \"ground truth\" label for the evaluator to incorporate\n * in its evaluation.\n * @param criteria - The criteria to use for the evaluator.\n * @param llm - The language model to use for the evaluator.\n * @returns The configuration for the evaluator.\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [LabeledCriteria(\"correctness\")],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [\n *     LabeledCriteria({\n *       \"mentionsAllFacts\": \"Does the include all facts provided in the reference?\"\n *     })\n *   ],\n * };\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [{\n *     evaluatorType: \"labeled_criteria\",\n *     criteria: \"correctness\",\n *     formatEvaluatorInputs: ...\n *   }],\n * };\n * ```\n * @example\n * ```ts\n * const evalConfig = {\n *   evaluators: [{\n *     evaluatorType: \"labeled_criteria\",\n *     criteria: { \"mentionsAllFacts\": \"Does the include all facts provided in the reference?\" },\n *     formatEvaluatorInputs: ...\n *   }],\n * };\n */\nexport type LabeledCriteria = EvalConfig & {\n  evaluatorType: \"labeled_criteria\";\n\n  /**\n   * The \"criteria\" to insert into the prompt template\n   * used for evaluation. See the prompt at\n   * https://smith.langchain.com/hub/langchain-ai/labeled-criteria\n   * for more information.\n   */\n  criteria?: CriteriaType | Record<string, string>;\n\n  /**\n   * The language model to use as the evaluator, defaults to GPT-4\n   */\n  llm?: BaseLanguageModel;\n};\n\nexport function LabeledCriteria(\n  criteria: CriteriaType | Record<string, string>,\n  config?: Pick<\n    Partial<LabeledCriteria>,\n    \"formatEvaluatorInputs\" | \"llm\" | \"feedbackKey\"\n  >\n): LabeledCriteria {\n  const formatEvaluatorInputs =\n    config?.formatEvaluatorInputs ??\n    ((payload) => ({\n      prediction: getSingleStringifiedValue(payload.rawPrediction),\n      input: getSingleStringifiedValue(payload.rawInput),\n      reference: getSingleStringifiedValue(payload.rawReferenceOutput),\n    }));\n\n  if (typeof criteria !== \"string\" && Object.keys(criteria).length !== 1) {\n    throw new Error(\n      \"Only one labeled criteria key is allowed when specifying custom criteria.\"\n    );\n  }\n\n  const criteriaKey =\n    typeof criteria === \"string\" ? criteria : Object.keys(criteria)[0];\n\n  return {\n    evaluatorType: \"labeled_criteria\",\n    criteria,\n    feedbackKey: config?.feedbackKey ?? criteriaKey,\n    llm: config?.llm,\n    formatEvaluatorInputs,\n  };\n}\n\n/**\n * Configuration to load a \"EmbeddingDistanceEvalChain\" evaluator,\n * which embeds distances to score semantic difference between\n * a prediction and reference.\n */\nexport type EmbeddingDistance = EvalConfig &\n  EmbeddingDistanceEvalChainInput & { evaluatorType: \"embedding_distance\" };\n\nexport function EmbeddingDistance(\n  distanceMetric: EmbeddingDistanceEvalChainInput[\"distanceMetric\"],\n  config?: Pick<\n    Partial<LabeledCriteria>,\n    \"formatEvaluatorInputs\" | \"embedding\" | \"feedbackKey\"\n  >\n): EmbeddingDistance {\n  const formatEvaluatorInputs =\n    config?.formatEvaluatorInputs ??\n    ((payload) => ({\n      prediction: getSingleStringifiedValue(payload.rawPrediction),\n      reference: getSingleStringifiedValue(payload.rawReferenceOutput),\n    }));\n\n  return {\n    evaluatorType: \"embedding_distance\",\n    embedding: config?.embedding,\n    distanceMetric,\n    feedbackKey: config?.feedbackKey ?? \"embedding_distance\",\n    formatEvaluatorInputs,\n  };\n}\n"],"mappings":";AA8DA,SAAgB,uBAGdA,WAA4D;AAC5D,QAAO,OAAO,cAAc,YAAY,mBAAmB;AAC5D;AAED,SAAgB,kBAGdA,WAA+C;AAC/C,QAAO,CAAC,uBAAuB,UAAU;AAC1C;AA6GD,MAAM,uBAAuB,CAC3BC,UAEA,OAAO,UAAU,YACjB,OAAO,UAAU,YACjB,OAAO,UAAU,aACjB,OAAO,UAAU;AAEnB,MAAM,4BAA4B,CAACA,UAAmB;AACpD,KAAI,qBAAqB,MAAM,CAC7B,QAAO,GAAG,OAAO;AAGnB,KAAI,OAAO,UAAU,YAAY,SAAS,QAAQ,CAAC,MAAM,QAAQ,MAAM,EAAE;EACvE,MAAM,UAAU,OAAO,QAAQ,MAAM;AAErC,MAAI,QAAQ,WAAW,KAAK,qBAAqB,QAAQ,GAAG,GAAG,CAC7D,QAAO,GAAG,QAAQ,GAAG,IAAI;CAE5B;CAED,QAAQ,KAAK,+CAA+C,MAAM;AAClE,QAAO,GAAG,OAAO;AAClB;AA+DD,SAAgB,SACdC,UACAC,QAIY;CACZ,MAAM,wBACJ,QAAQ,0BACP,CAAC,aAAa;EACb,YAAY,0BAA0B,QAAQ,cAAc;EAC5D,OAAO,0BAA0B,QAAQ,SAAS;CACnD;AAEH,KAAI,OAAO,aAAa,YAAY,OAAO,KAAK,SAAS,CAAC,WAAW,EACnE,OAAM,IAAI,MACR;CAIJ,MAAM,cACJ,OAAO,aAAa,WAAW,WAAW,OAAO,KAAK,SAAS,CAAC;AAElE,QAAO;EACL,eAAe;EACf;EACA,aAAa,QAAQ,eAAe;EACpC,KAAK,QAAQ;EACb;CACD;AACF;AA8DD,SAAgB,gBACdD,UACAC,QAIiB;CACjB,MAAM,wBACJ,QAAQ,0BACP,CAAC,aAAa;EACb,YAAY,0BAA0B,QAAQ,cAAc;EAC5D,OAAO,0BAA0B,QAAQ,SAAS;EAClD,WAAW,0BAA0B,QAAQ,mBAAmB;CACjE;AAEH,KAAI,OAAO,aAAa,YAAY,OAAO,KAAK,SAAS,CAAC,WAAW,EACnE,OAAM,IAAI,MACR;CAIJ,MAAM,cACJ,OAAO,aAAa,WAAW,WAAW,OAAO,KAAK,SAAS,CAAC;AAElE,QAAO;EACL,eAAe;EACf;EACA,aAAa,QAAQ,eAAe;EACpC,KAAK,QAAQ;EACb;CACD;AACF;AAUD,SAAgB,kBACdC,gBACAC,QAImB;CACnB,MAAM,wBACJ,QAAQ,0BACP,CAAC,aAAa;EACb,YAAY,0BAA0B,QAAQ,cAAc;EAC5D,WAAW,0BAA0B,QAAQ,mBAAmB;CACjE;AAEH,QAAO;EACL,eAAe;EACf,WAAW,QAAQ;EACnB;EACA,aAAa,QAAQ,eAAe;EACpC;CACD;AACF"}

package/dist/smith/index.cjs

@@ -0,0 +1,29 @@
+const require_rolldown_runtime = require('../_virtual/rolldown_runtime.cjs');
+const require_config = require('./config.cjs');
+const require_runner_utils = require('./runner_utils.cjs');
+
+//#region src/smith/index.ts
+var smith_exports = {};
+require_rolldown_runtime.__export(smith_exports, {
+	Criteria: () => require_config.Criteria,
+	EmbeddingDistance: () => require_config.EmbeddingDistance,
+	LabeledCriteria: () => require_config.LabeledCriteria,
+	isCustomEvaluator: () => require_config.isCustomEvaluator,
+	isOffTheShelfEvaluator: () => require_config.isOffTheShelfEvaluator,
+	runOnDataset: () => require_runner_utils.runOnDataset
+});
+
+//#endregion
+exports.Criteria = require_config.Criteria;
+exports.EmbeddingDistance = require_config.EmbeddingDistance;
+exports.LabeledCriteria = require_config.LabeledCriteria;
+exports.isCustomEvaluator = require_config.isCustomEvaluator;
+exports.isOffTheShelfEvaluator = require_config.isOffTheShelfEvaluator;
+exports.runOnDataset = require_runner_utils.runOnDataset;
+Object.defineProperty(exports, 'smith_exports', {
+  enumerable: true,
+  get: function () {
+    return smith_exports;
+  }
+});
+//# sourceMappingURL=index.cjs.map

package/dist/smith/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.cjs","names":[],"sources":["../../src/smith/index.ts"],"sourcesContent":["import {\n  type EvalResults,\n  type RunOnDatasetParams,\n  runOnDataset,\n} from \"./runner_utils.js\";\n\nexport { type EvalResults, type RunOnDatasetParams, runOnDataset };\nexport * from \"./config.js\";\n"],"mappings":""}

package/dist/smith/index.d.cts

@@ -0,0 +1,3 @@
+import { Criteria, CriteriaEvalChainConfig, DynamicRunEvaluatorParams, EmbeddingDistance, EvalConfig, EvaluatorInputFormatter, EvaluatorInputs, LabeledCriteria, RunEvalConfig, RunEvalType, RunEvaluatorLike, isCustomEvaluator, isOffTheShelfEvaluator } from "./config.cjs";
+import { EvalResults, RunOnDatasetParams, runOnDataset } from "./runner_utils.cjs";
+export { Criteria, CriteriaEvalChainConfig, DynamicRunEvaluatorParams, EmbeddingDistance, EvalConfig, type EvalResults, EvaluatorInputFormatter, EvaluatorInputs, LabeledCriteria, RunEvalConfig, RunEvalType, RunEvaluatorLike, type RunOnDatasetParams, isCustomEvaluator, isOffTheShelfEvaluator, runOnDataset };

package/dist/smith/index.d.ts

@@ -0,0 +1,3 @@
+import { Criteria, CriteriaEvalChainConfig, DynamicRunEvaluatorParams, EmbeddingDistance, EvalConfig, EvaluatorInputFormatter, EvaluatorInputs, LabeledCriteria, RunEvalConfig, RunEvalType, RunEvaluatorLike, isCustomEvaluator, isOffTheShelfEvaluator } from "./config.js";
+import { EvalResults, RunOnDatasetParams, runOnDataset } from "./runner_utils.js";
+export { Criteria, CriteriaEvalChainConfig, DynamicRunEvaluatorParams, EmbeddingDistance, EvalConfig, type EvalResults, EvaluatorInputFormatter, EvaluatorInputs, LabeledCriteria, RunEvalConfig, RunEvalType, RunEvaluatorLike, type RunOnDatasetParams, isCustomEvaluator, isOffTheShelfEvaluator, runOnDataset };

package/dist/smith/index.js

@@ -0,0 +1,18 @@
+import { __export } from "../_virtual/rolldown_runtime.js";
+import { Criteria, EmbeddingDistance, LabeledCriteria, isCustomEvaluator, isOffTheShelfEvaluator } from "./config.js";
+import { runOnDataset } from "./runner_utils.js";
+
+//#region src/smith/index.ts
+var smith_exports = {};
+__export(smith_exports, {
+	Criteria: () => Criteria,
+	EmbeddingDistance: () => EmbeddingDistance,
+	LabeledCriteria: () => LabeledCriteria,
+	isCustomEvaluator: () => isCustomEvaluator,
+	isOffTheShelfEvaluator: () => isOffTheShelfEvaluator,
+	runOnDataset: () => runOnDataset
+});
+
+//#endregion
+export { Criteria, EmbeddingDistance, LabeledCriteria, isCustomEvaluator, isOffTheShelfEvaluator, runOnDataset, smith_exports };
+//# sourceMappingURL=index.js.map

package/dist/smith/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":[],"sources":["../../src/smith/index.ts"],"sourcesContent":["import {\n  type EvalResults,\n  type RunOnDatasetParams,\n  runOnDataset,\n} from \"./runner_utils.js\";\n\nexport { type EvalResults, type RunOnDatasetParams, runOnDataset };\nexport * from \"./config.js\";\n"],"mappings":""}