langchain 0.1.5 → 0.1.6

package/dist/load/import_map.cjs

@@ -25,7 +25,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.util__document = exports.storage__in_memory = exports.storage__encoder_backed = exports.stores__message__in_memory = exports.stores__file__in_memory = exports.stores__doc__in_memory = exports.retrievers__vespa = exports.retrievers__score_threshold = exports.retrievers__hyde = exports.retrievers__document_compressors__embeddings_filter = exports.retrievers__document_compressors__chain_extract = exports.retrievers__time_weighted = exports.retrievers__parent_document = exports.retrievers__multi_vector = exports.retrievers__multi_query = exports.retrievers__document_compressors = exports.retrievers__contextual_compression = exports.retrievers__remote = exports.output_parsers = exports.schema__query_constructor = exports.schema__prompt_template = exports.chat_models__anthropic = exports.document_transformers__openai_functions = exports.document_loaders__web__sort_xyz_blockchain = exports.document_loaders__web__serpapi = exports.document_loaders__web__searchapi = exports.document_loaders__base = exports.text_splitter = exports.vectorstores__memory = exports.llms__fake = exports.embeddings__fake = exports.embeddings__cache_backed = exports.chains__retrieval = exports.chains__openai_functions = exports.chains__history_aware_retriever = exports.chains__combine_documents__reduce = exports.chains__combine_documents = exports.chains = exports.tools__retriever = exports.tools__render = exports.agents__openai__output_parser = exports.agents__xml__output_parser = exports.agents__react__output_parser = exports.agents__format_scratchpad__log_to_message = exports.agents__format_scratchpad__xml = exports.agents__format_scratchpad__log = exports.agents__format_scratchpad__openai_tools = exports.agents__format_scratchpad = exports.agents__toolkits = exports.agents = void 0;
-exports.llms__fireworks = exports.chat_models__fireworks = exports.schema__output = exports.schema__output_parser = exports.schema__runnable = exports.prompts__base = exports.prompts__pipeline = exports.prompts__chat = exports.schema__messages = exports.prompts__prompt = exports.embeddings__openai = exports.llms__openai = exports.chat_models__openai = exports.runnables__remote = exports.evaluation = exports.experimental__prompts__custom_format = exports.experimental__masking = exports.experimental__chains__violation_of_expectations = exports.experimental__chat_models__bittensor = exports.experimental__plan_and_execute = exports.experimental__generative_agents = exports.experimental__babyagi = exports.experimental__openai_files = exports.experimental__openai_assistant = exports.experimental__autogpt = exports.util__time = exports.util__math = void 0;
+exports.llms__fireworks = exports.chat_models__fireworks = exports.schema__output = exports.schema__output_parser = exports.schema__runnable = exports.prompts__base = exports.prompts__pipeline = exports.prompts__chat = exports.schema__messages = exports.prompts__prompt = exports.embeddings__openai = exports.llms__openai = exports.chat_models__openai = exports.runnables__remote = exports.smith = exports.evaluation = exports.experimental__prompts__custom_format = exports.experimental__masking = exports.experimental__chains__violation_of_expectations = exports.experimental__chat_models__bittensor = exports.experimental__plan_and_execute = exports.experimental__generative_agents = exports.experimental__babyagi = exports.experimental__openai_files = exports.experimental__openai_assistant = exports.experimental__autogpt = exports.util__time = exports.util__math = void 0;
 exports.agents = __importStar(require("../agents/index.cjs"));
 exports.agents__toolkits = __importStar(require("../agents/toolkits/index.cjs"));
 exports.agents__format_scratchpad = __importStar(require("../agents/format_scratchpad/openai_functions.cjs"));
@@ -89,6 +89,7 @@ exports.experimental__chains__violation_of_expectations = __importStar(require("
 exports.experimental__masking = __importStar(require("../experimental/masking/index.cjs"));
 exports.experimental__prompts__custom_format = __importStar(require("../experimental/prompts/custom_format.cjs"));
 exports.evaluation = __importStar(require("../evaluation/index.cjs"));
+exports.smith = __importStar(require("../smith/index.cjs"));
 exports.runnables__remote = __importStar(require("../runnables/remote.cjs"));
 const openai_1 = require("@langchain/openai");
 const prompts_1 = require("@langchain/core/prompts");

package/dist/load/import_map.d.ts

@@ -61,6 +61,7 @@ export * as experimental__chains__violation_of_expectations from "../experimenta
 export * as experimental__masking from "../experimental/masking/index.js";
 export * as experimental__prompts__custom_format from "../experimental/prompts/custom_format.js";
 export * as evaluation from "../evaluation/index.js";
+export * as smith from "../smith/index.js";
 export * as runnables__remote from "../runnables/remote.js";
 import { ChatOpenAI, OpenAI, OpenAIEmbeddings } from "@langchain/openai";
 import { PromptTemplate, AIMessagePromptTemplate, ChatMessagePromptTemplate, ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate, PipelinePromptTemplate } from "@langchain/core/prompts";

package/dist/load/import_map.js

@@ -62,6 +62,7 @@ export * as experimental__chains__violation_of_expectations from "../experimenta
 export * as experimental__masking from "../experimental/masking/index.js";
 export * as experimental__prompts__custom_format from "../experimental/prompts/custom_format.js";
 export * as evaluation from "../evaluation/index.js";
+export * as smith from "../smith/index.js";
 export * as runnables__remote from "../runnables/remote.js";
 import { ChatOpenAI, OpenAI, OpenAIEmbeddings } from "@langchain/openai";
 import { PromptTemplate, AIMessagePromptTemplate, ChatMessagePromptTemplate, ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder, SystemMessagePromptTemplate, PipelinePromptTemplate } from "@langchain/core/prompts";

package/dist/smith/config.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/smith/config.d.ts

@@ -0,0 +1,166 @@
+import { BaseLanguageModel } from "@langchain/core/language_models/base";
+import { Example, Run } from "langsmith";
+import { EvaluationResult, RunEvaluator } from "langsmith/evaluation";
+import { Criteria } from "../evaluation/index.js";
+import { LoadEvaluatorOptions } from "../evaluation/loader.js";
+import { EvaluatorType } from "../evaluation/types.js";
+export type EvaluatorInputs = {
+    input?: string | unknown;
+    prediction: string | unknown;
+    reference?: string | unknown;
+};
+export type EvaluatorInputFormatter = ({ rawInput, rawPrediction, rawReferenceOutput, run, }: {
+    rawInput: any;
+    rawPrediction: any;
+    rawReferenceOutput?: any;
+    run: Run;
+}) => EvaluatorInputs;
+/**
+ * Type of a function that can be coerced into a RunEvaluator function.
+ * While we have the class-based RunEvaluator, it's often more convenient to directly
+ * pass a function to the runner. This type allows us to do that.
+ */
+export type RunEvaluatorLike = (({ run, example, }: {
+    run: Run;
+    example?: Example;
+}) => Promise<EvaluationResult>) | (({ run, example }: {
+    run: Run;
+    example?: Example;
+}) => EvaluationResult);
+/**
+ * Configuration class for running evaluations on datasets.
+ *
+ * @remarks
+ * RunEvalConfig in LangSmith is a configuration class for running evaluations on datasets. Its primary purpose is to define the parameters and evaluators that will be applied during the evaluation of a dataset. This configuration can include various evaluators, custom evaluators, and different keys for inputs, predictions, and references.
+ *
+ * @typeparam T - The type of evaluators.
+ * @typeparam U - The type of custom evaluators.
+ */
+export type RunEvalConfig<T extends keyof EvaluatorType = keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = {
+    /**
+     * Custom evaluators to apply to a dataset run.
+     * Each evaluator is provided with a run trace containing the model
+     * outputs, as well as an "example" object representing a record
+     * in the dataset.
+     */
+    customEvaluators?: U[];
+    /**
+     * LangChain evaluators to apply to a dataset run.
+     * You can optionally specify these by name, or by
+     * configuring them with an EvalConfig object.
+     */
+    evaluators?: (T | EvalConfig)[];
+    /**
+     * Convert the evaluation data into a format that can be used by the evaluator.
+     * By default, we pass the first value of the run.inputs, run.outputs (predictions),
+     * and references (example.outputs)
+     *
+     * @returns The prepared data.
+     */
+    formatEvaluatorInputs?: EvaluatorInputFormatter;
+    /**
+     * The language model specification for evaluators that require one.
+     */
+    evalLlm?: string;
+};
+export interface EvalConfig extends LoadEvaluatorOptions {
+    /**
+     * The name of the evaluator to use.
+     * Example: labeled_criteria, criteria, etc.
+     */
+    evaluatorType: keyof EvaluatorType;
+    /**
+     * The feedback (or metric) name to use for the logged
+     * evaluation results. If none provided, we default to
+     * the evaluationName.
+     */
+    feedbackKey?: string;
+    /**
+     * Convert the evaluation data into a format that can be used by the evaluator.
+     * @param data The data to prepare.
+     * @returns The prepared data.
+     */
+    formatEvaluatorInputs: EvaluatorInputFormatter;
+}
+/**
+ * Configuration to load a "CriteriaEvalChain" evaluator,
+ * which prompts an LLM to determine whether the model's
+ * prediction complies with the provided criteria.
+ * @param criteria - The criteria to use for the evaluator.
+ * @param llm - The language model to use for the evaluator.
+ * @returns The configuration for the evaluator.
+ * @example
+ * ```ts
+ * const evalConfig = new RunEvalConfig(
+ *  [new RunEvalConfig.Criteria("helpfulness")],
+ * );
+ * ```
+ * @example
+ * ```ts
+ * const evalConfig = new RunEvalConfig(
+ * [new RunEvalConfig.Criteria(
+ *      { "isCompliant": "Does the submission comply with the requirements of XYZ"
+ *  })],
+ */
+export type CriteriaEvalChainConfig = EvalConfig & {
+    evaluatorType: "criteria";
+    /**
+     * The "criteria" to insert into the prompt template
+     * used for evaluation. See the prompt at
+     * https://smith.langchain.com/hub/langchain-ai/criteria-evaluator
+     * for more information.
+     */
+    criteria?: Criteria | Record<string, string>;
+    /**
+     * The feedback (or metric) name to use for the logged
+     * evaluation results. If none provided, we default to
+     * the evaluationName.
+     */
+    feedbackKey?: string;
+    /**
+     * The language model to use as the evaluator.
+     */
+    llm?: BaseLanguageModel;
+};
+/**
+ * Configuration to load a "LabeledCriteriaEvalChain" evaluator,
+ * which prompts an LLM to determine whether the model's
+ * prediction complies with the provided criteria and also
+ * provides a "ground truth" label for the evaluator to incorporate
+ * in its evaluation.
+ * @param criteria - The criteria to use for the evaluator.
+ * @param llm - The language model to use for the evaluator.
+ * @returns The configuration for the evaluator.
+ * @example
+ * ```ts
+ * const evalConfig = new RunEvalConfig(
+ *  [new RunEvalConfig.LabeledCriteria("correctness")],
+ * );
+ * ```
+ * @example
+ * ```ts
+ * const evalConfig = new RunEvalConfig(
+ * [new RunEvalConfig.Criteria(
+ *      { "mentionsAllFacts": "Does the include all facts provided in the reference?"
+ *  })],
+ */
+export type LabeledCriteria = EvalConfig & {
+    evaluatorType: "labeled_criteria";
+    /**
+     * The "criteria" to insert into the prompt template
+     * used for evaluation. See the prompt at
+     * https://smith.langchain.com/hub/langchain-ai/labeled-criteria
+     * for more information.
+     */
+    criteria?: Criteria | Record<string, string>;
+    /**
+     * The feedback (or metric) name to use for the logged
+     * evaluation results. If none provided, we default to
+     * the evaluationName.
+     */
+    feedbackKey?: string;
+    /**
+     * The language model to use as the evaluator.
+     */
+    llm?: BaseLanguageModel;
+};

package/dist/smith/config.js

@@ -0,0 +1 @@
+export {};

package/dist/smith/index.cjs

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runOnDataset = void 0;
+const runner_utils_js_1 = require("./runner_utils.cjs");
+Object.defineProperty(exports, "runOnDataset", { enumerable: true, get: function () { return runner_utils_js_1.runOnDataset; } });

package/dist/smith/index.d.ts

@@ -0,0 +1,3 @@
+import type { RunEvalConfig } from "./config.js";
+import { runOnDataset } from "./runner_utils.js";
+export { type RunEvalConfig, runOnDataset };

package/dist/smith/index.js

@@ -0,0 +1,2 @@
+import { runOnDataset } from "./runner_utils.js";
+export { runOnDataset };