langchain 0.1.19-rc.1 → 0.1.19-rc.2

package/dist/document_loaders/web/couchbase.cjs

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CouchbaseDocumentLoader = void 0;
+const documents_1 = require("@langchain/core/documents");
+const base_js_1 = require("../base.cjs");
+/**
+ * loader for couchbase document
+ */
+class CouchbaseDocumentLoader extends base_js_1.BaseDocumentLoader {
+    /**
+     * construct Couchbase document loader with a requirement for couchbase cluster client
+     * @param client { Cluster } [ couchbase connected client to connect to database ]
+     * @param query { string } [ query to get results from while loading the data ]
+     * @param pageContentFields { Array<string> } [ filters fields of the document and shows these only ]
+     * @param metadataFields { Array<string> } [ metadata fields required ]
+     */
+    constructor(client, query, pageContentFields, metadataFields) {
+        super();
+        Object.defineProperty(this, "cluster", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "query", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pageContentFields", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "metadataFields", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        if (!client) {
+            throw new Error("Couchbase client cluster must be provided.");
+        }
+        this.cluster = client;
+        this.query = query;
+        this.pageContentFields = pageContentFields;
+        this.metadataFields = metadataFields;
+    }
+    /**
+     * Function to load document based on query from couchbase
+     * @returns {Promise<Document[]>} [ Returns a promise of all the documents as array ]
+     */
+    async load() {
+        const documents = [];
+        for await (const doc of this.lazyLoad()) {
+            documents.push(doc);
+        }
+        return documents;
+    }
+    /**
+     * Function to load documents based on iterator rather than full load
+     * @returns {AsyncIterable<Document>} [ Returns an iterator to fetch documents ]
+     */
+    async *lazyLoad() {
+        // Run SQL++ Query
+        const result = await this.cluster.query(this.query);
+        for await (const row of result.rows) {
+            let { metadataFields, pageContentFields } = this;
+            if (!pageContentFields) {
+                pageContentFields = Object.keys(row);
+            }
+            if (!metadataFields) {
+                metadataFields = [];
+            }
+            const metadata = metadataFields.reduce((obj, field) => ({ ...obj, [field]: row[field] }), {});
+            const document = pageContentFields
+                .map((k) => `${k}: ${JSON.stringify(row[k])}`)
+                .join("\n");
+            yield new documents_1.Document({
+                pageContent: document,
+                metadata,
+            });
+        }
+    }
+}
+exports.CouchbaseDocumentLoader = CouchbaseDocumentLoader;

package/dist/document_loaders/web/couchbase.d.ts

@@ -0,0 +1,30 @@
+import { Cluster } from "couchbase";
+import { Document } from "@langchain/core/documents";
+import { BaseDocumentLoader, DocumentLoader } from "../base.js";
+/**
+ * loader for couchbase document
+ */
+export declare class CouchbaseDocumentLoader extends BaseDocumentLoader implements DocumentLoader {
+    private cluster;
+    private query;
+    private pageContentFields?;
+    private metadataFields?;
+    /**
+     * construct Couchbase document loader with a requirement for couchbase cluster client
+     * @param client { Cluster } [ couchbase connected client to connect to database ]
+     * @param query { string } [ query to get results from while loading the data ]
+     * @param pageContentFields { Array<string> } [ filters fields of the document and shows these only ]
+     * @param metadataFields { Array<string> } [ metadata fields required ]
+     */
+    constructor(client: Cluster, query: string, pageContentFields?: string[], metadataFields?: string[]);
+    /**
+     * Function to load document based on query from couchbase
+     * @returns {Promise<Document[]>} [ Returns a promise of all the documents as array ]
+     */
+    load(): Promise<Document[]>;
+    /**
+     * Function to load documents based on iterator rather than full load
+     * @returns {AsyncIterable<Document>} [ Returns an iterator to fetch documents ]
+     */
+    lazyLoad(): AsyncIterable<Document>;
+}

package/dist/document_loaders/web/couchbase.js

@@ -0,0 +1,84 @@
+import { Document } from "@langchain/core/documents";
+import { BaseDocumentLoader } from "../base.js";
+/**
+ * loader for couchbase document
+ */
+export class CouchbaseDocumentLoader extends BaseDocumentLoader {
+    /**
+     * construct Couchbase document loader with a requirement for couchbase cluster client
+     * @param client { Cluster } [ couchbase connected client to connect to database ]
+     * @param query { string } [ query to get results from while loading the data ]
+     * @param pageContentFields { Array<string> } [ filters fields of the document and shows these only ]
+     * @param metadataFields { Array<string> } [ metadata fields required ]
+     */
+    constructor(client, query, pageContentFields, metadataFields) {
+        super();
+        Object.defineProperty(this, "cluster", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "query", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pageContentFields", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "metadataFields", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        if (!client) {
+            throw new Error("Couchbase client cluster must be provided.");
+        }
+        this.cluster = client;
+        this.query = query;
+        this.pageContentFields = pageContentFields;
+        this.metadataFields = metadataFields;
+    }
+    /**
+     * Function to load document based on query from couchbase
+     * @returns {Promise<Document[]>} [ Returns a promise of all the documents as array ]
+     */
+    async load() {
+        const documents = [];
+        for await (const doc of this.lazyLoad()) {
+            documents.push(doc);
+        }
+        return documents;
+    }
+    /**
+     * Function to load documents based on iterator rather than full load
+     * @returns {AsyncIterable<Document>} [ Returns an iterator to fetch documents ]
+     */
+    async *lazyLoad() {
+        // Run SQL++ Query
+        const result = await this.cluster.query(this.query);
+        for await (const row of result.rows) {
+            let { metadataFields, pageContentFields } = this;
+            if (!pageContentFields) {
+                pageContentFields = Object.keys(row);
+            }
+            if (!metadataFields) {
+                metadataFields = [];
+            }
+            const metadata = metadataFields.reduce((obj, field) => ({ ...obj, [field]: row[field] }), {});
+            const document = pageContentFields
+                .map((k) => `${k}: ${JSON.stringify(row[k])}`)
+                .join("\n");
+            yield new Document({
+                pageContent: document,
+                metadata,
+            });
+        }
+    }
+}

package/dist/load/import_constants.cjs

@@ -102,6 +102,7 @@ exports.optionalImportEntrypoints = [
     "langchain/document_loaders/web/sitemap",
     "langchain/document_loaders/web/sonix_audio",
     "langchain/document_loaders/web/confluence",
+    "langchain/document_loaders/web/couchbase",
     "langchain/document_loaders/web/youtube",
     "langchain/document_loaders/fs/directory",
     "langchain/document_loaders/fs/buffer",

package/dist/load/import_constants.js

@@ -99,6 +99,7 @@ export const optionalImportEntrypoints = [
     "langchain/document_loaders/web/sitemap",
     "langchain/document_loaders/web/sonix_audio",
     "langchain/document_loaders/web/confluence",
+    "langchain/document_loaders/web/couchbase",
     "langchain/document_loaders/web/youtube",
     "langchain/document_loaders/fs/directory",
     "langchain/document_loaders/fs/buffer",

package/dist/smith/config.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LabeledCriteria = exports.Criteria = exports.isCustomEvaluator = exports.isOffTheShelfEvaluator = void 0;
+exports.EmbeddingDistance = exports.LabeledCriteria = exports.Criteria = exports.isCustomEvaluator = exports.isOffTheShelfEvaluator = void 0;
 function isOffTheShelfEvaluator(evaluator) {
     return typeof evaluator === "string" || "evaluatorType" in evaluator;
 }
@@ -36,6 +36,7 @@ function Criteria(criteria, config) {
         evaluatorType: "criteria",
         criteria,
         feedbackKey: config?.feedbackKey ?? criteria,
+        llm: config?.llm,
         formatEvaluatorInputs,
     };
 }
@@ -51,7 +52,23 @@ function LabeledCriteria(criteria, config) {
         evaluatorType: "labeled_criteria",
         criteria,
         feedbackKey: config?.feedbackKey ?? criteria,
+        llm: config?.llm,
         formatEvaluatorInputs,
     };
 }
 exports.LabeledCriteria = LabeledCriteria;
+function EmbeddingDistance(distanceMetric, config) {
+    const formatEvaluatorInputs = config?.formatEvaluatorInputs ??
+        ((payload) => ({
+            prediction: getSingleStringifiedValue(payload.rawPrediction),
+            reference: getSingleStringifiedValue(payload.rawReferenceOutput),
+        }));
+    return {
+        evaluatorType: "embedding_distance",
+        embedding: config?.embedding,
+        distanceMetric,
+        feedbackKey: config?.feedbackKey ?? "embedding_distance",
+        formatEvaluatorInputs,
+    };
+}
+exports.EmbeddingDistance = EmbeddingDistance;

package/dist/smith/config.d.ts

@@ -2,7 +2,7 @@ import { BaseLanguageModel } from "@langchain/core/language_models/base";
 import { RunnableConfig } from "@langchain/core/runnables";
 import { Example, Run } from "langsmith";
 import { EvaluationResult, RunEvaluator } from "langsmith/evaluation";
-import { Criteria as CriteriaType } from "../evaluation/index.js";
+import { Criteria as CriteriaType, type EmbeddingDistanceEvalChainInput } from "../evaluation/index.js";
 import { LoadEvaluatorOptions } from "../evaluation/loader.js";
 import { EvaluatorType } from "../evaluation/types.js";
 export type EvaluatorInputs = {
@@ -35,6 +35,7 @@ export type RunEvaluatorLike = ((props: DynamicRunEvaluatorParams, options?: {
 }) => EvaluationResult);
 export declare function isOffTheShelfEvaluator<T extends keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike>(evaluator: T | EvalConfig | U): evaluator is T | EvalConfig;
 export declare function isCustomEvaluator<T extends keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike>(evaluator: T | EvalConfig | U): evaluator is U;
+export type RunEvalType<T extends keyof EvaluatorType = "criteria" | "labeled_criteria" | "embedding_distance", U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = T | EvalConfig | U;
 /**
  * Configuration class for running evaluations on datasets.
  *
@@ -44,22 +45,13 @@ export declare function isCustomEvaluator<T extends keyof EvaluatorType, U exten
  * @typeparam T - The type of evaluators.
  * @typeparam U - The type of custom evaluators.
  */
-export type RunEvalConfig<T extends keyof EvaluatorType = keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = {
+export type RunEvalConfig<T extends keyof EvaluatorType = "criteria" | "labeled_criteria" | "embedding_distance", U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = {
     /**
-     * Custom evaluators to apply to a dataset run.
-     * Each evaluator is provided with a run trace containing the model
-     * outputs, as well as an "example" object representing a record
-     * in the dataset.
-     *
-     * @deprecated Use `evaluators` instead.
-     */
-    customEvaluators?: U[];
-    /**
-     * LangChain evaluators to apply to a dataset run.
+     * Evaluators to apply to a dataset run.
      * You can optionally specify these by name, or by
      * configuring them with an EvalConfig object.
      */
-    evaluators?: (T | EvalConfig | U)[];
+    evaluators?: RunEvalType<T, U>[];
     /**
      * Convert the evaluation data into formats that can be used by the evaluator.
      * This should most commonly be a string.
@@ -85,9 +77,14 @@ export type RunEvalConfig<T extends keyof EvaluatorType = keyof EvaluatorType, U
      */
     formatEvaluatorInputs?: EvaluatorInputFormatter;
     /**
-     * The language model specification for evaluators that require one.
+     * Custom evaluators to apply to a dataset run.
+     * Each evaluator is provided with a run trace containing the model
+     * outputs, as well as an "example" object representing a record
+     * in the dataset.
+     *
+     * @deprecated Use `evaluators` instead.
      */
-    evalLlm?: string;
+    customEvaluators?: U[];
 };
 export interface EvalConfig extends LoadEvaluatorOptions {
     /**
@@ -151,7 +148,7 @@ export interface EvalConfig extends LoadEvaluatorOptions {
  *   }]
  * };
  */
-export type CriteriaEvalChainConfig = EvalConfig & {
+export type Criteria = EvalConfig & {
     evaluatorType: "criteria";
     /**
      * The "criteria" to insert into the prompt template
@@ -161,16 +158,12 @@ export type CriteriaEvalChainConfig = EvalConfig & {
      */
     criteria?: CriteriaType | Record<string, string>;
     /**
-     * The feedback (or metric) name to use for the logged
-     * evaluation results. If none provided, we default to
-     * the evaluationName.
-     */
-    feedbackKey?: string;
-    /**
-     * The language model to use as the evaluator.
+     * The language model to use as the evaluator, defaults to GPT-4
      */
     llm?: BaseLanguageModel;
 };
+export type CriteriaEvalChainConfig = Criteria;
+export declare function Criteria(criteria: CriteriaType, config?: Pick<Partial<LabeledCriteria>, "formatEvaluatorInputs" | "llm" | "feedbackKey">): EvalConfig;
 /**
  * Configuration to load a "LabeledCriteriaEvalChain" evaluator,
  * which prompts an LLM to determine whether the model's
@@ -208,39 +201,17 @@ export type LabeledCriteria = EvalConfig & {
      */
     criteria?: CriteriaType | Record<string, string>;
     /**
-     * The feedback (or metric) name to use for the logged
-     * evaluation results. If none provided, we default to
-     * the evaluationName.
-     */
-    feedbackKey?: string;
-    /**
-     * The language model to use as the evaluator.
+     * The language model to use as the evaluator, defaults to GPT-4
      */
     llm?: BaseLanguageModel;
 };
-export declare function Criteria(criteria: CriteriaType, config?: {
-    formatEvaluatorInputs?: EvaluatorInputFormatter;
-    feedbackKey?: string;
-}): {
-    evaluatorType: "criteria";
-    criteria: CriteriaType;
-    feedbackKey: string;
-    formatEvaluatorInputs: EvaluatorInputFormatter | ((payload: {
-        rawInput: any;
-        rawPrediction: any;
-        rawReferenceOutput?: any;
-        run: Run;
-    }) => {
-        prediction: string;
-        input: string;
-    });
-};
-export declare function LabeledCriteria(criteria: CriteriaType, config?: {
-    formatEvaluatorInputs?: EvaluatorInputFormatter;
-    feedbackKey?: string;
-}): {
-    evaluatorType: "labeled_criteria";
-    criteria: CriteriaType;
-    feedbackKey: string;
-    formatEvaluatorInputs: EvaluatorInputFormatter;
+export declare function LabeledCriteria(criteria: CriteriaType, config?: Pick<Partial<LabeledCriteria>, "formatEvaluatorInputs" | "llm" | "feedbackKey">): LabeledCriteria;
+/**
+ * Configuration to load a "EmbeddingDistanceEvalChain" evaluator,
+ * which embeds distances to score semantic difference between
+ * a prediction and reference.
+ */
+export type EmbeddingDistance = EvalConfig & EmbeddingDistanceEvalChainInput & {
+    evaluatorType: "embedding_distance";
 };
+export declare function EmbeddingDistance(distanceMetric: EmbeddingDistanceEvalChainInput["distanceMetric"], config?: Pick<Partial<LabeledCriteria>, "formatEvaluatorInputs" | "embedding" | "feedbackKey">): EmbeddingDistance;

package/dist/smith/config.js

@@ -31,6 +31,7 @@ export function Criteria(criteria, config) {
         evaluatorType: "criteria",
         criteria,
         feedbackKey: config?.feedbackKey ?? criteria,
+        llm: config?.llm,
         formatEvaluatorInputs,
     };
 }
@@ -45,6 +46,21 @@ export function LabeledCriteria(criteria, config) {
         evaluatorType: "labeled_criteria",
         criteria,
         feedbackKey: config?.feedbackKey ?? criteria,
+        llm: config?.llm,
+        formatEvaluatorInputs,
+    };
+}
+export function EmbeddingDistance(distanceMetric, config) {
+    const formatEvaluatorInputs = config?.formatEvaluatorInputs ??
+        ((payload) => ({
+            prediction: getSingleStringifiedValue(payload.rawPrediction),
+            reference: getSingleStringifiedValue(payload.rawReferenceOutput),
+        }));
+    return {
+        evaluatorType: "embedding_distance",
+        embedding: config?.embedding,
+        distanceMetric,
+        feedbackKey: config?.feedbackKey ?? "embedding_distance",
         formatEvaluatorInputs,
     };
 }

package/dist/smith/runner_utils.cjs

@@ -115,6 +115,61 @@ class DynamicRunEvaluator {
 function isLLMStringEvaluator(evaluator) {
     return evaluator && typeof evaluator.evaluateStrings === "function";
 }
+/**
+ * Internal implementation of RunTree, which uses the
+ * provided callback manager instead of the internal LangSmith client.
+ *
+ * The goal of this class is to ensure seamless interop when intergrated
+ * with other Runnables.
+ */
+class CallbackManagerRunTree extends langsmith_1.RunTree {
+    constructor(config, callbackManager) {
+        super(config);
+        Object.defineProperty(this, "callbackManager", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "activeCallbackManager", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        this.callbackManager = callbackManager;
+    }
+    async createChild(config) {
+        const child = new CallbackManagerRunTree({
+            ...config,
+            parent_run: this,
+            project_name: this.project_name,
+            client: this.client,
+        }, this.activeCallbackManager?.getChild() ?? this.callbackManager);
+        this.child_runs.push(child);
+        return child;
+    }
+    async postRun() {
+        // how it is translated in comparison to basic RunTree?
+        this.activeCallbackManager = await this.callbackManager.handleChainStart(typeof this.serialized !== "object" &&
+            this.serialized != null &&
+            "lc" in this.serialized
+            ? this.serialized
+            : {
+                id: ["langchain", "smith", "CallbackManagerRunTree"],
+                lc: 1,
+                type: "not_implemented",
+            }, this.inputs, this.id, this.run_type, undefined, undefined, this.name);
+    }
+    async patchRun() {
+        if (this.error) {
+            await this.activeCallbackManager?.handleChainError(this.error, this.id, this.parent_run?.id, undefined, undefined);
+        }
+        else {
+            await this.activeCallbackManager?.handleChainEnd(this.outputs ?? {}, this.id, this.parent_run?.id, undefined, undefined);
+        }
+    }
+}
 class RunnableTraceable extends runnables_1.Runnable {
     constructor(fields) {
         super(fields);
@@ -147,12 +202,14 @@ class RunnableTraceable extends runnables_1.Runnable {
         const partialConfig = "langsmith:traceable" in this.func
             ? this.func["langsmith:traceable"]
             : { name: "<lambda>" };
-        const runTree = new langsmith_1.RunTree({
+        if (!callbackManager)
+            throw new Error("CallbackManager not found");
+        const runTree = new CallbackManagerRunTree({
             ...partialConfig,
             parent_run: callbackManager?._parentRunId
                 ? new langsmith_1.RunTree({ name: "<parent>", id: callbackManager?._parentRunId })
                 : undefined,
-        });
+        }, callbackManager);
         if (typeof input === "object" &&
             input != null &&
             Object.keys(input).length === 1) {
@@ -217,14 +274,14 @@ class PreparedRunEvaluator {
         const evalConfig = typeof config === "string" ? {} : config;
         const evaluator = await (0, loader_js_1.loadEvaluator)(evaluatorType, evalConfig);
         const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;
-        if (!feedbackKey) {
-            throw new Error(`Evaluator of type ${evaluatorType} must have an evaluationName` +
-                ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`);
-        }
         if (!isLLMStringEvaluator(evaluator)) {
             throw new Error(`Evaluator of type ${evaluatorType} not yet supported. ` +
                 "Please use a string evaluator, or implement your " +
-                "evaluation logic as a customEvaluator.");
+                "evaluation logic as a custom evaluator.");
+        }
+        if (!feedbackKey) {
+            throw new Error(`Evaluator of type ${evaluatorType} must have an evaluationName` +
+                ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`);
         }
         return new PreparedRunEvaluator(evaluator, feedbackKey, evalConfig?.formatEvaluatorInputs);
     }
@@ -261,7 +318,7 @@ class PreparedRunEvaluator {
         }
         throw new Error("Evaluator not yet supported. " +
             "Please use a string evaluator, or implement your " +
-            "evaluation logic as a customEvaluator.");
+            "evaluation logic as a custom evaluator.");
     }
 }
 class LoadedEvalConfig {
@@ -302,6 +359,10 @@ const createWrappedModel = async (modelOrFactory) => {
         return () => modelOrFactory;
     }
     if (typeof modelOrFactory === "function") {
+        if (isLangsmithTraceableFunction(modelOrFactory)) {
+            const wrappedModel = new RunnableTraceable({ func: modelOrFactory });
+            return () => wrappedModel;
+        }
         try {
             // If it works with no arguments, assume it's a factory
             let res = modelOrFactory();
@@ -313,11 +374,7 @@ const createWrappedModel = async (modelOrFactory) => {
         }
         catch (err) {
             // Otherwise, it's a custom UDF, and we'll wrap
-            // in a lambda or a traceable function
-            if (isLangsmithTraceableFunction(modelOrFactory)) {
-                const wrappedModel = new RunnableTraceable({ func: modelOrFactory });
-                return () => wrappedModel;
-            }
+            // the function in a lambda
             const wrappedModel = new runnables_1.RunnableLambda({ func: modelOrFactory });
             return () => wrappedModel;
         }
@@ -389,10 +446,65 @@ const getExamplesInputs = (examples, chainOrFactory, dataType) => {
     }
     return examples.map(({ inputs }) => inputs);
 };
+/**
+ * Evaluates a given model or chain against a specified LangSmith dataset.
+ *
+ * This function fetches example records from the specified dataset,
+ * runs the model or chain against each example, and returns the evaluation
+ * results.
+ *
+ * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a
+ * Runnable instance, a factory function that returns a Runnable, or a user-defined
+ * function or factory.
+ *
+ * @param datasetName - The name of the dataset against which the evaluation will be
+ * performed. This dataset should already be defined and contain the relevant data
+ * for evaluation.
+ *
+ * @param options - (Optional) Additional parameters for the evaluation process:
+ *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.
+ *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.
+ *   - `projectName` (string): Name of the project for logging and tracking.
+ *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.
+ *   - `client` (Client): Client instance for LangSmith service interaction.
+ *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.
+ *
+ * @returns A promise that resolves to an `EvalResults` object. This object includes
+ * detailed results of the evaluation, such as execution time, run IDs, and feedback
+ * for each entry in the dataset.
+ *
+ * @example
+ * ```typescript
+ * // Example usage for evaluating a model on a dataset
+ * async function evaluateModel() {
+ *   const chain = /* ...create your model or chain...*\//
+ *   const datasetName = 'example-dataset';
+ *   const client = new Client(/* ...config... *\//);
+ *
+ *   const results = await runOnDataset(chain, datasetName, {
+ *     evaluators: [/* ...evaluators... *\//],
+ *     client,
+ *   });
+ *
+ *   console.log('Evaluation Results:', results);
+ * }
+ *
+ * evaluateModel();
+ * ```
+ * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against
+ * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams["evaluators"]`, which can
+ * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.
+ * The function returns the evaluation results, which can be logged or further processed as needed.
+ */
 async function runOnDataset(chainOrFactory, datasetName, options) {
-    const { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, } = Array.isArray(options)
-        ? { evaluationConfig: { evaluators: options } }
-        : options ?? {};
+    const { projectName, projectMetadata, client, maxConcurrency, } = options ?? {};
+    const evaluationConfig = options?.evaluationConfig ??
+        (options?.evaluators != null
+            ? {
+                evaluators: options.evaluators,
+                formatEvaluatorInputs: options.formatEvaluatorInputs,
+            }
+            : undefined);
     const wrappedModel = await createWrappedModel(chainOrFactory);
     const testClient = client ?? new langsmith_1.Client();
     const testProjectName = projectName ?? (0, name_generation_js_1.randomName)();

package/dist/smith/runner_utils.d.ts

@@ -4,13 +4,28 @@ import type { TraceableFunction } from "langsmith/traceable";
 import { type RunEvalConfig } from "./config.js";
 export type ChainOrFactory = Runnable | (() => Runnable) | AnyTraceableFunction | ((obj: any) => any) | ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);
 type AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;
-export type RunOnDatasetParams = {
-    evaluationConfig?: RunEvalConfig;
-    projectMetadata?: Record<string, unknown>;
+export interface RunOnDatasetParams extends Omit<RunEvalConfig, "customEvaluators"> {
+    /**
+     * Name of the project for logging and tracking.
+     */
     projectName?: string;
+    /**
+     * Additional metadata for the project.
+     */
+    projectMetadata?: Record<string, unknown>;
+    /**
+     * Client instance for LangSmith service interaction.
+     */
     client?: Client;
+    /**
+     * Maximum concurrency level for dataset processing.
+     */
     maxConcurrency?: number;
-};
+    /**
+     * @deprecated Pass keys directly to the RunOnDatasetParams instead
+     */
+    evaluationConfig?: RunEvalConfig;
+}
 export type EvalResults = {
     projectName: string;
     results: {
@@ -37,11 +52,11 @@ export type EvalResults = {
  * for evaluation.
  *
  * @param options - (Optional) Additional parameters for the evaluation process:
- *   - `evaluationConfig` (RunEvalConfig): Configuration for the evaluation, including
- *     standard and custom evaluators.
+ *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.
+ *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.
  *   - `projectName` (string): Name of the project for logging and tracking.
  *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.
- *   - `client` (Client): Client instance for LangChain service interaction.
+ *   - `client` (Client): Client instance for LangSmith service interaction.
  *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.
  *
  * @returns A promise that resolves to an `EvalResults` object. This object includes
@@ -56,13 +71,8 @@ export type EvalResults = {
  *   const datasetName = 'example-dataset';
  *   const client = new Client(/* ...config... *\//);
  *
- *   const evaluationConfig = {
- *     evaluators: [/* ...evaluators... *\//],
- *     customEvaluators: [/* ...custom evaluators... *\//],
- *   };
- *
  *   const results = await runOnDataset(chain, datasetName, {
- *     evaluationConfig,
+ *     evaluators: [/* ...evaluators... *\//],
  *     client,
  *   });
  *
@@ -72,10 +82,9 @@ export type EvalResults = {
  * evaluateModel();
  * ```
  * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against
- * a dataset named 'example-dataset'. The evaluation process is configured using `RunEvalConfig`, which can
+ * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams["evaluators"]`, which can
  * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.
  * The function returns the evaluation results, which can be logged or further processed as needed.
  */
-export declare function runOnDataset(chainOrFactory: ChainOrFactory, datasetName: string, { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, }: RunOnDatasetParams): Promise<EvalResults>;
-export declare function runOnDataset(chainOrFactory: ChainOrFactory, datasetName: string, evaluators: RunEvalConfig["evaluators"]): Promise<EvalResults>;
+export declare function runOnDataset(chainOrFactory: ChainOrFactory, datasetName: string, options?: RunOnDatasetParams): Promise<EvalResults>;
 export {};

package/dist/smith/runner_utils.js

@@ -112,6 +112,61 @@ class DynamicRunEvaluator {
 function isLLMStringEvaluator(evaluator) {
     return evaluator && typeof evaluator.evaluateStrings === "function";
 }
+/**
+ * Internal implementation of RunTree, which uses the
+ * provided callback manager instead of the internal LangSmith client.
+ *
+ * The goal of this class is to ensure seamless interop when intergrated
+ * with other Runnables.
+ */
+class CallbackManagerRunTree extends RunTree {
+    constructor(config, callbackManager) {
+        super(config);
+        Object.defineProperty(this, "callbackManager", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "activeCallbackManager", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        this.callbackManager = callbackManager;
+    }
+    async createChild(config) {
+        const child = new CallbackManagerRunTree({
+            ...config,
+            parent_run: this,
+            project_name: this.project_name,
+            client: this.client,
+        }, this.activeCallbackManager?.getChild() ?? this.callbackManager);
+        this.child_runs.push(child);
+        return child;
+    }
+    async postRun() {
+        // how it is translated in comparison to basic RunTree?
+        this.activeCallbackManager = await this.callbackManager.handleChainStart(typeof this.serialized !== "object" &&
+            this.serialized != null &&
+            "lc" in this.serialized
+            ? this.serialized
+            : {
+                id: ["langchain", "smith", "CallbackManagerRunTree"],
+                lc: 1,
+                type: "not_implemented",
+            }, this.inputs, this.id, this.run_type, undefined, undefined, this.name);
+    }
+    async patchRun() {
+        if (this.error) {
+            await this.activeCallbackManager?.handleChainError(this.error, this.id, this.parent_run?.id, undefined, undefined);
+        }
+        else {
+            await this.activeCallbackManager?.handleChainEnd(this.outputs ?? {}, this.id, this.parent_run?.id, undefined, undefined);
+        }
+    }
+}
 class RunnableTraceable extends Runnable {
     constructor(fields) {
         super(fields);
@@ -144,12 +199,14 @@ class RunnableTraceable extends Runnable {
         const partialConfig = "langsmith:traceable" in this.func
             ? this.func["langsmith:traceable"]
             : { name: "<lambda>" };
-        const runTree = new RunTree({
+        if (!callbackManager)
+            throw new Error("CallbackManager not found");
+        const runTree = new CallbackManagerRunTree({
             ...partialConfig,
             parent_run: callbackManager?._parentRunId
                 ? new RunTree({ name: "<parent>", id: callbackManager?._parentRunId })
                 : undefined,
-        });
+        }, callbackManager);
         if (typeof input === "object" &&
             input != null &&
             Object.keys(input).length === 1) {
@@ -214,14 +271,14 @@ class PreparedRunEvaluator {
         const evalConfig = typeof config === "string" ? {} : config;
         const evaluator = await loadEvaluator(evaluatorType, evalConfig);
         const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;
-        if (!feedbackKey) {
-            throw new Error(`Evaluator of type ${evaluatorType} must have an evaluationName` +
-                ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`);
-        }
         if (!isLLMStringEvaluator(evaluator)) {
             throw new Error(`Evaluator of type ${evaluatorType} not yet supported. ` +
                 "Please use a string evaluator, or implement your " +
-                "evaluation logic as a customEvaluator.");
+                "evaluation logic as a custom evaluator.");
+        }
+        if (!feedbackKey) {
+            throw new Error(`Evaluator of type ${evaluatorType} must have an evaluationName` +
+                ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`);
         }
         return new PreparedRunEvaluator(evaluator, feedbackKey, evalConfig?.formatEvaluatorInputs);
     }
@@ -258,7 +315,7 @@ class PreparedRunEvaluator {
         }
         throw new Error("Evaluator not yet supported. " +
             "Please use a string evaluator, or implement your " +
-            "evaluation logic as a customEvaluator.");
+            "evaluation logic as a custom evaluator.");
     }
 }
 class LoadedEvalConfig {
@@ -299,6 +356,10 @@ const createWrappedModel = async (modelOrFactory) => {
         return () => modelOrFactory;
     }
     if (typeof modelOrFactory === "function") {
+        if (isLangsmithTraceableFunction(modelOrFactory)) {
+            const wrappedModel = new RunnableTraceable({ func: modelOrFactory });
+            return () => wrappedModel;
+        }
         try {
             // If it works with no arguments, assume it's a factory
             let res = modelOrFactory();
@@ -310,11 +371,7 @@ const createWrappedModel = async (modelOrFactory) => {
         }
         catch (err) {
             // Otherwise, it's a custom UDF, and we'll wrap
-            // in a lambda or a traceable function
-            if (isLangsmithTraceableFunction(modelOrFactory)) {
-                const wrappedModel = new RunnableTraceable({ func: modelOrFactory });
-                return () => wrappedModel;
-            }
+            // the function in a lambda
             const wrappedModel = new RunnableLambda({ func: modelOrFactory });
             return () => wrappedModel;
         }
@@ -386,10 +443,65 @@ const getExamplesInputs = (examples, chainOrFactory, dataType) => {
     }
     return examples.map(({ inputs }) => inputs);
 };
+/**
+ * Evaluates a given model or chain against a specified LangSmith dataset.
+ *
+ * This function fetches example records from the specified dataset,
+ * runs the model or chain against each example, and returns the evaluation
+ * results.
+ *
+ * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a
+ * Runnable instance, a factory function that returns a Runnable, or a user-defined
+ * function or factory.
+ *
+ * @param datasetName - The name of the dataset against which the evaluation will be
+ * performed. This dataset should already be defined and contain the relevant data
+ * for evaluation.
+ *
+ * @param options - (Optional) Additional parameters for the evaluation process:
+ *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.
+ *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.
+ *   - `projectName` (string): Name of the project for logging and tracking.
+ *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.
+ *   - `client` (Client): Client instance for LangSmith service interaction.
+ *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.
+ *
+ * @returns A promise that resolves to an `EvalResults` object. This object includes
+ * detailed results of the evaluation, such as execution time, run IDs, and feedback
+ * for each entry in the dataset.
+ *
+ * @example
+ * ```typescript
+ * // Example usage for evaluating a model on a dataset
+ * async function evaluateModel() {
+ *   const chain = /* ...create your model or chain...*\//
+ *   const datasetName = 'example-dataset';
+ *   const client = new Client(/* ...config... *\//);
+ *
+ *   const results = await runOnDataset(chain, datasetName, {
+ *     evaluators: [/* ...evaluators... *\//],
+ *     client,
+ *   });
+ *
+ *   console.log('Evaluation Results:', results);
+ * }
+ *
+ * evaluateModel();
+ * ```
+ * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against
+ * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams["evaluators"]`, which can
+ * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.
+ * The function returns the evaluation results, which can be logged or further processed as needed.
+ */
 export async function runOnDataset(chainOrFactory, datasetName, options) {
-    const { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, } = Array.isArray(options)
-        ? { evaluationConfig: { evaluators: options } }
-        : options ?? {};
+    const { projectName, projectMetadata, client, maxConcurrency, } = options ?? {};
+    const evaluationConfig = options?.evaluationConfig ??
+        (options?.evaluators != null
+            ? {
+                evaluators: options.evaluators,
+                formatEvaluatorInputs: options.formatEvaluatorInputs,
+            }
+            : undefined);
     const wrappedModel = await createWrappedModel(chainOrFactory);
     const testClient = client ?? new Client();
     const testProjectName = projectName ?? randomName();

package/document_loaders/web/couchbase.cjs

@@ -0,0 +1 @@
+module.exports = require('../../dist/document_loaders/web/couchbase.cjs');

package/document_loaders/web/couchbase.d.cts

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/web/couchbase.js'

package/document_loaders/web/couchbase.d.ts

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/web/couchbase.js'

package/document_loaders/web/couchbase.js

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/web/couchbase.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.1.19-rc.1",
+  "version": "0.1.19-rc.2",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {
@@ -614,6 +614,10 @@
     "document_loaders/web/confluence.js",
     "document_loaders/web/confluence.d.ts",
     "document_loaders/web/confluence.d.cts",
+    "document_loaders/web/couchbase.cjs",
+    "document_loaders/web/couchbase.js",
+    "document_loaders/web/couchbase.d.ts",
+    "document_loaders/web/couchbase.d.cts",
     "document_loaders/web/searchapi.cjs",
     "document_loaders/web/searchapi.js",
     "document_loaders/web/searchapi.d.ts",
@@ -1247,6 +1251,7 @@
     "cheerio": "^1.0.0-rc.12",
     "chromadb": "^1.5.3",
     "convex": "^1.3.1",
+    "couchbase": "^4.2.10",
     "d3-dsv": "^2.0.0",
     "dotenv": "^16.0.3",
     "dpdm": "^3.12.0",
@@ -1317,6 +1322,7 @@
     "cheerio": "^1.0.0-rc.12",
     "chromadb": "*",
     "convex": "^1.3.1",
+    "couchbase": "^4.2.10",
     "d3-dsv": "^2.0.0",
     "epub2": "^3.0.1",
     "fast-xml-parser": "^4.2.7",
@@ -1411,6 +1417,9 @@
     "convex": {
       "optional": true
     },
+    "couchbase": {
+      "optional": true
+    },
     "d3-dsv": {
       "optional": true
     },
@@ -1504,8 +1513,8 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.9.1",
-    "@langchain/community": "~0.0.28",
-    "@langchain/core": "~0.1.28",
+    "@langchain/community": "~0.0.29",
+    "@langchain/core": "~0.1.29",
     "@langchain/openai": "~0.0.14",
     "binary-extensions": "^2.2.0",
     "expr-eval": "^2.0.2",
@@ -2899,6 +2908,15 @@
       "import": "./document_loaders/web/confluence.js",
       "require": "./document_loaders/web/confluence.cjs"
     },
+    "./document_loaders/web/couchbase": {
+      "types": {
+        "import": "./document_loaders/web/couchbase.d.ts",
+        "require": "./document_loaders/web/couchbase.d.cts",
+        "default": "./document_loaders/web/couchbase.d.ts"
+      },
+      "import": "./document_loaders/web/couchbase.js",
+      "require": "./document_loaders/web/couchbase.cjs"
+    },
     "./document_loaders/web/searchapi": {
       "types": {
         "import": "./document_loaders/web/searchapi.d.ts",