langchain 0.1.4 → 0.1.6

package/dist/smith/runner_utils.js

@@ -0,0 +1,349 @@
+import { mapStoredMessagesToChatMessages } from "@langchain/core/messages";
+import { Runnable, RunnableLambda } from "@langchain/core/runnables";
+import { RunCollectorCallbackHandler } from "@langchain/core/tracers/run_collector";
+import { LangChainTracer } from "@langchain/core/tracers/tracer_langchain";
+import { Client } from "langsmith";
+import { loadEvaluator } from "../evaluation/loader.js";
+import { randomName } from "./name_generation.js";
+import { ProgressBar } from "./progress.js";
+/**
+ * Wraps an evaluator function + implements the RunEvaluator interface.
+ */
+class DynamicRunEvaluator {
+    constructor(evaluator) {
+        Object.defineProperty(this, "evaluator", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.evaluator = new RunnableLambda({ func: evaluator });
+    }
+    /**
+     * Evaluates a run with an optional example and returns the evaluation result.
+     * @param run The run to evaluate.
+     * @param example The optional example to use for evaluation.
+     * @returns A promise that resolves to the evaluation result.
+     */
+    async evaluateRun(run, example) {
+        return await this.evaluator.invoke({ run, example });
+    }
+}
+function isLLMStringEvaluator(evaluator) {
+    return evaluator && typeof evaluator.evaluateStrings === "function";
+}
+/**
+ * Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])
+ * and composes with a prepareData function so the user can prepare the trace and
+ * dataset data for the evaluator.
+ */
+class PreparedRunEvaluator {
+    constructor(evaluator, evaluationName, formatEvaluatorInputs) {
+        Object.defineProperty(this, "evaluator", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "formatEvaluatorInputs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "isStringEvaluator", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "evaluationName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.evaluator = evaluator;
+        this.isStringEvaluator = typeof evaluator?.evaluateStrings === "function";
+        this.evaluationName = evaluationName;
+        this.formatEvaluatorInputs = formatEvaluatorInputs;
+    }
+    static async fromEvalConfig(config) {
+        const evaluatorType = typeof config === "string" ? config : config.evaluatorType;
+        const evalConfig = typeof config === "string" ? {} : config;
+        const evaluator = await loadEvaluator(evaluatorType, evalConfig);
+        const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;
+        if (!feedbackKey) {
+            throw new Error(`Evaluator of type ${evaluatorType} must have an evaluationName` +
+                ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`);
+        }
+        if (!isLLMStringEvaluator(evaluator)) {
+            throw new Error(`Evaluator of type ${evaluatorType} not yet supported. ` +
+                "Please use a string evaluator, or implement your " +
+                "evaluation logic as a customEvaluator.");
+        }
+        return new PreparedRunEvaluator(evaluator, feedbackKey, evalConfig?.formatEvaluatorInputs);
+    }
+    /**
+     * Evaluates a run with an optional example and returns the evaluation result.
+     * @param run The run to evaluate.
+     * @param example The optional example to use for evaluation.
+     * @returns A promise that resolves to the evaluation result.
+     */
+    async evaluateRun(run, example) {
+        const { prediction, input, reference } = this.formatEvaluatorInputs({
+            rawInput: run.inputs,
+            rawPrediction: run.outputs,
+            rawReferenceOutput: example?.outputs,
+            run,
+        });
+        if (this.isStringEvaluator) {
+            const evalResult = await this.evaluator.evaluateStrings({
+                prediction: prediction,
+                reference: reference,
+                input: input,
+            });
+            return {
+                key: this.evaluationName,
+                comment: evalResult?.reasoning,
+                ...evalResult,
+            };
+        }
+        throw new Error("Evaluator not yet supported. " +
+            "Please use a string evaluator, or implement your " +
+            "evaluation logic as a customEvaluator.");
+    }
+}
+class LoadedEvalConfig {
+    constructor(evaluators) {
+        Object.defineProperty(this, "evaluators", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: evaluators
+        });
+    }
+    static async fromRunEvalConfig(config) {
+        // Custom evaluators are applied "as-is"
+        const customEvaluators = config?.customEvaluators?.map((evaluator) => {
+            if (typeof evaluator === "function") {
+                return new DynamicRunEvaluator(evaluator);
+            }
+            else {
+                return evaluator;
+            }
+        });
+        const offTheShelfEvaluators = await Promise.all(config?.evaluators?.map(async (evaluator) => await PreparedRunEvaluator.fromEvalConfig(evaluator)) ?? []);
+        return new LoadedEvalConfig((customEvaluators ?? []).concat(offTheShelfEvaluators ?? []));
+    }
+}
+/**
+ * Internals expect a constructor () -> Runnable. This function wraps/coerces
+ * the provided LangChain object, custom function, or factory function into
+ * a constructor of a runnable.
+ * @param modelOrFactory The model or factory to create a wrapped model from.
+ * @returns A function that returns the wrapped model.
+ * @throws Error if the modelOrFactory is invalid.
+ */
+const createWrappedModel = async (modelOrFactory) => {
+    if (Runnable.isRunnable(modelOrFactory)) {
+        return () => modelOrFactory;
+    }
+    if (typeof modelOrFactory === "function") {
+        try {
+            // If it works with no arguments, assume it's a factory
+            let res = modelOrFactory();
+            if (res &&
+                typeof res.then === "function") {
+                res = await res;
+            }
+            return modelOrFactory;
+        }
+        catch (err) {
+            // Otherwise, it's a custom UDF, and we'll wrap
+            // in a lambda
+            const wrappedModel = new RunnableLambda({ func: modelOrFactory });
+            return () => wrappedModel;
+        }
+    }
+    throw new Error("Invalid modelOrFactory");
+};
+const loadExamples = async ({ datasetName, client, projectName, }) => {
+    const exampleIterator = client.listExamples({ datasetName });
+    const configs = [];
+    const runCollectors = [];
+    const examples = [];
+    for await (const example of exampleIterator) {
+        const runCollector = new RunCollectorCallbackHandler({
+            exampleId: example.id,
+        });
+        configs.push({
+            callbacks: [
+                new LangChainTracer({ exampleId: example.id, projectName }),
+                runCollector,
+            ],
+        });
+        examples.push(example);
+        runCollectors.push(runCollector);
+    }
+    return {
+        configs,
+        examples,
+        runCollectors,
+    };
+};
+const applyEvaluators = async ({ evaluation, runs, examples, client, }) => {
+    // TODO: Parallelize and/or put in callbacks to speed up evals.
+    const { evaluators } = evaluation;
+    const progress = new ProgressBar({
+        total: examples.length,
+        format: "Running Evaluators: {bar} {percentage}% | {value}/{total}\n",
+    });
+    const results = {};
+    for (let i = 0; i < runs.length; i += 1) {
+        const run = runs[i];
+        const example = examples[i];
+        const evaluatorResults = await Promise.all(evaluators.map((evaluator) => client.evaluateRun(run, evaluator, {
+            referenceExample: example,
+            loadChildRuns: false,
+        })));
+        progress.increment();
+        results[example.id] = {
+            execution_time: run?.end_time && run.start_time
+                ? run.end_time - run.start_time
+                : undefined,
+            feedback: evaluatorResults,
+            run_id: run.id,
+        };
+    }
+    return results;
+};
+const getExamplesInputs = (examples, chainOrFactory, dataType) => {
+    if (dataType === "chat") {
+        // For some batty reason, we store the chat dataset differently.
+        // { type: "system", data: { content: inputs.input } },
+        // But we need to create AIMesage, SystemMessage, etc.
+        return examples.map(({ inputs }) => mapStoredMessagesToChatMessages(inputs.input));
+    }
+    // If it's a language model and ALL example inputs have a single value,
+    // then we can be friendly and flatten the inputs to a list of strings.
+    const isLanguageModel = typeof chainOrFactory === "object" &&
+        typeof chainOrFactory._llmType === "function";
+    if (isLanguageModel &&
+        examples.every(({ inputs }) => Object.keys(inputs).length === 1)) {
+        return examples.map(({ inputs }) => Object.values(inputs)[0]);
+    }
+    return examples.map(({ inputs }) => inputs);
+};
+/**
+ * Evaluates a given model or chain against a specified LangSmith dataset.
+ *
+ * This function fetches example records from the specified dataset,
+ * runs the model or chain against each example, and returns the evaluation
+ * results.
+ *
+ * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a
+ * Runnable instance, a factory function that returns a Runnable, or a user-defined
+ * function or factory.
+ *
+ * @param datasetName - The name of the dataset against which the evaluation will be
+ * performed. This dataset should already be defined and contain the relevant data
+ * for evaluation.
+ *
+ * @param options - (Optional) Additional parameters for the evaluation process:
+ *   - `evaluation` (RunEvalConfig): Configuration for the evaluation, including
+ *     standard and custom evaluators.
+ *   - `projectName` (string): Name of the project for logging and tracking.
+ *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.
+ *   - `client` (Client): Client instance for LangChain service interaction.
+ *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.
+ *
+ * @returns A promise that resolves to an `EvalResults` object. This object includes
+ * detailed results of the evaluation, such as execution time, run IDs, and feedback
+ * for each entry in the dataset.
+ *
+ * @example
+ * ```typescript
+ * // Example usage for evaluating a model on a dataset
+ * async function evaluateModel() {
+ *   const chain = /* ...create your model or chain...*\//
+ *   const datasetName = 'example-dataset';
+ *   const client = new Client(/* ...config... *\//);
+ *
+ *   const evaluationConfig = new RunEvalConfig({
+ *     evaluators: [/* ...evaluators... *\//],
+ *     customEvaluators: [/* ...custom evaluators... *\//],
+ *   });
+ *
+ *   const results = await runOnDataset(chain, datasetName, {
+ *     evaluationConfig,
+ *     client,
+ *   });
+ *
+ *   console.log('Evaluation Results:', results);
+ * }
+ *
+ * evaluateModel();
+ * ```
+ * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against
+ * a dataset named 'example-dataset'. The evaluation process is configured using `RunEvalConfig`, which can
+ * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.
+ * The function returns the evaluation results, which can be logged or further processed as needed.
+ */
+export const runOnDataset = async (chainOrFactory, datasetName, { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, }) => {
+    const wrappedModel = await createWrappedModel(chainOrFactory);
+    const testClient = client ?? new Client();
+    const testProjectName = projectName ?? randomName();
+    const dataset = await testClient.readDataset({ datasetName });
+    const datasetId = dataset.id;
+    const testConcurrency = maxConcurrency ?? 5;
+    const { configs, examples, runCollectors } = await loadExamples({
+        datasetName,
+        client: testClient,
+        projectName: testProjectName,
+        maxConcurrency: testConcurrency,
+    });
+    await testClient.createProject({
+        projectName: testProjectName,
+        referenceDatasetId: datasetId,
+        projectExtra: { metadata: { ...projectMetadata } },
+    });
+    const wrappedRunnable = new RunnableLambda({
+        func: wrappedModel,
+    }).withConfig({ runName: "evaluationRun" });
+    const runInputs = getExamplesInputs(examples, chainOrFactory, dataset.data_type);
+    const progress = new ProgressBar({
+        total: runInputs.length,
+        format: "Predicting: {bar} {percentage}% | {value}/{total}",
+    });
+    // TODO: Collect the runs as well.
+    await wrappedRunnable
+        .withListeners({
+        onEnd: () => progress.increment(),
+    })
+        // TODO: Insert evaluation inline for immediate feedback.
+        .batch(runInputs, configs, {
+        maxConcurrency,
+        returnExceptions: true,
+    });
+    progress.complete();
+    const runs = [];
+    for (let i = 0; i < examples.length; i += 1) {
+        runs.push(runCollectors[i].tracedRuns[0]);
+    }
+    let evalResults = {};
+    if (evaluationConfig) {
+        const loadedEvalConfig = await LoadedEvalConfig.fromRunEvalConfig(evaluationConfig);
+        evalResults = await applyEvaluators({
+            evaluation: loadedEvalConfig,
+            runs,
+            examples,
+            client: testClient,
+        });
+    }
+    const results = {
+        projectName: testProjectName,
+        results: evalResults ?? {},
+    };
+    return results;
+};

package/dist/text_splitter.cjs

@@ -691,13 +691,14 @@ class TokenTextSplitter extends TextSplitter {
         const splits = [];
         const input_ids = this.tokenizer.encode(text, this.allowedSpecial, this.disallowedSpecial);
         let start_idx = 0;
-        let cur_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
-        let chunk_ids = input_ids.slice(start_idx, cur_idx);
         while (start_idx < input_ids.length) {
+            if (start_idx > 0) {
+                start_idx -= this.chunkOverlap;
+            }
+            const end_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
+            const chunk_ids = input_ids.slice(start_idx, end_idx);
             splits.push(this.tokenizer.decode(chunk_ids));
-            start_idx += this.chunkSize - this.chunkOverlap;
-            cur_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
-            chunk_ids = input_ids.slice(start_idx, cur_idx);
+            start_idx = end_idx;
         }
         return splits;
     }

package/dist/text_splitter.js

@@ -685,13 +685,14 @@ export class TokenTextSplitter extends TextSplitter {
         const splits = [];
         const input_ids = this.tokenizer.encode(text, this.allowedSpecial, this.disallowedSpecial);
         let start_idx = 0;
-        let cur_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
-        let chunk_ids = input_ids.slice(start_idx, cur_idx);
         while (start_idx < input_ids.length) {
+            if (start_idx > 0) {
+                start_idx -= this.chunkOverlap;
+            }
+            const end_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
+            const chunk_ids = input_ids.slice(start_idx, end_idx);
             splits.push(this.tokenizer.decode(chunk_ids));
-            start_idx += this.chunkSize - this.chunkOverlap;
-            cur_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
-            chunk_ids = input_ids.slice(start_idx, cur_idx);
+            start_idx = end_idx;
         }
         return splits;
     }

package/dist/util/sql_utils.cjs

@@ -96,14 +96,15 @@ const getTableAndColumnsName = async (appDataSource) => {
         return formatToSqlTable(rep);
     }
     if (appDataSource.options.type === "mssql") {
-        sql =
-            "SELECT " +
-                "TABLE_NAME AS table_name, " +
-                "COLUMN_NAME AS column_name, " +
-                "DATA_TYPE AS data_type, " +
-                "IS_NULLABLE AS is_nullable " +
-                "FROM INFORMATION_SCHEMA.COLUMNS " +
-                "ORDER BY TABLE_NAME, ORDINAL_POSITION;";
+        const schema = appDataSource.options?.schema;
+        const sql = `SELECT
+    TABLE_NAME AS table_name,
+    COLUMN_NAME AS column_name,
+    DATA_TYPE AS data_type,
+    IS_NULLABLE AS is_nullable
+    FROM INFORMATION_SCHEMA.COLUMNS
+    ${schema && `WHERE TABLE_SCHEMA = '${schema}'`} 
+ORDER BY TABLE_NAME, ORDINAL_POSITION;`;
         const rep = await appDataSource.query(sql);
         return formatToSqlTable(rep);
     }
@@ -169,6 +170,9 @@ const generateTableInfoFromTables = async (tables, appDataSource, nbSampleRow, c
         if (appDataSource.options.type === "postgres") {
             schema = appDataSource.options?.schema ?? "public";
         }
+        else if (appDataSource.options.type === "mssql") {
+            schema = appDataSource.options?.schema;
+        }
         else if (appDataSource.options.type === "sap") {
             schema =
                 appDataSource.options?.schema ??
@@ -198,7 +202,10 @@ const generateTableInfoFromTables = async (tables, appDataSource, nbSampleRow, c
             sqlSelectInfoQuery = `SELECT * FROM "${schema}"."${currentTable.tableName}" LIMIT ${nbSampleRow};\n`;
         }
         else if (appDataSource.options.type === "mssql") {
-            sqlSelectInfoQuery = `SELECT TOP ${nbSampleRow} * FROM [${currentTable.tableName}];\n`;
+            const schema = appDataSource.options?.schema;
+            sqlSelectInfoQuery = schema
+                ? `SELECT TOP ${nbSampleRow} * FROM ${schema}.[${currentTable.tableName}];\n`
+                : `SELECT TOP ${nbSampleRow} * FROM [${currentTable.tableName}];\n`;
         }
         else if (appDataSource.options.type === "sap") {
             const schema = appDataSource.options?.schema ??

package/dist/util/sql_utils.js

@@ -90,14 +90,15 @@ export const getTableAndColumnsName = async (appDataSource) => {
         return formatToSqlTable(rep);
     }
     if (appDataSource.options.type === "mssql") {
-        sql =
-            "SELECT " +
-                "TABLE_NAME AS table_name, " +
-                "COLUMN_NAME AS column_name, " +
-                "DATA_TYPE AS data_type, " +
-                "IS_NULLABLE AS is_nullable " +
-                "FROM INFORMATION_SCHEMA.COLUMNS " +
-                "ORDER BY TABLE_NAME, ORDINAL_POSITION;";
+        const schema = appDataSource.options?.schema;
+        const sql = `SELECT
+    TABLE_NAME AS table_name,
+    COLUMN_NAME AS column_name,
+    DATA_TYPE AS data_type,
+    IS_NULLABLE AS is_nullable
+    FROM INFORMATION_SCHEMA.COLUMNS
+    ${schema && `WHERE TABLE_SCHEMA = '${schema}'`} 
+ORDER BY TABLE_NAME, ORDINAL_POSITION;`;
         const rep = await appDataSource.query(sql);
         return formatToSqlTable(rep);
     }
@@ -162,6 +163,9 @@ export const generateTableInfoFromTables = async (tables, appDataSource, nbSampl
         if (appDataSource.options.type === "postgres") {
             schema = appDataSource.options?.schema ?? "public";
         }
+        else if (appDataSource.options.type === "mssql") {
+            schema = appDataSource.options?.schema;
+        }
         else if (appDataSource.options.type === "sap") {
             schema =
                 appDataSource.options?.schema ??
@@ -191,7 +195,10 @@ export const generateTableInfoFromTables = async (tables, appDataSource, nbSampl
             sqlSelectInfoQuery = `SELECT * FROM "${schema}"."${currentTable.tableName}" LIMIT ${nbSampleRow};\n`;
         }
         else if (appDataSource.options.type === "mssql") {
-            sqlSelectInfoQuery = `SELECT TOP ${nbSampleRow} * FROM [${currentTable.tableName}];\n`;
+            const schema = appDataSource.options?.schema;
+            sqlSelectInfoQuery = schema
+                ? `SELECT TOP ${nbSampleRow} * FROM ${schema}.[${currentTable.tableName}];\n`
+                : `SELECT TOP ${nbSampleRow} * FROM [${currentTable.tableName}];\n`;
         }
         else if (appDataSource.options.type === "sap") {
             const schema = appDataSource.options?.schema ??

package/dist/vectorstores/pinecone.cjs

@@ -17,5 +17,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const entrypoint_deprecation_js_1 = require("../util/entrypoint_deprecation.cjs");
 /* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion010MigrationWarning)({
     oldEntrypointName: "vectorstores/pinecone",
+    newEntrypointName: "",
+    newPackageName: "@langchain/pinecone",
 });
 __exportStar(require("@langchain/community/vectorstores/pinecone"), exports);

package/dist/vectorstores/pinecone.js

@@ -1,5 +1,7 @@
 import { logVersion010MigrationWarning } from "../util/entrypoint_deprecation.js";
 /* #__PURE__ */ logVersion010MigrationWarning({
     oldEntrypointName: "vectorstores/pinecone",
+    newEntrypointName: "",
+    newPackageName: "@langchain/pinecone",
 });
 export * from "@langchain/community/vectorstores/pinecone";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {
@@ -865,6 +865,9 @@
     "evaluation.cjs",
     "evaluation.js",
     "evaluation.d.ts",
+    "smith.cjs",
+    "smith.js",
+    "smith.d.ts",
     "runnables.cjs",
     "runnables.js",
     "runnables.d.ts",
@@ -898,8 +901,8 @@
     "test:watch": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts",
     "test:integration": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%",
     "test:single": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000",
-    "format": "prettier --write \"src\"",
-    "format:check": "prettier --check \"src\""
+    "format": "prettier --config .prettierrc --write \"src\" \"scripts\"",
+    "format:check": "prettier --config .prettierrc --check \"src\" \"scripts\""
   },
   "author": "LangChain",
   "license": "MIT",
@@ -1001,7 +1004,7 @@
     "@google-ai/generativelanguage": "^0.2.1",
     "@google-cloud/storage": "^6.10.1",
     "@notionhq/client": "^2.2.10",
-    "@pinecone-database/pinecone": "^1.1.0",
+    "@pinecone-database/pinecone": "*",
     "@supabase/supabase-js": "^2.10.0",
     "@vercel/kv": "^0.2.3",
     "@xata.io/client": "^0.28.0",
@@ -1211,7 +1214,7 @@
     "js-yaml": "^4.1.0",
     "jsonpointer": "^5.0.1",
     "langchainhub": "~0.0.6",
-    "langsmith": "~0.0.48",
+    "langsmith": "~0.0.59",
     "ml-distance": "^4.0.0",
     "openapi-types": "^12.1.3",
     "p-retry": "4",
@@ -2663,6 +2666,11 @@
       "import": "./evaluation.js",
       "require": "./evaluation.cjs"
     },
+    "./smith": {
+      "types": "./smith.d.ts",
+      "import": "./smith.js",
+      "require": "./smith.cjs"
+    },
     "./runnables": {
       "types": "./runnables.d.ts",
       "import": "./runnables.js",

package/smith.cjs

@@ -0,0 +1 @@
+module.exports = require('./dist/smith/index.cjs');

package/smith.d.ts

@@ -0,0 +1 @@
+export * from './dist/smith/index.js'

package/smith.js

@@ -0,0 +1 @@
+export * from './dist/smith/index.js'