langchain 0.1.15 → 0.1.16-rc.0

package/dist/evaluation/base.cjs

@@ -143,9 +143,9 @@ class LLMStringEvaluator extends LLMEvalChain {
      * @param callOptions
      * @param config
      */
-    evaluateStrings(args, callOptions, config) {
+    evaluateStrings(args, config) {
         this.checkEvaluationArgs(args.reference, args.input);
-        return this._evaluateStrings(args, callOptions, config);
+        return this._evaluateStrings(args, config);
     }
 }
 exports.LLMStringEvaluator = LLMStringEvaluator;

package/dist/evaluation/base.d.ts

@@ -114,7 +114,7 @@ export declare abstract class LLMStringEvaluator<T extends EvalOutputType = Eval
      * @param callOptions
      * @param config
      */
-    abstract _evaluateStrings(args: StringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+    abstract _evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
     /**
      * Evaluate Chain or LLM output, based on optional input and label.
      * @returns The evaluation results containing the score or value. It is recommended that the dictionary contain the following keys:
@@ -125,7 +125,7 @@ export declare abstract class LLMStringEvaluator<T extends EvalOutputType = Eval
      * @param callOptions
      * @param config
      */
-    evaluateStrings(args: StringEvaluatorArgs, callOptions?: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+    evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
 }
 /**
  * Grade, tag, or otherwise evaluate predictions relative to their inputs

package/dist/evaluation/base.js

@@ -137,9 +137,9 @@ export class LLMStringEvaluator extends LLMEvalChain {
      * @param callOptions
      * @param config
      */
-    evaluateStrings(args, callOptions, config) {
+    evaluateStrings(args, config) {
         this.checkEvaluationArgs(args.reference, args.input);
-        return this._evaluateStrings(args, callOptions, config);
+        return this._evaluateStrings(args, config);
     }
 }
 /**

package/dist/evaluation/criteria/criteria.cjs

@@ -217,8 +217,8 @@ class CriteriaEvalChain extends base_js_1.LLMStringEvaluator {
         }
         return parsed;
     }
-    async _evaluateStrings(args, callOptions, config) {
-        const result = await this.call({ ...this.getEvalInput(args), ...callOptions }, config);
+    async _evaluateStrings(args, config) {
+        const result = await this.call({ ...this.getEvalInput(args) }, config);
         return this._prepareOutput(result);
     }
 }

package/dist/evaluation/criteria/criteria.d.ts

@@ -61,7 +61,7 @@ export declare class CriteriaEvalChain extends LLMStringEvaluator {
      * @param result
      */
     _prepareOutput(result: ChainValues): any;
-    _evaluateStrings(args: StringEvaluatorArgs, callOptions: ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
+    _evaluateStrings(args: StringEvaluatorArgs & ExtractLLMCallOptions<this["llm"]>, config?: Callbacks | BaseCallbackConfig): Promise<ChainValues>;
 }
 /**
  * Criteria evaluation chain that requires references.

package/dist/evaluation/criteria/criteria.js

@@ -213,8 +213,8 @@ export class CriteriaEvalChain extends LLMStringEvaluator {
         }
         return parsed;
     }
-    async _evaluateStrings(args, callOptions, config) {
-        const result = await this.call({ ...this.getEvalInput(args), ...callOptions }, config);
+    async _evaluateStrings(args, config) {
+        const result = await this.call({ ...this.getEvalInput(args) }, config);
         return this._prepareOutput(result);
     }
 }

package/dist/smith/runner_utils.cjs

@@ -3,13 +3,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.runOnDataset = void 0;
 const messages_1 = require("@langchain/core/messages");
 const runnables_1 = require("@langchain/core/runnables");
-const run_collector_1 = require("@langchain/core/tracers/run_collector");
 const tracer_langchain_1 = require("@langchain/core/tracers/tracer_langchain");
+const base_1 = require("@langchain/core/tracers/base");
 const langsmith_1 = require("langsmith");
 const loader_js_1 = require("../evaluation/loader.cjs");
 const name_generation_js_1 = require("./name_generation.cjs");
 const progress_js_1 = require("./progress.cjs");
-class RunIdExtractor {
+class SingleRunIdExtractor {
     constructor() {
         Object.defineProperty(this, "runIdPromiseResolver", {
             enumerable: true,
@@ -39,6 +39,39 @@ class RunIdExtractor {
         return this.runIdPromise;
     }
 }
+class SingleRunExtractor extends base_1.BaseTracer {
+    constructor() {
+        super();
+        Object.defineProperty(this, "runPromiseResolver", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "runPromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** The name of the callback handler. */
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "single_run_extractor"
+        });
+        this.runPromise = new Promise((extract) => {
+            this.runPromiseResolver = extract;
+        });
+    }
+    async persistRun(run) {
+        this.runPromiseResolver(run);
+    }
+    async extract() {
+        return this.runPromise;
+    }
+}
 /**
  * Wraps an evaluator function + implements the RunEvaluator interface.
  */
@@ -59,7 +92,7 @@ class DynamicRunEvaluator {
      * @returns A promise that extracts to the evaluation result.
      */
     async evaluateRun(run, example) {
-        const extractor = new RunIdExtractor();
+        const extractor = new SingleRunIdExtractor();
         const tracer = new tracer_langchain_1.LangChainTracer({ projectName: "evaluators" });
         const result = await this.evaluator.invoke({
             run,
@@ -146,7 +179,7 @@ class PreparedRunEvaluator {
             rawReferenceOutput: example?.outputs,
             run,
         });
-        const extractor = new RunIdExtractor();
+        const extractor = new SingleRunIdExtractor();
         const tracer = new tracer_langchain_1.LangChainTracer({ projectName: "evaluators" });
         if (this.isStringEvaluator) {
             const evalResult = await this.evaluator.evaluateStrings({
@@ -226,25 +259,23 @@ const createWrappedModel = async (modelOrFactory) => {
 const loadExamples = async ({ datasetName, client, projectName, }) => {
     const exampleIterator = client.listExamples({ datasetName });
     const configs = [];
-    const runCollectors = [];
+    const runExtractors = [];
     const examples = [];
     for await (const example of exampleIterator) {
-        const runCollector = new run_collector_1.RunCollectorCallbackHandler({
-            exampleId: example.id,
-        });
+        const runExtractor = new SingleRunExtractor();
         configs.push({
             callbacks: [
                 new tracer_langchain_1.LangChainTracer({ exampleId: example.id, projectName }),
-                runCollector,
+                runExtractor,
             ],
         });
         examples.push(example);
-        runCollectors.push(runCollector);
+        runExtractors.push(runExtractor);
     }
     return {
         configs,
         examples,
-        runCollectors,
+        runExtractors,
     };
 };
 const applyEvaluators = async ({ evaluation, runs, examples, client, }) => {
@@ -352,7 +383,7 @@ const runOnDataset = async (chainOrFactory, datasetName, { evaluationConfig, pro
     const dataset = await testClient.readDataset({ datasetName });
     const datasetId = dataset.id;
     const testConcurrency = maxConcurrency ?? 5;
-    const { configs, examples, runCollectors } = await loadExamples({
+    const { configs, examples, runExtractors } = await loadExamples({
         datasetName,
         client: testClient,
         projectName: testProjectName,
@@ -384,7 +415,7 @@ const runOnDataset = async (chainOrFactory, datasetName, { evaluationConfig, pro
     progress.complete();
     const runs = [];
     for (let i = 0; i < examples.length; i += 1) {
-        runs.push(runCollectors[i].tracedRuns[0]);
+        runs.push(await runExtractors[i].extract());
     }
     let evalResults = {};
     if (evaluationConfig) {

package/dist/smith/runner_utils.js

@@ -1,12 +1,12 @@
 import { mapStoredMessagesToChatMessages } from "@langchain/core/messages";
 import { Runnable, RunnableLambda, } from "@langchain/core/runnables";
-import { RunCollectorCallbackHandler } from "@langchain/core/tracers/run_collector";
 import { LangChainTracer } from "@langchain/core/tracers/tracer_langchain";
+import { BaseTracer } from "@langchain/core/tracers/base";
 import { Client } from "langsmith";
 import { loadEvaluator } from "../evaluation/loader.js";
 import { randomName } from "./name_generation.js";
 import { ProgressBar } from "./progress.js";
-class RunIdExtractor {
+class SingleRunIdExtractor {
     constructor() {
         Object.defineProperty(this, "runIdPromiseResolver", {
             enumerable: true,
@@ -36,6 +36,39 @@ class RunIdExtractor {
         return this.runIdPromise;
     }
 }
+class SingleRunExtractor extends BaseTracer {
+    constructor() {
+        super();
+        Object.defineProperty(this, "runPromiseResolver", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "runPromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** The name of the callback handler. */
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "single_run_extractor"
+        });
+        this.runPromise = new Promise((extract) => {
+            this.runPromiseResolver = extract;
+        });
+    }
+    async persistRun(run) {
+        this.runPromiseResolver(run);
+    }
+    async extract() {
+        return this.runPromise;
+    }
+}
 /**
  * Wraps an evaluator function + implements the RunEvaluator interface.
  */
@@ -56,7 +89,7 @@ class DynamicRunEvaluator {
      * @returns A promise that extracts to the evaluation result.
      */
     async evaluateRun(run, example) {
-        const extractor = new RunIdExtractor();
+        const extractor = new SingleRunIdExtractor();
         const tracer = new LangChainTracer({ projectName: "evaluators" });
         const result = await this.evaluator.invoke({
             run,
@@ -143,7 +176,7 @@ class PreparedRunEvaluator {
             rawReferenceOutput: example?.outputs,
             run,
         });
-        const extractor = new RunIdExtractor();
+        const extractor = new SingleRunIdExtractor();
         const tracer = new LangChainTracer({ projectName: "evaluators" });
         if (this.isStringEvaluator) {
             const evalResult = await this.evaluator.evaluateStrings({
@@ -223,25 +256,23 @@ const createWrappedModel = async (modelOrFactory) => {
 const loadExamples = async ({ datasetName, client, projectName, }) => {
     const exampleIterator = client.listExamples({ datasetName });
     const configs = [];
-    const runCollectors = [];
+    const runExtractors = [];
     const examples = [];
     for await (const example of exampleIterator) {
-        const runCollector = new RunCollectorCallbackHandler({
-            exampleId: example.id,
-        });
+        const runExtractor = new SingleRunExtractor();
         configs.push({
             callbacks: [
                 new LangChainTracer({ exampleId: example.id, projectName }),
-                runCollector,
+                runExtractor,
             ],
         });
         examples.push(example);
-        runCollectors.push(runCollector);
+        runExtractors.push(runExtractor);
     }
     return {
         configs,
         examples,
-        runCollectors,
+        runExtractors,
     };
 };
 const applyEvaluators = async ({ evaluation, runs, examples, client, }) => {
@@ -349,7 +380,7 @@ export const runOnDataset = async (chainOrFactory, datasetName, { evaluationConf
     const dataset = await testClient.readDataset({ datasetName });
     const datasetId = dataset.id;
     const testConcurrency = maxConcurrency ?? 5;
-    const { configs, examples, runCollectors } = await loadExamples({
+    const { configs, examples, runExtractors } = await loadExamples({
         datasetName,
         client: testClient,
         projectName: testProjectName,
@@ -381,7 +412,7 @@ export const runOnDataset = async (chainOrFactory, datasetName, { evaluationConf
     progress.complete();
     const runs = [];
     for (let i = 0; i < examples.length; i += 1) {
-        runs.push(runCollectors[i].tracedRuns[0]);
+        runs.push(await runExtractors[i].extract());
     }
     let evalResults = {};
     if (evaluationConfig) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.1.15",
+  "version": "0.1.16-rc.0",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {
@@ -1268,6 +1268,7 @@
     "node-llama-cpp": "2.7.3",
     "notion-to-md": "^3.1.0",
     "officeparser": "^4.0.4",
+    "openai": "^4.26.1",
     "pdf-parse": "1.1.1",
     "peggy": "^3.0.2",
     "playwright": "^1.32.1",
@@ -1286,6 +1287,7 @@
     "vectordb": "^0.1.4",
     "weaviate-ts-client": "^1.4.0",
     "web-auth-library": "^1.0.3",
+    "wikipedia": "^2.1.2",
     "youtube-transcript": "^1.0.6",
     "youtubei.js": "^5.8.0"
   },