langsmith 0.1.25 → 0.1.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.cjs +17 -2
- package/dist/client.d.ts +9 -2
- package/dist/client.js +17 -2
- package/dist/env.cjs +17 -0
- package/dist/env.d.ts +1 -0
- package/dist/env.js +13 -0
- package/dist/evaluation/_runner.cjs +29 -3
- package/dist/evaluation/_runner.d.ts +2 -1
- package/dist/evaluation/_runner.js +29 -3
- package/dist/index.cjs +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/langchain.cjs +105 -0
- package/dist/langchain.d.ts +29 -0
- package/dist/langchain.js +100 -0
- package/dist/run_trees.cjs +25 -16
- package/dist/run_trees.d.ts +1 -1
- package/dist/run_trees.js +25 -16
- package/dist/schemas.d.ts +20 -0
- package/dist/singletons/traceable.cjs +62 -0
- package/dist/singletons/traceable.d.ts +23 -0
- package/dist/singletons/traceable.js +57 -0
- package/dist/singletons/types.cjs +2 -0
- package/dist/singletons/types.d.ts +38 -0
- package/dist/singletons/types.js +1 -0
- package/dist/traceable.cjs +38 -72
- package/dist/traceable.d.ts +14 -47
- package/dist/traceable.js +32 -66
- package/dist/utils/error.cjs +25 -0
- package/dist/utils/error.d.ts +1 -0
- package/dist/utils/error.js +21 -0
- package/dist/wrappers/openai.cjs +32 -0
- package/dist/wrappers/openai.js +32 -0
- package/langchain.cjs +1 -0
- package/langchain.d.cts +1 -0
- package/langchain.d.ts +1 -0
- package/langchain.js +1 -0
- package/package.json +33 -3
- package/singletons/traceable.cjs +1 -0
- package/singletons/traceable.d.cts +1 -0
- package/singletons/traceable.d.ts +1 -0
- package/singletons/traceable.js +1 -0
package/dist/client.cjs
CHANGED
|
@@ -1185,6 +1185,14 @@ class Client {
|
|
|
1185
1185
|
const tenantId = await this._getTenantId();
|
|
1186
1186
|
return `${this.getHostUrl()}/o/${tenantId}/projects/p/${project.id}`;
|
|
1187
1187
|
}
|
|
1188
|
+
async getDatasetUrl({ datasetId, datasetName, }) {
|
|
1189
|
+
if (datasetId === undefined && datasetName === undefined) {
|
|
1190
|
+
throw new Error("Must provide either datasetName or datasetId");
|
|
1191
|
+
}
|
|
1192
|
+
const dataset = await this.readDataset({ datasetId, datasetName });
|
|
1193
|
+
const tenantId = await this._getTenantId();
|
|
1194
|
+
return `${this.getHostUrl()}/o/${tenantId}/datasets/${dataset.id}`;
|
|
1195
|
+
}
|
|
1188
1196
|
async _getTenantId() {
|
|
1189
1197
|
if (this._tenantId !== null) {
|
|
1190
1198
|
return this._tenantId;
|
|
@@ -1443,7 +1451,7 @@ class Client {
|
|
|
1443
1451
|
}
|
|
1444
1452
|
await response.json();
|
|
1445
1453
|
}
|
|
1446
|
-
async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId, metadata, }) {
|
|
1454
|
+
async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId, metadata, split, }) {
|
|
1447
1455
|
let datasetId_ = datasetId;
|
|
1448
1456
|
if (datasetId_ === undefined && datasetName === undefined) {
|
|
1449
1457
|
throw new Error("Must provide either datasetName or datasetId");
|
|
@@ -1463,6 +1471,7 @@ class Client {
|
|
|
1463
1471
|
created_at: createdAt_?.toISOString(),
|
|
1464
1472
|
id: exampleId,
|
|
1465
1473
|
metadata,
|
|
1474
|
+
split,
|
|
1466
1475
|
};
|
|
1467
1476
|
const response = await this.caller.call(fetch, `${this.apiUrl}/examples`, {
|
|
1468
1477
|
method: "POST",
|
|
@@ -1496,6 +1505,7 @@ class Client {
|
|
|
1496
1505
|
inputs: input,
|
|
1497
1506
|
outputs: outputs ? outputs[idx] : undefined,
|
|
1498
1507
|
metadata: metadata ? metadata[idx] : undefined,
|
|
1508
|
+
split: props.splits ? props.splits[idx] : undefined,
|
|
1499
1509
|
id: exampleIds ? exampleIds[idx] : undefined,
|
|
1500
1510
|
source_run_id: sourceRunIds ? sourceRunIds[idx] : undefined,
|
|
1501
1511
|
};
|
|
@@ -1533,7 +1543,7 @@ class Client {
|
|
|
1533
1543
|
const path = `/examples/${exampleId}`;
|
|
1534
1544
|
return await this._get(path);
|
|
1535
1545
|
}
|
|
1536
|
-
async *listExamples({ datasetId, datasetName, exampleIds, asOf, inlineS3Urls, metadata, } = {}) {
|
|
1546
|
+
async *listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, } = {}) {
|
|
1537
1547
|
let datasetId_;
|
|
1538
1548
|
if (datasetId !== undefined && datasetName !== undefined) {
|
|
1539
1549
|
throw new Error("Must provide either datasetName or datasetId, not both");
|
|
@@ -1564,6 +1574,11 @@ class Client {
|
|
|
1564
1574
|
params.append("id", id_);
|
|
1565
1575
|
}
|
|
1566
1576
|
}
|
|
1577
|
+
if (splits !== undefined) {
|
|
1578
|
+
for (const split of splits) {
|
|
1579
|
+
params.append("splits", split);
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1567
1582
|
if (metadata !== undefined) {
|
|
1568
1583
|
const serializedMetadata = JSON.stringify(metadata);
|
|
1569
1584
|
params.append("metadata", serializedMetadata);
|
package/dist/client.d.ts
CHANGED
|
@@ -145,6 +145,7 @@ export type CreateExampleOptions = {
|
|
|
145
145
|
createdAt?: Date;
|
|
146
146
|
exampleId?: string;
|
|
147
147
|
metadata?: KVMap;
|
|
148
|
+
split?: string | string[];
|
|
148
149
|
};
|
|
149
150
|
export declare class Queue<T> {
|
|
150
151
|
items: [T, () => void][];
|
|
@@ -338,6 +339,10 @@ export declare class Client {
|
|
|
338
339
|
projectId?: string;
|
|
339
340
|
projectName?: string;
|
|
340
341
|
}): Promise<string>;
|
|
342
|
+
getDatasetUrl({ datasetId, datasetName, }: {
|
|
343
|
+
datasetId?: string;
|
|
344
|
+
datasetName?: string;
|
|
345
|
+
}): Promise<string>;
|
|
341
346
|
private _getTenantId;
|
|
342
347
|
listProjects({ projectIds, name, nameContains, referenceDatasetId, referenceDatasetName, referenceFree, }?: {
|
|
343
348
|
projectIds?: string[];
|
|
@@ -385,11 +390,12 @@ export declare class Client {
|
|
|
385
390
|
datasetId?: string;
|
|
386
391
|
datasetName?: string;
|
|
387
392
|
}): Promise<void>;
|
|
388
|
-
createExample(inputs: KVMap, outputs: KVMap, { datasetId, datasetName, createdAt, exampleId, metadata, }: CreateExampleOptions): Promise<Example>;
|
|
393
|
+
createExample(inputs: KVMap, outputs: KVMap, { datasetId, datasetName, createdAt, exampleId, metadata, split, }: CreateExampleOptions): Promise<Example>;
|
|
389
394
|
createExamples(props: {
|
|
390
395
|
inputs: Array<KVMap>;
|
|
391
396
|
outputs?: Array<KVMap>;
|
|
392
397
|
metadata?: Array<KVMap>;
|
|
398
|
+
splits?: Array<string | Array<string>>;
|
|
393
399
|
sourceRunIds?: Array<string>;
|
|
394
400
|
exampleIds?: Array<string>;
|
|
395
401
|
datasetId?: string;
|
|
@@ -398,11 +404,12 @@ export declare class Client {
|
|
|
398
404
|
createLLMExample(input: string, generation: string | undefined, options: CreateExampleOptions): Promise<Example>;
|
|
399
405
|
createChatExample(input: KVMap[] | LangChainBaseMessage[], generations: KVMap | LangChainBaseMessage | undefined, options: CreateExampleOptions): Promise<Example>;
|
|
400
406
|
readExample(exampleId: string): Promise<Example>;
|
|
401
|
-
listExamples({ datasetId, datasetName, exampleIds, asOf, inlineS3Urls, metadata, }?: {
|
|
407
|
+
listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, }?: {
|
|
402
408
|
datasetId?: string;
|
|
403
409
|
datasetName?: string;
|
|
404
410
|
exampleIds?: string[];
|
|
405
411
|
asOf?: string | Date;
|
|
412
|
+
splits?: string[];
|
|
406
413
|
inlineS3Urls?: boolean;
|
|
407
414
|
metadata?: KVMap;
|
|
408
415
|
}): AsyncIterable<Example>;
|
package/dist/client.js
CHANGED
|
@@ -1158,6 +1158,14 @@ export class Client {
|
|
|
1158
1158
|
const tenantId = await this._getTenantId();
|
|
1159
1159
|
return `${this.getHostUrl()}/o/${tenantId}/projects/p/${project.id}`;
|
|
1160
1160
|
}
|
|
1161
|
+
async getDatasetUrl({ datasetId, datasetName, }) {
|
|
1162
|
+
if (datasetId === undefined && datasetName === undefined) {
|
|
1163
|
+
throw new Error("Must provide either datasetName or datasetId");
|
|
1164
|
+
}
|
|
1165
|
+
const dataset = await this.readDataset({ datasetId, datasetName });
|
|
1166
|
+
const tenantId = await this._getTenantId();
|
|
1167
|
+
return `${this.getHostUrl()}/o/${tenantId}/datasets/${dataset.id}`;
|
|
1168
|
+
}
|
|
1161
1169
|
async _getTenantId() {
|
|
1162
1170
|
if (this._tenantId !== null) {
|
|
1163
1171
|
return this._tenantId;
|
|
@@ -1416,7 +1424,7 @@ export class Client {
|
|
|
1416
1424
|
}
|
|
1417
1425
|
await response.json();
|
|
1418
1426
|
}
|
|
1419
|
-
async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId, metadata, }) {
|
|
1427
|
+
async createExample(inputs, outputs, { datasetId, datasetName, createdAt, exampleId, metadata, split, }) {
|
|
1420
1428
|
let datasetId_ = datasetId;
|
|
1421
1429
|
if (datasetId_ === undefined && datasetName === undefined) {
|
|
1422
1430
|
throw new Error("Must provide either datasetName or datasetId");
|
|
@@ -1436,6 +1444,7 @@ export class Client {
|
|
|
1436
1444
|
created_at: createdAt_?.toISOString(),
|
|
1437
1445
|
id: exampleId,
|
|
1438
1446
|
metadata,
|
|
1447
|
+
split,
|
|
1439
1448
|
};
|
|
1440
1449
|
const response = await this.caller.call(fetch, `${this.apiUrl}/examples`, {
|
|
1441
1450
|
method: "POST",
|
|
@@ -1469,6 +1478,7 @@ export class Client {
|
|
|
1469
1478
|
inputs: input,
|
|
1470
1479
|
outputs: outputs ? outputs[idx] : undefined,
|
|
1471
1480
|
metadata: metadata ? metadata[idx] : undefined,
|
|
1481
|
+
split: props.splits ? props.splits[idx] : undefined,
|
|
1472
1482
|
id: exampleIds ? exampleIds[idx] : undefined,
|
|
1473
1483
|
source_run_id: sourceRunIds ? sourceRunIds[idx] : undefined,
|
|
1474
1484
|
};
|
|
@@ -1506,7 +1516,7 @@ export class Client {
|
|
|
1506
1516
|
const path = `/examples/${exampleId}`;
|
|
1507
1517
|
return await this._get(path);
|
|
1508
1518
|
}
|
|
1509
|
-
async *listExamples({ datasetId, datasetName, exampleIds, asOf, inlineS3Urls, metadata, } = {}) {
|
|
1519
|
+
async *listExamples({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, } = {}) {
|
|
1510
1520
|
let datasetId_;
|
|
1511
1521
|
if (datasetId !== undefined && datasetName !== undefined) {
|
|
1512
1522
|
throw new Error("Must provide either datasetName or datasetId, not both");
|
|
@@ -1537,6 +1547,11 @@ export class Client {
|
|
|
1537
1547
|
params.append("id", id_);
|
|
1538
1548
|
}
|
|
1539
1549
|
}
|
|
1550
|
+
if (splits !== undefined) {
|
|
1551
|
+
for (const split of splits) {
|
|
1552
|
+
params.append("splits", split);
|
|
1553
|
+
}
|
|
1554
|
+
}
|
|
1540
1555
|
if (metadata !== undefined) {
|
|
1541
1556
|
const serializedMetadata = JSON.stringify(metadata);
|
|
1542
1557
|
params.append("metadata", serializedMetadata);
|
package/dist/env.cjs
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.isTracingEnabled = void 0;
|
|
4
|
+
const env_js_1 = require("./utils/env.cjs");
|
|
5
|
+
const isTracingEnabled = (tracingEnabled) => {
|
|
6
|
+
if (tracingEnabled !== undefined) {
|
|
7
|
+
return tracingEnabled;
|
|
8
|
+
}
|
|
9
|
+
const envVars = [
|
|
10
|
+
"LANGSMITH_TRACING_V2",
|
|
11
|
+
"LANGCHAIN_TRACING_V2",
|
|
12
|
+
"LANGSMITH_TRACING",
|
|
13
|
+
"LANGCHAIN_TRACING",
|
|
14
|
+
];
|
|
15
|
+
return !!envVars.find((envVar) => (0, env_js_1.getEnvironmentVariable)(envVar) === "true");
|
|
16
|
+
};
|
|
17
|
+
exports.isTracingEnabled = isTracingEnabled;
|
package/dist/env.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const isTracingEnabled: (tracingEnabled?: boolean) => boolean;
|
package/dist/env.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { getEnvironmentVariable } from "./utils/env.js";
|
|
2
|
+
export const isTracingEnabled = (tracingEnabled) => {
|
|
3
|
+
if (tracingEnabled !== undefined) {
|
|
4
|
+
return tracingEnabled;
|
|
5
|
+
}
|
|
6
|
+
const envVars = [
|
|
7
|
+
"LANGSMITH_TRACING_V2",
|
|
8
|
+
"LANGCHAIN_TRACING_V2",
|
|
9
|
+
"LANGSMITH_TRACING",
|
|
10
|
+
"LANGCHAIN_TRACING",
|
|
11
|
+
];
|
|
12
|
+
return !!envVars.find((envVar) => getEnvironmentVariable(envVar) === "true");
|
|
13
|
+
};
|
|
@@ -8,6 +8,7 @@ const _uuid_js_1 = require("../utils/_uuid.cjs");
|
|
|
8
8
|
const async_caller_js_1 = require("../utils/async_caller.cjs");
|
|
9
9
|
const atee_js_1 = require("../utils/atee.cjs");
|
|
10
10
|
const env_js_1 = require("../utils/env.cjs");
|
|
11
|
+
const error_js_1 = require("../utils/error.cjs");
|
|
11
12
|
const _random_name_js_1 = require("./_random_name.cjs");
|
|
12
13
|
const evaluator_js_1 = require("./evaluator.cjs");
|
|
13
14
|
const uuid_1 = require("uuid");
|
|
@@ -238,15 +239,21 @@ class _ExperimentManager {
|
|
|
238
239
|
}
|
|
239
240
|
return project;
|
|
240
241
|
}
|
|
241
|
-
_printExperimentStart() {
|
|
242
|
-
// @TODO log with experiment URL
|
|
242
|
+
async _printExperimentStart() {
|
|
243
243
|
console.log(`Starting evaluation of experiment: ${this.experimentName}`);
|
|
244
|
+
const firstExample = this._examples?.[0];
|
|
245
|
+
const datasetId = firstExample?.dataset_id;
|
|
246
|
+
if (!datasetId || !this._experiment)
|
|
247
|
+
return;
|
|
248
|
+
const datasetUrl = await this.client.getDatasetUrl({ datasetId });
|
|
249
|
+
const compareUrl = `${datasetUrl}/compare?selectedSessions=${this._experiment.id}`;
|
|
250
|
+
console.log(`View results at ${compareUrl}`);
|
|
244
251
|
}
|
|
245
252
|
async start() {
|
|
246
253
|
const examples = await this.getExamples();
|
|
247
254
|
const firstExample = examples[0];
|
|
248
255
|
const project = await this._getProject(firstExample);
|
|
249
|
-
this._printExperimentStart();
|
|
256
|
+
await this._printExperimentStart();
|
|
250
257
|
return new _ExperimentManager({
|
|
251
258
|
examples,
|
|
252
259
|
experiment: project,
|
|
@@ -390,6 +397,7 @@ class _ExperimentManager {
|
|
|
390
397
|
}
|
|
391
398
|
catch (e) {
|
|
392
399
|
console.error(`Error running evaluator ${evaluator.evaluateRun.name} on run ${run.id}: ${e}`);
|
|
400
|
+
(0, error_js_1.printErrorStackTrace)(e);
|
|
393
401
|
}
|
|
394
402
|
}
|
|
395
403
|
return {
|
|
@@ -458,6 +466,7 @@ class _ExperimentManager {
|
|
|
458
466
|
}
|
|
459
467
|
catch (e) {
|
|
460
468
|
console.error(`Error running summary evaluator ${evaluator.name}: ${JSON.stringify(e, null, 2)}`);
|
|
469
|
+
(0, error_js_1.printErrorStackTrace)(e);
|
|
461
470
|
}
|
|
462
471
|
}
|
|
463
472
|
yield {
|
|
@@ -493,6 +502,21 @@ class _ExperimentManager {
|
|
|
493
502
|
return undefined;
|
|
494
503
|
return modifiedAtTime.reduce((max, current) => (current.time > max.time ? current : max), modifiedAtTime[0]).date;
|
|
495
504
|
}
|
|
505
|
+
async _getDatasetSplits() {
|
|
506
|
+
const examples = await this.getExamples();
|
|
507
|
+
const allSplits = examples.reduce((acc, ex) => {
|
|
508
|
+
if (ex.metadata && ex.metadata.dataset_split) {
|
|
509
|
+
if (Array.isArray(ex.metadata.dataset_split)) {
|
|
510
|
+
ex.metadata.dataset_split.forEach((split) => acc.add(split));
|
|
511
|
+
}
|
|
512
|
+
else if (typeof ex.metadata.dataset_split === "string") {
|
|
513
|
+
acc.add(ex.metadata.dataset_split);
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
return acc;
|
|
517
|
+
}, new Set());
|
|
518
|
+
return allSplits.size ? Array.from(allSplits) : undefined;
|
|
519
|
+
}
|
|
496
520
|
async _end() {
|
|
497
521
|
const experiment = this._experiment;
|
|
498
522
|
if (!experiment) {
|
|
@@ -500,6 +524,7 @@ class _ExperimentManager {
|
|
|
500
524
|
}
|
|
501
525
|
const projectMetadata = await this._getExperimentMetadata();
|
|
502
526
|
projectMetadata["dataset_version"] = await this._getDatasetVersion();
|
|
527
|
+
projectMetadata["dataset_splits"] = await this._getDatasetSplits();
|
|
503
528
|
// Update revision_id if not already set
|
|
504
529
|
if (!projectMetadata["revision_id"]) {
|
|
505
530
|
projectMetadata["revision_id"] = await (0, _git_js_1.getDefaultRevisionId)();
|
|
@@ -631,6 +656,7 @@ async function _forward(fn, example, experimentName, metadata, client) {
|
|
|
631
656
|
}
|
|
632
657
|
catch (e) {
|
|
633
658
|
console.error(`Error running target function: ${e}`);
|
|
659
|
+
(0, error_js_1.printErrorStackTrace)(e);
|
|
634
660
|
}
|
|
635
661
|
if (!run) {
|
|
636
662
|
throw new Error(`Run not created by target function.
|
|
@@ -104,7 +104,7 @@ declare class _ExperimentManager {
|
|
|
104
104
|
_getExperiment(): TracerSession;
|
|
105
105
|
_getExperimentMetadata(): Promise<KVMap>;
|
|
106
106
|
_getProject(firstExample: Example): Promise<TracerSession>;
|
|
107
|
-
_printExperimentStart(): void
|
|
107
|
+
protected _printExperimentStart(): Promise<void>;
|
|
108
108
|
start(): Promise<_ExperimentManager>;
|
|
109
109
|
withPredictions(target: TargetNoInvoke, options?: {
|
|
110
110
|
maxConcurrency?: number;
|
|
@@ -139,6 +139,7 @@ declare class _ExperimentManager {
|
|
|
139
139
|
}): AsyncGenerator<ExperimentResultRow>;
|
|
140
140
|
_applySummaryEvaluators(summaryEvaluators: Array<SummaryEvaluatorT>): AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults>>;
|
|
141
141
|
_getDatasetVersion(): Promise<string | undefined>;
|
|
142
|
+
_getDatasetSplits(): Promise<string[] | undefined>;
|
|
142
143
|
_end(): Promise<void>;
|
|
143
144
|
}
|
|
144
145
|
/**
|
|
@@ -5,6 +5,7 @@ import { assertUuid } from "../utils/_uuid.js";
|
|
|
5
5
|
import { AsyncCaller } from "../utils/async_caller.js";
|
|
6
6
|
import { atee } from "../utils/atee.js";
|
|
7
7
|
import { getLangChainEnvVarsMetadata } from "../utils/env.js";
|
|
8
|
+
import { printErrorStackTrace } from "../utils/error.js";
|
|
8
9
|
import { randomName } from "./_random_name.js";
|
|
9
10
|
import { runEvaluator, } from "./evaluator.js";
|
|
10
11
|
import { v4 as uuidv4 } from "uuid";
|
|
@@ -234,15 +235,21 @@ class _ExperimentManager {
|
|
|
234
235
|
}
|
|
235
236
|
return project;
|
|
236
237
|
}
|
|
237
|
-
_printExperimentStart() {
|
|
238
|
-
// @TODO log with experiment URL
|
|
238
|
+
async _printExperimentStart() {
|
|
239
239
|
console.log(`Starting evaluation of experiment: ${this.experimentName}`);
|
|
240
|
+
const firstExample = this._examples?.[0];
|
|
241
|
+
const datasetId = firstExample?.dataset_id;
|
|
242
|
+
if (!datasetId || !this._experiment)
|
|
243
|
+
return;
|
|
244
|
+
const datasetUrl = await this.client.getDatasetUrl({ datasetId });
|
|
245
|
+
const compareUrl = `${datasetUrl}/compare?selectedSessions=${this._experiment.id}`;
|
|
246
|
+
console.log(`View results at ${compareUrl}`);
|
|
240
247
|
}
|
|
241
248
|
async start() {
|
|
242
249
|
const examples = await this.getExamples();
|
|
243
250
|
const firstExample = examples[0];
|
|
244
251
|
const project = await this._getProject(firstExample);
|
|
245
|
-
this._printExperimentStart();
|
|
252
|
+
await this._printExperimentStart();
|
|
246
253
|
return new _ExperimentManager({
|
|
247
254
|
examples,
|
|
248
255
|
experiment: project,
|
|
@@ -386,6 +393,7 @@ class _ExperimentManager {
|
|
|
386
393
|
}
|
|
387
394
|
catch (e) {
|
|
388
395
|
console.error(`Error running evaluator ${evaluator.evaluateRun.name} on run ${run.id}: ${e}`);
|
|
396
|
+
printErrorStackTrace(e);
|
|
389
397
|
}
|
|
390
398
|
}
|
|
391
399
|
return {
|
|
@@ -454,6 +462,7 @@ class _ExperimentManager {
|
|
|
454
462
|
}
|
|
455
463
|
catch (e) {
|
|
456
464
|
console.error(`Error running summary evaluator ${evaluator.name}: ${JSON.stringify(e, null, 2)}`);
|
|
465
|
+
printErrorStackTrace(e);
|
|
457
466
|
}
|
|
458
467
|
}
|
|
459
468
|
yield {
|
|
@@ -489,6 +498,21 @@ class _ExperimentManager {
|
|
|
489
498
|
return undefined;
|
|
490
499
|
return modifiedAtTime.reduce((max, current) => (current.time > max.time ? current : max), modifiedAtTime[0]).date;
|
|
491
500
|
}
|
|
501
|
+
async _getDatasetSplits() {
|
|
502
|
+
const examples = await this.getExamples();
|
|
503
|
+
const allSplits = examples.reduce((acc, ex) => {
|
|
504
|
+
if (ex.metadata && ex.metadata.dataset_split) {
|
|
505
|
+
if (Array.isArray(ex.metadata.dataset_split)) {
|
|
506
|
+
ex.metadata.dataset_split.forEach((split) => acc.add(split));
|
|
507
|
+
}
|
|
508
|
+
else if (typeof ex.metadata.dataset_split === "string") {
|
|
509
|
+
acc.add(ex.metadata.dataset_split);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
return acc;
|
|
513
|
+
}, new Set());
|
|
514
|
+
return allSplits.size ? Array.from(allSplits) : undefined;
|
|
515
|
+
}
|
|
492
516
|
async _end() {
|
|
493
517
|
const experiment = this._experiment;
|
|
494
518
|
if (!experiment) {
|
|
@@ -496,6 +520,7 @@ class _ExperimentManager {
|
|
|
496
520
|
}
|
|
497
521
|
const projectMetadata = await this._getExperimentMetadata();
|
|
498
522
|
projectMetadata["dataset_version"] = await this._getDatasetVersion();
|
|
523
|
+
projectMetadata["dataset_splits"] = await this._getDatasetSplits();
|
|
499
524
|
// Update revision_id if not already set
|
|
500
525
|
if (!projectMetadata["revision_id"]) {
|
|
501
526
|
projectMetadata["revision_id"] = await getDefaultRevisionId();
|
|
@@ -627,6 +652,7 @@ async function _forward(fn, example, experimentName, metadata, client) {
|
|
|
627
652
|
}
|
|
628
653
|
catch (e) {
|
|
629
654
|
console.error(`Error running target function: ${e}`);
|
|
655
|
+
printErrorStackTrace(e);
|
|
630
656
|
}
|
|
631
657
|
if (!run) {
|
|
632
658
|
throw new Error(`Run not created by target function.
|
package/dist/index.cjs
CHANGED
|
@@ -6,4 +6,4 @@ Object.defineProperty(exports, "Client", { enumerable: true, get: function () {
|
|
|
6
6
|
var run_trees_js_1 = require("./run_trees.cjs");
|
|
7
7
|
Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () { return run_trees_js_1.RunTree; } });
|
|
8
8
|
// Update using yarn bump-version
|
|
9
|
-
exports.__version__ = "0.1.
|
|
9
|
+
exports.__version__ = "0.1.27";
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { Client } from "./client.js";
|
|
2
|
-
export type { Dataset, Example, TracerSession, Run, Feedback, } from "./schemas.js";
|
|
2
|
+
export type { Dataset, Example, TracerSession, Run, Feedback, RetrieverOutput, } from "./schemas.js";
|
|
3
3
|
export { RunTree, type RunTreeConfig } from "./run_trees.js";
|
|
4
|
-
export declare const __version__ = "0.1.
|
|
4
|
+
export declare const __version__ = "0.1.27";
|
package/dist/index.js
CHANGED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RunnableTraceable = exports.getLangchainCallbacks = void 0;
|
|
4
|
+
const manager_1 = require("@langchain/core/callbacks/manager");
|
|
5
|
+
const tracer_langchain_1 = require("@langchain/core/tracers/tracer_langchain");
|
|
6
|
+
const runnables_1 = require("@langchain/core/runnables");
|
|
7
|
+
const traceable_js_1 = require("./traceable.cjs");
|
|
8
|
+
/**
|
|
9
|
+
* Converts the current run tree active within a traceable-wrapped function
|
|
10
|
+
* into a LangChain compatible callback manager. This is useful to handoff tracing
|
|
11
|
+
* from LangSmith to LangChain Runnables and LLMs.
|
|
12
|
+
*
|
|
13
|
+
* @param {RunTree | undefined} currentRunTree Current RunTree from within a traceable-wrapped function. If not provided, the current run tree will be inferred from AsyncLocalStorage.
|
|
14
|
+
* @returns {CallbackManager | undefined} Callback manager used by LangChain Runnable objects.
|
|
15
|
+
*/
|
|
16
|
+
async function getLangchainCallbacks(currentRunTree) {
|
|
17
|
+
const runTree = currentRunTree ?? (0, traceable_js_1.getCurrentRunTree)();
|
|
18
|
+
if (!runTree)
|
|
19
|
+
return undefined;
|
|
20
|
+
// TODO: CallbackManager.configure() is only async due to LangChainTracer
|
|
21
|
+
// factory being unnecessarily async.
|
|
22
|
+
let callbacks = await manager_1.CallbackManager.configure();
|
|
23
|
+
if (!callbacks && runTree.tracingEnabled) {
|
|
24
|
+
callbacks = new manager_1.CallbackManager();
|
|
25
|
+
}
|
|
26
|
+
let langChainTracer = callbacks?.handlers.find((handler) => handler?.name === "langchain_tracer");
|
|
27
|
+
if (!langChainTracer && runTree.tracingEnabled) {
|
|
28
|
+
langChainTracer = new tracer_langchain_1.LangChainTracer();
|
|
29
|
+
callbacks?.addHandler(langChainTracer);
|
|
30
|
+
}
|
|
31
|
+
const runMap = new Map();
|
|
32
|
+
// find upward root run
|
|
33
|
+
let rootRun = runTree;
|
|
34
|
+
const rootVisited = new Set();
|
|
35
|
+
while (rootRun.parent_run) {
|
|
36
|
+
if (rootVisited.has(rootRun.id))
|
|
37
|
+
break;
|
|
38
|
+
rootVisited.add(rootRun.id);
|
|
39
|
+
rootRun = rootRun.parent_run;
|
|
40
|
+
}
|
|
41
|
+
const queue = [rootRun];
|
|
42
|
+
const visited = new Set();
|
|
43
|
+
while (queue.length > 0) {
|
|
44
|
+
const current = queue.shift();
|
|
45
|
+
if (!current || visited.has(current.id))
|
|
46
|
+
continue;
|
|
47
|
+
visited.add(current.id);
|
|
48
|
+
runMap.set(current.id, current);
|
|
49
|
+
if (current.child_runs) {
|
|
50
|
+
queue.push(...current.child_runs);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (callbacks != null) {
|
|
54
|
+
Object.assign(callbacks, { _parentRunId: runTree.id });
|
|
55
|
+
}
|
|
56
|
+
if (langChainTracer != null) {
|
|
57
|
+
Object.assign(langChainTracer, {
|
|
58
|
+
runMap,
|
|
59
|
+
client: runTree.client,
|
|
60
|
+
projectName: runTree.project_name || langChainTracer.projectName,
|
|
61
|
+
exampleId: runTree.reference_example_id || langChainTracer.exampleId,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
return callbacks;
|
|
65
|
+
}
|
|
66
|
+
exports.getLangchainCallbacks = getLangchainCallbacks;
|
|
67
|
+
/**
|
|
68
|
+
* RunnableTraceable is a Runnable that wraps a traceable function.
|
|
69
|
+
* This allows adding Langsmith traced functions into LangChain sequences.
|
|
70
|
+
*/
|
|
71
|
+
class RunnableTraceable extends runnables_1.Runnable {
|
|
72
|
+
constructor(fields) {
|
|
73
|
+
super(fields);
|
|
74
|
+
Object.defineProperty(this, "lc_serializable", {
|
|
75
|
+
enumerable: true,
|
|
76
|
+
configurable: true,
|
|
77
|
+
writable: true,
|
|
78
|
+
value: false
|
|
79
|
+
});
|
|
80
|
+
Object.defineProperty(this, "lc_namespace", {
|
|
81
|
+
enumerable: true,
|
|
82
|
+
configurable: true,
|
|
83
|
+
writable: true,
|
|
84
|
+
value: ["langchain_core", "runnables"]
|
|
85
|
+
});
|
|
86
|
+
Object.defineProperty(this, "func", {
|
|
87
|
+
enumerable: true,
|
|
88
|
+
configurable: true,
|
|
89
|
+
writable: true,
|
|
90
|
+
value: void 0
|
|
91
|
+
});
|
|
92
|
+
if (!(0, traceable_js_1.isTraceableFunction)(fields.func)) {
|
|
93
|
+
throw new Error("RunnableTraceable requires a function that is wrapped in traceable higher-order function");
|
|
94
|
+
}
|
|
95
|
+
this.func = fields.func;
|
|
96
|
+
}
|
|
97
|
+
async invoke(input, options) {
|
|
98
|
+
const [config] = this._getOptionsList(options ?? {}, 1);
|
|
99
|
+
return (await this.func(config, input));
|
|
100
|
+
}
|
|
101
|
+
static from(func) {
|
|
102
|
+
return new RunnableTraceable({ func });
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
exports.RunnableTraceable = RunnableTraceable;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { CallbackManager } from "@langchain/core/callbacks/manager";
|
|
2
|
+
import { Runnable, RunnableConfig } from "@langchain/core/runnables";
|
|
3
|
+
import { RunTree } from "./run_trees.js";
|
|
4
|
+
import { TraceableFunction } from "./traceable.js";
|
|
5
|
+
/**
|
|
6
|
+
* Converts the current run tree active within a traceable-wrapped function
|
|
7
|
+
* into a LangChain compatible callback manager. This is useful to handoff tracing
|
|
8
|
+
* from LangSmith to LangChain Runnables and LLMs.
|
|
9
|
+
*
|
|
10
|
+
* @param {RunTree | undefined} currentRunTree Current RunTree from within a traceable-wrapped function. If not provided, the current run tree will be inferred from AsyncLocalStorage.
|
|
11
|
+
* @returns {CallbackManager | undefined} Callback manager used by LangChain Runnable objects.
|
|
12
|
+
*/
|
|
13
|
+
export declare function getLangchainCallbacks(currentRunTree?: RunTree | undefined): Promise<CallbackManager | undefined>;
|
|
14
|
+
type AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;
|
|
15
|
+
/**
|
|
16
|
+
* RunnableTraceable is a Runnable that wraps a traceable function.
|
|
17
|
+
* This allows adding Langsmith traced functions into LangChain sequences.
|
|
18
|
+
*/
|
|
19
|
+
export declare class RunnableTraceable<RunInput, RunOutput> extends Runnable<RunInput, RunOutput> {
|
|
20
|
+
lc_serializable: boolean;
|
|
21
|
+
lc_namespace: string[];
|
|
22
|
+
protected func: AnyTraceableFunction;
|
|
23
|
+
constructor(fields: {
|
|
24
|
+
func: AnyTraceableFunction;
|
|
25
|
+
});
|
|
26
|
+
invoke(input: RunInput, options?: Partial<RunnableConfig>): Promise<RunOutput>;
|
|
27
|
+
static from(func: AnyTraceableFunction): RunnableTraceable<unknown, unknown>;
|
|
28
|
+
}
|
|
29
|
+
export {};
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { CallbackManager } from "@langchain/core/callbacks/manager";
|
|
2
|
+
import { LangChainTracer } from "@langchain/core/tracers/tracer_langchain";
|
|
3
|
+
import { Runnable } from "@langchain/core/runnables";
|
|
4
|
+
import { getCurrentRunTree, isTraceableFunction, } from "./traceable.js";
|
|
5
|
+
/**
|
|
6
|
+
* Converts the current run tree active within a traceable-wrapped function
|
|
7
|
+
* into a LangChain compatible callback manager. This is useful to handoff tracing
|
|
8
|
+
* from LangSmith to LangChain Runnables and LLMs.
|
|
9
|
+
*
|
|
10
|
+
* @param {RunTree | undefined} currentRunTree Current RunTree from within a traceable-wrapped function. If not provided, the current run tree will be inferred from AsyncLocalStorage.
|
|
11
|
+
* @returns {CallbackManager | undefined} Callback manager used by LangChain Runnable objects.
|
|
12
|
+
*/
|
|
13
|
+
export async function getLangchainCallbacks(currentRunTree) {
|
|
14
|
+
const runTree = currentRunTree ?? getCurrentRunTree();
|
|
15
|
+
if (!runTree)
|
|
16
|
+
return undefined;
|
|
17
|
+
// TODO: CallbackManager.configure() is only async due to LangChainTracer
|
|
18
|
+
// factory being unnecessarily async.
|
|
19
|
+
let callbacks = await CallbackManager.configure();
|
|
20
|
+
if (!callbacks && runTree.tracingEnabled) {
|
|
21
|
+
callbacks = new CallbackManager();
|
|
22
|
+
}
|
|
23
|
+
let langChainTracer = callbacks?.handlers.find((handler) => handler?.name === "langchain_tracer");
|
|
24
|
+
if (!langChainTracer && runTree.tracingEnabled) {
|
|
25
|
+
langChainTracer = new LangChainTracer();
|
|
26
|
+
callbacks?.addHandler(langChainTracer);
|
|
27
|
+
}
|
|
28
|
+
const runMap = new Map();
|
|
29
|
+
// find upward root run
|
|
30
|
+
let rootRun = runTree;
|
|
31
|
+
const rootVisited = new Set();
|
|
32
|
+
while (rootRun.parent_run) {
|
|
33
|
+
if (rootVisited.has(rootRun.id))
|
|
34
|
+
break;
|
|
35
|
+
rootVisited.add(rootRun.id);
|
|
36
|
+
rootRun = rootRun.parent_run;
|
|
37
|
+
}
|
|
38
|
+
const queue = [rootRun];
|
|
39
|
+
const visited = new Set();
|
|
40
|
+
while (queue.length > 0) {
|
|
41
|
+
const current = queue.shift();
|
|
42
|
+
if (!current || visited.has(current.id))
|
|
43
|
+
continue;
|
|
44
|
+
visited.add(current.id);
|
|
45
|
+
runMap.set(current.id, current);
|
|
46
|
+
if (current.child_runs) {
|
|
47
|
+
queue.push(...current.child_runs);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
if (callbacks != null) {
|
|
51
|
+
Object.assign(callbacks, { _parentRunId: runTree.id });
|
|
52
|
+
}
|
|
53
|
+
if (langChainTracer != null) {
|
|
54
|
+
Object.assign(langChainTracer, {
|
|
55
|
+
runMap,
|
|
56
|
+
client: runTree.client,
|
|
57
|
+
projectName: runTree.project_name || langChainTracer.projectName,
|
|
58
|
+
exampleId: runTree.reference_example_id || langChainTracer.exampleId,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
return callbacks;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* RunnableTraceable is a Runnable that wraps a traceable function.
|
|
65
|
+
* This allows adding Langsmith traced functions into LangChain sequences.
|
|
66
|
+
*/
|
|
67
|
+
export class RunnableTraceable extends Runnable {
|
|
68
|
+
constructor(fields) {
|
|
69
|
+
super(fields);
|
|
70
|
+
Object.defineProperty(this, "lc_serializable", {
|
|
71
|
+
enumerable: true,
|
|
72
|
+
configurable: true,
|
|
73
|
+
writable: true,
|
|
74
|
+
value: false
|
|
75
|
+
});
|
|
76
|
+
Object.defineProperty(this, "lc_namespace", {
|
|
77
|
+
enumerable: true,
|
|
78
|
+
configurable: true,
|
|
79
|
+
writable: true,
|
|
80
|
+
value: ["langchain_core", "runnables"]
|
|
81
|
+
});
|
|
82
|
+
Object.defineProperty(this, "func", {
|
|
83
|
+
enumerable: true,
|
|
84
|
+
configurable: true,
|
|
85
|
+
writable: true,
|
|
86
|
+
value: void 0
|
|
87
|
+
});
|
|
88
|
+
if (!isTraceableFunction(fields.func)) {
|
|
89
|
+
throw new Error("RunnableTraceable requires a function that is wrapped in traceable higher-order function");
|
|
90
|
+
}
|
|
91
|
+
this.func = fields.func;
|
|
92
|
+
}
|
|
93
|
+
async invoke(input, options) {
|
|
94
|
+
const [config] = this._getOptionsList(options ?? {}, 1);
|
|
95
|
+
return (await this.func(config, input));
|
|
96
|
+
}
|
|
97
|
+
static from(func) {
|
|
98
|
+
return new RunnableTraceable({ func });
|
|
99
|
+
}
|
|
100
|
+
}
|