langsmith 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.cjs +138 -8
- package/dist/client.d.ts +148 -1
- package/dist/client.js +137 -7
- package/dist/index.cjs +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/package.json +1 -1
package/dist/client.cjs
CHANGED
|
@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.Client = exports.Queue = void 0;
|
|
26
|
+
exports.Client = exports.DEFAULT_BATCH_SIZE_LIMIT_BYTES = exports.Queue = void 0;
|
|
27
27
|
const uuid = __importStar(require("uuid"));
|
|
28
28
|
const async_caller_js_1 = require("./utils/async_caller.cjs");
|
|
29
29
|
const messages_js_1 = require("./utils/messages.cjs");
|
|
@@ -147,6 +147,8 @@ class Queue {
|
|
|
147
147
|
}
|
|
148
148
|
}
|
|
149
149
|
exports.Queue = Queue;
|
|
150
|
+
// 20 MB
|
|
151
|
+
exports.DEFAULT_BATCH_SIZE_LIMIT_BYTES = 20971520;
|
|
150
152
|
class Client {
|
|
151
153
|
constructor(config = {}) {
|
|
152
154
|
Object.defineProperty(this, "apiKey", {
|
|
@@ -257,6 +259,12 @@ class Client {
|
|
|
257
259
|
writable: true,
|
|
258
260
|
value: 50
|
|
259
261
|
});
|
|
262
|
+
Object.defineProperty(this, "serverInfo", {
|
|
263
|
+
enumerable: true,
|
|
264
|
+
configurable: true,
|
|
265
|
+
writable: true,
|
|
266
|
+
value: void 0
|
|
267
|
+
});
|
|
260
268
|
const defaultConfig = Client.getDefaultClientConfig();
|
|
261
269
|
this.tracingSampleRate = getTracingSamplingRate();
|
|
262
270
|
this.apiUrl = trimQuotes(config.apiUrl ?? defaultConfig.apiUrl) ?? "";
|
|
@@ -479,7 +487,7 @@ class Client {
|
|
|
479
487
|
}
|
|
480
488
|
return itemPromise;
|
|
481
489
|
}
|
|
482
|
-
async
|
|
490
|
+
async _getServerInfo() {
|
|
483
491
|
const response = await fetch(`${this.apiUrl}/info`, {
|
|
484
492
|
method: "GET",
|
|
485
493
|
headers: { Accept: "application/json" },
|
|
@@ -489,6 +497,15 @@ class Client {
|
|
|
489
497
|
// consume the response body to release the connection
|
|
490
498
|
// https://undici.nodejs.org/#/?id=garbage-collection
|
|
491
499
|
await response.text();
|
|
500
|
+
throw new Error("Failed to retrieve server info.");
|
|
501
|
+
}
|
|
502
|
+
return response.json();
|
|
503
|
+
}
|
|
504
|
+
async batchEndpointIsSupported() {
|
|
505
|
+
try {
|
|
506
|
+
this.serverInfo = await this._getServerInfo();
|
|
507
|
+
}
|
|
508
|
+
catch (e) {
|
|
492
509
|
return false;
|
|
493
510
|
}
|
|
494
511
|
return true;
|
|
@@ -558,11 +575,11 @@ class Client {
|
|
|
558
575
|
preparedCreateParams = Object.values(createById);
|
|
559
576
|
preparedUpdateParams = standaloneUpdates;
|
|
560
577
|
}
|
|
561
|
-
const
|
|
578
|
+
const rawBatch = {
|
|
562
579
|
post: this._filterForSampling(preparedCreateParams),
|
|
563
580
|
patch: this._filterForSampling(preparedUpdateParams, true),
|
|
564
581
|
};
|
|
565
|
-
if (!
|
|
582
|
+
if (!rawBatch.post.length && !rawBatch.patch.length) {
|
|
566
583
|
return;
|
|
567
584
|
}
|
|
568
585
|
preparedCreateParams = await mergeRuntimeEnvIntoRunCreates(preparedCreateParams);
|
|
@@ -571,16 +588,46 @@ class Client {
|
|
|
571
588
|
}
|
|
572
589
|
if (!this.batchEndpointSupported) {
|
|
573
590
|
this.autoBatchTracing = false;
|
|
574
|
-
for (const preparedCreateParam of
|
|
591
|
+
for (const preparedCreateParam of rawBatch.post) {
|
|
575
592
|
await this.createRun(preparedCreateParam);
|
|
576
593
|
}
|
|
577
|
-
for (const preparedUpdateParam of
|
|
594
|
+
for (const preparedUpdateParam of rawBatch.patch) {
|
|
578
595
|
if (preparedUpdateParam.id !== undefined) {
|
|
579
596
|
await this.updateRun(preparedUpdateParam.id, preparedUpdateParam);
|
|
580
597
|
}
|
|
581
598
|
}
|
|
582
599
|
return;
|
|
583
600
|
}
|
|
601
|
+
const sizeLimitBytes = this.serverInfo?.batch_ingest_config?.size_limit_bytes ??
|
|
602
|
+
exports.DEFAULT_BATCH_SIZE_LIMIT_BYTES;
|
|
603
|
+
const batchChunks = {
|
|
604
|
+
post: [],
|
|
605
|
+
patch: [],
|
|
606
|
+
};
|
|
607
|
+
let currentBatchSizeBytes = 0;
|
|
608
|
+
for (const k of ["post", "patch"]) {
|
|
609
|
+
const key = k;
|
|
610
|
+
const batchItems = rawBatch[key].reverse();
|
|
611
|
+
let batchItem = batchItems.pop();
|
|
612
|
+
while (batchItem !== undefined) {
|
|
613
|
+
const stringifiedBatchItem = JSON.stringify(batchItem);
|
|
614
|
+
if (currentBatchSizeBytes > 0 &&
|
|
615
|
+
currentBatchSizeBytes + stringifiedBatchItem.length > sizeLimitBytes) {
|
|
616
|
+
await this._postBatchIngestRuns(JSON.stringify(batchChunks));
|
|
617
|
+
currentBatchSizeBytes = 0;
|
|
618
|
+
batchChunks.post = [];
|
|
619
|
+
batchChunks.patch = [];
|
|
620
|
+
}
|
|
621
|
+
currentBatchSizeBytes += stringifiedBatchItem.length;
|
|
622
|
+
batchChunks[key].push(batchItem);
|
|
623
|
+
batchItem = batchItems.pop();
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
if (batchChunks.post.length > 0 || batchChunks.patch.length > 0) {
|
|
627
|
+
await this._postBatchIngestRuns(JSON.stringify(batchChunks));
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
async _postBatchIngestRuns(body) {
|
|
584
631
|
const headers = {
|
|
585
632
|
...this.headers,
|
|
586
633
|
"Content-Type": "application/json",
|
|
@@ -589,7 +636,7 @@ class Client {
|
|
|
589
636
|
const response = await this.batchIngestCaller.call(fetch, `${this.apiUrl}/runs/batch`, {
|
|
590
637
|
method: "POST",
|
|
591
638
|
headers,
|
|
592
|
-
body:
|
|
639
|
+
body: body,
|
|
593
640
|
signal: AbortSignal.timeout(this.timeout_ms),
|
|
594
641
|
});
|
|
595
642
|
await raiseForStatus(response, "batch create run");
|
|
@@ -696,7 +743,89 @@ class Client {
|
|
|
696
743
|
}
|
|
697
744
|
return run;
|
|
698
745
|
}
|
|
699
|
-
|
|
746
|
+
/**
|
|
747
|
+
* List runs from the LangSmith server.
|
|
748
|
+
* @param projectId - The ID of the project to filter by.
|
|
749
|
+
* @param projectName - The name of the project to filter by.
|
|
750
|
+
* @param parentRunId - The ID of the parent run to filter by.
|
|
751
|
+
* @param traceId - The ID of the trace to filter by.
|
|
752
|
+
* @param referenceExampleId - The ID of the reference example to filter by.
|
|
753
|
+
* @param startTime - The start time to filter by.
|
|
754
|
+
* @param executionOrder - The execution order to filter by.
|
|
755
|
+
* @param runType - The run type to filter by.
|
|
756
|
+
* @param error - Indicates whether to filter by error runs.
|
|
757
|
+
* @param id - The ID of the run to filter by.
|
|
758
|
+
* @param query - The query string to filter by.
|
|
759
|
+
* @param filter - The filter string to apply to the run spans.
|
|
760
|
+
* @param traceFilter - The filter string to apply on the root run of the trace.
|
|
761
|
+
* @param limit - The maximum number of runs to retrieve.
|
|
762
|
+
* @returns {AsyncIterable<Run>} - The runs.
|
|
763
|
+
*
|
|
764
|
+
* @example
|
|
765
|
+
* // List all runs in a project
|
|
766
|
+
* const projectRuns = client.listRuns({ projectName: "<your_project>" });
|
|
767
|
+
*
|
|
768
|
+
* @example
|
|
769
|
+
* // List LLM and Chat runs in the last 24 hours
|
|
770
|
+
* const todaysLLMRuns = client.listRuns({
|
|
771
|
+
* projectName: "<your_project>",
|
|
772
|
+
* start_time: new Date(Date.now() - 24 * 60 * 60 * 1000),
|
|
773
|
+
* run_type: "llm",
|
|
774
|
+
* });
|
|
775
|
+
*
|
|
776
|
+
* @example
|
|
777
|
+
* // List traces in a project
|
|
778
|
+
* const rootRuns = client.listRuns({
|
|
779
|
+
* projectName: "<your_project>",
|
|
780
|
+
* execution_order: 1,
|
|
781
|
+
* });
|
|
782
|
+
*
|
|
783
|
+
* @example
|
|
784
|
+
* // List runs without errors
|
|
785
|
+
* const correctRuns = client.listRuns({
|
|
786
|
+
* projectName: "<your_project>",
|
|
787
|
+
* error: false,
|
|
788
|
+
* });
|
|
789
|
+
*
|
|
790
|
+
* @example
|
|
791
|
+
* // List runs by run ID
|
|
792
|
+
* const runIds = [
|
|
793
|
+
* "a36092d2-4ad5-4fb4-9c0d-0dba9a2ed836",
|
|
794
|
+
* "9398e6be-964f-4aa4-8ae9-ad78cd4b7074",
|
|
795
|
+
* ];
|
|
796
|
+
* const selectedRuns = client.listRuns({ run_ids: runIds });
|
|
797
|
+
*
|
|
798
|
+
* @example
|
|
799
|
+
* // List all "chain" type runs that took more than 10 seconds and had `total_tokens` greater than 5000
|
|
800
|
+
* const chainRuns = client.listRuns({
|
|
801
|
+
* projectName: "<your_project>",
|
|
802
|
+
* filter: 'and(eq(run_type, "chain"), gt(latency, 10), gt(total_tokens, 5000))',
|
|
803
|
+
* });
|
|
804
|
+
*
|
|
805
|
+
* @example
|
|
806
|
+
* // List all runs called "extractor" whose root of the trace was assigned feedback "user_score" score of 1
|
|
807
|
+
* const goodExtractorRuns = client.listRuns({
|
|
808
|
+
* projectName: "<your_project>",
|
|
809
|
+
* filter: 'eq(name, "extractor")',
|
|
810
|
+
* traceFilter: 'and(eq(feedback_key, "user_score"), eq(feedback_score, 1))',
|
|
811
|
+
* });
|
|
812
|
+
*
|
|
813
|
+
* @example
|
|
814
|
+
* // List all runs that started after a specific timestamp and either have "error" not equal to null or a "Correctness" feedback score equal to 0
|
|
815
|
+
* const complexRuns = client.listRuns({
|
|
816
|
+
* projectName: "<your_project>",
|
|
817
|
+
* filter: 'and(gt(start_time, "2023-07-15T12:34:56Z"), or(neq(error, null), and(eq(feedback_key, "Correctness"), eq(feedback_score, 0.0))))',
|
|
818
|
+
* });
|
|
819
|
+
*
|
|
820
|
+
* @example
|
|
821
|
+
* // List all runs where `tags` include "experimental" or "beta" and `latency` is greater than 2 seconds
|
|
822
|
+
* const taggedRuns = client.listRuns({
|
|
823
|
+
* projectName: "<your_project>",
|
|
824
|
+
* filter: 'and(or(has(tags, "experimental"), has(tags, "beta")), gt(latency, 2))',
|
|
825
|
+
* });
|
|
826
|
+
*/
|
|
827
|
+
async *listRuns(props) {
|
|
828
|
+
const { projectId, projectName, parentRunId, traceId, referenceExampleId, startTime, executionOrder, runType, error, id, query, filter, traceFilter, limit, } = props;
|
|
700
829
|
let projectIds = [];
|
|
701
830
|
if (projectId) {
|
|
702
831
|
projectIds = Array.isArray(projectId) ? projectId : [projectId];
|
|
@@ -714,6 +843,7 @@ class Client {
|
|
|
714
843
|
reference_example: referenceExampleId,
|
|
715
844
|
query,
|
|
716
845
|
filter,
|
|
846
|
+
trace_filter: traceFilter,
|
|
717
847
|
execution_order: executionOrder,
|
|
718
848
|
parent_run: parentRunId ? [parentRunId] : null,
|
|
719
849
|
start_time: startTime ? startTime.toISOString() : null,
|
package/dist/client.d.ts
CHANGED
|
@@ -12,20 +12,82 @@ interface ClientConfig {
|
|
|
12
12
|
autoBatchTracing?: boolean;
|
|
13
13
|
pendingAutoBatchedRunLimit?: number;
|
|
14
14
|
}
|
|
15
|
+
/**
|
|
16
|
+
* Represents the parameters for listing runs (spans) from the Langsmith server.
|
|
17
|
+
*/
|
|
15
18
|
interface ListRunsParams {
|
|
19
|
+
/**
|
|
20
|
+
* The ID or IDs of the project(s) to filter by.
|
|
21
|
+
*/
|
|
16
22
|
projectId?: string | string[];
|
|
23
|
+
/**
|
|
24
|
+
* The name or names of the project(s) to filter by.
|
|
25
|
+
*/
|
|
17
26
|
projectName?: string | string[];
|
|
27
|
+
/**
|
|
28
|
+
* The ID of the trace to filter by.
|
|
29
|
+
*/
|
|
18
30
|
traceId?: string;
|
|
31
|
+
/**
|
|
32
|
+
* The execution order to filter by.
|
|
33
|
+
*/
|
|
19
34
|
executionOrder?: number;
|
|
35
|
+
/**
|
|
36
|
+
* The ID of the parent run to filter by.
|
|
37
|
+
*/
|
|
20
38
|
parentRunId?: string;
|
|
39
|
+
/**
|
|
40
|
+
* The ID of the reference example to filter by.
|
|
41
|
+
*/
|
|
21
42
|
referenceExampleId?: string;
|
|
43
|
+
/**
|
|
44
|
+
* The start time to filter by.
|
|
45
|
+
*/
|
|
22
46
|
startTime?: Date;
|
|
47
|
+
/**
|
|
48
|
+
* The run type to filter by.
|
|
49
|
+
*/
|
|
23
50
|
runType?: string;
|
|
51
|
+
/**
|
|
52
|
+
* Indicates whether to filter by error runs.
|
|
53
|
+
*/
|
|
24
54
|
error?: boolean;
|
|
55
|
+
/**
|
|
56
|
+
* The ID or IDs of the runs to filter by.
|
|
57
|
+
*/
|
|
25
58
|
id?: string[];
|
|
59
|
+
/**
|
|
60
|
+
* The maximum number of runs to retrieve.
|
|
61
|
+
*/
|
|
26
62
|
limit?: number;
|
|
63
|
+
/**
|
|
64
|
+
* The query string to filter by.
|
|
65
|
+
*/
|
|
27
66
|
query?: string;
|
|
67
|
+
/**
|
|
68
|
+
* The filter string to apply.
|
|
69
|
+
*
|
|
70
|
+
* Run Filtering:
|
|
71
|
+
* Listing runs with query params is useful for simple queries, but doesn't support many common needs, such as filtering by metadata, tags, or other fields.
|
|
72
|
+
* LangSmith supports a filter query language to permit more complex filtering operations when fetching runs. This guide will provide a high level overview of the grammar as well as a few examples of when it can be useful.
|
|
73
|
+
* If you'd prefer a more visual guide, you can get a taste of the language by viewing the table of runs on any of your projects' pages. We provide some recommended filters to get you started that you can copy and use the SDK.
|
|
74
|
+
*
|
|
75
|
+
* Grammar:
|
|
76
|
+
* The filtering grammar is based on common comparators on fields in the run object. Supported comparators include:
|
|
77
|
+
* - gte (greater than or equal to)
|
|
78
|
+
* - gt (greater than)
|
|
79
|
+
* - lte (less than or equal to)
|
|
80
|
+
* - lt (less than)
|
|
81
|
+
* - eq (equal to)
|
|
82
|
+
* - neq (not equal to)
|
|
83
|
+
* - has (check if run contains a tag or metadata json blob)
|
|
84
|
+
* - search (search for a substring in a string field)
|
|
85
|
+
*/
|
|
28
86
|
filter?: string;
|
|
87
|
+
/**
|
|
88
|
+
* The trace filter string to apply. Uses the same grammar as the filter string.
|
|
89
|
+
*/
|
|
90
|
+
traceFilter?: string;
|
|
29
91
|
}
|
|
30
92
|
interface UploadCSVParams {
|
|
31
93
|
csvFile: Blob;
|
|
@@ -72,6 +134,7 @@ export declare class Queue<T> {
|
|
|
72
134
|
push(item: T): Promise<void>;
|
|
73
135
|
pop(upToN: number): [T[], () => void];
|
|
74
136
|
}
|
|
137
|
+
export declare const DEFAULT_BATCH_SIZE_LIMIT_BYTES = 20971520;
|
|
75
138
|
export declare class Client {
|
|
76
139
|
private apiKey?;
|
|
77
140
|
private apiUrl;
|
|
@@ -91,6 +154,7 @@ export declare class Client {
|
|
|
91
154
|
private autoBatchTimeout;
|
|
92
155
|
private autoBatchInitialDelayMs;
|
|
93
156
|
private autoBatchAggregationDelayMs;
|
|
157
|
+
private serverInfo;
|
|
94
158
|
constructor(config?: ClientConfig);
|
|
95
159
|
static getDefaultClientConfig(): {
|
|
96
160
|
apiUrl: string;
|
|
@@ -111,6 +175,7 @@ export declare class Client {
|
|
|
111
175
|
private _filterForSampling;
|
|
112
176
|
private drainAutoBatchQueue;
|
|
113
177
|
private processRunOperation;
|
|
178
|
+
protected _getServerInfo(): Promise<any>;
|
|
114
179
|
protected batchEndpointIsSupported(): Promise<boolean>;
|
|
115
180
|
createRun(run: CreateRunParams): Promise<void>;
|
|
116
181
|
/**
|
|
@@ -121,6 +186,7 @@ export declare class Client {
|
|
|
121
186
|
runCreates?: RunCreate[];
|
|
122
187
|
runUpdates?: RunUpdate[];
|
|
123
188
|
}): Promise<void>;
|
|
189
|
+
private _postBatchIngestRuns;
|
|
124
190
|
updateRun(runId: string, run: RunUpdate): Promise<void>;
|
|
125
191
|
readRun(runId: string, { loadChildRuns }?: {
|
|
126
192
|
loadChildRuns: boolean;
|
|
@@ -131,7 +197,88 @@ export declare class Client {
|
|
|
131
197
|
projectOpts?: ProjectOptions;
|
|
132
198
|
}): Promise<string>;
|
|
133
199
|
private _loadChildRuns;
|
|
134
|
-
|
|
200
|
+
/**
|
|
201
|
+
* List runs from the LangSmith server.
|
|
202
|
+
* @param projectId - The ID of the project to filter by.
|
|
203
|
+
* @param projectName - The name of the project to filter by.
|
|
204
|
+
* @param parentRunId - The ID of the parent run to filter by.
|
|
205
|
+
* @param traceId - The ID of the trace to filter by.
|
|
206
|
+
* @param referenceExampleId - The ID of the reference example to filter by.
|
|
207
|
+
* @param startTime - The start time to filter by.
|
|
208
|
+
* @param executionOrder - The execution order to filter by.
|
|
209
|
+
* @param runType - The run type to filter by.
|
|
210
|
+
* @param error - Indicates whether to filter by error runs.
|
|
211
|
+
* @param id - The ID of the run to filter by.
|
|
212
|
+
* @param query - The query string to filter by.
|
|
213
|
+
* @param filter - The filter string to apply to the run spans.
|
|
214
|
+
* @param traceFilter - The filter string to apply on the root run of the trace.
|
|
215
|
+
* @param limit - The maximum number of runs to retrieve.
|
|
216
|
+
* @returns {AsyncIterable<Run>} - The runs.
|
|
217
|
+
*
|
|
218
|
+
* @example
|
|
219
|
+
* // List all runs in a project
|
|
220
|
+
* const projectRuns = client.listRuns({ projectName: "<your_project>" });
|
|
221
|
+
*
|
|
222
|
+
* @example
|
|
223
|
+
* // List LLM and Chat runs in the last 24 hours
|
|
224
|
+
* const todaysLLMRuns = client.listRuns({
|
|
225
|
+
* projectName: "<your_project>",
|
|
226
|
+
* start_time: new Date(Date.now() - 24 * 60 * 60 * 1000),
|
|
227
|
+
* run_type: "llm",
|
|
228
|
+
* });
|
|
229
|
+
*
|
|
230
|
+
* @example
|
|
231
|
+
* // List traces in a project
|
|
232
|
+
* const rootRuns = client.listRuns({
|
|
233
|
+
* projectName: "<your_project>",
|
|
234
|
+
* execution_order: 1,
|
|
235
|
+
* });
|
|
236
|
+
*
|
|
237
|
+
* @example
|
|
238
|
+
* // List runs without errors
|
|
239
|
+
* const correctRuns = client.listRuns({
|
|
240
|
+
* projectName: "<your_project>",
|
|
241
|
+
* error: false,
|
|
242
|
+
* });
|
|
243
|
+
*
|
|
244
|
+
* @example
|
|
245
|
+
* // List runs by run ID
|
|
246
|
+
* const runIds = [
|
|
247
|
+
* "a36092d2-4ad5-4fb4-9c0d-0dba9a2ed836",
|
|
248
|
+
* "9398e6be-964f-4aa4-8ae9-ad78cd4b7074",
|
|
249
|
+
* ];
|
|
250
|
+
* const selectedRuns = client.listRuns({ run_ids: runIds });
|
|
251
|
+
*
|
|
252
|
+
* @example
|
|
253
|
+
* // List all "chain" type runs that took more than 10 seconds and had `total_tokens` greater than 5000
|
|
254
|
+
* const chainRuns = client.listRuns({
|
|
255
|
+
* projectName: "<your_project>",
|
|
256
|
+
* filter: 'and(eq(run_type, "chain"), gt(latency, 10), gt(total_tokens, 5000))',
|
|
257
|
+
* });
|
|
258
|
+
*
|
|
259
|
+
* @example
|
|
260
|
+
* // List all runs called "extractor" whose root of the trace was assigned feedback "user_score" score of 1
|
|
261
|
+
* const goodExtractorRuns = client.listRuns({
|
|
262
|
+
* projectName: "<your_project>",
|
|
263
|
+
* filter: 'eq(name, "extractor")',
|
|
264
|
+
* traceFilter: 'and(eq(feedback_key, "user_score"), eq(feedback_score, 1))',
|
|
265
|
+
* });
|
|
266
|
+
*
|
|
267
|
+
* @example
|
|
268
|
+
* // List all runs that started after a specific timestamp and either have "error" not equal to null or a "Correctness" feedback score equal to 0
|
|
269
|
+
* const complexRuns = client.listRuns({
|
|
270
|
+
* projectName: "<your_project>",
|
|
271
|
+
* filter: 'and(gt(start_time, "2023-07-15T12:34:56Z"), or(neq(error, null), and(eq(feedback_key, "Correctness"), eq(feedback_score, 0.0))))',
|
|
272
|
+
* });
|
|
273
|
+
*
|
|
274
|
+
* @example
|
|
275
|
+
* // List all runs where `tags` include "experimental" or "beta" and `latency` is greater than 2 seconds
|
|
276
|
+
* const taggedRuns = client.listRuns({
|
|
277
|
+
* projectName: "<your_project>",
|
|
278
|
+
* filter: 'and(or(has(tags, "experimental"), has(tags, "beta")), gt(latency, 2))',
|
|
279
|
+
* });
|
|
280
|
+
*/
|
|
281
|
+
listRuns(props: ListRunsParams): AsyncIterable<Run>;
|
|
135
282
|
shareRun(runId: string, { shareId }?: {
|
|
136
283
|
shareId?: string;
|
|
137
284
|
}): Promise<string>;
|
package/dist/client.js
CHANGED
|
@@ -120,6 +120,8 @@ export class Queue {
|
|
|
120
120
|
return [popped.map((it) => it[0]), () => popped.forEach((it) => it[1]())];
|
|
121
121
|
}
|
|
122
122
|
}
|
|
123
|
+
// 20 MB
|
|
124
|
+
export const DEFAULT_BATCH_SIZE_LIMIT_BYTES = 20971520;
|
|
123
125
|
export class Client {
|
|
124
126
|
constructor(config = {}) {
|
|
125
127
|
Object.defineProperty(this, "apiKey", {
|
|
@@ -230,6 +232,12 @@ export class Client {
|
|
|
230
232
|
writable: true,
|
|
231
233
|
value: 50
|
|
232
234
|
});
|
|
235
|
+
Object.defineProperty(this, "serverInfo", {
|
|
236
|
+
enumerable: true,
|
|
237
|
+
configurable: true,
|
|
238
|
+
writable: true,
|
|
239
|
+
value: void 0
|
|
240
|
+
});
|
|
233
241
|
const defaultConfig = Client.getDefaultClientConfig();
|
|
234
242
|
this.tracingSampleRate = getTracingSamplingRate();
|
|
235
243
|
this.apiUrl = trimQuotes(config.apiUrl ?? defaultConfig.apiUrl) ?? "";
|
|
@@ -452,7 +460,7 @@ export class Client {
|
|
|
452
460
|
}
|
|
453
461
|
return itemPromise;
|
|
454
462
|
}
|
|
455
|
-
async
|
|
463
|
+
async _getServerInfo() {
|
|
456
464
|
const response = await fetch(`${this.apiUrl}/info`, {
|
|
457
465
|
method: "GET",
|
|
458
466
|
headers: { Accept: "application/json" },
|
|
@@ -462,6 +470,15 @@ export class Client {
|
|
|
462
470
|
// consume the response body to release the connection
|
|
463
471
|
// https://undici.nodejs.org/#/?id=garbage-collection
|
|
464
472
|
await response.text();
|
|
473
|
+
throw new Error("Failed to retrieve server info.");
|
|
474
|
+
}
|
|
475
|
+
return response.json();
|
|
476
|
+
}
|
|
477
|
+
async batchEndpointIsSupported() {
|
|
478
|
+
try {
|
|
479
|
+
this.serverInfo = await this._getServerInfo();
|
|
480
|
+
}
|
|
481
|
+
catch (e) {
|
|
465
482
|
return false;
|
|
466
483
|
}
|
|
467
484
|
return true;
|
|
@@ -531,11 +548,11 @@ export class Client {
|
|
|
531
548
|
preparedCreateParams = Object.values(createById);
|
|
532
549
|
preparedUpdateParams = standaloneUpdates;
|
|
533
550
|
}
|
|
534
|
-
const
|
|
551
|
+
const rawBatch = {
|
|
535
552
|
post: this._filterForSampling(preparedCreateParams),
|
|
536
553
|
patch: this._filterForSampling(preparedUpdateParams, true),
|
|
537
554
|
};
|
|
538
|
-
if (!
|
|
555
|
+
if (!rawBatch.post.length && !rawBatch.patch.length) {
|
|
539
556
|
return;
|
|
540
557
|
}
|
|
541
558
|
preparedCreateParams = await mergeRuntimeEnvIntoRunCreates(preparedCreateParams);
|
|
@@ -544,16 +561,46 @@ export class Client {
|
|
|
544
561
|
}
|
|
545
562
|
if (!this.batchEndpointSupported) {
|
|
546
563
|
this.autoBatchTracing = false;
|
|
547
|
-
for (const preparedCreateParam of
|
|
564
|
+
for (const preparedCreateParam of rawBatch.post) {
|
|
548
565
|
await this.createRun(preparedCreateParam);
|
|
549
566
|
}
|
|
550
|
-
for (const preparedUpdateParam of
|
|
567
|
+
for (const preparedUpdateParam of rawBatch.patch) {
|
|
551
568
|
if (preparedUpdateParam.id !== undefined) {
|
|
552
569
|
await this.updateRun(preparedUpdateParam.id, preparedUpdateParam);
|
|
553
570
|
}
|
|
554
571
|
}
|
|
555
572
|
return;
|
|
556
573
|
}
|
|
574
|
+
const sizeLimitBytes = this.serverInfo?.batch_ingest_config?.size_limit_bytes ??
|
|
575
|
+
DEFAULT_BATCH_SIZE_LIMIT_BYTES;
|
|
576
|
+
const batchChunks = {
|
|
577
|
+
post: [],
|
|
578
|
+
patch: [],
|
|
579
|
+
};
|
|
580
|
+
let currentBatchSizeBytes = 0;
|
|
581
|
+
for (const k of ["post", "patch"]) {
|
|
582
|
+
const key = k;
|
|
583
|
+
const batchItems = rawBatch[key].reverse();
|
|
584
|
+
let batchItem = batchItems.pop();
|
|
585
|
+
while (batchItem !== undefined) {
|
|
586
|
+
const stringifiedBatchItem = JSON.stringify(batchItem);
|
|
587
|
+
if (currentBatchSizeBytes > 0 &&
|
|
588
|
+
currentBatchSizeBytes + stringifiedBatchItem.length > sizeLimitBytes) {
|
|
589
|
+
await this._postBatchIngestRuns(JSON.stringify(batchChunks));
|
|
590
|
+
currentBatchSizeBytes = 0;
|
|
591
|
+
batchChunks.post = [];
|
|
592
|
+
batchChunks.patch = [];
|
|
593
|
+
}
|
|
594
|
+
currentBatchSizeBytes += stringifiedBatchItem.length;
|
|
595
|
+
batchChunks[key].push(batchItem);
|
|
596
|
+
batchItem = batchItems.pop();
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
if (batchChunks.post.length > 0 || batchChunks.patch.length > 0) {
|
|
600
|
+
await this._postBatchIngestRuns(JSON.stringify(batchChunks));
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
async _postBatchIngestRuns(body) {
|
|
557
604
|
const headers = {
|
|
558
605
|
...this.headers,
|
|
559
606
|
"Content-Type": "application/json",
|
|
@@ -562,7 +609,7 @@ export class Client {
|
|
|
562
609
|
const response = await this.batchIngestCaller.call(fetch, `${this.apiUrl}/runs/batch`, {
|
|
563
610
|
method: "POST",
|
|
564
611
|
headers,
|
|
565
|
-
body:
|
|
612
|
+
body: body,
|
|
566
613
|
signal: AbortSignal.timeout(this.timeout_ms),
|
|
567
614
|
});
|
|
568
615
|
await raiseForStatus(response, "batch create run");
|
|
@@ -669,7 +716,89 @@ export class Client {
|
|
|
669
716
|
}
|
|
670
717
|
return run;
|
|
671
718
|
}
|
|
672
|
-
|
|
719
|
+
/**
|
|
720
|
+
* List runs from the LangSmith server.
|
|
721
|
+
* @param projectId - The ID of the project to filter by.
|
|
722
|
+
* @param projectName - The name of the project to filter by.
|
|
723
|
+
* @param parentRunId - The ID of the parent run to filter by.
|
|
724
|
+
* @param traceId - The ID of the trace to filter by.
|
|
725
|
+
* @param referenceExampleId - The ID of the reference example to filter by.
|
|
726
|
+
* @param startTime - The start time to filter by.
|
|
727
|
+
* @param executionOrder - The execution order to filter by.
|
|
728
|
+
* @param runType - The run type to filter by.
|
|
729
|
+
* @param error - Indicates whether to filter by error runs.
|
|
730
|
+
* @param id - The ID of the run to filter by.
|
|
731
|
+
* @param query - The query string to filter by.
|
|
732
|
+
* @param filter - The filter string to apply to the run spans.
|
|
733
|
+
* @param traceFilter - The filter string to apply on the root run of the trace.
|
|
734
|
+
* @param limit - The maximum number of runs to retrieve.
|
|
735
|
+
* @returns {AsyncIterable<Run>} - The runs.
|
|
736
|
+
*
|
|
737
|
+
* @example
|
|
738
|
+
* // List all runs in a project
|
|
739
|
+
* const projectRuns = client.listRuns({ projectName: "<your_project>" });
|
|
740
|
+
*
|
|
741
|
+
* @example
|
|
742
|
+
* // List LLM and Chat runs in the last 24 hours
|
|
743
|
+
* const todaysLLMRuns = client.listRuns({
|
|
744
|
+
* projectName: "<your_project>",
|
|
745
|
+
* start_time: new Date(Date.now() - 24 * 60 * 60 * 1000),
|
|
746
|
+
* run_type: "llm",
|
|
747
|
+
* });
|
|
748
|
+
*
|
|
749
|
+
* @example
|
|
750
|
+
* // List traces in a project
|
|
751
|
+
* const rootRuns = client.listRuns({
|
|
752
|
+
* projectName: "<your_project>",
|
|
753
|
+
* execution_order: 1,
|
|
754
|
+
* });
|
|
755
|
+
*
|
|
756
|
+
* @example
|
|
757
|
+
* // List runs without errors
|
|
758
|
+
* const correctRuns = client.listRuns({
|
|
759
|
+
* projectName: "<your_project>",
|
|
760
|
+
* error: false,
|
|
761
|
+
* });
|
|
762
|
+
*
|
|
763
|
+
* @example
|
|
764
|
+
* // List runs by run ID
|
|
765
|
+
* const runIds = [
|
|
766
|
+
* "a36092d2-4ad5-4fb4-9c0d-0dba9a2ed836",
|
|
767
|
+
* "9398e6be-964f-4aa4-8ae9-ad78cd4b7074",
|
|
768
|
+
* ];
|
|
769
|
+
* const selectedRuns = client.listRuns({ run_ids: runIds });
|
|
770
|
+
*
|
|
771
|
+
* @example
|
|
772
|
+
* // List all "chain" type runs that took more than 10 seconds and had `total_tokens` greater than 5000
|
|
773
|
+
* const chainRuns = client.listRuns({
|
|
774
|
+
* projectName: "<your_project>",
|
|
775
|
+
* filter: 'and(eq(run_type, "chain"), gt(latency, 10), gt(total_tokens, 5000))',
|
|
776
|
+
* });
|
|
777
|
+
*
|
|
778
|
+
* @example
|
|
779
|
+
* // List all runs called "extractor" whose root of the trace was assigned feedback "user_score" score of 1
|
|
780
|
+
* const goodExtractorRuns = client.listRuns({
|
|
781
|
+
* projectName: "<your_project>",
|
|
782
|
+
* filter: 'eq(name, "extractor")',
|
|
783
|
+
* traceFilter: 'and(eq(feedback_key, "user_score"), eq(feedback_score, 1))',
|
|
784
|
+
* });
|
|
785
|
+
*
|
|
786
|
+
* @example
|
|
787
|
+
* // List all runs that started after a specific timestamp and either have "error" not equal to null or a "Correctness" feedback score equal to 0
|
|
788
|
+
* const complexRuns = client.listRuns({
|
|
789
|
+
* projectName: "<your_project>",
|
|
790
|
+
* filter: 'and(gt(start_time, "2023-07-15T12:34:56Z"), or(neq(error, null), and(eq(feedback_key, "Correctness"), eq(feedback_score, 0.0))))',
|
|
791
|
+
* });
|
|
792
|
+
*
|
|
793
|
+
* @example
|
|
794
|
+
* // List all runs where `tags` include "experimental" or "beta" and `latency` is greater than 2 seconds
|
|
795
|
+
* const taggedRuns = client.listRuns({
|
|
796
|
+
* projectName: "<your_project>",
|
|
797
|
+
* filter: 'and(or(has(tags, "experimental"), has(tags, "beta")), gt(latency, 2))',
|
|
798
|
+
* });
|
|
799
|
+
*/
|
|
800
|
+
async *listRuns(props) {
|
|
801
|
+
const { projectId, projectName, parentRunId, traceId, referenceExampleId, startTime, executionOrder, runType, error, id, query, filter, traceFilter, limit, } = props;
|
|
673
802
|
let projectIds = [];
|
|
674
803
|
if (projectId) {
|
|
675
804
|
projectIds = Array.isArray(projectId) ? projectId : [projectId];
|
|
@@ -687,6 +816,7 @@ export class Client {
|
|
|
687
816
|
reference_example: referenceExampleId,
|
|
688
817
|
query,
|
|
689
818
|
filter,
|
|
819
|
+
trace_filter: traceFilter,
|
|
690
820
|
execution_order: executionOrder,
|
|
691
821
|
parent_run: parentRunId ? [parentRunId] : null,
|
|
692
822
|
start_time: startTime ? startTime.toISOString() : null,
|
package/dist/index.cjs
CHANGED
|
@@ -6,4 +6,4 @@ Object.defineProperty(exports, "Client", { enumerable: true, get: function () {
|
|
|
6
6
|
var run_trees_js_1 = require("./run_trees.cjs");
|
|
7
7
|
Object.defineProperty(exports, "RunTree", { enumerable: true, get: function () { return run_trees_js_1.RunTree; } });
|
|
8
8
|
// Update using yarn bump-version
|
|
9
|
-
exports.__version__ = "0.1.
|
|
9
|
+
exports.__version__ = "0.1.8";
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { Client } from "./client.js";
|
|
2
2
|
export type { Dataset, Example, TracerSession, Run, Feedback, } from "./schemas.js";
|
|
3
3
|
export { RunTree, type RunTreeConfig } from "./run_trees.js";
|
|
4
|
-
export declare const __version__ = "0.1.
|
|
4
|
+
export declare const __version__ = "0.1.8";
|
package/dist/index.js
CHANGED