langsmith 0.10.3 → 0.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.cjs +19 -4
- package/dist/client.d.ts +2 -1
- package/dist/client.js +19 -4
- package/dist/evaluation/_runner.cjs +47 -5
- package/dist/evaluation/_runner.d.ts +12 -0
- package/dist/evaluation/_runner.js +47 -5
- package/dist/index.cjs +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/sandbox/access_delegation.cjs +24 -0
- package/dist/sandbox/access_delegation.d.ts +15 -0
- package/dist/sandbox/access_delegation.js +21 -0
- package/dist/sandbox/client.cjs +70 -5
- package/dist/sandbox/client.d.ts +18 -0
- package/dist/sandbox/client.js +70 -5
- package/dist/sandbox/command_handle.cjs +34 -0
- package/dist/sandbox/command_handle.d.ts +10 -0
- package/dist/sandbox/command_handle.js +34 -0
- package/dist/sandbox/helpers.cjs +129 -0
- package/dist/sandbox/helpers.d.ts +29 -0
- package/dist/sandbox/helpers.js +123 -0
- package/dist/sandbox/index.cjs +7 -1
- package/dist/sandbox/index.d.ts +5 -1
- package/dist/sandbox/index.js +2 -0
- package/dist/sandbox/mounts.cjs +21 -3
- package/dist/sandbox/mounts.d.ts +7 -1
- package/dist/sandbox/mounts.js +21 -3
- package/dist/sandbox/proxy_config.cjs +41 -10
- package/dist/sandbox/proxy_config.d.ts +33 -6
- package/dist/sandbox/proxy_config.js +40 -10
- package/dist/sandbox/sandbox.cjs +160 -4
- package/dist/sandbox/sandbox.d.ts +71 -1
- package/dist/sandbox/sandbox.js +161 -5
- package/dist/sandbox/service_url.cjs +125 -0
- package/dist/sandbox/service_url.d.ts +50 -0
- package/dist/sandbox/service_url.js +120 -0
- package/dist/sandbox/types.d.ts +251 -8
- package/dist/sandbox/ws_execute.cjs +31 -1
- package/dist/sandbox/ws_execute.d.ts +9 -0
- package/dist/sandbox/ws_execute.js +31 -1
- package/dist/utils/error.cjs +17 -22
- package/dist/utils/error.js +17 -22
- package/dist/wrappers/openai_agents.cjs +37 -28
- package/dist/wrappers/openai_agents.d.ts +0 -1
- package/dist/wrappers/openai_agents.js +37 -28
- package/package.json +2 -2
package/dist/client.cjs
CHANGED
|
@@ -82,7 +82,7 @@ function _normalizeRunTimestamps(run) {
|
|
|
82
82
|
end_time: _ensureUTCTimestamp(run.end_time),
|
|
83
83
|
};
|
|
84
84
|
}
|
|
85
|
-
function mergeRuntimeEnvIntoRun(run, cachedEnvVars, omitTracedRuntimeInfo) {
|
|
85
|
+
function mergeRuntimeEnvIntoRun(run, cachedEnvVars, omitTracedRuntimeInfo, tracingSampleRate) {
|
|
86
86
|
if (omitTracedRuntimeInfo) {
|
|
87
87
|
return run;
|
|
88
88
|
}
|
|
@@ -105,6 +105,9 @@ function mergeRuntimeEnvIntoRun(run, cachedEnvVars, omitTracedRuntimeInfo) {
|
|
|
105
105
|
}
|
|
106
106
|
: {}),
|
|
107
107
|
...metadata,
|
|
108
|
+
...(tracingSampleRate !== undefined
|
|
109
|
+
? { ls_tracing_sample_rate: tracingSampleRate }
|
|
110
|
+
: {}),
|
|
108
111
|
},
|
|
109
112
|
};
|
|
110
113
|
return run;
|
|
@@ -863,6 +866,10 @@ class Client {
|
|
|
863
866
|
throw new Error("Trace batch concurrency must be positive.");
|
|
864
867
|
}
|
|
865
868
|
this.debug = config.debug ?? this.debug;
|
|
869
|
+
if (this.debug) {
|
|
870
|
+
const source = config.apiUrl != null ? "apiUrl option" : defaultConfig.apiUrlSource;
|
|
871
|
+
console.log(`LangSmith API URL ${this.apiUrl} resolved from ${source}`);
|
|
872
|
+
}
|
|
866
873
|
this.fetchImplementation = config.fetchImplementation;
|
|
867
874
|
// Failed trace dump configuration
|
|
868
875
|
this.failedTracesDir =
|
|
@@ -949,12 +956,18 @@ class Client {
|
|
|
949
956
|
const envWorkspaceId = (0, env_js_1.getLangSmithEnvironmentVariable)("WORKSPACE_ID");
|
|
950
957
|
const envAuthSet = (0, profiles_js_1.hasValue)(envApiKey);
|
|
951
958
|
const apiUrl = envApiUrl ?? profileConfig.apiUrl ?? profiles_js_1.DEFAULT_API_URL;
|
|
959
|
+
const apiUrlSource = envApiUrl != null
|
|
960
|
+
? "LANGSMITH_ENDPOINT / LANGCHAIN_ENDPOINT environment variable"
|
|
961
|
+
: profileConfig.apiUrl != null
|
|
962
|
+
? "profile config"
|
|
963
|
+
: "built-in default";
|
|
952
964
|
const workspaceId = envWorkspaceId ?? profileConfig.workspaceId;
|
|
953
965
|
const hideInputs = (0, env_js_1.getLangSmithEnvironmentVariable)("HIDE_INPUTS") === "true";
|
|
954
966
|
const hideOutputs = (0, env_js_1.getLangSmithEnvironmentVariable)("HIDE_OUTPUTS") === "true";
|
|
955
967
|
const hideMetadata = (0, env_js_1.getLangSmithEnvironmentVariable)("HIDE_METADATA") === "true";
|
|
956
968
|
return {
|
|
957
969
|
apiUrl: apiUrl,
|
|
970
|
+
apiUrlSource,
|
|
958
971
|
apiKey: envApiKey,
|
|
959
972
|
webUrl: undefined,
|
|
960
973
|
hideInputs: hideInputs,
|
|
@@ -1617,7 +1630,7 @@ class Client {
|
|
|
1617
1630
|
};
|
|
1618
1631
|
}
|
|
1619
1632
|
async _mergeRuntimeEnvAndMaskMetadata(run) {
|
|
1620
|
-
const merged = mergeRuntimeEnvIntoRun(run, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo);
|
|
1633
|
+
const merged = mergeRuntimeEnvIntoRun(run, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo, this.tracingSampleRate);
|
|
1621
1634
|
if (this.omitTracedRuntimeInfo) {
|
|
1622
1635
|
return merged;
|
|
1623
1636
|
}
|
|
@@ -1626,7 +1639,7 @@ class Client {
|
|
|
1626
1639
|
async processRunOperation(item) {
|
|
1627
1640
|
clearTimeout(this.autoBatchTimeout);
|
|
1628
1641
|
this.autoBatchTimeout = undefined;
|
|
1629
|
-
item.item = mergeRuntimeEnvIntoRun(item.item, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo);
|
|
1642
|
+
item.item = mergeRuntimeEnvIntoRun(item.item, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo, this.tracingSampleRate);
|
|
1630
1643
|
const itemPromise = this.autoBatchQueue.push(item);
|
|
1631
1644
|
if (this.manualFlushMode) {
|
|
1632
1645
|
// Rely on manual flushing in serverless environments
|
|
@@ -2227,7 +2240,9 @@ class Client {
|
|
|
2227
2240
|
if (options?.workspaceId !== undefined) {
|
|
2228
2241
|
headers["x-tenant-id"] = options.workspaceId;
|
|
2229
2242
|
}
|
|
2230
|
-
const body = (0, index_js_4.serialize)(run
|
|
2243
|
+
const body = (0, index_js_4.serialize)(run.extra
|
|
2244
|
+
? mergeRuntimeEnvIntoRun(run, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo, this.tracingSampleRate)
|
|
2245
|
+
: run, `Serializing payload to update run with id: ${runId}`);
|
|
2231
2246
|
await this.caller.call(async () => {
|
|
2232
2247
|
const res = await this._fetch(`${options?.apiUrl ?? this.apiUrl}/runs/${runId}`, {
|
|
2233
2248
|
method: "PATCH",
|
package/dist/client.d.ts
CHANGED
|
@@ -300,6 +300,7 @@ interface UploadCSVParams {
|
|
|
300
300
|
}
|
|
301
301
|
type DefaultClientConfig = {
|
|
302
302
|
apiUrl: string;
|
|
303
|
+
apiUrlSource: string;
|
|
303
304
|
apiKey?: string;
|
|
304
305
|
webUrl?: string;
|
|
305
306
|
hideInputs?: boolean;
|
|
@@ -445,7 +446,7 @@ type Thread = {
|
|
|
445
446
|
export interface ListThreadsItem extends Thread {
|
|
446
447
|
runs: Run[];
|
|
447
448
|
}
|
|
448
|
-
export declare function mergeRuntimeEnvIntoRun<T extends RunCreate | RunUpdate>(run: T, cachedEnvVars?: Record<string, string>, omitTracedRuntimeInfo?: boolean): T;
|
|
449
|
+
export declare function mergeRuntimeEnvIntoRun<T extends RunCreate | RunUpdate>(run: T, cachedEnvVars?: Record<string, string>, omitTracedRuntimeInfo?: boolean, tracingSampleRate?: number): T;
|
|
449
450
|
export declare function _checkBackendVersion(backendVersion: string, minVersion: string): void;
|
|
450
451
|
export declare const DEFAULT_UNCOMPRESSED_BATCH_SIZE_LIMIT_BYTES: number;
|
|
451
452
|
/** Default maximum memory (1GB) for queue size limits. */
|
package/dist/client.js
CHANGED
|
@@ -44,7 +44,7 @@ function _normalizeRunTimestamps(run) {
|
|
|
44
44
|
end_time: _ensureUTCTimestamp(run.end_time),
|
|
45
45
|
};
|
|
46
46
|
}
|
|
47
|
-
export function mergeRuntimeEnvIntoRun(run, cachedEnvVars, omitTracedRuntimeInfo) {
|
|
47
|
+
export function mergeRuntimeEnvIntoRun(run, cachedEnvVars, omitTracedRuntimeInfo, tracingSampleRate) {
|
|
48
48
|
if (omitTracedRuntimeInfo) {
|
|
49
49
|
return run;
|
|
50
50
|
}
|
|
@@ -67,6 +67,9 @@ export function mergeRuntimeEnvIntoRun(run, cachedEnvVars, omitTracedRuntimeInfo
|
|
|
67
67
|
}
|
|
68
68
|
: {}),
|
|
69
69
|
...metadata,
|
|
70
|
+
...(tracingSampleRate !== undefined
|
|
71
|
+
? { ls_tracing_sample_rate: tracingSampleRate }
|
|
72
|
+
: {}),
|
|
70
73
|
},
|
|
71
74
|
};
|
|
72
75
|
return run;
|
|
@@ -824,6 +827,10 @@ export class Client {
|
|
|
824
827
|
throw new Error("Trace batch concurrency must be positive.");
|
|
825
828
|
}
|
|
826
829
|
this.debug = config.debug ?? this.debug;
|
|
830
|
+
if (this.debug) {
|
|
831
|
+
const source = config.apiUrl != null ? "apiUrl option" : defaultConfig.apiUrlSource;
|
|
832
|
+
console.log(`LangSmith API URL ${this.apiUrl} resolved from ${source}`);
|
|
833
|
+
}
|
|
827
834
|
this.fetchImplementation = config.fetchImplementation;
|
|
828
835
|
// Failed trace dump configuration
|
|
829
836
|
this.failedTracesDir =
|
|
@@ -910,12 +917,18 @@ export class Client {
|
|
|
910
917
|
const envWorkspaceId = getLangSmithEnvironmentVariable("WORKSPACE_ID");
|
|
911
918
|
const envAuthSet = hasValue(envApiKey);
|
|
912
919
|
const apiUrl = envApiUrl ?? profileConfig.apiUrl ?? DEFAULT_API_URL;
|
|
920
|
+
const apiUrlSource = envApiUrl != null
|
|
921
|
+
? "LANGSMITH_ENDPOINT / LANGCHAIN_ENDPOINT environment variable"
|
|
922
|
+
: profileConfig.apiUrl != null
|
|
923
|
+
? "profile config"
|
|
924
|
+
: "built-in default";
|
|
913
925
|
const workspaceId = envWorkspaceId ?? profileConfig.workspaceId;
|
|
914
926
|
const hideInputs = getLangSmithEnvironmentVariable("HIDE_INPUTS") === "true";
|
|
915
927
|
const hideOutputs = getLangSmithEnvironmentVariable("HIDE_OUTPUTS") === "true";
|
|
916
928
|
const hideMetadata = getLangSmithEnvironmentVariable("HIDE_METADATA") === "true";
|
|
917
929
|
return {
|
|
918
930
|
apiUrl: apiUrl,
|
|
931
|
+
apiUrlSource,
|
|
919
932
|
apiKey: envApiKey,
|
|
920
933
|
webUrl: undefined,
|
|
921
934
|
hideInputs: hideInputs,
|
|
@@ -1578,7 +1591,7 @@ export class Client {
|
|
|
1578
1591
|
};
|
|
1579
1592
|
}
|
|
1580
1593
|
async _mergeRuntimeEnvAndMaskMetadata(run) {
|
|
1581
|
-
const merged = mergeRuntimeEnvIntoRun(run, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo);
|
|
1594
|
+
const merged = mergeRuntimeEnvIntoRun(run, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo, this.tracingSampleRate);
|
|
1582
1595
|
if (this.omitTracedRuntimeInfo) {
|
|
1583
1596
|
return merged;
|
|
1584
1597
|
}
|
|
@@ -1587,7 +1600,7 @@ export class Client {
|
|
|
1587
1600
|
async processRunOperation(item) {
|
|
1588
1601
|
clearTimeout(this.autoBatchTimeout);
|
|
1589
1602
|
this.autoBatchTimeout = undefined;
|
|
1590
|
-
item.item = mergeRuntimeEnvIntoRun(item.item, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo);
|
|
1603
|
+
item.item = mergeRuntimeEnvIntoRun(item.item, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo, this.tracingSampleRate);
|
|
1591
1604
|
const itemPromise = this.autoBatchQueue.push(item);
|
|
1592
1605
|
if (this.manualFlushMode) {
|
|
1593
1606
|
// Rely on manual flushing in serverless environments
|
|
@@ -2188,7 +2201,9 @@ export class Client {
|
|
|
2188
2201
|
if (options?.workspaceId !== undefined) {
|
|
2189
2202
|
headers["x-tenant-id"] = options.workspaceId;
|
|
2190
2203
|
}
|
|
2191
|
-
const body = serializePayloadForTracing(run
|
|
2204
|
+
const body = serializePayloadForTracing(run.extra
|
|
2205
|
+
? mergeRuntimeEnvIntoRun(run, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo, this.tracingSampleRate)
|
|
2206
|
+
: run, `Serializing payload to update run with id: ${runId}`);
|
|
2192
2207
|
await this.caller.call(async () => {
|
|
2193
2208
|
const res = await this._fetch(`${options?.apiUrl ?? this.apiUrl}/runs/${runId}`, {
|
|
2194
2209
|
method: "PATCH",
|
|
@@ -211,6 +211,12 @@ class _ExperimentManager {
|
|
|
211
211
|
writable: true,
|
|
212
212
|
value: void 0
|
|
213
213
|
});
|
|
214
|
+
Object.defineProperty(this, "_disableEvaluatorTracing", {
|
|
215
|
+
enumerable: true,
|
|
216
|
+
configurable: true,
|
|
217
|
+
writable: true,
|
|
218
|
+
value: void 0
|
|
219
|
+
});
|
|
214
220
|
this.client = args.client ?? new index_js_1.Client();
|
|
215
221
|
if (!args.experiment) {
|
|
216
222
|
this._experimentName = (0, _random_name_js_1.randomName)();
|
|
@@ -248,6 +254,7 @@ class _ExperimentManager {
|
|
|
248
254
|
this._numExamples = args.numExamples;
|
|
249
255
|
this._evaluatorKeys = args.evaluatorKeys;
|
|
250
256
|
this._includeAttachments = args.includeAttachments;
|
|
257
|
+
this._disableEvaluatorTracing = args.disableEvaluatorTracing;
|
|
251
258
|
}
|
|
252
259
|
_getExperiment() {
|
|
253
260
|
if (!this._experiment) {
|
|
@@ -350,6 +357,7 @@ class _ExperimentManager {
|
|
|
350
357
|
numExamples: this._numExamples,
|
|
351
358
|
evaluatorKeys: this._evaluatorKeys,
|
|
352
359
|
includeAttachments: this._includeAttachments,
|
|
360
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
353
361
|
});
|
|
354
362
|
}
|
|
355
363
|
async withPredictions(target, options) {
|
|
@@ -379,6 +387,7 @@ class _ExperimentManager {
|
|
|
379
387
|
numExamples: this._numExamples,
|
|
380
388
|
evaluatorKeys: this._evaluatorKeys,
|
|
381
389
|
includeAttachments: this._includeAttachments,
|
|
390
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
382
391
|
});
|
|
383
392
|
}
|
|
384
393
|
async withEvaluators(evaluators, options) {
|
|
@@ -406,6 +415,7 @@ class _ExperimentManager {
|
|
|
406
415
|
numExamples: this._numExamples,
|
|
407
416
|
evaluatorKeys: this._evaluatorKeys,
|
|
408
417
|
includeAttachments: this._includeAttachments,
|
|
418
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
409
419
|
});
|
|
410
420
|
}
|
|
411
421
|
async withSummaryEvaluators(summaryEvaluators) {
|
|
@@ -424,6 +434,7 @@ class _ExperimentManager {
|
|
|
424
434
|
numExamples: this._numExamples,
|
|
425
435
|
evaluatorKeys: this._evaluatorKeys,
|
|
426
436
|
includeAttachments: this._includeAttachments,
|
|
437
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
427
438
|
});
|
|
428
439
|
}
|
|
429
440
|
async *getResults() {
|
|
@@ -541,9 +552,14 @@ class _ExperimentManager {
|
|
|
541
552
|
: new Date(example.created_at).toISOString(),
|
|
542
553
|
},
|
|
543
554
|
client: fields.client,
|
|
544
|
-
tracingEnabled:
|
|
555
|
+
tracingEnabled: !this._disableEvaluatorTracing,
|
|
545
556
|
};
|
|
546
|
-
const
|
|
557
|
+
const rawEvaluatorResponse = await evaluator.evaluateRun(run, example, options);
|
|
558
|
+
// With tracing off there is no evaluator run for the feedback to point
|
|
559
|
+
// at, so drop the link rather than leaving it dangling.
|
|
560
|
+
const evaluatorResponse = this._disableEvaluatorTracing
|
|
561
|
+
? _withoutSourceRunIds(rawEvaluatorResponse)
|
|
562
|
+
: rawEvaluatorResponse;
|
|
547
563
|
evaluationResults.results.push(...(await fields.client.logEvaluationFeedback({
|
|
548
564
|
evaluatorResponse,
|
|
549
565
|
run,
|
|
@@ -590,7 +606,10 @@ class _ExperimentManager {
|
|
|
590
606
|
experiment: this.experimentName,
|
|
591
607
|
projectId: projectId,
|
|
592
608
|
}));
|
|
593
|
-
const wrappedEvaluators = await wrapSummaryEvaluators(summaryEvaluators, options
|
|
609
|
+
const wrappedEvaluators = await wrapSummaryEvaluators(summaryEvaluators, options,
|
|
610
|
+
// Only override when disabling. Passing `true` here would force tracing
|
|
611
|
+
// on for callers who have it switched off via the environment.
|
|
612
|
+
this._disableEvaluatorTracing ? false : undefined);
|
|
594
613
|
yield async function* (runsArray) {
|
|
595
614
|
const aggregateFeedback = [];
|
|
596
615
|
for (const evaluator of wrappedEvaluators) {
|
|
@@ -777,6 +796,7 @@ async function _evaluate(target, fields) {
|
|
|
777
796
|
numRepetitions: fields.numRepetitions ?? 1,
|
|
778
797
|
evaluatorKeys: _collectEvaluatorKeys(standardFields.evaluators),
|
|
779
798
|
includeAttachments: standardFields.includeAttachments,
|
|
799
|
+
disableEvaluatorTracing: standardFields.disableEvaluatorTracing,
|
|
780
800
|
}).start();
|
|
781
801
|
const targetConcurrency = standardFields.targetConcurrency ?? standardFields.maxConcurrency ?? 0;
|
|
782
802
|
const evaluationConcurrency = standardFields.evaluationConcurrency ?? standardFields.maxConcurrency ?? 0;
|
|
@@ -988,7 +1008,25 @@ function _resolveData(data, options) {
|
|
|
988
1008
|
}
|
|
989
1009
|
return data;
|
|
990
1010
|
}
|
|
991
|
-
|
|
1011
|
+
/**
|
|
1012
|
+
* Strip evaluator-run links from an evaluator response.
|
|
1013
|
+
*
|
|
1014
|
+
* Used when evaluator tracing is disabled: no evaluator run is created, so a
|
|
1015
|
+
* `sourceRunId` on the feedback would point at a run that does not exist.
|
|
1016
|
+
*/
|
|
1017
|
+
function _withoutSourceRunIds(response) {
|
|
1018
|
+
if ("results" in response) {
|
|
1019
|
+
return {
|
|
1020
|
+
...response,
|
|
1021
|
+
results: response.results.map((result) => ({
|
|
1022
|
+
...result,
|
|
1023
|
+
sourceRunId: undefined,
|
|
1024
|
+
})),
|
|
1025
|
+
};
|
|
1026
|
+
}
|
|
1027
|
+
return { ...response, sourceRunId: undefined };
|
|
1028
|
+
}
|
|
1029
|
+
async function wrapSummaryEvaluators(evaluators, optionsArray, tracingEnabled) {
|
|
992
1030
|
async function _wrap(evaluator) {
|
|
993
1031
|
const evalName = evaluator.name || "BatchEvaluator";
|
|
994
1032
|
const wrapperInner = (runs, examples) => {
|
|
@@ -1008,7 +1046,11 @@ async function wrapSummaryEvaluators(evaluators, optionsArray) {
|
|
|
1008
1046
|
}
|
|
1009
1047
|
// Otherwise use the traditional (runs, examples) signature
|
|
1010
1048
|
return Promise.resolve(evaluator(runs, examples));
|
|
1011
|
-
}, {
|
|
1049
|
+
}, {
|
|
1050
|
+
...optionsArray,
|
|
1051
|
+
name: evalName,
|
|
1052
|
+
...(tracingEnabled === undefined ? {} : { tracingEnabled }),
|
|
1053
|
+
});
|
|
1012
1054
|
return Promise.resolve(wrapperSuperInner(`Runs[] (Length=${runs.length})`, `Examples[] (Length=${examples.length})`));
|
|
1013
1055
|
};
|
|
1014
1056
|
return wrapperInner;
|
|
@@ -73,6 +73,7 @@ interface _ExperimentManagerArgs {
|
|
|
73
73
|
_runsArray?: Run[];
|
|
74
74
|
resultRows?: AsyncGenerator<_ExperimentResultRowWithIndex>;
|
|
75
75
|
includeAttachments?: boolean;
|
|
76
|
+
disableEvaluatorTracing?: boolean;
|
|
76
77
|
}
|
|
77
78
|
type BaseEvaluateOptions = {
|
|
78
79
|
/**
|
|
@@ -140,6 +141,16 @@ export interface EvaluateOptions extends BaseEvaluateOptions {
|
|
|
140
141
|
* @default false
|
|
141
142
|
*/
|
|
142
143
|
includeAttachments?: boolean;
|
|
144
|
+
/**
|
|
145
|
+
* Whether to skip tracing evaluator invocations to the `evaluators` project.
|
|
146
|
+
*
|
|
147
|
+
* When `true`, evaluators run without creating evaluator traces. Feedback is
|
|
148
|
+
* still created and attached to the experiment runs, but it won't link back
|
|
149
|
+
* to an evaluator run, which also means it can't be corrected from the UI.
|
|
150
|
+
* Tracing of the target itself is unaffected.
|
|
151
|
+
* @default false
|
|
152
|
+
*/
|
|
153
|
+
disableEvaluatorTracing?: boolean;
|
|
143
154
|
}
|
|
144
155
|
export interface ComparativeEvaluateOptions extends BaseEvaluateOptions {
|
|
145
156
|
/**
|
|
@@ -194,6 +205,7 @@ export declare class _ExperimentManager {
|
|
|
194
205
|
_metadata: KVMap;
|
|
195
206
|
_description?: string;
|
|
196
207
|
_includeAttachments?: boolean;
|
|
208
|
+
_disableEvaluatorTracing?: boolean;
|
|
197
209
|
get experimentName(): string;
|
|
198
210
|
getExamples(): Promise<Array<Example>>;
|
|
199
211
|
setExamples(examples: Example[]): void;
|
|
@@ -203,6 +203,12 @@ export class _ExperimentManager {
|
|
|
203
203
|
writable: true,
|
|
204
204
|
value: void 0
|
|
205
205
|
});
|
|
206
|
+
Object.defineProperty(this, "_disableEvaluatorTracing", {
|
|
207
|
+
enumerable: true,
|
|
208
|
+
configurable: true,
|
|
209
|
+
writable: true,
|
|
210
|
+
value: void 0
|
|
211
|
+
});
|
|
206
212
|
this.client = args.client ?? new Client();
|
|
207
213
|
if (!args.experiment) {
|
|
208
214
|
this._experimentName = randomName();
|
|
@@ -240,6 +246,7 @@ export class _ExperimentManager {
|
|
|
240
246
|
this._numExamples = args.numExamples;
|
|
241
247
|
this._evaluatorKeys = args.evaluatorKeys;
|
|
242
248
|
this._includeAttachments = args.includeAttachments;
|
|
249
|
+
this._disableEvaluatorTracing = args.disableEvaluatorTracing;
|
|
243
250
|
}
|
|
244
251
|
_getExperiment() {
|
|
245
252
|
if (!this._experiment) {
|
|
@@ -342,6 +349,7 @@ export class _ExperimentManager {
|
|
|
342
349
|
numExamples: this._numExamples,
|
|
343
350
|
evaluatorKeys: this._evaluatorKeys,
|
|
344
351
|
includeAttachments: this._includeAttachments,
|
|
352
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
345
353
|
});
|
|
346
354
|
}
|
|
347
355
|
async withPredictions(target, options) {
|
|
@@ -371,6 +379,7 @@ export class _ExperimentManager {
|
|
|
371
379
|
numExamples: this._numExamples,
|
|
372
380
|
evaluatorKeys: this._evaluatorKeys,
|
|
373
381
|
includeAttachments: this._includeAttachments,
|
|
382
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
374
383
|
});
|
|
375
384
|
}
|
|
376
385
|
async withEvaluators(evaluators, options) {
|
|
@@ -398,6 +407,7 @@ export class _ExperimentManager {
|
|
|
398
407
|
numExamples: this._numExamples,
|
|
399
408
|
evaluatorKeys: this._evaluatorKeys,
|
|
400
409
|
includeAttachments: this._includeAttachments,
|
|
410
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
401
411
|
});
|
|
402
412
|
}
|
|
403
413
|
async withSummaryEvaluators(summaryEvaluators) {
|
|
@@ -416,6 +426,7 @@ export class _ExperimentManager {
|
|
|
416
426
|
numExamples: this._numExamples,
|
|
417
427
|
evaluatorKeys: this._evaluatorKeys,
|
|
418
428
|
includeAttachments: this._includeAttachments,
|
|
429
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
419
430
|
});
|
|
420
431
|
}
|
|
421
432
|
async *getResults() {
|
|
@@ -533,9 +544,14 @@ export class _ExperimentManager {
|
|
|
533
544
|
: new Date(example.created_at).toISOString(),
|
|
534
545
|
},
|
|
535
546
|
client: fields.client,
|
|
536
|
-
tracingEnabled:
|
|
547
|
+
tracingEnabled: !this._disableEvaluatorTracing,
|
|
537
548
|
};
|
|
538
|
-
const
|
|
549
|
+
const rawEvaluatorResponse = await evaluator.evaluateRun(run, example, options);
|
|
550
|
+
// With tracing off there is no evaluator run for the feedback to point
|
|
551
|
+
// at, so drop the link rather than leaving it dangling.
|
|
552
|
+
const evaluatorResponse = this._disableEvaluatorTracing
|
|
553
|
+
? _withoutSourceRunIds(rawEvaluatorResponse)
|
|
554
|
+
: rawEvaluatorResponse;
|
|
539
555
|
evaluationResults.results.push(...(await fields.client.logEvaluationFeedback({
|
|
540
556
|
evaluatorResponse,
|
|
541
557
|
run,
|
|
@@ -582,7 +598,10 @@ export class _ExperimentManager {
|
|
|
582
598
|
experiment: this.experimentName,
|
|
583
599
|
projectId: projectId,
|
|
584
600
|
}));
|
|
585
|
-
const wrappedEvaluators = await wrapSummaryEvaluators(summaryEvaluators, options
|
|
601
|
+
const wrappedEvaluators = await wrapSummaryEvaluators(summaryEvaluators, options,
|
|
602
|
+
// Only override when disabling. Passing `true` here would force tracing
|
|
603
|
+
// on for callers who have it switched off via the environment.
|
|
604
|
+
this._disableEvaluatorTracing ? false : undefined);
|
|
586
605
|
yield async function* (runsArray) {
|
|
587
606
|
const aggregateFeedback = [];
|
|
588
607
|
for (const evaluator of wrappedEvaluators) {
|
|
@@ -768,6 +787,7 @@ async function _evaluate(target, fields) {
|
|
|
768
787
|
numRepetitions: fields.numRepetitions ?? 1,
|
|
769
788
|
evaluatorKeys: _collectEvaluatorKeys(standardFields.evaluators),
|
|
770
789
|
includeAttachments: standardFields.includeAttachments,
|
|
790
|
+
disableEvaluatorTracing: standardFields.disableEvaluatorTracing,
|
|
771
791
|
}).start();
|
|
772
792
|
const targetConcurrency = standardFields.targetConcurrency ?? standardFields.maxConcurrency ?? 0;
|
|
773
793
|
const evaluationConcurrency = standardFields.evaluationConcurrency ?? standardFields.maxConcurrency ?? 0;
|
|
@@ -979,7 +999,25 @@ function _resolveData(data, options) {
|
|
|
979
999
|
}
|
|
980
1000
|
return data;
|
|
981
1001
|
}
|
|
982
|
-
|
|
1002
|
+
/**
|
|
1003
|
+
* Strip evaluator-run links from an evaluator response.
|
|
1004
|
+
*
|
|
1005
|
+
* Used when evaluator tracing is disabled: no evaluator run is created, so a
|
|
1006
|
+
* `sourceRunId` on the feedback would point at a run that does not exist.
|
|
1007
|
+
*/
|
|
1008
|
+
function _withoutSourceRunIds(response) {
|
|
1009
|
+
if ("results" in response) {
|
|
1010
|
+
return {
|
|
1011
|
+
...response,
|
|
1012
|
+
results: response.results.map((result) => ({
|
|
1013
|
+
...result,
|
|
1014
|
+
sourceRunId: undefined,
|
|
1015
|
+
})),
|
|
1016
|
+
};
|
|
1017
|
+
}
|
|
1018
|
+
return { ...response, sourceRunId: undefined };
|
|
1019
|
+
}
|
|
1020
|
+
async function wrapSummaryEvaluators(evaluators, optionsArray, tracingEnabled) {
|
|
983
1021
|
async function _wrap(evaluator) {
|
|
984
1022
|
const evalName = evaluator.name || "BatchEvaluator";
|
|
985
1023
|
const wrapperInner = (runs, examples) => {
|
|
@@ -999,7 +1037,11 @@ async function wrapSummaryEvaluators(evaluators, optionsArray) {
|
|
|
999
1037
|
}
|
|
1000
1038
|
// Otherwise use the traditional (runs, examples) signature
|
|
1001
1039
|
return Promise.resolve(evaluator(runs, examples));
|
|
1002
|
-
}, {
|
|
1040
|
+
}, {
|
|
1041
|
+
...optionsArray,
|
|
1042
|
+
name: evalName,
|
|
1043
|
+
...(tracingEnabled === undefined ? {} : { tracingEnabled }),
|
|
1044
|
+
});
|
|
1003
1045
|
return Promise.resolve(wrapperSuperInner(`Runs[] (Length=${runs.length})`, `Examples[] (Length=${examples.length})`));
|
|
1004
1046
|
};
|
|
1005
1047
|
return wrapperInner;
|
package/dist/index.cjs
CHANGED
|
@@ -35,6 +35,6 @@ Object.defineProperty(exports, "UnprocessableEntityError", { enumerable: true, g
|
|
|
35
35
|
Object.defineProperty(exports, "RateLimitError", { enumerable: true, get: function () { return error_js_1.RateLimitError; } });
|
|
36
36
|
Object.defineProperty(exports, "InternalServerError", { enumerable: true, get: function () { return error_js_1.InternalServerError; } });
|
|
37
37
|
// Update using pnpm bump-version
|
|
38
|
-
exports.__version__ = "0.10.
|
|
38
|
+
exports.__version__ = "0.10.5";
|
|
39
39
|
// Metadata key to hide a traced run from LangSmith's Messages View.
|
|
40
40
|
exports.LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
|
package/dist/index.d.ts
CHANGED
|
@@ -7,5 +7,5 @@ export { computeRunIdForSecondaryReplica, uuid7, uuid7FromTime, } from "./uuid.j
|
|
|
7
7
|
export { isTracingEnabled } from "./utils/guard.js";
|
|
8
8
|
export { Cache, PromptCache, type CacheConfig, type CacheMetrics, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
|
|
9
9
|
export { LangsmithError, APIError, APIUserAbortError, APIConnectionError, APIConnectionTimeoutError, BadRequestError, AuthenticationError, PermissionDeniedError, NotFoundError, ConflictError, UnprocessableEntityError, RateLimitError, InternalServerError, } from "./_openapi_client/core/error.js";
|
|
10
|
-
export declare const __version__ = "0.10.
|
|
10
|
+
export declare const __version__ = "0.10.5";
|
|
11
11
|
export declare const LS_MESSAGE_VIEW_EXCLUDE: "ls_message_view_exclude";
|
package/dist/index.js
CHANGED
|
@@ -7,6 +7,6 @@ export { isTracingEnabled } from "./utils/guard.js";
|
|
|
7
7
|
export { Cache, PromptCache, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
|
|
8
8
|
export { LangsmithError, APIError, APIUserAbortError, APIConnectionError, APIConnectionTimeoutError, BadRequestError, AuthenticationError, PermissionDeniedError, NotFoundError, ConflictError, UnprocessableEntityError, RateLimitError, InternalServerError, } from "./_openapi_client/core/error.js";
|
|
9
9
|
// Update using pnpm bump-version
|
|
10
|
-
export const __version__ = "0.10.
|
|
10
|
+
export const __version__ = "0.10.5";
|
|
11
11
|
// Metadata key to hide a traced run from LangSmith's Messages View.
|
|
12
12
|
export const LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.validateAccessDelegation = validateAccessDelegation;
|
|
4
|
+
const errors_js_1 = require("./errors.cjs");
|
|
5
|
+
/** Rejects grants the server would refuse, without the round trip. */
|
|
6
|
+
function validateAccessDelegation(value) {
|
|
7
|
+
if (value.mode !== "INHERIT" && value.mode !== "EXPLICIT") {
|
|
8
|
+
throw new errors_js_1.LangSmithValidationError('accessDelegation.mode must be "INHERIT" or "EXPLICIT"', "accessDelegation");
|
|
9
|
+
}
|
|
10
|
+
const { permissions } = value;
|
|
11
|
+
if (permissions !== undefined && !Array.isArray(permissions)) {
|
|
12
|
+
throw new errors_js_1.LangSmithValidationError("accessDelegation.permissions must be an array of strings", "accessDelegation");
|
|
13
|
+
}
|
|
14
|
+
if (value.mode === "INHERIT") {
|
|
15
|
+
if (permissions !== undefined && permissions.length > 0) {
|
|
16
|
+
throw new errors_js_1.LangSmithValidationError('accessDelegation.permissions is not allowed with mode "INHERIT"', "accessDelegation");
|
|
17
|
+
}
|
|
18
|
+
return { mode: "INHERIT" };
|
|
19
|
+
}
|
|
20
|
+
if (permissions === undefined || permissions.length === 0) {
|
|
21
|
+
throw new errors_js_1.LangSmithValidationError('accessDelegation.permissions is required with mode "EXPLICIT"', "accessDelegation");
|
|
22
|
+
}
|
|
23
|
+
return { mode: "EXPLICIT", permissions: [...permissions] };
|
|
24
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** How a delegation grant derives its permission set. */
|
|
2
|
+
export type AccessDelegationMode = "INHERIT" | "EXPLICIT";
|
|
3
|
+
/**
|
|
4
|
+
* Grant letting code inside a sandbox call the LangSmith API as the creator.
|
|
5
|
+
*
|
|
6
|
+
* `INHERIT` tracks everything the creator can do; `EXPLICIT` is capped to
|
|
7
|
+
* `permissions`. Permissions are a ceiling re-checked on every request rather
|
|
8
|
+
* than a snapshot, so access the creator loses is lost here too.
|
|
9
|
+
*/
|
|
10
|
+
export interface AccessDelegation {
|
|
11
|
+
mode: AccessDelegationMode;
|
|
12
|
+
permissions?: string[];
|
|
13
|
+
}
|
|
14
|
+
/** Rejects grants the server would refuse, without the round trip. */
|
|
15
|
+
export declare function validateAccessDelegation(value: AccessDelegation): AccessDelegation;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { LangSmithValidationError } from "./errors.js";
|
|
2
|
+
/** Rejects grants the server would refuse, without the round trip. */
|
|
3
|
+
export function validateAccessDelegation(value) {
|
|
4
|
+
if (value.mode !== "INHERIT" && value.mode !== "EXPLICIT") {
|
|
5
|
+
throw new LangSmithValidationError('accessDelegation.mode must be "INHERIT" or "EXPLICIT"', "accessDelegation");
|
|
6
|
+
}
|
|
7
|
+
const { permissions } = value;
|
|
8
|
+
if (permissions !== undefined && !Array.isArray(permissions)) {
|
|
9
|
+
throw new LangSmithValidationError("accessDelegation.permissions must be an array of strings", "accessDelegation");
|
|
10
|
+
}
|
|
11
|
+
if (value.mode === "INHERIT") {
|
|
12
|
+
if (permissions !== undefined && permissions.length > 0) {
|
|
13
|
+
throw new LangSmithValidationError('accessDelegation.permissions is not allowed with mode "INHERIT"', "accessDelegation");
|
|
14
|
+
}
|
|
15
|
+
return { mode: "INHERIT" };
|
|
16
|
+
}
|
|
17
|
+
if (permissions === undefined || permissions.length === 0) {
|
|
18
|
+
throw new LangSmithValidationError('accessDelegation.permissions is required with mode "EXPLICIT"', "accessDelegation");
|
|
19
|
+
}
|
|
20
|
+
return { mode: "EXPLICIT", permissions: [...permissions] };
|
|
21
|
+
}
|