langsmith 0.10.4 → 0.10.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.cjs +10 -0
- package/dist/client.d.ts +1 -0
- package/dist/client.js +10 -0
- package/dist/evaluation/_runner.cjs +47 -5
- package/dist/evaluation/_runner.d.ts +12 -0
- package/dist/evaluation/_runner.js +47 -5
- package/dist/index.cjs +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/sandbox/access_delegation.cjs +24 -0
- package/dist/sandbox/access_delegation.d.ts +15 -0
- package/dist/sandbox/access_delegation.js +21 -0
- package/dist/sandbox/client.cjs +70 -5
- package/dist/sandbox/client.d.ts +18 -0
- package/dist/sandbox/client.js +70 -5
- package/dist/sandbox/command_handle.cjs +34 -0
- package/dist/sandbox/command_handle.d.ts +10 -0
- package/dist/sandbox/command_handle.js +34 -0
- package/dist/sandbox/helpers.cjs +129 -0
- package/dist/sandbox/helpers.d.ts +29 -0
- package/dist/sandbox/helpers.js +123 -0
- package/dist/sandbox/index.cjs +7 -1
- package/dist/sandbox/index.d.ts +5 -1
- package/dist/sandbox/index.js +2 -0
- package/dist/sandbox/mounts.cjs +21 -3
- package/dist/sandbox/mounts.d.ts +7 -1
- package/dist/sandbox/mounts.js +21 -3
- package/dist/sandbox/proxy_config.cjs +41 -10
- package/dist/sandbox/proxy_config.d.ts +33 -6
- package/dist/sandbox/proxy_config.js +40 -10
- package/dist/sandbox/sandbox.cjs +160 -4
- package/dist/sandbox/sandbox.d.ts +71 -1
- package/dist/sandbox/sandbox.js +161 -5
- package/dist/sandbox/service_url.cjs +125 -0
- package/dist/sandbox/service_url.d.ts +50 -0
- package/dist/sandbox/service_url.js +120 -0
- package/dist/sandbox/types.d.ts +251 -8
- package/dist/sandbox/ws_execute.cjs +31 -1
- package/dist/sandbox/ws_execute.d.ts +9 -0
- package/dist/sandbox/ws_execute.js +31 -1
- package/dist/utils/error.cjs +17 -22
- package/dist/utils/error.js +17 -22
- package/package.json +1 -1
package/dist/client.cjs
CHANGED
|
@@ -866,6 +866,10 @@ class Client {
|
|
|
866
866
|
throw new Error("Trace batch concurrency must be positive.");
|
|
867
867
|
}
|
|
868
868
|
this.debug = config.debug ?? this.debug;
|
|
869
|
+
if (this.debug) {
|
|
870
|
+
const source = config.apiUrl != null ? "apiUrl option" : defaultConfig.apiUrlSource;
|
|
871
|
+
console.log(`LangSmith API URL ${this.apiUrl} resolved from ${source}`);
|
|
872
|
+
}
|
|
869
873
|
this.fetchImplementation = config.fetchImplementation;
|
|
870
874
|
// Failed trace dump configuration
|
|
871
875
|
this.failedTracesDir =
|
|
@@ -952,12 +956,18 @@ class Client {
|
|
|
952
956
|
const envWorkspaceId = (0, env_js_1.getLangSmithEnvironmentVariable)("WORKSPACE_ID");
|
|
953
957
|
const envAuthSet = (0, profiles_js_1.hasValue)(envApiKey);
|
|
954
958
|
const apiUrl = envApiUrl ?? profileConfig.apiUrl ?? profiles_js_1.DEFAULT_API_URL;
|
|
959
|
+
const apiUrlSource = envApiUrl != null
|
|
960
|
+
? "LANGSMITH_ENDPOINT / LANGCHAIN_ENDPOINT environment variable"
|
|
961
|
+
: profileConfig.apiUrl != null
|
|
962
|
+
? "profile config"
|
|
963
|
+
: "built-in default";
|
|
955
964
|
const workspaceId = envWorkspaceId ?? profileConfig.workspaceId;
|
|
956
965
|
const hideInputs = (0, env_js_1.getLangSmithEnvironmentVariable)("HIDE_INPUTS") === "true";
|
|
957
966
|
const hideOutputs = (0, env_js_1.getLangSmithEnvironmentVariable)("HIDE_OUTPUTS") === "true";
|
|
958
967
|
const hideMetadata = (0, env_js_1.getLangSmithEnvironmentVariable)("HIDE_METADATA") === "true";
|
|
959
968
|
return {
|
|
960
969
|
apiUrl: apiUrl,
|
|
970
|
+
apiUrlSource,
|
|
961
971
|
apiKey: envApiKey,
|
|
962
972
|
webUrl: undefined,
|
|
963
973
|
hideInputs: hideInputs,
|
package/dist/client.d.ts
CHANGED
package/dist/client.js
CHANGED
|
@@ -827,6 +827,10 @@ export class Client {
|
|
|
827
827
|
throw new Error("Trace batch concurrency must be positive.");
|
|
828
828
|
}
|
|
829
829
|
this.debug = config.debug ?? this.debug;
|
|
830
|
+
if (this.debug) {
|
|
831
|
+
const source = config.apiUrl != null ? "apiUrl option" : defaultConfig.apiUrlSource;
|
|
832
|
+
console.log(`LangSmith API URL ${this.apiUrl} resolved from ${source}`);
|
|
833
|
+
}
|
|
830
834
|
this.fetchImplementation = config.fetchImplementation;
|
|
831
835
|
// Failed trace dump configuration
|
|
832
836
|
this.failedTracesDir =
|
|
@@ -913,12 +917,18 @@ export class Client {
|
|
|
913
917
|
const envWorkspaceId = getLangSmithEnvironmentVariable("WORKSPACE_ID");
|
|
914
918
|
const envAuthSet = hasValue(envApiKey);
|
|
915
919
|
const apiUrl = envApiUrl ?? profileConfig.apiUrl ?? DEFAULT_API_URL;
|
|
920
|
+
const apiUrlSource = envApiUrl != null
|
|
921
|
+
? "LANGSMITH_ENDPOINT / LANGCHAIN_ENDPOINT environment variable"
|
|
922
|
+
: profileConfig.apiUrl != null
|
|
923
|
+
? "profile config"
|
|
924
|
+
: "built-in default";
|
|
916
925
|
const workspaceId = envWorkspaceId ?? profileConfig.workspaceId;
|
|
917
926
|
const hideInputs = getLangSmithEnvironmentVariable("HIDE_INPUTS") === "true";
|
|
918
927
|
const hideOutputs = getLangSmithEnvironmentVariable("HIDE_OUTPUTS") === "true";
|
|
919
928
|
const hideMetadata = getLangSmithEnvironmentVariable("HIDE_METADATA") === "true";
|
|
920
929
|
return {
|
|
921
930
|
apiUrl: apiUrl,
|
|
931
|
+
apiUrlSource,
|
|
922
932
|
apiKey: envApiKey,
|
|
923
933
|
webUrl: undefined,
|
|
924
934
|
hideInputs: hideInputs,
|
|
@@ -211,6 +211,12 @@ class _ExperimentManager {
|
|
|
211
211
|
writable: true,
|
|
212
212
|
value: void 0
|
|
213
213
|
});
|
|
214
|
+
Object.defineProperty(this, "_disableEvaluatorTracing", {
|
|
215
|
+
enumerable: true,
|
|
216
|
+
configurable: true,
|
|
217
|
+
writable: true,
|
|
218
|
+
value: void 0
|
|
219
|
+
});
|
|
214
220
|
this.client = args.client ?? new index_js_1.Client();
|
|
215
221
|
if (!args.experiment) {
|
|
216
222
|
this._experimentName = (0, _random_name_js_1.randomName)();
|
|
@@ -248,6 +254,7 @@ class _ExperimentManager {
|
|
|
248
254
|
this._numExamples = args.numExamples;
|
|
249
255
|
this._evaluatorKeys = args.evaluatorKeys;
|
|
250
256
|
this._includeAttachments = args.includeAttachments;
|
|
257
|
+
this._disableEvaluatorTracing = args.disableEvaluatorTracing;
|
|
251
258
|
}
|
|
252
259
|
_getExperiment() {
|
|
253
260
|
if (!this._experiment) {
|
|
@@ -350,6 +357,7 @@ class _ExperimentManager {
|
|
|
350
357
|
numExamples: this._numExamples,
|
|
351
358
|
evaluatorKeys: this._evaluatorKeys,
|
|
352
359
|
includeAttachments: this._includeAttachments,
|
|
360
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
353
361
|
});
|
|
354
362
|
}
|
|
355
363
|
async withPredictions(target, options) {
|
|
@@ -379,6 +387,7 @@ class _ExperimentManager {
|
|
|
379
387
|
numExamples: this._numExamples,
|
|
380
388
|
evaluatorKeys: this._evaluatorKeys,
|
|
381
389
|
includeAttachments: this._includeAttachments,
|
|
390
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
382
391
|
});
|
|
383
392
|
}
|
|
384
393
|
async withEvaluators(evaluators, options) {
|
|
@@ -406,6 +415,7 @@ class _ExperimentManager {
|
|
|
406
415
|
numExamples: this._numExamples,
|
|
407
416
|
evaluatorKeys: this._evaluatorKeys,
|
|
408
417
|
includeAttachments: this._includeAttachments,
|
|
418
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
409
419
|
});
|
|
410
420
|
}
|
|
411
421
|
async withSummaryEvaluators(summaryEvaluators) {
|
|
@@ -424,6 +434,7 @@ class _ExperimentManager {
|
|
|
424
434
|
numExamples: this._numExamples,
|
|
425
435
|
evaluatorKeys: this._evaluatorKeys,
|
|
426
436
|
includeAttachments: this._includeAttachments,
|
|
437
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
427
438
|
});
|
|
428
439
|
}
|
|
429
440
|
async *getResults() {
|
|
@@ -541,9 +552,14 @@ class _ExperimentManager {
|
|
|
541
552
|
: new Date(example.created_at).toISOString(),
|
|
542
553
|
},
|
|
543
554
|
client: fields.client,
|
|
544
|
-
tracingEnabled:
|
|
555
|
+
tracingEnabled: !this._disableEvaluatorTracing,
|
|
545
556
|
};
|
|
546
|
-
const
|
|
557
|
+
const rawEvaluatorResponse = await evaluator.evaluateRun(run, example, options);
|
|
558
|
+
// With tracing off there is no evaluator run for the feedback to point
|
|
559
|
+
// at, so drop the link rather than leaving it dangling.
|
|
560
|
+
const evaluatorResponse = this._disableEvaluatorTracing
|
|
561
|
+
? _withoutSourceRunIds(rawEvaluatorResponse)
|
|
562
|
+
: rawEvaluatorResponse;
|
|
547
563
|
evaluationResults.results.push(...(await fields.client.logEvaluationFeedback({
|
|
548
564
|
evaluatorResponse,
|
|
549
565
|
run,
|
|
@@ -590,7 +606,10 @@ class _ExperimentManager {
|
|
|
590
606
|
experiment: this.experimentName,
|
|
591
607
|
projectId: projectId,
|
|
592
608
|
}));
|
|
593
|
-
const wrappedEvaluators = await wrapSummaryEvaluators(summaryEvaluators, options
|
|
609
|
+
const wrappedEvaluators = await wrapSummaryEvaluators(summaryEvaluators, options,
|
|
610
|
+
// Only override when disabling. Passing `true` here would force tracing
|
|
611
|
+
// on for callers who have it switched off via the environment.
|
|
612
|
+
this._disableEvaluatorTracing ? false : undefined);
|
|
594
613
|
yield async function* (runsArray) {
|
|
595
614
|
const aggregateFeedback = [];
|
|
596
615
|
for (const evaluator of wrappedEvaluators) {
|
|
@@ -777,6 +796,7 @@ async function _evaluate(target, fields) {
|
|
|
777
796
|
numRepetitions: fields.numRepetitions ?? 1,
|
|
778
797
|
evaluatorKeys: _collectEvaluatorKeys(standardFields.evaluators),
|
|
779
798
|
includeAttachments: standardFields.includeAttachments,
|
|
799
|
+
disableEvaluatorTracing: standardFields.disableEvaluatorTracing,
|
|
780
800
|
}).start();
|
|
781
801
|
const targetConcurrency = standardFields.targetConcurrency ?? standardFields.maxConcurrency ?? 0;
|
|
782
802
|
const evaluationConcurrency = standardFields.evaluationConcurrency ?? standardFields.maxConcurrency ?? 0;
|
|
@@ -988,7 +1008,25 @@ function _resolveData(data, options) {
|
|
|
988
1008
|
}
|
|
989
1009
|
return data;
|
|
990
1010
|
}
|
|
991
|
-
|
|
1011
|
+
/**
|
|
1012
|
+
* Strip evaluator-run links from an evaluator response.
|
|
1013
|
+
*
|
|
1014
|
+
* Used when evaluator tracing is disabled: no evaluator run is created, so a
|
|
1015
|
+
* `sourceRunId` on the feedback would point at a run that does not exist.
|
|
1016
|
+
*/
|
|
1017
|
+
function _withoutSourceRunIds(response) {
|
|
1018
|
+
if ("results" in response) {
|
|
1019
|
+
return {
|
|
1020
|
+
...response,
|
|
1021
|
+
results: response.results.map((result) => ({
|
|
1022
|
+
...result,
|
|
1023
|
+
sourceRunId: undefined,
|
|
1024
|
+
})),
|
|
1025
|
+
};
|
|
1026
|
+
}
|
|
1027
|
+
return { ...response, sourceRunId: undefined };
|
|
1028
|
+
}
|
|
1029
|
+
async function wrapSummaryEvaluators(evaluators, optionsArray, tracingEnabled) {
|
|
992
1030
|
async function _wrap(evaluator) {
|
|
993
1031
|
const evalName = evaluator.name || "BatchEvaluator";
|
|
994
1032
|
const wrapperInner = (runs, examples) => {
|
|
@@ -1008,7 +1046,11 @@ async function wrapSummaryEvaluators(evaluators, optionsArray) {
|
|
|
1008
1046
|
}
|
|
1009
1047
|
// Otherwise use the traditional (runs, examples) signature
|
|
1010
1048
|
return Promise.resolve(evaluator(runs, examples));
|
|
1011
|
-
}, {
|
|
1049
|
+
}, {
|
|
1050
|
+
...optionsArray,
|
|
1051
|
+
name: evalName,
|
|
1052
|
+
...(tracingEnabled === undefined ? {} : { tracingEnabled }),
|
|
1053
|
+
});
|
|
1012
1054
|
return Promise.resolve(wrapperSuperInner(`Runs[] (Length=${runs.length})`, `Examples[] (Length=${examples.length})`));
|
|
1013
1055
|
};
|
|
1014
1056
|
return wrapperInner;
|
|
@@ -73,6 +73,7 @@ interface _ExperimentManagerArgs {
|
|
|
73
73
|
_runsArray?: Run[];
|
|
74
74
|
resultRows?: AsyncGenerator<_ExperimentResultRowWithIndex>;
|
|
75
75
|
includeAttachments?: boolean;
|
|
76
|
+
disableEvaluatorTracing?: boolean;
|
|
76
77
|
}
|
|
77
78
|
type BaseEvaluateOptions = {
|
|
78
79
|
/**
|
|
@@ -140,6 +141,16 @@ export interface EvaluateOptions extends BaseEvaluateOptions {
|
|
|
140
141
|
* @default false
|
|
141
142
|
*/
|
|
142
143
|
includeAttachments?: boolean;
|
|
144
|
+
/**
|
|
145
|
+
* Whether to skip tracing evaluator invocations to the `evaluators` project.
|
|
146
|
+
*
|
|
147
|
+
* When `true`, evaluators run without creating evaluator traces. Feedback is
|
|
148
|
+
* still created and attached to the experiment runs, but it won't link back
|
|
149
|
+
* to an evaluator run, which also means it can't be corrected from the UI.
|
|
150
|
+
* Tracing of the target itself is unaffected.
|
|
151
|
+
* @default false
|
|
152
|
+
*/
|
|
153
|
+
disableEvaluatorTracing?: boolean;
|
|
143
154
|
}
|
|
144
155
|
export interface ComparativeEvaluateOptions extends BaseEvaluateOptions {
|
|
145
156
|
/**
|
|
@@ -194,6 +205,7 @@ export declare class _ExperimentManager {
|
|
|
194
205
|
_metadata: KVMap;
|
|
195
206
|
_description?: string;
|
|
196
207
|
_includeAttachments?: boolean;
|
|
208
|
+
_disableEvaluatorTracing?: boolean;
|
|
197
209
|
get experimentName(): string;
|
|
198
210
|
getExamples(): Promise<Array<Example>>;
|
|
199
211
|
setExamples(examples: Example[]): void;
|
|
@@ -203,6 +203,12 @@ export class _ExperimentManager {
|
|
|
203
203
|
writable: true,
|
|
204
204
|
value: void 0
|
|
205
205
|
});
|
|
206
|
+
Object.defineProperty(this, "_disableEvaluatorTracing", {
|
|
207
|
+
enumerable: true,
|
|
208
|
+
configurable: true,
|
|
209
|
+
writable: true,
|
|
210
|
+
value: void 0
|
|
211
|
+
});
|
|
206
212
|
this.client = args.client ?? new Client();
|
|
207
213
|
if (!args.experiment) {
|
|
208
214
|
this._experimentName = randomName();
|
|
@@ -240,6 +246,7 @@ export class _ExperimentManager {
|
|
|
240
246
|
this._numExamples = args.numExamples;
|
|
241
247
|
this._evaluatorKeys = args.evaluatorKeys;
|
|
242
248
|
this._includeAttachments = args.includeAttachments;
|
|
249
|
+
this._disableEvaluatorTracing = args.disableEvaluatorTracing;
|
|
243
250
|
}
|
|
244
251
|
_getExperiment() {
|
|
245
252
|
if (!this._experiment) {
|
|
@@ -342,6 +349,7 @@ export class _ExperimentManager {
|
|
|
342
349
|
numExamples: this._numExamples,
|
|
343
350
|
evaluatorKeys: this._evaluatorKeys,
|
|
344
351
|
includeAttachments: this._includeAttachments,
|
|
352
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
345
353
|
});
|
|
346
354
|
}
|
|
347
355
|
async withPredictions(target, options) {
|
|
@@ -371,6 +379,7 @@ export class _ExperimentManager {
|
|
|
371
379
|
numExamples: this._numExamples,
|
|
372
380
|
evaluatorKeys: this._evaluatorKeys,
|
|
373
381
|
includeAttachments: this._includeAttachments,
|
|
382
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
374
383
|
});
|
|
375
384
|
}
|
|
376
385
|
async withEvaluators(evaluators, options) {
|
|
@@ -398,6 +407,7 @@ export class _ExperimentManager {
|
|
|
398
407
|
numExamples: this._numExamples,
|
|
399
408
|
evaluatorKeys: this._evaluatorKeys,
|
|
400
409
|
includeAttachments: this._includeAttachments,
|
|
410
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
401
411
|
});
|
|
402
412
|
}
|
|
403
413
|
async withSummaryEvaluators(summaryEvaluators) {
|
|
@@ -416,6 +426,7 @@ export class _ExperimentManager {
|
|
|
416
426
|
numExamples: this._numExamples,
|
|
417
427
|
evaluatorKeys: this._evaluatorKeys,
|
|
418
428
|
includeAttachments: this._includeAttachments,
|
|
429
|
+
disableEvaluatorTracing: this._disableEvaluatorTracing,
|
|
419
430
|
});
|
|
420
431
|
}
|
|
421
432
|
async *getResults() {
|
|
@@ -533,9 +544,14 @@ export class _ExperimentManager {
|
|
|
533
544
|
: new Date(example.created_at).toISOString(),
|
|
534
545
|
},
|
|
535
546
|
client: fields.client,
|
|
536
|
-
tracingEnabled:
|
|
547
|
+
tracingEnabled: !this._disableEvaluatorTracing,
|
|
537
548
|
};
|
|
538
|
-
const
|
|
549
|
+
const rawEvaluatorResponse = await evaluator.evaluateRun(run, example, options);
|
|
550
|
+
// With tracing off there is no evaluator run for the feedback to point
|
|
551
|
+
// at, so drop the link rather than leaving it dangling.
|
|
552
|
+
const evaluatorResponse = this._disableEvaluatorTracing
|
|
553
|
+
? _withoutSourceRunIds(rawEvaluatorResponse)
|
|
554
|
+
: rawEvaluatorResponse;
|
|
539
555
|
evaluationResults.results.push(...(await fields.client.logEvaluationFeedback({
|
|
540
556
|
evaluatorResponse,
|
|
541
557
|
run,
|
|
@@ -582,7 +598,10 @@ export class _ExperimentManager {
|
|
|
582
598
|
experiment: this.experimentName,
|
|
583
599
|
projectId: projectId,
|
|
584
600
|
}));
|
|
585
|
-
const wrappedEvaluators = await wrapSummaryEvaluators(summaryEvaluators, options
|
|
601
|
+
const wrappedEvaluators = await wrapSummaryEvaluators(summaryEvaluators, options,
|
|
602
|
+
// Only override when disabling. Passing `true` here would force tracing
|
|
603
|
+
// on for callers who have it switched off via the environment.
|
|
604
|
+
this._disableEvaluatorTracing ? false : undefined);
|
|
586
605
|
yield async function* (runsArray) {
|
|
587
606
|
const aggregateFeedback = [];
|
|
588
607
|
for (const evaluator of wrappedEvaluators) {
|
|
@@ -768,6 +787,7 @@ async function _evaluate(target, fields) {
|
|
|
768
787
|
numRepetitions: fields.numRepetitions ?? 1,
|
|
769
788
|
evaluatorKeys: _collectEvaluatorKeys(standardFields.evaluators),
|
|
770
789
|
includeAttachments: standardFields.includeAttachments,
|
|
790
|
+
disableEvaluatorTracing: standardFields.disableEvaluatorTracing,
|
|
771
791
|
}).start();
|
|
772
792
|
const targetConcurrency = standardFields.targetConcurrency ?? standardFields.maxConcurrency ?? 0;
|
|
773
793
|
const evaluationConcurrency = standardFields.evaluationConcurrency ?? standardFields.maxConcurrency ?? 0;
|
|
@@ -979,7 +999,25 @@ function _resolveData(data, options) {
|
|
|
979
999
|
}
|
|
980
1000
|
return data;
|
|
981
1001
|
}
|
|
982
|
-
|
|
1002
|
+
/**
|
|
1003
|
+
* Strip evaluator-run links from an evaluator response.
|
|
1004
|
+
*
|
|
1005
|
+
* Used when evaluator tracing is disabled: no evaluator run is created, so a
|
|
1006
|
+
* `sourceRunId` on the feedback would point at a run that does not exist.
|
|
1007
|
+
*/
|
|
1008
|
+
function _withoutSourceRunIds(response) {
|
|
1009
|
+
if ("results" in response) {
|
|
1010
|
+
return {
|
|
1011
|
+
...response,
|
|
1012
|
+
results: response.results.map((result) => ({
|
|
1013
|
+
...result,
|
|
1014
|
+
sourceRunId: undefined,
|
|
1015
|
+
})),
|
|
1016
|
+
};
|
|
1017
|
+
}
|
|
1018
|
+
return { ...response, sourceRunId: undefined };
|
|
1019
|
+
}
|
|
1020
|
+
async function wrapSummaryEvaluators(evaluators, optionsArray, tracingEnabled) {
|
|
983
1021
|
async function _wrap(evaluator) {
|
|
984
1022
|
const evalName = evaluator.name || "BatchEvaluator";
|
|
985
1023
|
const wrapperInner = (runs, examples) => {
|
|
@@ -999,7 +1037,11 @@ async function wrapSummaryEvaluators(evaluators, optionsArray) {
|
|
|
999
1037
|
}
|
|
1000
1038
|
// Otherwise use the traditional (runs, examples) signature
|
|
1001
1039
|
return Promise.resolve(evaluator(runs, examples));
|
|
1002
|
-
}, {
|
|
1040
|
+
}, {
|
|
1041
|
+
...optionsArray,
|
|
1042
|
+
name: evalName,
|
|
1043
|
+
...(tracingEnabled === undefined ? {} : { tracingEnabled }),
|
|
1044
|
+
});
|
|
1003
1045
|
return Promise.resolve(wrapperSuperInner(`Runs[] (Length=${runs.length})`, `Examples[] (Length=${examples.length})`));
|
|
1004
1046
|
};
|
|
1005
1047
|
return wrapperInner;
|
package/dist/index.cjs
CHANGED
|
@@ -35,6 +35,6 @@ Object.defineProperty(exports, "UnprocessableEntityError", { enumerable: true, g
|
|
|
35
35
|
Object.defineProperty(exports, "RateLimitError", { enumerable: true, get: function () { return error_js_1.RateLimitError; } });
|
|
36
36
|
Object.defineProperty(exports, "InternalServerError", { enumerable: true, get: function () { return error_js_1.InternalServerError; } });
|
|
37
37
|
// Update using pnpm bump-version
|
|
38
|
-
exports.__version__ = "0.10.
|
|
38
|
+
exports.__version__ = "0.10.5";
|
|
39
39
|
// Metadata key to hide a traced run from LangSmith's Messages View.
|
|
40
40
|
exports.LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
|
package/dist/index.d.ts
CHANGED
|
@@ -7,5 +7,5 @@ export { computeRunIdForSecondaryReplica, uuid7, uuid7FromTime, } from "./uuid.j
|
|
|
7
7
|
export { isTracingEnabled } from "./utils/guard.js";
|
|
8
8
|
export { Cache, PromptCache, type CacheConfig, type CacheMetrics, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
|
|
9
9
|
export { LangsmithError, APIError, APIUserAbortError, APIConnectionError, APIConnectionTimeoutError, BadRequestError, AuthenticationError, PermissionDeniedError, NotFoundError, ConflictError, UnprocessableEntityError, RateLimitError, InternalServerError, } from "./_openapi_client/core/error.js";
|
|
10
|
-
export declare const __version__ = "0.10.
|
|
10
|
+
export declare const __version__ = "0.10.5";
|
|
11
11
|
export declare const LS_MESSAGE_VIEW_EXCLUDE: "ls_message_view_exclude";
|
package/dist/index.js
CHANGED
|
@@ -7,6 +7,6 @@ export { isTracingEnabled } from "./utils/guard.js";
|
|
|
7
7
|
export { Cache, PromptCache, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
|
|
8
8
|
export { LangsmithError, APIError, APIUserAbortError, APIConnectionError, APIConnectionTimeoutError, BadRequestError, AuthenticationError, PermissionDeniedError, NotFoundError, ConflictError, UnprocessableEntityError, RateLimitError, InternalServerError, } from "./_openapi_client/core/error.js";
|
|
9
9
|
// Update using pnpm bump-version
|
|
10
|
-
export const __version__ = "0.10.
|
|
10
|
+
export const __version__ = "0.10.5";
|
|
11
11
|
// Metadata key to hide a traced run from LangSmith's Messages View.
|
|
12
12
|
export const LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.validateAccessDelegation = validateAccessDelegation;
|
|
4
|
+
const errors_js_1 = require("./errors.cjs");
|
|
5
|
+
/** Rejects grants the server would refuse, without the round trip. */
|
|
6
|
+
function validateAccessDelegation(value) {
|
|
7
|
+
if (value.mode !== "INHERIT" && value.mode !== "EXPLICIT") {
|
|
8
|
+
throw new errors_js_1.LangSmithValidationError('accessDelegation.mode must be "INHERIT" or "EXPLICIT"', "accessDelegation");
|
|
9
|
+
}
|
|
10
|
+
const { permissions } = value;
|
|
11
|
+
if (permissions !== undefined && !Array.isArray(permissions)) {
|
|
12
|
+
throw new errors_js_1.LangSmithValidationError("accessDelegation.permissions must be an array of strings", "accessDelegation");
|
|
13
|
+
}
|
|
14
|
+
if (value.mode === "INHERIT") {
|
|
15
|
+
if (permissions !== undefined && permissions.length > 0) {
|
|
16
|
+
throw new errors_js_1.LangSmithValidationError('accessDelegation.permissions is not allowed with mode "INHERIT"', "accessDelegation");
|
|
17
|
+
}
|
|
18
|
+
return { mode: "INHERIT" };
|
|
19
|
+
}
|
|
20
|
+
if (permissions === undefined || permissions.length === 0) {
|
|
21
|
+
throw new errors_js_1.LangSmithValidationError('accessDelegation.permissions is required with mode "EXPLICIT"', "accessDelegation");
|
|
22
|
+
}
|
|
23
|
+
return { mode: "EXPLICIT", permissions: [...permissions] };
|
|
24
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** How a delegation grant derives its permission set. */
|
|
2
|
+
export type AccessDelegationMode = "INHERIT" | "EXPLICIT";
|
|
3
|
+
/**
|
|
4
|
+
* Grant letting code inside a sandbox call the LangSmith API as the creator.
|
|
5
|
+
*
|
|
6
|
+
* `INHERIT` tracks everything the creator can do; `EXPLICIT` is capped to
|
|
7
|
+
* `permissions`. Permissions are a ceiling re-checked on every request rather
|
|
8
|
+
* than a snapshot, so access the creator loses is lost here too.
|
|
9
|
+
*/
|
|
10
|
+
export interface AccessDelegation {
|
|
11
|
+
mode: AccessDelegationMode;
|
|
12
|
+
permissions?: string[];
|
|
13
|
+
}
|
|
14
|
+
/** Rejects grants the server would refuse, without the round trip. */
|
|
15
|
+
export declare function validateAccessDelegation(value: AccessDelegation): AccessDelegation;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { LangSmithValidationError } from "./errors.js";
|
|
2
|
+
/** Rejects grants the server would refuse, without the round trip. */
|
|
3
|
+
export function validateAccessDelegation(value) {
|
|
4
|
+
if (value.mode !== "INHERIT" && value.mode !== "EXPLICIT") {
|
|
5
|
+
throw new LangSmithValidationError('accessDelegation.mode must be "INHERIT" or "EXPLICIT"', "accessDelegation");
|
|
6
|
+
}
|
|
7
|
+
const { permissions } = value;
|
|
8
|
+
if (permissions !== undefined && !Array.isArray(permissions)) {
|
|
9
|
+
throw new LangSmithValidationError("accessDelegation.permissions must be an array of strings", "accessDelegation");
|
|
10
|
+
}
|
|
11
|
+
if (value.mode === "INHERIT") {
|
|
12
|
+
if (permissions !== undefined && permissions.length > 0) {
|
|
13
|
+
throw new LangSmithValidationError('accessDelegation.permissions is not allowed with mode "INHERIT"', "accessDelegation");
|
|
14
|
+
}
|
|
15
|
+
return { mode: "INHERIT" };
|
|
16
|
+
}
|
|
17
|
+
if (permissions === undefined || permissions.length === 0) {
|
|
18
|
+
throw new LangSmithValidationError('accessDelegation.permissions is required with mode "EXPLICIT"', "accessDelegation");
|
|
19
|
+
}
|
|
20
|
+
return { mode: "EXPLICIT", permissions: [...permissions] };
|
|
21
|
+
}
|
package/dist/sandbox/client.cjs
CHANGED
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
*/
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.SandboxClient = void 0;
|
|
7
|
+
const access_delegation_js_1 = require("./access_delegation.cjs");
|
|
8
|
+
const service_url_js_1 = require("./service_url.cjs");
|
|
7
9
|
const env_js_1 = require("../utils/env.cjs");
|
|
8
10
|
const fetch_js_1 = require("../singletons/fetch.cjs");
|
|
9
11
|
const async_caller_js_1 = require("../utils/async_caller.cjs");
|
|
@@ -407,6 +409,53 @@ class SandboxClient {
|
|
|
407
409
|
const suffix = segments.length ? `/${segments.join("/")}` : "";
|
|
408
410
|
return `${this._baseUrl}/boxes/${encodeURIComponent(name)}${suffix}`;
|
|
409
411
|
}
|
|
412
|
+
/**
|
|
413
|
+
* Generate an authenticated URL for an HTTP service running in a sandbox.
|
|
414
|
+
*
|
|
415
|
+
* Without `access`, mints a short-lived token and returns a {@link ServiceUrl}
|
|
416
|
+
* that refreshes it as it nears expiry. With `access` set to `"restricted"`
|
|
417
|
+
* or `"workspace"` the URL is gated by LangSmith login instead and a
|
|
418
|
+
* {@link ServiceLoginUrl} is returned: no token, no expiry, browser only.
|
|
419
|
+
*
|
|
420
|
+
* A login grant is durable, so token mode is refused with 409 while one is
|
|
421
|
+
* in place, and `expiresInSeconds` does not apply to the login modes.
|
|
422
|
+
*/
|
|
423
|
+
async serviceUrl(name, options) {
|
|
424
|
+
const { port, expiresInSeconds, access, signal } = options;
|
|
425
|
+
if (!Number.isInteger(port) || port < 1 || port > 65535) {
|
|
426
|
+
throw new errors_js_1.LangSmithValidationError(`port must be between 1 and 65535 (got ${port})`, "port");
|
|
427
|
+
}
|
|
428
|
+
if (access !== undefined &&
|
|
429
|
+
access !== "restricted" &&
|
|
430
|
+
access !== "workspace") {
|
|
431
|
+
throw new errors_js_1.LangSmithValidationError(`access must be "restricted" or "workspace" (got ${String(access)})`, "access");
|
|
432
|
+
}
|
|
433
|
+
const loginMode = access !== undefined;
|
|
434
|
+
if (expiresInSeconds !== undefined && loginMode) {
|
|
435
|
+
throw new errors_js_1.LangSmithValidationError(`expiresInSeconds does not apply to access "${access}": a LangSmith ` +
|
|
436
|
+
"login URL carries no token and does not expire", "expiresInSeconds");
|
|
437
|
+
}
|
|
438
|
+
const url = this._boxUrl(name, "service-url");
|
|
439
|
+
const payload = { port };
|
|
440
|
+
if (!loginMode && expiresInSeconds !== undefined) {
|
|
441
|
+
payload.expires_in_seconds = expiresInSeconds;
|
|
442
|
+
}
|
|
443
|
+
if (access !== undefined) {
|
|
444
|
+
payload.access = access;
|
|
445
|
+
}
|
|
446
|
+
const response = await this._postJson(url, payload, { signal });
|
|
447
|
+
const data = await response.json();
|
|
448
|
+
if (loginMode) {
|
|
449
|
+
return new service_url_js_1.ServiceLoginUrl(data);
|
|
450
|
+
}
|
|
451
|
+
// Refresh with the token parameters only. Carrying options.signal over
|
|
452
|
+
// would tie every later refresh to the first request's lifetime, so a
|
|
453
|
+
// caller-supplied timeout signal would leave the URL unable to refresh.
|
|
454
|
+
return new service_url_js_1.ServiceUrl(data, () => this.serviceUrl(name, {
|
|
455
|
+
port,
|
|
456
|
+
expiresInSeconds,
|
|
457
|
+
}));
|
|
458
|
+
}
|
|
410
459
|
async _postJson(url, body, options) {
|
|
411
460
|
const response = await this._fetch(url, {
|
|
412
461
|
method: "POST",
|
|
@@ -424,7 +473,7 @@ class SandboxClient {
|
|
|
424
473
|
const resolvedOptions = typeof snapshotIdOrOptions === "object" && snapshotIdOrOptions !== null
|
|
425
474
|
? snapshotIdOrOptions
|
|
426
475
|
: options;
|
|
427
|
-
const { snapshotName, name, timeout = 30, waitForReady = true, idleTtlSeconds, deleteAfterStopSeconds, vCpus, memBytes, fsCapacityBytes, mountConfig, proxyConfig, } = resolvedOptions;
|
|
476
|
+
const { snapshotName, name, timeout = 30, waitForReady = true, idleTtlSeconds, deleteAfterStopSeconds, vCpus, memBytes, fsCapacityBytes, mountConfig, proxyConfig, runConfig, accessDelegation, } = resolvedOptions;
|
|
428
477
|
if (snapshotId && snapshotName) {
|
|
429
478
|
throw new errors_js_1.LangSmithValidationError("At most one of snapshotId or options.snapshotName may be set", "snapshotId");
|
|
430
479
|
}
|
|
@@ -468,6 +517,12 @@ class SandboxClient {
|
|
|
468
517
|
if (proxyConfig !== undefined) {
|
|
469
518
|
payload.proxy_config = proxyConfig;
|
|
470
519
|
}
|
|
520
|
+
if (runConfig !== undefined) {
|
|
521
|
+
payload.run_config = runConfig;
|
|
522
|
+
}
|
|
523
|
+
if (accessDelegation !== undefined) {
|
|
524
|
+
payload.access_delegation = (0, access_delegation_js_1.validateAccessDelegation)(accessDelegation);
|
|
525
|
+
}
|
|
471
526
|
const httpTimeout = waitForReady ? (timeout + 30) * 1000 : 30 * 1000;
|
|
472
527
|
const response = await this._fetch(url, {
|
|
473
528
|
method: "POST",
|
|
@@ -523,13 +578,14 @@ class SandboxClient {
|
|
|
523
578
|
const options = typeof newNameOrOptions === "string"
|
|
524
579
|
? { newName: newNameOrOptions }
|
|
525
580
|
: newNameOrOptions;
|
|
526
|
-
const { newName, idleTtlSeconds, deleteAfterStopSeconds, proxyConfig } = options;
|
|
581
|
+
const { newName, idleTtlSeconds, deleteAfterStopSeconds, proxyConfig, runConfig, } = options;
|
|
527
582
|
(0, helpers_js_1.validateTtl)(idleTtlSeconds, "idleTtlSeconds");
|
|
528
583
|
(0, helpers_js_1.validateTtl)(deleteAfterStopSeconds, "deleteAfterStopSeconds");
|
|
529
584
|
if (newName === undefined &&
|
|
530
585
|
idleTtlSeconds === undefined &&
|
|
531
586
|
deleteAfterStopSeconds === undefined &&
|
|
532
|
-
proxyConfig === undefined
|
|
587
|
+
proxyConfig === undefined &&
|
|
588
|
+
runConfig === undefined) {
|
|
533
589
|
return this.getSandbox(name);
|
|
534
590
|
}
|
|
535
591
|
const url = this._boxUrl(name);
|
|
@@ -546,6 +602,9 @@ class SandboxClient {
|
|
|
546
602
|
if (proxyConfig !== undefined) {
|
|
547
603
|
payload.proxy_config = proxyConfig;
|
|
548
604
|
}
|
|
605
|
+
if (runConfig !== undefined) {
|
|
606
|
+
payload.run_config = runConfig;
|
|
607
|
+
}
|
|
549
608
|
const response = await this._fetch(url, {
|
|
550
609
|
method: "PATCH",
|
|
551
610
|
headers: { "Content-Type": "application/json" },
|
|
@@ -730,7 +789,7 @@ class SandboxClient {
|
|
|
730
789
|
* @returns Snapshot in "ready" status.
|
|
731
790
|
*/
|
|
732
791
|
async createSnapshot(name, dockerImage, fsCapacityBytes, options = {}) {
|
|
733
|
-
const { registryId, timeout = 60, signal } = options;
|
|
792
|
+
const { registryId, runConfig, timeout = 60, signal } = options;
|
|
734
793
|
const url = `${this._baseUrl}/snapshots`;
|
|
735
794
|
const payload = {
|
|
736
795
|
name,
|
|
@@ -740,6 +799,9 @@ class SandboxClient {
|
|
|
740
799
|
if (registryId !== undefined) {
|
|
741
800
|
payload.registry_id = registryId;
|
|
742
801
|
}
|
|
802
|
+
if (runConfig !== undefined) {
|
|
803
|
+
payload.run_config = runConfig;
|
|
804
|
+
}
|
|
743
805
|
const response = await this._postJson(url, payload, { signal });
|
|
744
806
|
const snapshot = (await response.json());
|
|
745
807
|
return this.waitForSnapshot(snapshot.id, { timeout, signal });
|
|
@@ -832,7 +894,7 @@ class SandboxClient {
|
|
|
832
894
|
* @returns Snapshot in "ready" status.
|
|
833
895
|
*/
|
|
834
896
|
async captureSnapshot(sandboxName, name, options = {}) {
|
|
835
|
-
const { dockerImage, fsCapacityBytes, timeout = 60, signal } = options;
|
|
897
|
+
const { dockerImage, fsCapacityBytes, runConfig, timeout = 60, signal, } = options;
|
|
836
898
|
const url = this._boxUrl(sandboxName, "snapshot");
|
|
837
899
|
const payload = { name };
|
|
838
900
|
if (dockerImage !== undefined) {
|
|
@@ -841,6 +903,9 @@ class SandboxClient {
|
|
|
841
903
|
if (fsCapacityBytes !== undefined) {
|
|
842
904
|
payload.fs_capacity_bytes = fsCapacityBytes;
|
|
843
905
|
}
|
|
906
|
+
if (runConfig !== undefined) {
|
|
907
|
+
payload.run_config = runConfig;
|
|
908
|
+
}
|
|
844
909
|
const response = await this._postJson(url, payload, { signal });
|
|
845
910
|
const snapshot = (await response.json());
|
|
846
911
|
return this.waitForSnapshot(snapshot.id, { timeout, signal });
|
package/dist/sandbox/client.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Main SandboxClient class for interacting with the sandbox server API.
|
|
3
3
|
*/
|
|
4
|
+
import { ServiceLoginUrl, ServiceUrl, type ServiceAccess } from "./service_url.js";
|
|
4
5
|
import { Registries } from "../_openapi_client/resources/sandboxes/registries.js";
|
|
5
6
|
import type { CaptureSnapshotOptions, CreateDockerfileSnapshotOptions, CreateSandboxOptions, CreateSnapshotOptions, DownloadURL, GenerateDownloadURLOptions, ListSnapshotsOptions, ResourceStatus, SandboxClientConfig, Snapshot, StartSandboxOptions, UpdateSandboxOptions, WaitForSandboxOptions, WaitForSnapshotOptions } from "./types.js";
|
|
6
7
|
import { Sandbox } from "./sandbox.js";
|
|
@@ -64,6 +65,23 @@ export declare class SandboxClient {
|
|
|
64
65
|
* ```
|
|
65
66
|
*/
|
|
66
67
|
get registries(): Registries;
|
|
68
|
+
/**
|
|
69
|
+
* Generate an authenticated URL for an HTTP service running in a sandbox.
|
|
70
|
+
*
|
|
71
|
+
* Without `access`, mints a short-lived token and returns a {@link ServiceUrl}
|
|
72
|
+
* that refreshes it as it nears expiry. With `access` set to `"restricted"`
|
|
73
|
+
* or `"workspace"` the URL is gated by LangSmith login instead and a
|
|
74
|
+
* {@link ServiceLoginUrl} is returned: no token, no expiry, browser only.
|
|
75
|
+
*
|
|
76
|
+
* A login grant is durable, so token mode is refused with 409 while one is
|
|
77
|
+
* in place, and `expiresInSeconds` does not apply to the login modes.
|
|
78
|
+
*/
|
|
79
|
+
serviceUrl(name: string, options: {
|
|
80
|
+
port: number;
|
|
81
|
+
expiresInSeconds?: number;
|
|
82
|
+
access?: ServiceAccess;
|
|
83
|
+
signal?: AbortSignal;
|
|
84
|
+
}): Promise<ServiceUrl | ServiceLoginUrl>;
|
|
67
85
|
private _postJson;
|
|
68
86
|
/**
|
|
69
87
|
* Create a new Sandbox.
|