langsmith 0.10.3 → 0.10.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/client.cjs +19 -4
  2. package/dist/client.d.ts +2 -1
  3. package/dist/client.js +19 -4
  4. package/dist/evaluation/_runner.cjs +47 -5
  5. package/dist/evaluation/_runner.d.ts +12 -0
  6. package/dist/evaluation/_runner.js +47 -5
  7. package/dist/index.cjs +1 -1
  8. package/dist/index.d.ts +1 -1
  9. package/dist/index.js +1 -1
  10. package/dist/sandbox/access_delegation.cjs +24 -0
  11. package/dist/sandbox/access_delegation.d.ts +15 -0
  12. package/dist/sandbox/access_delegation.js +21 -0
  13. package/dist/sandbox/client.cjs +70 -5
  14. package/dist/sandbox/client.d.ts +18 -0
  15. package/dist/sandbox/client.js +70 -5
  16. package/dist/sandbox/command_handle.cjs +34 -0
  17. package/dist/sandbox/command_handle.d.ts +10 -0
  18. package/dist/sandbox/command_handle.js +34 -0
  19. package/dist/sandbox/helpers.cjs +129 -0
  20. package/dist/sandbox/helpers.d.ts +29 -0
  21. package/dist/sandbox/helpers.js +123 -0
  22. package/dist/sandbox/index.cjs +7 -1
  23. package/dist/sandbox/index.d.ts +5 -1
  24. package/dist/sandbox/index.js +2 -0
  25. package/dist/sandbox/mounts.cjs +21 -3
  26. package/dist/sandbox/mounts.d.ts +7 -1
  27. package/dist/sandbox/mounts.js +21 -3
  28. package/dist/sandbox/proxy_config.cjs +41 -10
  29. package/dist/sandbox/proxy_config.d.ts +33 -6
  30. package/dist/sandbox/proxy_config.js +40 -10
  31. package/dist/sandbox/sandbox.cjs +160 -4
  32. package/dist/sandbox/sandbox.d.ts +71 -1
  33. package/dist/sandbox/sandbox.js +161 -5
  34. package/dist/sandbox/service_url.cjs +125 -0
  35. package/dist/sandbox/service_url.d.ts +50 -0
  36. package/dist/sandbox/service_url.js +120 -0
  37. package/dist/sandbox/types.d.ts +251 -8
  38. package/dist/sandbox/ws_execute.cjs +31 -1
  39. package/dist/sandbox/ws_execute.d.ts +9 -0
  40. package/dist/sandbox/ws_execute.js +31 -1
  41. package/dist/utils/error.cjs +17 -22
  42. package/dist/utils/error.js +17 -22
  43. package/dist/wrappers/openai_agents.cjs +37 -28
  44. package/dist/wrappers/openai_agents.d.ts +0 -1
  45. package/dist/wrappers/openai_agents.js +37 -28
  46. package/package.json +2 -2
package/dist/client.cjs CHANGED
@@ -82,7 +82,7 @@ function _normalizeRunTimestamps(run) {
82
82
  end_time: _ensureUTCTimestamp(run.end_time),
83
83
  };
84
84
  }
85
- function mergeRuntimeEnvIntoRun(run, cachedEnvVars, omitTracedRuntimeInfo) {
85
+ function mergeRuntimeEnvIntoRun(run, cachedEnvVars, omitTracedRuntimeInfo, tracingSampleRate) {
86
86
  if (omitTracedRuntimeInfo) {
87
87
  return run;
88
88
  }
@@ -105,6 +105,9 @@ function mergeRuntimeEnvIntoRun(run, cachedEnvVars, omitTracedRuntimeInfo) {
105
105
  }
106
106
  : {}),
107
107
  ...metadata,
108
+ ...(tracingSampleRate !== undefined
109
+ ? { ls_tracing_sample_rate: tracingSampleRate }
110
+ : {}),
108
111
  },
109
112
  };
110
113
  return run;
@@ -863,6 +866,10 @@ class Client {
863
866
  throw new Error("Trace batch concurrency must be positive.");
864
867
  }
865
868
  this.debug = config.debug ?? this.debug;
869
+ if (this.debug) {
870
+ const source = config.apiUrl != null ? "apiUrl option" : defaultConfig.apiUrlSource;
871
+ console.log(`LangSmith API URL ${this.apiUrl} resolved from ${source}`);
872
+ }
866
873
  this.fetchImplementation = config.fetchImplementation;
867
874
  // Failed trace dump configuration
868
875
  this.failedTracesDir =
@@ -949,12 +956,18 @@ class Client {
949
956
  const envWorkspaceId = (0, env_js_1.getLangSmithEnvironmentVariable)("WORKSPACE_ID");
950
957
  const envAuthSet = (0, profiles_js_1.hasValue)(envApiKey);
951
958
  const apiUrl = envApiUrl ?? profileConfig.apiUrl ?? profiles_js_1.DEFAULT_API_URL;
959
+ const apiUrlSource = envApiUrl != null
960
+ ? "LANGSMITH_ENDPOINT / LANGCHAIN_ENDPOINT environment variable"
961
+ : profileConfig.apiUrl != null
962
+ ? "profile config"
963
+ : "built-in default";
952
964
  const workspaceId = envWorkspaceId ?? profileConfig.workspaceId;
953
965
  const hideInputs = (0, env_js_1.getLangSmithEnvironmentVariable)("HIDE_INPUTS") === "true";
954
966
  const hideOutputs = (0, env_js_1.getLangSmithEnvironmentVariable)("HIDE_OUTPUTS") === "true";
955
967
  const hideMetadata = (0, env_js_1.getLangSmithEnvironmentVariable)("HIDE_METADATA") === "true";
956
968
  return {
957
969
  apiUrl: apiUrl,
970
+ apiUrlSource,
958
971
  apiKey: envApiKey,
959
972
  webUrl: undefined,
960
973
  hideInputs: hideInputs,
@@ -1617,7 +1630,7 @@ class Client {
1617
1630
  };
1618
1631
  }
1619
1632
  async _mergeRuntimeEnvAndMaskMetadata(run) {
1620
- const merged = mergeRuntimeEnvIntoRun(run, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo);
1633
+ const merged = mergeRuntimeEnvIntoRun(run, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo, this.tracingSampleRate);
1621
1634
  if (this.omitTracedRuntimeInfo) {
1622
1635
  return merged;
1623
1636
  }
@@ -1626,7 +1639,7 @@ class Client {
1626
1639
  async processRunOperation(item) {
1627
1640
  clearTimeout(this.autoBatchTimeout);
1628
1641
  this.autoBatchTimeout = undefined;
1629
- item.item = mergeRuntimeEnvIntoRun(item.item, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo);
1642
+ item.item = mergeRuntimeEnvIntoRun(item.item, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo, this.tracingSampleRate);
1630
1643
  const itemPromise = this.autoBatchQueue.push(item);
1631
1644
  if (this.manualFlushMode) {
1632
1645
  // Rely on manual flushing in serverless environments
@@ -2227,7 +2240,9 @@ class Client {
2227
2240
  if (options?.workspaceId !== undefined) {
2228
2241
  headers["x-tenant-id"] = options.workspaceId;
2229
2242
  }
2230
- const body = (0, index_js_4.serialize)(run, `Serializing payload to update run with id: ${runId}`);
2243
+ const body = (0, index_js_4.serialize)(run.extra
2244
+ ? mergeRuntimeEnvIntoRun(run, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo, this.tracingSampleRate)
2245
+ : run, `Serializing payload to update run with id: ${runId}`);
2231
2246
  await this.caller.call(async () => {
2232
2247
  const res = await this._fetch(`${options?.apiUrl ?? this.apiUrl}/runs/${runId}`, {
2233
2248
  method: "PATCH",
package/dist/client.d.ts CHANGED
@@ -300,6 +300,7 @@ interface UploadCSVParams {
300
300
  }
301
301
  type DefaultClientConfig = {
302
302
  apiUrl: string;
303
+ apiUrlSource: string;
303
304
  apiKey?: string;
304
305
  webUrl?: string;
305
306
  hideInputs?: boolean;
@@ -445,7 +446,7 @@ type Thread = {
445
446
  export interface ListThreadsItem extends Thread {
446
447
  runs: Run[];
447
448
  }
448
- export declare function mergeRuntimeEnvIntoRun<T extends RunCreate | RunUpdate>(run: T, cachedEnvVars?: Record<string, string>, omitTracedRuntimeInfo?: boolean): T;
449
+ export declare function mergeRuntimeEnvIntoRun<T extends RunCreate | RunUpdate>(run: T, cachedEnvVars?: Record<string, string>, omitTracedRuntimeInfo?: boolean, tracingSampleRate?: number): T;
449
450
  export declare function _checkBackendVersion(backendVersion: string, minVersion: string): void;
450
451
  export declare const DEFAULT_UNCOMPRESSED_BATCH_SIZE_LIMIT_BYTES: number;
451
452
  /** Default maximum memory (1GB) for queue size limits. */
package/dist/client.js CHANGED
@@ -44,7 +44,7 @@ function _normalizeRunTimestamps(run) {
44
44
  end_time: _ensureUTCTimestamp(run.end_time),
45
45
  };
46
46
  }
47
- export function mergeRuntimeEnvIntoRun(run, cachedEnvVars, omitTracedRuntimeInfo) {
47
+ export function mergeRuntimeEnvIntoRun(run, cachedEnvVars, omitTracedRuntimeInfo, tracingSampleRate) {
48
48
  if (omitTracedRuntimeInfo) {
49
49
  return run;
50
50
  }
@@ -67,6 +67,9 @@ export function mergeRuntimeEnvIntoRun(run, cachedEnvVars, omitTracedRuntimeInfo
67
67
  }
68
68
  : {}),
69
69
  ...metadata,
70
+ ...(tracingSampleRate !== undefined
71
+ ? { ls_tracing_sample_rate: tracingSampleRate }
72
+ : {}),
70
73
  },
71
74
  };
72
75
  return run;
@@ -824,6 +827,10 @@ export class Client {
824
827
  throw new Error("Trace batch concurrency must be positive.");
825
828
  }
826
829
  this.debug = config.debug ?? this.debug;
830
+ if (this.debug) {
831
+ const source = config.apiUrl != null ? "apiUrl option" : defaultConfig.apiUrlSource;
832
+ console.log(`LangSmith API URL ${this.apiUrl} resolved from ${source}`);
833
+ }
827
834
  this.fetchImplementation = config.fetchImplementation;
828
835
  // Failed trace dump configuration
829
836
  this.failedTracesDir =
@@ -910,12 +917,18 @@ export class Client {
910
917
  const envWorkspaceId = getLangSmithEnvironmentVariable("WORKSPACE_ID");
911
918
  const envAuthSet = hasValue(envApiKey);
912
919
  const apiUrl = envApiUrl ?? profileConfig.apiUrl ?? DEFAULT_API_URL;
920
+ const apiUrlSource = envApiUrl != null
921
+ ? "LANGSMITH_ENDPOINT / LANGCHAIN_ENDPOINT environment variable"
922
+ : profileConfig.apiUrl != null
923
+ ? "profile config"
924
+ : "built-in default";
913
925
  const workspaceId = envWorkspaceId ?? profileConfig.workspaceId;
914
926
  const hideInputs = getLangSmithEnvironmentVariable("HIDE_INPUTS") === "true";
915
927
  const hideOutputs = getLangSmithEnvironmentVariable("HIDE_OUTPUTS") === "true";
916
928
  const hideMetadata = getLangSmithEnvironmentVariable("HIDE_METADATA") === "true";
917
929
  return {
918
930
  apiUrl: apiUrl,
931
+ apiUrlSource,
919
932
  apiKey: envApiKey,
920
933
  webUrl: undefined,
921
934
  hideInputs: hideInputs,
@@ -1578,7 +1591,7 @@ export class Client {
1578
1591
  };
1579
1592
  }
1580
1593
  async _mergeRuntimeEnvAndMaskMetadata(run) {
1581
- const merged = mergeRuntimeEnvIntoRun(run, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo);
1594
+ const merged = mergeRuntimeEnvIntoRun(run, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo, this.tracingSampleRate);
1582
1595
  if (this.omitTracedRuntimeInfo) {
1583
1596
  return merged;
1584
1597
  }
@@ -1587,7 +1600,7 @@ export class Client {
1587
1600
  async processRunOperation(item) {
1588
1601
  clearTimeout(this.autoBatchTimeout);
1589
1602
  this.autoBatchTimeout = undefined;
1590
- item.item = mergeRuntimeEnvIntoRun(item.item, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo);
1603
+ item.item = mergeRuntimeEnvIntoRun(item.item, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo, this.tracingSampleRate);
1591
1604
  const itemPromise = this.autoBatchQueue.push(item);
1592
1605
  if (this.manualFlushMode) {
1593
1606
  // Rely on manual flushing in serverless environments
@@ -2188,7 +2201,9 @@ export class Client {
2188
2201
  if (options?.workspaceId !== undefined) {
2189
2202
  headers["x-tenant-id"] = options.workspaceId;
2190
2203
  }
2191
- const body = serializePayloadForTracing(run, `Serializing payload to update run with id: ${runId}`);
2204
+ const body = serializePayloadForTracing(run.extra
2205
+ ? mergeRuntimeEnvIntoRun(run, this.cachedLSEnvVarsForMetadata, this.omitTracedRuntimeInfo, this.tracingSampleRate)
2206
+ : run, `Serializing payload to update run with id: ${runId}`);
2192
2207
  await this.caller.call(async () => {
2193
2208
  const res = await this._fetch(`${options?.apiUrl ?? this.apiUrl}/runs/${runId}`, {
2194
2209
  method: "PATCH",
@@ -211,6 +211,12 @@ class _ExperimentManager {
211
211
  writable: true,
212
212
  value: void 0
213
213
  });
214
+ Object.defineProperty(this, "_disableEvaluatorTracing", {
215
+ enumerable: true,
216
+ configurable: true,
217
+ writable: true,
218
+ value: void 0
219
+ });
214
220
  this.client = args.client ?? new index_js_1.Client();
215
221
  if (!args.experiment) {
216
222
  this._experimentName = (0, _random_name_js_1.randomName)();
@@ -248,6 +254,7 @@ class _ExperimentManager {
248
254
  this._numExamples = args.numExamples;
249
255
  this._evaluatorKeys = args.evaluatorKeys;
250
256
  this._includeAttachments = args.includeAttachments;
257
+ this._disableEvaluatorTracing = args.disableEvaluatorTracing;
251
258
  }
252
259
  _getExperiment() {
253
260
  if (!this._experiment) {
@@ -350,6 +357,7 @@ class _ExperimentManager {
350
357
  numExamples: this._numExamples,
351
358
  evaluatorKeys: this._evaluatorKeys,
352
359
  includeAttachments: this._includeAttachments,
360
+ disableEvaluatorTracing: this._disableEvaluatorTracing,
353
361
  });
354
362
  }
355
363
  async withPredictions(target, options) {
@@ -379,6 +387,7 @@ class _ExperimentManager {
379
387
  numExamples: this._numExamples,
380
388
  evaluatorKeys: this._evaluatorKeys,
381
389
  includeAttachments: this._includeAttachments,
390
+ disableEvaluatorTracing: this._disableEvaluatorTracing,
382
391
  });
383
392
  }
384
393
  async withEvaluators(evaluators, options) {
@@ -406,6 +415,7 @@ class _ExperimentManager {
406
415
  numExamples: this._numExamples,
407
416
  evaluatorKeys: this._evaluatorKeys,
408
417
  includeAttachments: this._includeAttachments,
418
+ disableEvaluatorTracing: this._disableEvaluatorTracing,
409
419
  });
410
420
  }
411
421
  async withSummaryEvaluators(summaryEvaluators) {
@@ -424,6 +434,7 @@ class _ExperimentManager {
424
434
  numExamples: this._numExamples,
425
435
  evaluatorKeys: this._evaluatorKeys,
426
436
  includeAttachments: this._includeAttachments,
437
+ disableEvaluatorTracing: this._disableEvaluatorTracing,
427
438
  });
428
439
  }
429
440
  async *getResults() {
@@ -541,9 +552,14 @@ class _ExperimentManager {
541
552
  : new Date(example.created_at).toISOString(),
542
553
  },
543
554
  client: fields.client,
544
- tracingEnabled: true,
555
+ tracingEnabled: !this._disableEvaluatorTracing,
545
556
  };
546
- const evaluatorResponse = await evaluator.evaluateRun(run, example, options);
557
+ const rawEvaluatorResponse = await evaluator.evaluateRun(run, example, options);
558
+ // With tracing off there is no evaluator run for the feedback to point
559
+ // at, so drop the link rather than leaving it dangling.
560
+ const evaluatorResponse = this._disableEvaluatorTracing
561
+ ? _withoutSourceRunIds(rawEvaluatorResponse)
562
+ : rawEvaluatorResponse;
547
563
  evaluationResults.results.push(...(await fields.client.logEvaluationFeedback({
548
564
  evaluatorResponse,
549
565
  run,
@@ -590,7 +606,10 @@ class _ExperimentManager {
590
606
  experiment: this.experimentName,
591
607
  projectId: projectId,
592
608
  }));
593
- const wrappedEvaluators = await wrapSummaryEvaluators(summaryEvaluators, options);
609
+ const wrappedEvaluators = await wrapSummaryEvaluators(summaryEvaluators, options,
610
+ // Only override when disabling. Passing `true` here would force tracing
611
+ // on for callers who have it switched off via the environment.
612
+ this._disableEvaluatorTracing ? false : undefined);
594
613
  yield async function* (runsArray) {
595
614
  const aggregateFeedback = [];
596
615
  for (const evaluator of wrappedEvaluators) {
@@ -777,6 +796,7 @@ async function _evaluate(target, fields) {
777
796
  numRepetitions: fields.numRepetitions ?? 1,
778
797
  evaluatorKeys: _collectEvaluatorKeys(standardFields.evaluators),
779
798
  includeAttachments: standardFields.includeAttachments,
799
+ disableEvaluatorTracing: standardFields.disableEvaluatorTracing,
780
800
  }).start();
781
801
  const targetConcurrency = standardFields.targetConcurrency ?? standardFields.maxConcurrency ?? 0;
782
802
  const evaluationConcurrency = standardFields.evaluationConcurrency ?? standardFields.maxConcurrency ?? 0;
@@ -988,7 +1008,25 @@ function _resolveData(data, options) {
988
1008
  }
989
1009
  return data;
990
1010
  }
991
- async function wrapSummaryEvaluators(evaluators, optionsArray) {
1011
+ /**
1012
+ * Strip evaluator-run links from an evaluator response.
1013
+ *
1014
+ * Used when evaluator tracing is disabled: no evaluator run is created, so a
1015
+ * `sourceRunId` on the feedback would point at a run that does not exist.
1016
+ */
1017
+ function _withoutSourceRunIds(response) {
1018
+ if ("results" in response) {
1019
+ return {
1020
+ ...response,
1021
+ results: response.results.map((result) => ({
1022
+ ...result,
1023
+ sourceRunId: undefined,
1024
+ })),
1025
+ };
1026
+ }
1027
+ return { ...response, sourceRunId: undefined };
1028
+ }
1029
+ async function wrapSummaryEvaluators(evaluators, optionsArray, tracingEnabled) {
992
1030
  async function _wrap(evaluator) {
993
1031
  const evalName = evaluator.name || "BatchEvaluator";
994
1032
  const wrapperInner = (runs, examples) => {
@@ -1008,7 +1046,11 @@ async function wrapSummaryEvaluators(evaluators, optionsArray) {
1008
1046
  }
1009
1047
  // Otherwise use the traditional (runs, examples) signature
1010
1048
  return Promise.resolve(evaluator(runs, examples));
1011
- }, { ...optionsArray, name: evalName });
1049
+ }, {
1050
+ ...optionsArray,
1051
+ name: evalName,
1052
+ ...(tracingEnabled === undefined ? {} : { tracingEnabled }),
1053
+ });
1012
1054
  return Promise.resolve(wrapperSuperInner(`Runs[] (Length=${runs.length})`, `Examples[] (Length=${examples.length})`));
1013
1055
  };
1014
1056
  return wrapperInner;
@@ -73,6 +73,7 @@ interface _ExperimentManagerArgs {
73
73
  _runsArray?: Run[];
74
74
  resultRows?: AsyncGenerator<_ExperimentResultRowWithIndex>;
75
75
  includeAttachments?: boolean;
76
+ disableEvaluatorTracing?: boolean;
76
77
  }
77
78
  type BaseEvaluateOptions = {
78
79
  /**
@@ -140,6 +141,16 @@ export interface EvaluateOptions extends BaseEvaluateOptions {
140
141
  * @default false
141
142
  */
142
143
  includeAttachments?: boolean;
144
+ /**
145
+ * Whether to skip tracing evaluator invocations to the `evaluators` project.
146
+ *
147
+ * When `true`, evaluators run without creating evaluator traces. Feedback is
148
+ * still created and attached to the experiment runs, but it won't link back
149
+ * to an evaluator run, which also means it can't be corrected from the UI.
150
+ * Tracing of the target itself is unaffected.
151
+ * @default false
152
+ */
153
+ disableEvaluatorTracing?: boolean;
143
154
  }
144
155
  export interface ComparativeEvaluateOptions extends BaseEvaluateOptions {
145
156
  /**
@@ -194,6 +205,7 @@ export declare class _ExperimentManager {
194
205
  _metadata: KVMap;
195
206
  _description?: string;
196
207
  _includeAttachments?: boolean;
208
+ _disableEvaluatorTracing?: boolean;
197
209
  get experimentName(): string;
198
210
  getExamples(): Promise<Array<Example>>;
199
211
  setExamples(examples: Example[]): void;
@@ -203,6 +203,12 @@ export class _ExperimentManager {
203
203
  writable: true,
204
204
  value: void 0
205
205
  });
206
+ Object.defineProperty(this, "_disableEvaluatorTracing", {
207
+ enumerable: true,
208
+ configurable: true,
209
+ writable: true,
210
+ value: void 0
211
+ });
206
212
  this.client = args.client ?? new Client();
207
213
  if (!args.experiment) {
208
214
  this._experimentName = randomName();
@@ -240,6 +246,7 @@ export class _ExperimentManager {
240
246
  this._numExamples = args.numExamples;
241
247
  this._evaluatorKeys = args.evaluatorKeys;
242
248
  this._includeAttachments = args.includeAttachments;
249
+ this._disableEvaluatorTracing = args.disableEvaluatorTracing;
243
250
  }
244
251
  _getExperiment() {
245
252
  if (!this._experiment) {
@@ -342,6 +349,7 @@ export class _ExperimentManager {
342
349
  numExamples: this._numExamples,
343
350
  evaluatorKeys: this._evaluatorKeys,
344
351
  includeAttachments: this._includeAttachments,
352
+ disableEvaluatorTracing: this._disableEvaluatorTracing,
345
353
  });
346
354
  }
347
355
  async withPredictions(target, options) {
@@ -371,6 +379,7 @@ export class _ExperimentManager {
371
379
  numExamples: this._numExamples,
372
380
  evaluatorKeys: this._evaluatorKeys,
373
381
  includeAttachments: this._includeAttachments,
382
+ disableEvaluatorTracing: this._disableEvaluatorTracing,
374
383
  });
375
384
  }
376
385
  async withEvaluators(evaluators, options) {
@@ -398,6 +407,7 @@ export class _ExperimentManager {
398
407
  numExamples: this._numExamples,
399
408
  evaluatorKeys: this._evaluatorKeys,
400
409
  includeAttachments: this._includeAttachments,
410
+ disableEvaluatorTracing: this._disableEvaluatorTracing,
401
411
  });
402
412
  }
403
413
  async withSummaryEvaluators(summaryEvaluators) {
@@ -416,6 +426,7 @@ export class _ExperimentManager {
416
426
  numExamples: this._numExamples,
417
427
  evaluatorKeys: this._evaluatorKeys,
418
428
  includeAttachments: this._includeAttachments,
429
+ disableEvaluatorTracing: this._disableEvaluatorTracing,
419
430
  });
420
431
  }
421
432
  async *getResults() {
@@ -533,9 +544,14 @@ export class _ExperimentManager {
533
544
  : new Date(example.created_at).toISOString(),
534
545
  },
535
546
  client: fields.client,
536
- tracingEnabled: true,
547
+ tracingEnabled: !this._disableEvaluatorTracing,
537
548
  };
538
- const evaluatorResponse = await evaluator.evaluateRun(run, example, options);
549
+ const rawEvaluatorResponse = await evaluator.evaluateRun(run, example, options);
550
+ // With tracing off there is no evaluator run for the feedback to point
551
+ // at, so drop the link rather than leaving it dangling.
552
+ const evaluatorResponse = this._disableEvaluatorTracing
553
+ ? _withoutSourceRunIds(rawEvaluatorResponse)
554
+ : rawEvaluatorResponse;
539
555
  evaluationResults.results.push(...(await fields.client.logEvaluationFeedback({
540
556
  evaluatorResponse,
541
557
  run,
@@ -582,7 +598,10 @@ export class _ExperimentManager {
582
598
  experiment: this.experimentName,
583
599
  projectId: projectId,
584
600
  }));
585
- const wrappedEvaluators = await wrapSummaryEvaluators(summaryEvaluators, options);
601
+ const wrappedEvaluators = await wrapSummaryEvaluators(summaryEvaluators, options,
602
+ // Only override when disabling. Passing `true` here would force tracing
603
+ // on for callers who have it switched off via the environment.
604
+ this._disableEvaluatorTracing ? false : undefined);
586
605
  yield async function* (runsArray) {
587
606
  const aggregateFeedback = [];
588
607
  for (const evaluator of wrappedEvaluators) {
@@ -768,6 +787,7 @@ async function _evaluate(target, fields) {
768
787
  numRepetitions: fields.numRepetitions ?? 1,
769
788
  evaluatorKeys: _collectEvaluatorKeys(standardFields.evaluators),
770
789
  includeAttachments: standardFields.includeAttachments,
790
+ disableEvaluatorTracing: standardFields.disableEvaluatorTracing,
771
791
  }).start();
772
792
  const targetConcurrency = standardFields.targetConcurrency ?? standardFields.maxConcurrency ?? 0;
773
793
  const evaluationConcurrency = standardFields.evaluationConcurrency ?? standardFields.maxConcurrency ?? 0;
@@ -979,7 +999,25 @@ function _resolveData(data, options) {
979
999
  }
980
1000
  return data;
981
1001
  }
982
- async function wrapSummaryEvaluators(evaluators, optionsArray) {
1002
+ /**
1003
+ * Strip evaluator-run links from an evaluator response.
1004
+ *
1005
+ * Used when evaluator tracing is disabled: no evaluator run is created, so a
1006
+ * `sourceRunId` on the feedback would point at a run that does not exist.
1007
+ */
1008
+ function _withoutSourceRunIds(response) {
1009
+ if ("results" in response) {
1010
+ return {
1011
+ ...response,
1012
+ results: response.results.map((result) => ({
1013
+ ...result,
1014
+ sourceRunId: undefined,
1015
+ })),
1016
+ };
1017
+ }
1018
+ return { ...response, sourceRunId: undefined };
1019
+ }
1020
+ async function wrapSummaryEvaluators(evaluators, optionsArray, tracingEnabled) {
983
1021
  async function _wrap(evaluator) {
984
1022
  const evalName = evaluator.name || "BatchEvaluator";
985
1023
  const wrapperInner = (runs, examples) => {
@@ -999,7 +1037,11 @@ async function wrapSummaryEvaluators(evaluators, optionsArray) {
999
1037
  }
1000
1038
  // Otherwise use the traditional (runs, examples) signature
1001
1039
  return Promise.resolve(evaluator(runs, examples));
1002
- }, { ...optionsArray, name: evalName });
1040
+ }, {
1041
+ ...optionsArray,
1042
+ name: evalName,
1043
+ ...(tracingEnabled === undefined ? {} : { tracingEnabled }),
1044
+ });
1003
1045
  return Promise.resolve(wrapperSuperInner(`Runs[] (Length=${runs.length})`, `Examples[] (Length=${examples.length})`));
1004
1046
  };
1005
1047
  return wrapperInner;
package/dist/index.cjs CHANGED
@@ -35,6 +35,6 @@ Object.defineProperty(exports, "UnprocessableEntityError", { enumerable: true, g
35
35
  Object.defineProperty(exports, "RateLimitError", { enumerable: true, get: function () { return error_js_1.RateLimitError; } });
36
36
  Object.defineProperty(exports, "InternalServerError", { enumerable: true, get: function () { return error_js_1.InternalServerError; } });
37
37
  // Update using pnpm bump-version
38
- exports.__version__ = "0.10.3";
38
+ exports.__version__ = "0.10.5";
39
39
  // Metadata key to hide a traced run from LangSmith's Messages View.
40
40
  exports.LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
package/dist/index.d.ts CHANGED
@@ -7,5 +7,5 @@ export { computeRunIdForSecondaryReplica, uuid7, uuid7FromTime, } from "./uuid.j
7
7
  export { isTracingEnabled } from "./utils/guard.js";
8
8
  export { Cache, PromptCache, type CacheConfig, type CacheMetrics, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
9
9
  export { LangsmithError, APIError, APIUserAbortError, APIConnectionError, APIConnectionTimeoutError, BadRequestError, AuthenticationError, PermissionDeniedError, NotFoundError, ConflictError, UnprocessableEntityError, RateLimitError, InternalServerError, } from "./_openapi_client/core/error.js";
10
- export declare const __version__ = "0.10.3";
10
+ export declare const __version__ = "0.10.5";
11
11
  export declare const LS_MESSAGE_VIEW_EXCLUDE: "ls_message_view_exclude";
package/dist/index.js CHANGED
@@ -7,6 +7,6 @@ export { isTracingEnabled } from "./utils/guard.js";
7
7
  export { Cache, PromptCache, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
8
8
  export { LangsmithError, APIError, APIUserAbortError, APIConnectionError, APIConnectionTimeoutError, BadRequestError, AuthenticationError, PermissionDeniedError, NotFoundError, ConflictError, UnprocessableEntityError, RateLimitError, InternalServerError, } from "./_openapi_client/core/error.js";
9
9
  // Update using pnpm bump-version
10
- export const __version__ = "0.10.3";
10
+ export const __version__ = "0.10.5";
11
11
  // Metadata key to hide a traced run from LangSmith's Messages View.
12
12
  export const LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
@@ -0,0 +1,24 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.validateAccessDelegation = validateAccessDelegation;
4
+ const errors_js_1 = require("./errors.cjs");
5
+ /** Rejects grants the server would refuse, without the round trip. */
6
+ function validateAccessDelegation(value) {
7
+ if (value.mode !== "INHERIT" && value.mode !== "EXPLICIT") {
8
+ throw new errors_js_1.LangSmithValidationError('accessDelegation.mode must be "INHERIT" or "EXPLICIT"', "accessDelegation");
9
+ }
10
+ const { permissions } = value;
11
+ if (permissions !== undefined && !Array.isArray(permissions)) {
12
+ throw new errors_js_1.LangSmithValidationError("accessDelegation.permissions must be an array of strings", "accessDelegation");
13
+ }
14
+ if (value.mode === "INHERIT") {
15
+ if (permissions !== undefined && permissions.length > 0) {
16
+ throw new errors_js_1.LangSmithValidationError('accessDelegation.permissions is not allowed with mode "INHERIT"', "accessDelegation");
17
+ }
18
+ return { mode: "INHERIT" };
19
+ }
20
+ if (permissions === undefined || permissions.length === 0) {
21
+ throw new errors_js_1.LangSmithValidationError('accessDelegation.permissions is required with mode "EXPLICIT"', "accessDelegation");
22
+ }
23
+ return { mode: "EXPLICIT", permissions: [...permissions] };
24
+ }
@@ -0,0 +1,15 @@
1
+ /** How a delegation grant derives its permission set. */
2
+ export type AccessDelegationMode = "INHERIT" | "EXPLICIT";
3
+ /**
4
+ * Grant letting code inside a sandbox call the LangSmith API as the creator.
5
+ *
6
+ * `INHERIT` tracks everything the creator can do; `EXPLICIT` is capped to
7
+ * `permissions`. Permissions are a ceiling re-checked on every request rather
8
+ * than a snapshot, so access the creator loses is lost here too.
9
+ */
10
+ export interface AccessDelegation {
11
+ mode: AccessDelegationMode;
12
+ permissions?: string[];
13
+ }
14
+ /** Rejects grants the server would refuse, without the round trip. */
15
+ export declare function validateAccessDelegation(value: AccessDelegation): AccessDelegation;
@@ -0,0 +1,21 @@
1
+ import { LangSmithValidationError } from "./errors.js";
2
+ /** Rejects grants the server would refuse, without the round trip. */
3
+ export function validateAccessDelegation(value) {
4
+ if (value.mode !== "INHERIT" && value.mode !== "EXPLICIT") {
5
+ throw new LangSmithValidationError('accessDelegation.mode must be "INHERIT" or "EXPLICIT"', "accessDelegation");
6
+ }
7
+ const { permissions } = value;
8
+ if (permissions !== undefined && !Array.isArray(permissions)) {
9
+ throw new LangSmithValidationError("accessDelegation.permissions must be an array of strings", "accessDelegation");
10
+ }
11
+ if (value.mode === "INHERIT") {
12
+ if (permissions !== undefined && permissions.length > 0) {
13
+ throw new LangSmithValidationError('accessDelegation.permissions is not allowed with mode "INHERIT"', "accessDelegation");
14
+ }
15
+ return { mode: "INHERIT" };
16
+ }
17
+ if (permissions === undefined || permissions.length === 0) {
18
+ throw new LangSmithValidationError('accessDelegation.permissions is required with mode "EXPLICIT"', "accessDelegation");
19
+ }
20
+ return { mode: "EXPLICIT", permissions: [...permissions] };
21
+ }