langsmith 0.7.5 → 0.7.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +41 -0
  2. package/dist/client.cjs +13 -4
  3. package/dist/client.d.ts +4 -1
  4. package/dist/client.js +13 -4
  5. package/dist/evaluation/_runner.cjs +77 -0
  6. package/dist/evaluation/_runner.d.ts +4 -0
  7. package/dist/evaluation/_runner.js +77 -0
  8. package/dist/index.cjs +1 -1
  9. package/dist/index.d.ts +1 -1
  10. package/dist/index.js +1 -1
  11. package/dist/sandbox/command_handle.cjs +16 -0
  12. package/dist/sandbox/command_handle.d.ts +2 -0
  13. package/dist/sandbox/command_handle.js +16 -0
  14. package/dist/sandbox/index.cjs +5 -1
  15. package/dist/sandbox/index.d.ts +2 -1
  16. package/dist/sandbox/index.js +1 -0
  17. package/dist/sandbox/proxy_config.cjs +47 -0
  18. package/dist/sandbox/proxy_config.d.ts +12 -0
  19. package/dist/sandbox/proxy_config.js +42 -0
  20. package/dist/sandbox/sandbox.cjs +6 -4
  21. package/dist/sandbox/sandbox.js +6 -4
  22. package/dist/sandbox/types.d.ts +29 -1
  23. package/dist/utils/fs.browser.cjs +11 -0
  24. package/dist/utils/fs.browser.d.ts +3 -0
  25. package/dist/utils/fs.browser.js +8 -0
  26. package/dist/utils/fs.cjs +22 -0
  27. package/dist/utils/fs.d.ts +3 -0
  28. package/dist/utils/fs.js +19 -0
  29. package/dist/utils/profile-lock.cjs +140 -0
  30. package/dist/utils/profile-lock.d.ts +20 -0
  31. package/dist/utils/profile-lock.js +103 -0
  32. package/dist/utils/profiles.cjs +28 -2
  33. package/dist/utils/profiles.d.ts +1 -0
  34. package/dist/utils/profiles.js +28 -2
  35. package/dist/wrappers/gemini.cjs +3 -40
  36. package/dist/wrappers/gemini.js +3 -40
  37. package/dist/wrappers/gemini.utils.cjs +41 -0
  38. package/dist/wrappers/gemini.utils.d.ts +3 -0
  39. package/dist/wrappers/gemini.utils.js +37 -0
  40. package/package.json +4 -4
package/README.md CHANGED
@@ -25,6 +25,47 @@ A typical workflow looks like:
25
25
 
26
26
  We'll walk through these steps in more detail below.
27
27
 
28
+ ## Sandbox AWS auth proxy
29
+
30
+ When you create a LangSmith sandbox that needs to call AWS services, use the
31
+ sandbox AWS auth proxy helpers. The proxy keeps the real AWS credentials outside
32
+ the sandbox and signs supported AWS HTTPS requests with SigV4, so code in the
33
+ sandbox can use AWS SDKs normally without storing long-lived AWS keys in files,
34
+ environment variables, shell history, or logs.
35
+
36
+ First, store `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` as LangSmith
37
+ workspace secrets. Then create the sandbox with an AWS auth proxy config:
38
+
39
+ ```ts
40
+ import {
41
+ SandboxClient,
42
+ awsAuthProxyConfig,
43
+ workspaceSecret,
44
+ } from "langsmith/sandbox";
45
+
46
+ const client = new SandboxClient();
47
+
48
+ const sandbox = await client.createSandbox({
49
+ name: "aws-sandbox",
50
+ proxyConfig: awsAuthProxyConfig({
51
+ accessKeyId: workspaceSecret("AWS_ACCESS_KEY_ID"),
52
+ secretAccessKey: workspaceSecret("AWS_SECRET_ACCESS_KEY"),
53
+ }),
54
+ });
55
+
56
+ try {
57
+ // Your sandbox code can use the AWS SDK, the AWS CLI, or other AWS tooling normally.
58
+ const result = await sandbox.run("node your-aws-script.js");
59
+ console.log(result.stdout);
60
+ } finally {
61
+ await sandbox.delete();
62
+ }
63
+ ```
64
+
65
+ Use `opaqueSecret("...")` instead of `workspaceSecret(...)` when your application
66
+ needs to pass short-lived write-only AWS credentials at sandbox creation time.
67
+ Plaintext AWS credential values are not supported.
68
+
28
69
  ## 1. Connect to LangSmith
29
70
 
30
71
  Sign up for [LangSmith](https://smith.langchain.com/) using your GitHub, Discord accounts, or an email address and password. If you sign up with an email, make sure to verify your email address before logging in.
package/dist/client.cjs CHANGED
@@ -2557,7 +2557,7 @@ class Client {
2557
2557
  _hostUrl: this.getHostUrl(),
2558
2558
  }));
2559
2559
  }
2560
- async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, }) {
2560
+ async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, numExamples = null, numRepetitions = null, evaluatorKeys = null, }) {
2561
2561
  const upsert_ = upsert ? `?upsert=true` : "";
2562
2562
  const endpoint = `${this.apiUrl}/sessions${upsert_}`;
2563
2563
  const extra = projectExtra || {};
@@ -2572,6 +2572,15 @@ class Client {
2572
2572
  if (referenceDatasetId !== null) {
2573
2573
  body["reference_dataset_id"] = referenceDatasetId;
2574
2574
  }
2575
+ if (numExamples != null) {
2576
+ body["num_examples"] = numExamples;
2577
+ }
2578
+ if (numRepetitions != null) {
2579
+ body["num_repetitions"] = numRepetitions;
2580
+ }
2581
+ if (evaluatorKeys != null && evaluatorKeys.length > 0) {
2582
+ body["evaluator_keys"] = evaluatorKeys;
2583
+ }
2575
2584
  const serializedBody = JSON.stringify(body);
2576
2585
  const response = await this.caller.call(async () => {
2577
2586
  const res = await this._fetch(endpoint, {
@@ -4891,7 +4900,7 @@ class Client {
4891
4900
  async _pullDirectory(identifier, repoType, version) {
4892
4901
  const [owner, name, parsedVersion] = (0, prompts_js_1.parseHubIdentifier)(identifier);
4893
4902
  const resolvedVersion = version ?? (parsedVersion !== "latest" ? parsedVersion : undefined);
4894
- const url = new URL(`${this.apiUrl}/v1/platform/hub/repos/${owner}/${name}/directories`);
4903
+ const url = new URL(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories`)}`);
4895
4904
  url.searchParams.set("repo_type", repoType);
4896
4905
  if (resolvedVersion) {
4897
4906
  url.searchParams.set("commit", resolvedVersion);
@@ -4937,7 +4946,7 @@ class Client {
4937
4946
  body.parent_commit = options.parentCommit;
4938
4947
  }
4939
4948
  const response = await this.caller.call(async () => {
4940
- const res = await this._fetch(`${this.apiUrl}/v1/platform/hub/repos/${owner}/${name}/directories/commits`, {
4949
+ const res = await this._fetch(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories/commits`)}`, {
4941
4950
  method: "POST",
4942
4951
  headers: {
4943
4952
  ...this._mergedHeaders,
@@ -4962,7 +4971,7 @@ class Client {
4962
4971
  throw await this._ownerConflictError("delete", owner);
4963
4972
  }
4964
4973
  await this.caller.call(async () => {
4965
- const res = await this._fetch(`${this.apiUrl}/v1/platform/hub/repos/${owner}/${name}/directories`, {
4974
+ const res = await this._fetch(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories`)}`, {
4966
4975
  method: "DELETE",
4967
4976
  headers: this._mergedHeaders,
4968
4977
  signal: AbortSignal.timeout(this.timeout_ms),
package/dist/client.d.ts CHANGED
@@ -349,6 +349,9 @@ export type CreateProjectParams = {
349
349
  upsert?: boolean;
350
350
  projectExtra?: RecordStringAny | null;
351
351
  referenceDatasetId?: string | null;
352
+ numExamples?: number | null;
353
+ numRepetitions?: number | null;
354
+ evaluatorKeys?: string[] | null;
352
355
  };
353
356
  type AutoBatchQueueItem = {
354
357
  action: "create" | "update";
@@ -714,7 +717,7 @@ export declare class Client implements LangSmithTracingClientInterface {
714
717
  listSharedExamples(shareToken: string, options?: {
715
718
  exampleIds?: string[];
716
719
  }): Promise<Example[]>;
717
- createProject({ projectName, description, metadata, upsert, projectExtra, referenceDatasetId, }: CreateProjectParams): Promise<TracerSession>;
720
+ createProject({ projectName, description, metadata, upsert, projectExtra, referenceDatasetId, numExamples, numRepetitions, evaluatorKeys, }: CreateProjectParams): Promise<TracerSession>;
718
721
  updateProject(projectId: string, { name, description, metadata, projectExtra, endTime, }: {
719
722
  name?: string | null;
720
723
  description?: string | null;
package/dist/client.js CHANGED
@@ -2519,7 +2519,7 @@ export class Client {
2519
2519
  _hostUrl: this.getHostUrl(),
2520
2520
  }));
2521
2521
  }
2522
- async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, }) {
2522
+ async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, numExamples = null, numRepetitions = null, evaluatorKeys = null, }) {
2523
2523
  const upsert_ = upsert ? `?upsert=true` : "";
2524
2524
  const endpoint = `${this.apiUrl}/sessions${upsert_}`;
2525
2525
  const extra = projectExtra || {};
@@ -2534,6 +2534,15 @@ export class Client {
2534
2534
  if (referenceDatasetId !== null) {
2535
2535
  body["reference_dataset_id"] = referenceDatasetId;
2536
2536
  }
2537
+ if (numExamples != null) {
2538
+ body["num_examples"] = numExamples;
2539
+ }
2540
+ if (numRepetitions != null) {
2541
+ body["num_repetitions"] = numRepetitions;
2542
+ }
2543
+ if (evaluatorKeys != null && evaluatorKeys.length > 0) {
2544
+ body["evaluator_keys"] = evaluatorKeys;
2545
+ }
2537
2546
  const serializedBody = JSON.stringify(body);
2538
2547
  const response = await this.caller.call(async () => {
2539
2548
  const res = await this._fetch(endpoint, {
@@ -4853,7 +4862,7 @@ export class Client {
4853
4862
  async _pullDirectory(identifier, repoType, version) {
4854
4863
  const [owner, name, parsedVersion] = parseHubIdentifier(identifier);
4855
4864
  const resolvedVersion = version ?? (parsedVersion !== "latest" ? parsedVersion : undefined);
4856
- const url = new URL(`${this.apiUrl}/v1/platform/hub/repos/${owner}/${name}/directories`);
4865
+ const url = new URL(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories`)}`);
4857
4866
  url.searchParams.set("repo_type", repoType);
4858
4867
  if (resolvedVersion) {
4859
4868
  url.searchParams.set("commit", resolvedVersion);
@@ -4899,7 +4908,7 @@ export class Client {
4899
4908
  body.parent_commit = options.parentCommit;
4900
4909
  }
4901
4910
  const response = await this.caller.call(async () => {
4902
- const res = await this._fetch(`${this.apiUrl}/v1/platform/hub/repos/${owner}/${name}/directories/commits`, {
4911
+ const res = await this._fetch(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories/commits`)}`, {
4903
4912
  method: "POST",
4904
4913
  headers: {
4905
4914
  ...this._mergedHeaders,
@@ -4924,7 +4933,7 @@ export class Client {
4924
4933
  throw await this._ownerConflictError("delete", owner);
4925
4934
  }
4926
4935
  await this.caller.call(async () => {
4927
- const res = await this._fetch(`${this.apiUrl}/v1/platform/hub/repos/${owner}/${name}/directories`, {
4936
+ const res = await this._fetch(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories`)}`, {
4928
4937
  method: "DELETE",
4929
4938
  headers: this._mergedHeaders,
4930
4939
  signal: AbortSignal.timeout(this.timeout_ms),
@@ -149,12 +149,24 @@ class _ExperimentManager {
149
149
  writable: true,
150
150
  value: void 0
151
151
  });
152
+ Object.defineProperty(this, "_numExamples", {
153
+ enumerable: true,
154
+ configurable: true,
155
+ writable: true,
156
+ value: void 0
157
+ });
152
158
  Object.defineProperty(this, "_numRepetitions", {
153
159
  enumerable: true,
154
160
  configurable: true,
155
161
  writable: true,
156
162
  value: void 0
157
163
  });
164
+ Object.defineProperty(this, "_evaluatorKeys", {
165
+ enumerable: true,
166
+ configurable: true,
167
+ writable: true,
168
+ value: void 0
169
+ });
158
170
  Object.defineProperty(this, "_runsArray", {
159
171
  enumerable: true,
160
172
  configurable: true,
@@ -231,6 +243,8 @@ class _ExperimentManager {
231
243
  this._summaryResults = args.summaryResults;
232
244
  this._resultRows = args.resultRows;
233
245
  this._numRepetitions = args.numRepetitions;
246
+ this._numExamples = args.numExamples;
247
+ this._evaluatorKeys = args.evaluatorKeys;
234
248
  this._includeAttachments = args.includeAttachments;
235
249
  }
236
250
  _getExperiment() {
@@ -263,6 +277,12 @@ class _ExperimentManager {
263
277
  // Create the project, updating the experimentName until we find a unique one.
264
278
  let project;
265
279
  const originalExperimentName = this._experimentName;
280
+ if (this._numExamples === undefined) {
281
+ this._numExamples = await _resolveNumExamples(this._data, this.client);
282
+ }
283
+ const numExamples = this._numExamples ?? null;
284
+ const numRepetitions = this._numRepetitions ?? null;
285
+ const evaluatorKeys = this._evaluatorKeys ?? null;
266
286
  for (let i = 0; i < 10; i++) {
267
287
  try {
268
288
  project = await this.client.createProject({
@@ -270,6 +290,9 @@ class _ExperimentManager {
270
290
  referenceDatasetId: firstExample.dataset_id,
271
291
  metadata: projectMetadata,
272
292
  description: this._description,
293
+ numExamples,
294
+ numRepetitions,
295
+ evaluatorKeys,
273
296
  });
274
297
  return project;
275
298
  }
@@ -319,6 +342,9 @@ class _ExperimentManager {
319
342
  client: this.client,
320
343
  evaluationResults: this._evaluationResults,
321
344
  summaryResults: this._summaryResults,
345
+ numRepetitions: this._numRepetitions,
346
+ numExamples: this._numExamples,
347
+ evaluatorKeys: this._evaluatorKeys,
322
348
  includeAttachments: this._includeAttachments,
323
349
  });
324
350
  }
@@ -345,6 +371,9 @@ class _ExperimentManager {
345
371
  yield pred.run;
346
372
  }
347
373
  })(),
374
+ numRepetitions: this._numRepetitions,
375
+ numExamples: this._numExamples,
376
+ evaluatorKeys: this._evaluatorKeys,
348
377
  includeAttachments: this._includeAttachments,
349
378
  });
350
379
  }
@@ -369,6 +398,9 @@ class _ExperimentManager {
369
398
  }
370
399
  })(),
371
400
  summaryResults: this._summaryResults,
401
+ numRepetitions: this._numRepetitions,
402
+ numExamples: this._numExamples,
403
+ evaluatorKeys: this._evaluatorKeys,
372
404
  includeAttachments: this._includeAttachments,
373
405
  });
374
406
  }
@@ -384,6 +416,9 @@ class _ExperimentManager {
384
416
  evaluationResults: this._evaluationResults,
385
417
  resultRows: this._resultRows,
386
418
  summaryResults: aggregateFeedbackGen,
419
+ numRepetitions: this._numRepetitions,
420
+ numExamples: this._numExamples,
421
+ evaluatorKeys: this._evaluatorKeys,
387
422
  includeAttachments: this._includeAttachments,
388
423
  });
389
424
  }
@@ -732,6 +767,7 @@ async function _evaluate(target, fields) {
732
767
  experiment: experiment_ ?? fields.experimentPrefix,
733
768
  runs: newRuns ?? undefined,
734
769
  numRepetitions: fields.numRepetitions ?? 1,
770
+ evaluatorKeys: _collectEvaluatorKeys(standardFields.evaluators),
735
771
  includeAttachments: standardFields.includeAttachments,
736
772
  }).start();
737
773
  const targetConcurrency = standardFields.targetConcurrency ?? standardFields.maxConcurrency ?? 0;
@@ -852,6 +888,47 @@ Try setting "LANGSMITH_TRACING=true" in your environment.`);
852
888
  example,
853
889
  };
854
890
  }
891
+ function _collectEvaluatorKeys(evaluators) {
892
+ if (!evaluators)
893
+ return [];
894
+ const keys = [];
895
+ for (const ev of evaluators) {
896
+ const name = ev.name;
897
+ if (name && name.length > 0)
898
+ keys.push(name);
899
+ }
900
+ return keys;
901
+ }
902
+ async function _resolveNumExamples(data, client) {
903
+ // Best-effort dataset size for the experiment progress hint. Returns
904
+ // undefined for lazy async iterators where size cannot be inferred without
905
+ // consuming.
906
+ if (data === undefined)
907
+ return undefined;
908
+ try {
909
+ if (Array.isArray(data)) {
910
+ return data.length;
911
+ }
912
+ if (typeof data === "string") {
913
+ let isUUID = false;
914
+ try {
915
+ (0, _uuid_js_1.assertUuid)(data);
916
+ isUUID = true;
917
+ }
918
+ catch (_) {
919
+ isUUID = false;
920
+ }
921
+ const dataset = isUUID
922
+ ? await client.readDataset({ datasetId: data })
923
+ : await client.readDataset({ datasetName: data });
924
+ return dataset.example_count ?? undefined;
925
+ }
926
+ return undefined;
927
+ }
928
+ catch (_) {
929
+ return undefined;
930
+ }
931
+ }
855
932
  function _resolveData(data, options) {
856
933
  let isUUID = false;
857
934
  try {
@@ -68,6 +68,8 @@ interface _ExperimentManagerArgs {
68
68
  summaryResults?: AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults, any, unknown>, any, unknown>;
69
69
  examples?: Example[];
70
70
  numRepetitions?: number;
71
+ numExamples?: number;
72
+ evaluatorKeys?: string[];
71
73
  _runsArray?: Run[];
72
74
  resultRows?: AsyncGenerator<_ExperimentResultRowWithIndex>;
73
75
  includeAttachments?: boolean;
@@ -182,7 +184,9 @@ export declare class _ExperimentManager {
182
184
  _summaryResults?: AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults, any, unknown>, any, unknown>;
183
185
  _resultRows?: AsyncGenerator<_ExperimentResultRowWithIndex>;
184
186
  _examples?: Example[];
187
+ _numExamples?: number;
185
188
  _numRepetitions?: number;
189
+ _evaluatorKeys?: string[];
186
190
  _runsArray?: Run[];
187
191
  client: Client;
188
192
  _experiment?: TracerSession;
@@ -143,12 +143,24 @@ export class _ExperimentManager {
143
143
  writable: true,
144
144
  value: void 0
145
145
  });
146
+ Object.defineProperty(this, "_numExamples", {
147
+ enumerable: true,
148
+ configurable: true,
149
+ writable: true,
150
+ value: void 0
151
+ });
146
152
  Object.defineProperty(this, "_numRepetitions", {
147
153
  enumerable: true,
148
154
  configurable: true,
149
155
  writable: true,
150
156
  value: void 0
151
157
  });
158
+ Object.defineProperty(this, "_evaluatorKeys", {
159
+ enumerable: true,
160
+ configurable: true,
161
+ writable: true,
162
+ value: void 0
163
+ });
152
164
  Object.defineProperty(this, "_runsArray", {
153
165
  enumerable: true,
154
166
  configurable: true,
@@ -225,6 +237,8 @@ export class _ExperimentManager {
225
237
  this._summaryResults = args.summaryResults;
226
238
  this._resultRows = args.resultRows;
227
239
  this._numRepetitions = args.numRepetitions;
240
+ this._numExamples = args.numExamples;
241
+ this._evaluatorKeys = args.evaluatorKeys;
228
242
  this._includeAttachments = args.includeAttachments;
229
243
  }
230
244
  _getExperiment() {
@@ -257,6 +271,12 @@ export class _ExperimentManager {
257
271
  // Create the project, updating the experimentName until we find a unique one.
258
272
  let project;
259
273
  const originalExperimentName = this._experimentName;
274
+ if (this._numExamples === undefined) {
275
+ this._numExamples = await _resolveNumExamples(this._data, this.client);
276
+ }
277
+ const numExamples = this._numExamples ?? null;
278
+ const numRepetitions = this._numRepetitions ?? null;
279
+ const evaluatorKeys = this._evaluatorKeys ?? null;
260
280
  for (let i = 0; i < 10; i++) {
261
281
  try {
262
282
  project = await this.client.createProject({
@@ -264,6 +284,9 @@ export class _ExperimentManager {
264
284
  referenceDatasetId: firstExample.dataset_id,
265
285
  metadata: projectMetadata,
266
286
  description: this._description,
287
+ numExamples,
288
+ numRepetitions,
289
+ evaluatorKeys,
267
290
  });
268
291
  return project;
269
292
  }
@@ -313,6 +336,9 @@ export class _ExperimentManager {
313
336
  client: this.client,
314
337
  evaluationResults: this._evaluationResults,
315
338
  summaryResults: this._summaryResults,
339
+ numRepetitions: this._numRepetitions,
340
+ numExamples: this._numExamples,
341
+ evaluatorKeys: this._evaluatorKeys,
316
342
  includeAttachments: this._includeAttachments,
317
343
  });
318
344
  }
@@ -339,6 +365,9 @@ export class _ExperimentManager {
339
365
  yield pred.run;
340
366
  }
341
367
  })(),
368
+ numRepetitions: this._numRepetitions,
369
+ numExamples: this._numExamples,
370
+ evaluatorKeys: this._evaluatorKeys,
342
371
  includeAttachments: this._includeAttachments,
343
372
  });
344
373
  }
@@ -363,6 +392,9 @@ export class _ExperimentManager {
363
392
  }
364
393
  })(),
365
394
  summaryResults: this._summaryResults,
395
+ numRepetitions: this._numRepetitions,
396
+ numExamples: this._numExamples,
397
+ evaluatorKeys: this._evaluatorKeys,
366
398
  includeAttachments: this._includeAttachments,
367
399
  });
368
400
  }
@@ -378,6 +410,9 @@ export class _ExperimentManager {
378
410
  evaluationResults: this._evaluationResults,
379
411
  resultRows: this._resultRows,
380
412
  summaryResults: aggregateFeedbackGen,
413
+ numRepetitions: this._numRepetitions,
414
+ numExamples: this._numExamples,
415
+ evaluatorKeys: this._evaluatorKeys,
381
416
  includeAttachments: this._includeAttachments,
382
417
  });
383
418
  }
@@ -725,6 +760,7 @@ async function _evaluate(target, fields) {
725
760
  experiment: experiment_ ?? fields.experimentPrefix,
726
761
  runs: newRuns ?? undefined,
727
762
  numRepetitions: fields.numRepetitions ?? 1,
763
+ evaluatorKeys: _collectEvaluatorKeys(standardFields.evaluators),
728
764
  includeAttachments: standardFields.includeAttachments,
729
765
  }).start();
730
766
  const targetConcurrency = standardFields.targetConcurrency ?? standardFields.maxConcurrency ?? 0;
@@ -845,6 +881,47 @@ Try setting "LANGSMITH_TRACING=true" in your environment.`);
845
881
  example,
846
882
  };
847
883
  }
884
+ function _collectEvaluatorKeys(evaluators) {
885
+ if (!evaluators)
886
+ return [];
887
+ const keys = [];
888
+ for (const ev of evaluators) {
889
+ const name = ev.name;
890
+ if (name && name.length > 0)
891
+ keys.push(name);
892
+ }
893
+ return keys;
894
+ }
895
+ async function _resolveNumExamples(data, client) {
896
+ // Best-effort dataset size for the experiment progress hint. Returns
897
+ // undefined for lazy async iterators where size cannot be inferred without
898
+ // consuming.
899
+ if (data === undefined)
900
+ return undefined;
901
+ try {
902
+ if (Array.isArray(data)) {
903
+ return data.length;
904
+ }
905
+ if (typeof data === "string") {
906
+ let isUUID = false;
907
+ try {
908
+ assertUuid(data);
909
+ isUUID = true;
910
+ }
911
+ catch (_) {
912
+ isUUID = false;
913
+ }
914
+ const dataset = isUUID
915
+ ? await client.readDataset({ datasetId: data })
916
+ : await client.readDataset({ datasetName: data });
917
+ return dataset.example_count ?? undefined;
918
+ }
919
+ return undefined;
920
+ }
921
+ catch (_) {
922
+ return undefined;
923
+ }
924
+ }
848
925
  function _resolveData(data, options) {
849
926
  let isUUID = false;
850
927
  try {
package/dist/index.cjs CHANGED
@@ -18,6 +18,6 @@ Object.defineProperty(exports, "PromptCache", { enumerable: true, get: function
18
18
  Object.defineProperty(exports, "configureGlobalPromptCache", { enumerable: true, get: function () { return index_js_1.configureGlobalPromptCache; } });
19
19
  Object.defineProperty(exports, "promptCacheSingleton", { enumerable: true, get: function () { return index_js_1.promptCacheSingleton; } });
20
20
  // Update using pnpm bump-version
21
- exports.__version__ = "0.7.5";
21
+ exports.__version__ = "0.7.7";
22
22
  // Metadata key to hide a traced run from LangSmith's Messages View.
23
23
  exports.LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
package/dist/index.d.ts CHANGED
@@ -5,5 +5,5 @@ export { overrideFetchImplementation } from "./singletons/fetch.js";
5
5
  export { getDefaultProjectName } from "./utils/project.js";
6
6
  export { uuid7, uuid7FromTime } from "./uuid.js";
7
7
  export { Cache, PromptCache, type CacheConfig, type CacheMetrics, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
8
- export declare const __version__ = "0.7.5";
8
+ export declare const __version__ = "0.7.7";
9
9
  export declare const LS_MESSAGE_VIEW_EXCLUDE: "ls_message_view_exclude";
package/dist/index.js CHANGED
@@ -5,6 +5,6 @@ export { getDefaultProjectName } from "./utils/project.js";
5
5
  export { uuid7, uuid7FromTime } from "./uuid.js";
6
6
  export { Cache, PromptCache, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
7
7
  // Update using pnpm bump-version
8
- export const __version__ = "0.7.5";
8
+ export const __version__ = "0.7.7";
9
9
  // Metadata key to hide a traced run from LangSmith's Messages View.
10
10
  export const LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
@@ -107,11 +107,25 @@ class CommandHandle {
107
107
  writable: true,
108
108
  value: void 0
109
109
  });
110
+ Object.defineProperty(this, "_onStdout", {
111
+ enumerable: true,
112
+ configurable: true,
113
+ writable: true,
114
+ value: void 0
115
+ });
116
+ Object.defineProperty(this, "_onStderr", {
117
+ enumerable: true,
118
+ configurable: true,
119
+ writable: true,
120
+ value: void 0
121
+ });
110
122
  this._stream = messageStream;
111
123
  this._control = control;
112
124
  this._sandbox = sandbox;
113
125
  this._lastStdoutOffset = options?.stdoutOffset ?? 0;
114
126
  this._lastStderrOffset = options?.stderrOffset ?? 0;
127
+ this._onStdout = options?.onStdout;
128
+ this._onStderr = options?.onStderr;
115
129
  // New executions (no commandId): _ensureStarted reads "started".
116
130
  // Reconnections (commandId set): skip since reconnect streams
117
131
  // don't send a "started" message.
@@ -221,10 +235,12 @@ class CommandHandle {
221
235
  if (chunk.stream === "stdout") {
222
236
  this._lastStdoutOffset =
223
237
  chunk.offset + new TextEncoder().encode(chunk.data).length;
238
+ this._onStdout?.(chunk.data);
224
239
  }
225
240
  else {
226
241
  this._lastStderrOffset =
227
242
  chunk.offset + new TextEncoder().encode(chunk.data).length;
243
+ this._onStderr?.(chunk.data);
228
244
  }
229
245
  yield chunk;
230
246
  }
@@ -45,6 +45,8 @@ export declare class CommandHandle {
45
45
  private _lastStdoutOffset;
46
46
  private _lastStderrOffset;
47
47
  private _started;
48
+ private _onStdout?;
49
+ private _onStderr?;
48
50
  /**
49
51
  * Read the 'started' message to populate commandId and pid.
50
52
  *
@@ -104,11 +104,25 @@ export class CommandHandle {
104
104
  writable: true,
105
105
  value: void 0
106
106
  });
107
+ Object.defineProperty(this, "_onStdout", {
108
+ enumerable: true,
109
+ configurable: true,
110
+ writable: true,
111
+ value: void 0
112
+ });
113
+ Object.defineProperty(this, "_onStderr", {
114
+ enumerable: true,
115
+ configurable: true,
116
+ writable: true,
117
+ value: void 0
118
+ });
107
119
  this._stream = messageStream;
108
120
  this._control = control;
109
121
  this._sandbox = sandbox;
110
122
  this._lastStdoutOffset = options?.stdoutOffset ?? 0;
111
123
  this._lastStderrOffset = options?.stderrOffset ?? 0;
124
+ this._onStdout = options?.onStdout;
125
+ this._onStderr = options?.onStderr;
112
126
  // New executions (no commandId): _ensureStarted reads "started".
113
127
  // Reconnections (commandId set): skip since reconnect streams
114
128
  // don't send a "started" message.
@@ -218,10 +232,12 @@ export class CommandHandle {
218
232
  if (chunk.stream === "stdout") {
219
233
  this._lastStdoutOffset =
220
234
  chunk.offset + new TextEncoder().encode(chunk.data).length;
235
+ this._onStdout?.(chunk.data);
221
236
  }
222
237
  else {
223
238
  this._lastStderrOffset =
224
239
  chunk.offset + new TextEncoder().encode(chunk.data).length;
240
+ this._onStderr?.(chunk.data);
225
241
  }
226
242
  yield chunk;
227
243
  }
@@ -29,7 +29,7 @@
29
29
  * @packageDocumentation
30
30
  */
31
31
  Object.defineProperty(exports, "__esModule", { value: true });
32
- exports.LangSmithDataplaneNotConfiguredError = exports.LangSmithCommandTimeoutError = exports.LangSmithSandboxOperationError = exports.LangSmithSandboxNotReadyError = exports.LangSmithSandboxCreationError = exports.LangSmithResourceCreationError = exports.LangSmithQuotaExceededError = exports.LangSmithValidationError = exports.LangSmithResourceNameConflictError = exports.LangSmithResourceAlreadyExistsError = exports.LangSmithResourceInUseError = exports.LangSmithResourceTimeoutError = exports.LangSmithResourceNotFoundError = exports.LangSmithSandboxServerReloadError = exports.LangSmithSandboxConnectionError = exports.LangSmithSandboxAuthenticationError = exports.LangSmithSandboxAPIError = exports.LangSmithSandboxError = exports.CommandHandle = exports.Sandbox = exports.SandboxClient = void 0;
32
+ exports.LangSmithDataplaneNotConfiguredError = exports.LangSmithCommandTimeoutError = exports.LangSmithSandboxOperationError = exports.LangSmithSandboxNotReadyError = exports.LangSmithSandboxCreationError = exports.LangSmithResourceCreationError = exports.LangSmithQuotaExceededError = exports.LangSmithValidationError = exports.LangSmithResourceNameConflictError = exports.LangSmithResourceAlreadyExistsError = exports.LangSmithResourceInUseError = exports.LangSmithResourceTimeoutError = exports.LangSmithResourceNotFoundError = exports.LangSmithSandboxServerReloadError = exports.LangSmithSandboxConnectionError = exports.LangSmithSandboxAuthenticationError = exports.LangSmithSandboxAPIError = exports.LangSmithSandboxError = exports.workspaceSecret = exports.opaqueSecret = exports.awsAuthProxyConfig = exports.CommandHandle = exports.Sandbox = exports.SandboxClient = void 0;
33
33
  // Main classes
34
34
  var client_js_1 = require("./client.cjs");
35
35
  Object.defineProperty(exports, "SandboxClient", { enumerable: true, get: function () { return client_js_1.SandboxClient; } });
@@ -37,6 +37,10 @@ var sandbox_js_1 = require("./sandbox.cjs");
37
37
  Object.defineProperty(exports, "Sandbox", { enumerable: true, get: function () { return sandbox_js_1.Sandbox; } });
38
38
  var command_handle_js_1 = require("./command_handle.cjs");
39
39
  Object.defineProperty(exports, "CommandHandle", { enumerable: true, get: function () { return command_handle_js_1.CommandHandle; } });
40
+ var proxy_config_js_1 = require("./proxy_config.cjs");
41
+ Object.defineProperty(exports, "awsAuthProxyConfig", { enumerable: true, get: function () { return proxy_config_js_1.awsAuthProxyConfig; } });
42
+ Object.defineProperty(exports, "opaqueSecret", { enumerable: true, get: function () { return proxy_config_js_1.opaqueSecret; } });
43
+ Object.defineProperty(exports, "workspaceSecret", { enumerable: true, get: function () { return proxy_config_js_1.workspaceSecret; } });
40
44
  // Errors
41
45
  var errors_js_1 = require("./errors.cjs");
42
46
  // Base and connection errors
@@ -30,5 +30,6 @@
30
30
  export { SandboxClient } from "./client.js";
31
31
  export { Sandbox } from "./sandbox.js";
32
32
  export { CommandHandle } from "./command_handle.js";
33
- export type { ExecutionResult, OutputChunk, WsMessage, WsRunOptions, ResourceStatus, Snapshot, SandboxData, SandboxClientConfig, RunOptions, CreateSandboxOptions, SandboxAccessControl, SandboxProxyConfig, CreateSnapshotOptions, CreateDockerfileSnapshotOptions, CaptureSnapshotOptions, ListSnapshotsOptions, WaitForSnapshotOptions, StartSandboxOptions, UpdateSandboxOptions, WaitForSandboxOptions, } from "./types.js";
33
+ export { awsAuthProxyConfig, opaqueSecret, workspaceSecret, } from "./proxy_config.js";
34
+ export type { ExecutionResult, OutputChunk, WsMessage, WsRunOptions, ResourceStatus, Snapshot, SandboxData, SandboxClientConfig, RunOptions, CreateSandboxOptions, SandboxAccessControl, SandboxAwsAuthRule, SandboxProxyConfig, SandboxProxySecret, CreateSnapshotOptions, CreateDockerfileSnapshotOptions, CaptureSnapshotOptions, ListSnapshotsOptions, WaitForSnapshotOptions, StartSandboxOptions, UpdateSandboxOptions, WaitForSandboxOptions, } from "./types.js";
34
35
  export { LangSmithSandboxError, LangSmithSandboxAPIError, LangSmithSandboxAuthenticationError, LangSmithSandboxConnectionError, LangSmithSandboxServerReloadError, LangSmithResourceNotFoundError, LangSmithResourceTimeoutError, LangSmithResourceInUseError, LangSmithResourceAlreadyExistsError, LangSmithResourceNameConflictError, LangSmithValidationError, LangSmithQuotaExceededError, LangSmithResourceCreationError, LangSmithSandboxCreationError, LangSmithSandboxNotReadyError, LangSmithSandboxOperationError, LangSmithCommandTimeoutError, LangSmithDataplaneNotConfiguredError, } from "./errors.js";
@@ -31,6 +31,7 @@
31
31
  export { SandboxClient } from "./client.js";
32
32
  export { Sandbox } from "./sandbox.js";
33
33
  export { CommandHandle } from "./command_handle.js";
34
+ export { awsAuthProxyConfig, opaqueSecret, workspaceSecret, } from "./proxy_config.js";
34
35
  // Errors
35
36
  export {
36
37
  // Base and connection errors