langsmith 0.7.5 → 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -0
- package/dist/client.cjs +13 -4
- package/dist/client.d.ts +4 -1
- package/dist/client.js +13 -4
- package/dist/evaluation/_runner.cjs +77 -0
- package/dist/evaluation/_runner.d.ts +4 -0
- package/dist/evaluation/_runner.js +77 -0
- package/dist/index.cjs +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/sandbox/command_handle.cjs +16 -0
- package/dist/sandbox/command_handle.d.ts +2 -0
- package/dist/sandbox/command_handle.js +16 -0
- package/dist/sandbox/index.cjs +5 -1
- package/dist/sandbox/index.d.ts +2 -1
- package/dist/sandbox/index.js +1 -0
- package/dist/sandbox/proxy_config.cjs +47 -0
- package/dist/sandbox/proxy_config.d.ts +12 -0
- package/dist/sandbox/proxy_config.js +42 -0
- package/dist/sandbox/sandbox.cjs +6 -4
- package/dist/sandbox/sandbox.js +6 -4
- package/dist/sandbox/types.d.ts +29 -1
- package/dist/utils/fs.browser.cjs +11 -0
- package/dist/utils/fs.browser.d.ts +3 -0
- package/dist/utils/fs.browser.js +8 -0
- package/dist/utils/fs.cjs +22 -0
- package/dist/utils/fs.d.ts +3 -0
- package/dist/utils/fs.js +19 -0
- package/dist/utils/profile-lock.cjs +140 -0
- package/dist/utils/profile-lock.d.ts +20 -0
- package/dist/utils/profile-lock.js +103 -0
- package/dist/utils/profiles.cjs +28 -2
- package/dist/utils/profiles.d.ts +1 -0
- package/dist/utils/profiles.js +28 -2
- package/dist/wrappers/gemini.cjs +3 -40
- package/dist/wrappers/gemini.js +3 -40
- package/dist/wrappers/gemini.utils.cjs +41 -0
- package/dist/wrappers/gemini.utils.d.ts +3 -0
- package/dist/wrappers/gemini.utils.js +37 -0
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -25,6 +25,47 @@ A typical workflow looks like:
|
|
|
25
25
|
|
|
26
26
|
We'll walk through these steps in more detail below.
|
|
27
27
|
|
|
28
|
+
## Sandbox AWS auth proxy
|
|
29
|
+
|
|
30
|
+
When you create a LangSmith sandbox that needs to call AWS services, use the
|
|
31
|
+
sandbox AWS auth proxy helpers. The proxy keeps the real AWS credentials outside
|
|
32
|
+
the sandbox and signs supported AWS HTTPS requests with SigV4, so code in the
|
|
33
|
+
sandbox can use AWS SDKs normally without storing long-lived AWS keys in files,
|
|
34
|
+
environment variables, shell history, or logs.
|
|
35
|
+
|
|
36
|
+
First, store `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` as LangSmith
|
|
37
|
+
workspace secrets. Then create the sandbox with an AWS auth proxy config:
|
|
38
|
+
|
|
39
|
+
```ts
|
|
40
|
+
import {
|
|
41
|
+
SandboxClient,
|
|
42
|
+
awsAuthProxyConfig,
|
|
43
|
+
workspaceSecret,
|
|
44
|
+
} from "langsmith/sandbox";
|
|
45
|
+
|
|
46
|
+
const client = new SandboxClient();
|
|
47
|
+
|
|
48
|
+
const sandbox = await client.createSandbox({
|
|
49
|
+
name: "aws-sandbox",
|
|
50
|
+
proxyConfig: awsAuthProxyConfig({
|
|
51
|
+
accessKeyId: workspaceSecret("AWS_ACCESS_KEY_ID"),
|
|
52
|
+
secretAccessKey: workspaceSecret("AWS_SECRET_ACCESS_KEY"),
|
|
53
|
+
}),
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
try {
|
|
57
|
+
// Your sandbox code can use the AWS SDK, the AWS CLI, or other AWS tooling normally.
|
|
58
|
+
const result = await sandbox.run("node your-aws-script.js");
|
|
59
|
+
console.log(result.stdout);
|
|
60
|
+
} finally {
|
|
61
|
+
await sandbox.delete();
|
|
62
|
+
}
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Use `opaqueSecret("...")` instead of `workspaceSecret(...)` when your application
|
|
66
|
+
needs to pass short-lived write-only AWS credentials at sandbox creation time.
|
|
67
|
+
Plaintext AWS credential values are not supported.
|
|
68
|
+
|
|
28
69
|
## 1. Connect to LangSmith
|
|
29
70
|
|
|
30
71
|
Sign up for [LangSmith](https://smith.langchain.com/) using your GitHub, Discord accounts, or an email address and password. If you sign up with an email, make sure to verify your email address before logging in.
|
package/dist/client.cjs
CHANGED
|
@@ -2557,7 +2557,7 @@ class Client {
|
|
|
2557
2557
|
_hostUrl: this.getHostUrl(),
|
|
2558
2558
|
}));
|
|
2559
2559
|
}
|
|
2560
|
-
async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, }) {
|
|
2560
|
+
async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, numExamples = null, numRepetitions = null, evaluatorKeys = null, }) {
|
|
2561
2561
|
const upsert_ = upsert ? `?upsert=true` : "";
|
|
2562
2562
|
const endpoint = `${this.apiUrl}/sessions${upsert_}`;
|
|
2563
2563
|
const extra = projectExtra || {};
|
|
@@ -2572,6 +2572,15 @@ class Client {
|
|
|
2572
2572
|
if (referenceDatasetId !== null) {
|
|
2573
2573
|
body["reference_dataset_id"] = referenceDatasetId;
|
|
2574
2574
|
}
|
|
2575
|
+
if (numExamples != null) {
|
|
2576
|
+
body["num_examples"] = numExamples;
|
|
2577
|
+
}
|
|
2578
|
+
if (numRepetitions != null) {
|
|
2579
|
+
body["num_repetitions"] = numRepetitions;
|
|
2580
|
+
}
|
|
2581
|
+
if (evaluatorKeys != null && evaluatorKeys.length > 0) {
|
|
2582
|
+
body["evaluator_keys"] = evaluatorKeys;
|
|
2583
|
+
}
|
|
2575
2584
|
const serializedBody = JSON.stringify(body);
|
|
2576
2585
|
const response = await this.caller.call(async () => {
|
|
2577
2586
|
const res = await this._fetch(endpoint, {
|
|
@@ -4891,7 +4900,7 @@ class Client {
|
|
|
4891
4900
|
async _pullDirectory(identifier, repoType, version) {
|
|
4892
4901
|
const [owner, name, parsedVersion] = (0, prompts_js_1.parseHubIdentifier)(identifier);
|
|
4893
4902
|
const resolvedVersion = version ?? (parsedVersion !== "latest" ? parsedVersion : undefined);
|
|
4894
|
-
const url = new URL(`${this.apiUrl}
|
|
4903
|
+
const url = new URL(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories`)}`);
|
|
4895
4904
|
url.searchParams.set("repo_type", repoType);
|
|
4896
4905
|
if (resolvedVersion) {
|
|
4897
4906
|
url.searchParams.set("commit", resolvedVersion);
|
|
@@ -4937,7 +4946,7 @@ class Client {
|
|
|
4937
4946
|
body.parent_commit = options.parentCommit;
|
|
4938
4947
|
}
|
|
4939
4948
|
const response = await this.caller.call(async () => {
|
|
4940
|
-
const res = await this._fetch(`${this.apiUrl}
|
|
4949
|
+
const res = await this._fetch(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories/commits`)}`, {
|
|
4941
4950
|
method: "POST",
|
|
4942
4951
|
headers: {
|
|
4943
4952
|
...this._mergedHeaders,
|
|
@@ -4962,7 +4971,7 @@ class Client {
|
|
|
4962
4971
|
throw await this._ownerConflictError("delete", owner);
|
|
4963
4972
|
}
|
|
4964
4973
|
await this.caller.call(async () => {
|
|
4965
|
-
const res = await this._fetch(`${this.apiUrl}
|
|
4974
|
+
const res = await this._fetch(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories`)}`, {
|
|
4966
4975
|
method: "DELETE",
|
|
4967
4976
|
headers: this._mergedHeaders,
|
|
4968
4977
|
signal: AbortSignal.timeout(this.timeout_ms),
|
package/dist/client.d.ts
CHANGED
|
@@ -349,6 +349,9 @@ export type CreateProjectParams = {
|
|
|
349
349
|
upsert?: boolean;
|
|
350
350
|
projectExtra?: RecordStringAny | null;
|
|
351
351
|
referenceDatasetId?: string | null;
|
|
352
|
+
numExamples?: number | null;
|
|
353
|
+
numRepetitions?: number | null;
|
|
354
|
+
evaluatorKeys?: string[] | null;
|
|
352
355
|
};
|
|
353
356
|
type AutoBatchQueueItem = {
|
|
354
357
|
action: "create" | "update";
|
|
@@ -714,7 +717,7 @@ export declare class Client implements LangSmithTracingClientInterface {
|
|
|
714
717
|
listSharedExamples(shareToken: string, options?: {
|
|
715
718
|
exampleIds?: string[];
|
|
716
719
|
}): Promise<Example[]>;
|
|
717
|
-
createProject({ projectName, description, metadata, upsert, projectExtra, referenceDatasetId, }: CreateProjectParams): Promise<TracerSession>;
|
|
720
|
+
createProject({ projectName, description, metadata, upsert, projectExtra, referenceDatasetId, numExamples, numRepetitions, evaluatorKeys, }: CreateProjectParams): Promise<TracerSession>;
|
|
718
721
|
updateProject(projectId: string, { name, description, metadata, projectExtra, endTime, }: {
|
|
719
722
|
name?: string | null;
|
|
720
723
|
description?: string | null;
|
package/dist/client.js
CHANGED
|
@@ -2519,7 +2519,7 @@ export class Client {
|
|
|
2519
2519
|
_hostUrl: this.getHostUrl(),
|
|
2520
2520
|
}));
|
|
2521
2521
|
}
|
|
2522
|
-
async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, }) {
|
|
2522
|
+
async createProject({ projectName, description = null, metadata = null, upsert = false, projectExtra = null, referenceDatasetId = null, numExamples = null, numRepetitions = null, evaluatorKeys = null, }) {
|
|
2523
2523
|
const upsert_ = upsert ? `?upsert=true` : "";
|
|
2524
2524
|
const endpoint = `${this.apiUrl}/sessions${upsert_}`;
|
|
2525
2525
|
const extra = projectExtra || {};
|
|
@@ -2534,6 +2534,15 @@ export class Client {
|
|
|
2534
2534
|
if (referenceDatasetId !== null) {
|
|
2535
2535
|
body["reference_dataset_id"] = referenceDatasetId;
|
|
2536
2536
|
}
|
|
2537
|
+
if (numExamples != null) {
|
|
2538
|
+
body["num_examples"] = numExamples;
|
|
2539
|
+
}
|
|
2540
|
+
if (numRepetitions != null) {
|
|
2541
|
+
body["num_repetitions"] = numRepetitions;
|
|
2542
|
+
}
|
|
2543
|
+
if (evaluatorKeys != null && evaluatorKeys.length > 0) {
|
|
2544
|
+
body["evaluator_keys"] = evaluatorKeys;
|
|
2545
|
+
}
|
|
2537
2546
|
const serializedBody = JSON.stringify(body);
|
|
2538
2547
|
const response = await this.caller.call(async () => {
|
|
2539
2548
|
const res = await this._fetch(endpoint, {
|
|
@@ -4853,7 +4862,7 @@ export class Client {
|
|
|
4853
4862
|
async _pullDirectory(identifier, repoType, version) {
|
|
4854
4863
|
const [owner, name, parsedVersion] = parseHubIdentifier(identifier);
|
|
4855
4864
|
const resolvedVersion = version ?? (parsedVersion !== "latest" ? parsedVersion : undefined);
|
|
4856
|
-
const url = new URL(`${this.apiUrl}
|
|
4865
|
+
const url = new URL(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories`)}`);
|
|
4857
4866
|
url.searchParams.set("repo_type", repoType);
|
|
4858
4867
|
if (resolvedVersion) {
|
|
4859
4868
|
url.searchParams.set("commit", resolvedVersion);
|
|
@@ -4899,7 +4908,7 @@ export class Client {
|
|
|
4899
4908
|
body.parent_commit = options.parentCommit;
|
|
4900
4909
|
}
|
|
4901
4910
|
const response = await this.caller.call(async () => {
|
|
4902
|
-
const res = await this._fetch(`${this.apiUrl}
|
|
4911
|
+
const res = await this._fetch(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories/commits`)}`, {
|
|
4903
4912
|
method: "POST",
|
|
4904
4913
|
headers: {
|
|
4905
4914
|
...this._mergedHeaders,
|
|
@@ -4924,7 +4933,7 @@ export class Client {
|
|
|
4924
4933
|
throw await this._ownerConflictError("delete", owner);
|
|
4925
4934
|
}
|
|
4926
4935
|
await this.caller.call(async () => {
|
|
4927
|
-
const res = await this._fetch(`${this.apiUrl}
|
|
4936
|
+
const res = await this._fetch(`${this.apiUrl}${this._getPlatformEndpointPath(`hub/repos/${owner}/${name}/directories`)}`, {
|
|
4928
4937
|
method: "DELETE",
|
|
4929
4938
|
headers: this._mergedHeaders,
|
|
4930
4939
|
signal: AbortSignal.timeout(this.timeout_ms),
|
|
@@ -149,12 +149,24 @@ class _ExperimentManager {
|
|
|
149
149
|
writable: true,
|
|
150
150
|
value: void 0
|
|
151
151
|
});
|
|
152
|
+
Object.defineProperty(this, "_numExamples", {
|
|
153
|
+
enumerable: true,
|
|
154
|
+
configurable: true,
|
|
155
|
+
writable: true,
|
|
156
|
+
value: void 0
|
|
157
|
+
});
|
|
152
158
|
Object.defineProperty(this, "_numRepetitions", {
|
|
153
159
|
enumerable: true,
|
|
154
160
|
configurable: true,
|
|
155
161
|
writable: true,
|
|
156
162
|
value: void 0
|
|
157
163
|
});
|
|
164
|
+
Object.defineProperty(this, "_evaluatorKeys", {
|
|
165
|
+
enumerable: true,
|
|
166
|
+
configurable: true,
|
|
167
|
+
writable: true,
|
|
168
|
+
value: void 0
|
|
169
|
+
});
|
|
158
170
|
Object.defineProperty(this, "_runsArray", {
|
|
159
171
|
enumerable: true,
|
|
160
172
|
configurable: true,
|
|
@@ -231,6 +243,8 @@ class _ExperimentManager {
|
|
|
231
243
|
this._summaryResults = args.summaryResults;
|
|
232
244
|
this._resultRows = args.resultRows;
|
|
233
245
|
this._numRepetitions = args.numRepetitions;
|
|
246
|
+
this._numExamples = args.numExamples;
|
|
247
|
+
this._evaluatorKeys = args.evaluatorKeys;
|
|
234
248
|
this._includeAttachments = args.includeAttachments;
|
|
235
249
|
}
|
|
236
250
|
_getExperiment() {
|
|
@@ -263,6 +277,12 @@ class _ExperimentManager {
|
|
|
263
277
|
// Create the project, updating the experimentName until we find a unique one.
|
|
264
278
|
let project;
|
|
265
279
|
const originalExperimentName = this._experimentName;
|
|
280
|
+
if (this._numExamples === undefined) {
|
|
281
|
+
this._numExamples = await _resolveNumExamples(this._data, this.client);
|
|
282
|
+
}
|
|
283
|
+
const numExamples = this._numExamples ?? null;
|
|
284
|
+
const numRepetitions = this._numRepetitions ?? null;
|
|
285
|
+
const evaluatorKeys = this._evaluatorKeys ?? null;
|
|
266
286
|
for (let i = 0; i < 10; i++) {
|
|
267
287
|
try {
|
|
268
288
|
project = await this.client.createProject({
|
|
@@ -270,6 +290,9 @@ class _ExperimentManager {
|
|
|
270
290
|
referenceDatasetId: firstExample.dataset_id,
|
|
271
291
|
metadata: projectMetadata,
|
|
272
292
|
description: this._description,
|
|
293
|
+
numExamples,
|
|
294
|
+
numRepetitions,
|
|
295
|
+
evaluatorKeys,
|
|
273
296
|
});
|
|
274
297
|
return project;
|
|
275
298
|
}
|
|
@@ -319,6 +342,9 @@ class _ExperimentManager {
|
|
|
319
342
|
client: this.client,
|
|
320
343
|
evaluationResults: this._evaluationResults,
|
|
321
344
|
summaryResults: this._summaryResults,
|
|
345
|
+
numRepetitions: this._numRepetitions,
|
|
346
|
+
numExamples: this._numExamples,
|
|
347
|
+
evaluatorKeys: this._evaluatorKeys,
|
|
322
348
|
includeAttachments: this._includeAttachments,
|
|
323
349
|
});
|
|
324
350
|
}
|
|
@@ -345,6 +371,9 @@ class _ExperimentManager {
|
|
|
345
371
|
yield pred.run;
|
|
346
372
|
}
|
|
347
373
|
})(),
|
|
374
|
+
numRepetitions: this._numRepetitions,
|
|
375
|
+
numExamples: this._numExamples,
|
|
376
|
+
evaluatorKeys: this._evaluatorKeys,
|
|
348
377
|
includeAttachments: this._includeAttachments,
|
|
349
378
|
});
|
|
350
379
|
}
|
|
@@ -369,6 +398,9 @@ class _ExperimentManager {
|
|
|
369
398
|
}
|
|
370
399
|
})(),
|
|
371
400
|
summaryResults: this._summaryResults,
|
|
401
|
+
numRepetitions: this._numRepetitions,
|
|
402
|
+
numExamples: this._numExamples,
|
|
403
|
+
evaluatorKeys: this._evaluatorKeys,
|
|
372
404
|
includeAttachments: this._includeAttachments,
|
|
373
405
|
});
|
|
374
406
|
}
|
|
@@ -384,6 +416,9 @@ class _ExperimentManager {
|
|
|
384
416
|
evaluationResults: this._evaluationResults,
|
|
385
417
|
resultRows: this._resultRows,
|
|
386
418
|
summaryResults: aggregateFeedbackGen,
|
|
419
|
+
numRepetitions: this._numRepetitions,
|
|
420
|
+
numExamples: this._numExamples,
|
|
421
|
+
evaluatorKeys: this._evaluatorKeys,
|
|
387
422
|
includeAttachments: this._includeAttachments,
|
|
388
423
|
});
|
|
389
424
|
}
|
|
@@ -732,6 +767,7 @@ async function _evaluate(target, fields) {
|
|
|
732
767
|
experiment: experiment_ ?? fields.experimentPrefix,
|
|
733
768
|
runs: newRuns ?? undefined,
|
|
734
769
|
numRepetitions: fields.numRepetitions ?? 1,
|
|
770
|
+
evaluatorKeys: _collectEvaluatorKeys(standardFields.evaluators),
|
|
735
771
|
includeAttachments: standardFields.includeAttachments,
|
|
736
772
|
}).start();
|
|
737
773
|
const targetConcurrency = standardFields.targetConcurrency ?? standardFields.maxConcurrency ?? 0;
|
|
@@ -852,6 +888,47 @@ Try setting "LANGSMITH_TRACING=true" in your environment.`);
|
|
|
852
888
|
example,
|
|
853
889
|
};
|
|
854
890
|
}
|
|
891
|
+
function _collectEvaluatorKeys(evaluators) {
|
|
892
|
+
if (!evaluators)
|
|
893
|
+
return [];
|
|
894
|
+
const keys = [];
|
|
895
|
+
for (const ev of evaluators) {
|
|
896
|
+
const name = ev.name;
|
|
897
|
+
if (name && name.length > 0)
|
|
898
|
+
keys.push(name);
|
|
899
|
+
}
|
|
900
|
+
return keys;
|
|
901
|
+
}
|
|
902
|
+
async function _resolveNumExamples(data, client) {
|
|
903
|
+
// Best-effort dataset size for the experiment progress hint. Returns
|
|
904
|
+
// undefined for lazy async iterators where size cannot be inferred without
|
|
905
|
+
// consuming.
|
|
906
|
+
if (data === undefined)
|
|
907
|
+
return undefined;
|
|
908
|
+
try {
|
|
909
|
+
if (Array.isArray(data)) {
|
|
910
|
+
return data.length;
|
|
911
|
+
}
|
|
912
|
+
if (typeof data === "string") {
|
|
913
|
+
let isUUID = false;
|
|
914
|
+
try {
|
|
915
|
+
(0, _uuid_js_1.assertUuid)(data);
|
|
916
|
+
isUUID = true;
|
|
917
|
+
}
|
|
918
|
+
catch (_) {
|
|
919
|
+
isUUID = false;
|
|
920
|
+
}
|
|
921
|
+
const dataset = isUUID
|
|
922
|
+
? await client.readDataset({ datasetId: data })
|
|
923
|
+
: await client.readDataset({ datasetName: data });
|
|
924
|
+
return dataset.example_count ?? undefined;
|
|
925
|
+
}
|
|
926
|
+
return undefined;
|
|
927
|
+
}
|
|
928
|
+
catch (_) {
|
|
929
|
+
return undefined;
|
|
930
|
+
}
|
|
931
|
+
}
|
|
855
932
|
function _resolveData(data, options) {
|
|
856
933
|
let isUUID = false;
|
|
857
934
|
try {
|
|
@@ -68,6 +68,8 @@ interface _ExperimentManagerArgs {
|
|
|
68
68
|
summaryResults?: AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults, any, unknown>, any, unknown>;
|
|
69
69
|
examples?: Example[];
|
|
70
70
|
numRepetitions?: number;
|
|
71
|
+
numExamples?: number;
|
|
72
|
+
evaluatorKeys?: string[];
|
|
71
73
|
_runsArray?: Run[];
|
|
72
74
|
resultRows?: AsyncGenerator<_ExperimentResultRowWithIndex>;
|
|
73
75
|
includeAttachments?: boolean;
|
|
@@ -182,7 +184,9 @@ export declare class _ExperimentManager {
|
|
|
182
184
|
_summaryResults?: AsyncGenerator<(runsArray: Run[]) => AsyncGenerator<EvaluationResults, any, unknown>, any, unknown>;
|
|
183
185
|
_resultRows?: AsyncGenerator<_ExperimentResultRowWithIndex>;
|
|
184
186
|
_examples?: Example[];
|
|
187
|
+
_numExamples?: number;
|
|
185
188
|
_numRepetitions?: number;
|
|
189
|
+
_evaluatorKeys?: string[];
|
|
186
190
|
_runsArray?: Run[];
|
|
187
191
|
client: Client;
|
|
188
192
|
_experiment?: TracerSession;
|
|
@@ -143,12 +143,24 @@ export class _ExperimentManager {
|
|
|
143
143
|
writable: true,
|
|
144
144
|
value: void 0
|
|
145
145
|
});
|
|
146
|
+
Object.defineProperty(this, "_numExamples", {
|
|
147
|
+
enumerable: true,
|
|
148
|
+
configurable: true,
|
|
149
|
+
writable: true,
|
|
150
|
+
value: void 0
|
|
151
|
+
});
|
|
146
152
|
Object.defineProperty(this, "_numRepetitions", {
|
|
147
153
|
enumerable: true,
|
|
148
154
|
configurable: true,
|
|
149
155
|
writable: true,
|
|
150
156
|
value: void 0
|
|
151
157
|
});
|
|
158
|
+
Object.defineProperty(this, "_evaluatorKeys", {
|
|
159
|
+
enumerable: true,
|
|
160
|
+
configurable: true,
|
|
161
|
+
writable: true,
|
|
162
|
+
value: void 0
|
|
163
|
+
});
|
|
152
164
|
Object.defineProperty(this, "_runsArray", {
|
|
153
165
|
enumerable: true,
|
|
154
166
|
configurable: true,
|
|
@@ -225,6 +237,8 @@ export class _ExperimentManager {
|
|
|
225
237
|
this._summaryResults = args.summaryResults;
|
|
226
238
|
this._resultRows = args.resultRows;
|
|
227
239
|
this._numRepetitions = args.numRepetitions;
|
|
240
|
+
this._numExamples = args.numExamples;
|
|
241
|
+
this._evaluatorKeys = args.evaluatorKeys;
|
|
228
242
|
this._includeAttachments = args.includeAttachments;
|
|
229
243
|
}
|
|
230
244
|
_getExperiment() {
|
|
@@ -257,6 +271,12 @@ export class _ExperimentManager {
|
|
|
257
271
|
// Create the project, updating the experimentName until we find a unique one.
|
|
258
272
|
let project;
|
|
259
273
|
const originalExperimentName = this._experimentName;
|
|
274
|
+
if (this._numExamples === undefined) {
|
|
275
|
+
this._numExamples = await _resolveNumExamples(this._data, this.client);
|
|
276
|
+
}
|
|
277
|
+
const numExamples = this._numExamples ?? null;
|
|
278
|
+
const numRepetitions = this._numRepetitions ?? null;
|
|
279
|
+
const evaluatorKeys = this._evaluatorKeys ?? null;
|
|
260
280
|
for (let i = 0; i < 10; i++) {
|
|
261
281
|
try {
|
|
262
282
|
project = await this.client.createProject({
|
|
@@ -264,6 +284,9 @@ export class _ExperimentManager {
|
|
|
264
284
|
referenceDatasetId: firstExample.dataset_id,
|
|
265
285
|
metadata: projectMetadata,
|
|
266
286
|
description: this._description,
|
|
287
|
+
numExamples,
|
|
288
|
+
numRepetitions,
|
|
289
|
+
evaluatorKeys,
|
|
267
290
|
});
|
|
268
291
|
return project;
|
|
269
292
|
}
|
|
@@ -313,6 +336,9 @@ export class _ExperimentManager {
|
|
|
313
336
|
client: this.client,
|
|
314
337
|
evaluationResults: this._evaluationResults,
|
|
315
338
|
summaryResults: this._summaryResults,
|
|
339
|
+
numRepetitions: this._numRepetitions,
|
|
340
|
+
numExamples: this._numExamples,
|
|
341
|
+
evaluatorKeys: this._evaluatorKeys,
|
|
316
342
|
includeAttachments: this._includeAttachments,
|
|
317
343
|
});
|
|
318
344
|
}
|
|
@@ -339,6 +365,9 @@ export class _ExperimentManager {
|
|
|
339
365
|
yield pred.run;
|
|
340
366
|
}
|
|
341
367
|
})(),
|
|
368
|
+
numRepetitions: this._numRepetitions,
|
|
369
|
+
numExamples: this._numExamples,
|
|
370
|
+
evaluatorKeys: this._evaluatorKeys,
|
|
342
371
|
includeAttachments: this._includeAttachments,
|
|
343
372
|
});
|
|
344
373
|
}
|
|
@@ -363,6 +392,9 @@ export class _ExperimentManager {
|
|
|
363
392
|
}
|
|
364
393
|
})(),
|
|
365
394
|
summaryResults: this._summaryResults,
|
|
395
|
+
numRepetitions: this._numRepetitions,
|
|
396
|
+
numExamples: this._numExamples,
|
|
397
|
+
evaluatorKeys: this._evaluatorKeys,
|
|
366
398
|
includeAttachments: this._includeAttachments,
|
|
367
399
|
});
|
|
368
400
|
}
|
|
@@ -378,6 +410,9 @@ export class _ExperimentManager {
|
|
|
378
410
|
evaluationResults: this._evaluationResults,
|
|
379
411
|
resultRows: this._resultRows,
|
|
380
412
|
summaryResults: aggregateFeedbackGen,
|
|
413
|
+
numRepetitions: this._numRepetitions,
|
|
414
|
+
numExamples: this._numExamples,
|
|
415
|
+
evaluatorKeys: this._evaluatorKeys,
|
|
381
416
|
includeAttachments: this._includeAttachments,
|
|
382
417
|
});
|
|
383
418
|
}
|
|
@@ -725,6 +760,7 @@ async function _evaluate(target, fields) {
|
|
|
725
760
|
experiment: experiment_ ?? fields.experimentPrefix,
|
|
726
761
|
runs: newRuns ?? undefined,
|
|
727
762
|
numRepetitions: fields.numRepetitions ?? 1,
|
|
763
|
+
evaluatorKeys: _collectEvaluatorKeys(standardFields.evaluators),
|
|
728
764
|
includeAttachments: standardFields.includeAttachments,
|
|
729
765
|
}).start();
|
|
730
766
|
const targetConcurrency = standardFields.targetConcurrency ?? standardFields.maxConcurrency ?? 0;
|
|
@@ -845,6 +881,47 @@ Try setting "LANGSMITH_TRACING=true" in your environment.`);
|
|
|
845
881
|
example,
|
|
846
882
|
};
|
|
847
883
|
}
|
|
884
|
+
function _collectEvaluatorKeys(evaluators) {
|
|
885
|
+
if (!evaluators)
|
|
886
|
+
return [];
|
|
887
|
+
const keys = [];
|
|
888
|
+
for (const ev of evaluators) {
|
|
889
|
+
const name = ev.name;
|
|
890
|
+
if (name && name.length > 0)
|
|
891
|
+
keys.push(name);
|
|
892
|
+
}
|
|
893
|
+
return keys;
|
|
894
|
+
}
|
|
895
|
+
async function _resolveNumExamples(data, client) {
|
|
896
|
+
// Best-effort dataset size for the experiment progress hint. Returns
|
|
897
|
+
// undefined for lazy async iterators where size cannot be inferred without
|
|
898
|
+
// consuming.
|
|
899
|
+
if (data === undefined)
|
|
900
|
+
return undefined;
|
|
901
|
+
try {
|
|
902
|
+
if (Array.isArray(data)) {
|
|
903
|
+
return data.length;
|
|
904
|
+
}
|
|
905
|
+
if (typeof data === "string") {
|
|
906
|
+
let isUUID = false;
|
|
907
|
+
try {
|
|
908
|
+
assertUuid(data);
|
|
909
|
+
isUUID = true;
|
|
910
|
+
}
|
|
911
|
+
catch (_) {
|
|
912
|
+
isUUID = false;
|
|
913
|
+
}
|
|
914
|
+
const dataset = isUUID
|
|
915
|
+
? await client.readDataset({ datasetId: data })
|
|
916
|
+
: await client.readDataset({ datasetName: data });
|
|
917
|
+
return dataset.example_count ?? undefined;
|
|
918
|
+
}
|
|
919
|
+
return undefined;
|
|
920
|
+
}
|
|
921
|
+
catch (_) {
|
|
922
|
+
return undefined;
|
|
923
|
+
}
|
|
924
|
+
}
|
|
848
925
|
function _resolveData(data, options) {
|
|
849
926
|
let isUUID = false;
|
|
850
927
|
try {
|
package/dist/index.cjs
CHANGED
|
@@ -18,6 +18,6 @@ Object.defineProperty(exports, "PromptCache", { enumerable: true, get: function
|
|
|
18
18
|
Object.defineProperty(exports, "configureGlobalPromptCache", { enumerable: true, get: function () { return index_js_1.configureGlobalPromptCache; } });
|
|
19
19
|
Object.defineProperty(exports, "promptCacheSingleton", { enumerable: true, get: function () { return index_js_1.promptCacheSingleton; } });
|
|
20
20
|
// Update using pnpm bump-version
|
|
21
|
-
exports.__version__ = "0.7.
|
|
21
|
+
exports.__version__ = "0.7.7";
|
|
22
22
|
// Metadata key to hide a traced run from LangSmith's Messages View.
|
|
23
23
|
exports.LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
|
package/dist/index.d.ts
CHANGED
|
@@ -5,5 +5,5 @@ export { overrideFetchImplementation } from "./singletons/fetch.js";
|
|
|
5
5
|
export { getDefaultProjectName } from "./utils/project.js";
|
|
6
6
|
export { uuid7, uuid7FromTime } from "./uuid.js";
|
|
7
7
|
export { Cache, PromptCache, type CacheConfig, type CacheMetrics, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
|
|
8
|
-
export declare const __version__ = "0.7.
|
|
8
|
+
export declare const __version__ = "0.7.7";
|
|
9
9
|
export declare const LS_MESSAGE_VIEW_EXCLUDE: "ls_message_view_exclude";
|
package/dist/index.js
CHANGED
|
@@ -5,6 +5,6 @@ export { getDefaultProjectName } from "./utils/project.js";
|
|
|
5
5
|
export { uuid7, uuid7FromTime } from "./uuid.js";
|
|
6
6
|
export { Cache, PromptCache, configureGlobalPromptCache, promptCacheSingleton, } from "./utils/prompt_cache/index.js";
|
|
7
7
|
// Update using pnpm bump-version
|
|
8
|
-
export const __version__ = "0.7.
|
|
8
|
+
export const __version__ = "0.7.7";
|
|
9
9
|
// Metadata key to hide a traced run from LangSmith's Messages View.
|
|
10
10
|
export const LS_MESSAGE_VIEW_EXCLUDE = "ls_message_view_exclude";
|
|
@@ -107,11 +107,25 @@ class CommandHandle {
|
|
|
107
107
|
writable: true,
|
|
108
108
|
value: void 0
|
|
109
109
|
});
|
|
110
|
+
Object.defineProperty(this, "_onStdout", {
|
|
111
|
+
enumerable: true,
|
|
112
|
+
configurable: true,
|
|
113
|
+
writable: true,
|
|
114
|
+
value: void 0
|
|
115
|
+
});
|
|
116
|
+
Object.defineProperty(this, "_onStderr", {
|
|
117
|
+
enumerable: true,
|
|
118
|
+
configurable: true,
|
|
119
|
+
writable: true,
|
|
120
|
+
value: void 0
|
|
121
|
+
});
|
|
110
122
|
this._stream = messageStream;
|
|
111
123
|
this._control = control;
|
|
112
124
|
this._sandbox = sandbox;
|
|
113
125
|
this._lastStdoutOffset = options?.stdoutOffset ?? 0;
|
|
114
126
|
this._lastStderrOffset = options?.stderrOffset ?? 0;
|
|
127
|
+
this._onStdout = options?.onStdout;
|
|
128
|
+
this._onStderr = options?.onStderr;
|
|
115
129
|
// New executions (no commandId): _ensureStarted reads "started".
|
|
116
130
|
// Reconnections (commandId set): skip since reconnect streams
|
|
117
131
|
// don't send a "started" message.
|
|
@@ -221,10 +235,12 @@ class CommandHandle {
|
|
|
221
235
|
if (chunk.stream === "stdout") {
|
|
222
236
|
this._lastStdoutOffset =
|
|
223
237
|
chunk.offset + new TextEncoder().encode(chunk.data).length;
|
|
238
|
+
this._onStdout?.(chunk.data);
|
|
224
239
|
}
|
|
225
240
|
else {
|
|
226
241
|
this._lastStderrOffset =
|
|
227
242
|
chunk.offset + new TextEncoder().encode(chunk.data).length;
|
|
243
|
+
this._onStderr?.(chunk.data);
|
|
228
244
|
}
|
|
229
245
|
yield chunk;
|
|
230
246
|
}
|
|
@@ -104,11 +104,25 @@ export class CommandHandle {
|
|
|
104
104
|
writable: true,
|
|
105
105
|
value: void 0
|
|
106
106
|
});
|
|
107
|
+
Object.defineProperty(this, "_onStdout", {
|
|
108
|
+
enumerable: true,
|
|
109
|
+
configurable: true,
|
|
110
|
+
writable: true,
|
|
111
|
+
value: void 0
|
|
112
|
+
});
|
|
113
|
+
Object.defineProperty(this, "_onStderr", {
|
|
114
|
+
enumerable: true,
|
|
115
|
+
configurable: true,
|
|
116
|
+
writable: true,
|
|
117
|
+
value: void 0
|
|
118
|
+
});
|
|
107
119
|
this._stream = messageStream;
|
|
108
120
|
this._control = control;
|
|
109
121
|
this._sandbox = sandbox;
|
|
110
122
|
this._lastStdoutOffset = options?.stdoutOffset ?? 0;
|
|
111
123
|
this._lastStderrOffset = options?.stderrOffset ?? 0;
|
|
124
|
+
this._onStdout = options?.onStdout;
|
|
125
|
+
this._onStderr = options?.onStderr;
|
|
112
126
|
// New executions (no commandId): _ensureStarted reads "started".
|
|
113
127
|
// Reconnections (commandId set): skip since reconnect streams
|
|
114
128
|
// don't send a "started" message.
|
|
@@ -218,10 +232,12 @@ export class CommandHandle {
|
|
|
218
232
|
if (chunk.stream === "stdout") {
|
|
219
233
|
this._lastStdoutOffset =
|
|
220
234
|
chunk.offset + new TextEncoder().encode(chunk.data).length;
|
|
235
|
+
this._onStdout?.(chunk.data);
|
|
221
236
|
}
|
|
222
237
|
else {
|
|
223
238
|
this._lastStderrOffset =
|
|
224
239
|
chunk.offset + new TextEncoder().encode(chunk.data).length;
|
|
240
|
+
this._onStderr?.(chunk.data);
|
|
225
241
|
}
|
|
226
242
|
yield chunk;
|
|
227
243
|
}
|
package/dist/sandbox/index.cjs
CHANGED
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
* @packageDocumentation
|
|
30
30
|
*/
|
|
31
31
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
32
|
-
exports.LangSmithDataplaneNotConfiguredError = exports.LangSmithCommandTimeoutError = exports.LangSmithSandboxOperationError = exports.LangSmithSandboxNotReadyError = exports.LangSmithSandboxCreationError = exports.LangSmithResourceCreationError = exports.LangSmithQuotaExceededError = exports.LangSmithValidationError = exports.LangSmithResourceNameConflictError = exports.LangSmithResourceAlreadyExistsError = exports.LangSmithResourceInUseError = exports.LangSmithResourceTimeoutError = exports.LangSmithResourceNotFoundError = exports.LangSmithSandboxServerReloadError = exports.LangSmithSandboxConnectionError = exports.LangSmithSandboxAuthenticationError = exports.LangSmithSandboxAPIError = exports.LangSmithSandboxError = exports.CommandHandle = exports.Sandbox = exports.SandboxClient = void 0;
|
|
32
|
+
exports.LangSmithDataplaneNotConfiguredError = exports.LangSmithCommandTimeoutError = exports.LangSmithSandboxOperationError = exports.LangSmithSandboxNotReadyError = exports.LangSmithSandboxCreationError = exports.LangSmithResourceCreationError = exports.LangSmithQuotaExceededError = exports.LangSmithValidationError = exports.LangSmithResourceNameConflictError = exports.LangSmithResourceAlreadyExistsError = exports.LangSmithResourceInUseError = exports.LangSmithResourceTimeoutError = exports.LangSmithResourceNotFoundError = exports.LangSmithSandboxServerReloadError = exports.LangSmithSandboxConnectionError = exports.LangSmithSandboxAuthenticationError = exports.LangSmithSandboxAPIError = exports.LangSmithSandboxError = exports.workspaceSecret = exports.opaqueSecret = exports.awsAuthProxyConfig = exports.CommandHandle = exports.Sandbox = exports.SandboxClient = void 0;
|
|
33
33
|
// Main classes
|
|
34
34
|
var client_js_1 = require("./client.cjs");
|
|
35
35
|
Object.defineProperty(exports, "SandboxClient", { enumerable: true, get: function () { return client_js_1.SandboxClient; } });
|
|
@@ -37,6 +37,10 @@ var sandbox_js_1 = require("./sandbox.cjs");
|
|
|
37
37
|
Object.defineProperty(exports, "Sandbox", { enumerable: true, get: function () { return sandbox_js_1.Sandbox; } });
|
|
38
38
|
var command_handle_js_1 = require("./command_handle.cjs");
|
|
39
39
|
Object.defineProperty(exports, "CommandHandle", { enumerable: true, get: function () { return command_handle_js_1.CommandHandle; } });
|
|
40
|
+
var proxy_config_js_1 = require("./proxy_config.cjs");
|
|
41
|
+
Object.defineProperty(exports, "awsAuthProxyConfig", { enumerable: true, get: function () { return proxy_config_js_1.awsAuthProxyConfig; } });
|
|
42
|
+
Object.defineProperty(exports, "opaqueSecret", { enumerable: true, get: function () { return proxy_config_js_1.opaqueSecret; } });
|
|
43
|
+
Object.defineProperty(exports, "workspaceSecret", { enumerable: true, get: function () { return proxy_config_js_1.workspaceSecret; } });
|
|
40
44
|
// Errors
|
|
41
45
|
var errors_js_1 = require("./errors.cjs");
|
|
42
46
|
// Base and connection errors
|
package/dist/sandbox/index.d.ts
CHANGED
|
@@ -30,5 +30,6 @@
|
|
|
30
30
|
export { SandboxClient } from "./client.js";
|
|
31
31
|
export { Sandbox } from "./sandbox.js";
|
|
32
32
|
export { CommandHandle } from "./command_handle.js";
|
|
33
|
-
export
|
|
33
|
+
export { awsAuthProxyConfig, opaqueSecret, workspaceSecret, } from "./proxy_config.js";
|
|
34
|
+
export type { ExecutionResult, OutputChunk, WsMessage, WsRunOptions, ResourceStatus, Snapshot, SandboxData, SandboxClientConfig, RunOptions, CreateSandboxOptions, SandboxAccessControl, SandboxAwsAuthRule, SandboxProxyConfig, SandboxProxySecret, CreateSnapshotOptions, CreateDockerfileSnapshotOptions, CaptureSnapshotOptions, ListSnapshotsOptions, WaitForSnapshotOptions, StartSandboxOptions, UpdateSandboxOptions, WaitForSandboxOptions, } from "./types.js";
|
|
34
35
|
export { LangSmithSandboxError, LangSmithSandboxAPIError, LangSmithSandboxAuthenticationError, LangSmithSandboxConnectionError, LangSmithSandboxServerReloadError, LangSmithResourceNotFoundError, LangSmithResourceTimeoutError, LangSmithResourceInUseError, LangSmithResourceAlreadyExistsError, LangSmithResourceNameConflictError, LangSmithValidationError, LangSmithQuotaExceededError, LangSmithResourceCreationError, LangSmithSandboxCreationError, LangSmithSandboxNotReadyError, LangSmithSandboxOperationError, LangSmithCommandTimeoutError, LangSmithDataplaneNotConfiguredError, } from "./errors.js";
|
package/dist/sandbox/index.js
CHANGED
|
@@ -31,6 +31,7 @@
|
|
|
31
31
|
export { SandboxClient } from "./client.js";
|
|
32
32
|
export { Sandbox } from "./sandbox.js";
|
|
33
33
|
export { CommandHandle } from "./command_handle.js";
|
|
34
|
+
export { awsAuthProxyConfig, opaqueSecret, workspaceSecret, } from "./proxy_config.js";
|
|
34
35
|
// Errors
|
|
35
36
|
export {
|
|
36
37
|
// Base and connection errors
|