langsmith 0.7.4 → 0.7.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +41 -0
  2. package/dist/client.cjs +42 -8
  3. package/dist/client.d.ts +11 -3
  4. package/dist/client.js +42 -8
  5. package/dist/evaluation/_runner.cjs +77 -0
  6. package/dist/evaluation/_runner.d.ts +4 -0
  7. package/dist/evaluation/_runner.js +77 -0
  8. package/dist/evaluation/evaluate_comparative.cjs +1 -1
  9. package/dist/evaluation/evaluate_comparative.d.ts +7 -1
  10. package/dist/evaluation/evaluate_comparative.js +1 -1
  11. package/dist/index.cjs +4 -2
  12. package/dist/index.d.ts +2 -1
  13. package/dist/index.js +3 -1
  14. package/dist/sandbox/index.cjs +5 -1
  15. package/dist/sandbox/index.d.ts +2 -1
  16. package/dist/sandbox/index.js +1 -0
  17. package/dist/sandbox/proxy_config.cjs +47 -0
  18. package/dist/sandbox/proxy_config.d.ts +12 -0
  19. package/dist/sandbox/proxy_config.js +42 -0
  20. package/dist/sandbox/types.d.ts +23 -1
  21. package/dist/utils/fs.browser.cjs +11 -0
  22. package/dist/utils/fs.browser.d.ts +3 -0
  23. package/dist/utils/fs.browser.js +8 -0
  24. package/dist/utils/fs.cjs +22 -0
  25. package/dist/utils/fs.d.ts +3 -0
  26. package/dist/utils/fs.js +19 -0
  27. package/dist/utils/profile-lock.cjs +140 -0
  28. package/dist/utils/profile-lock.d.ts +20 -0
  29. package/dist/utils/profile-lock.js +103 -0
  30. package/dist/utils/profiles.cjs +28 -2
  31. package/dist/utils/profiles.d.ts +1 -0
  32. package/dist/utils/profiles.js +28 -2
  33. package/dist/wrappers/gemini.cjs +3 -40
  34. package/dist/wrappers/gemini.js +3 -40
  35. package/dist/wrappers/gemini.utils.cjs +41 -0
  36. package/dist/wrappers/gemini.utils.d.ts +3 -0
  37. package/dist/wrappers/gemini.utils.js +37 -0
  38. package/package.json +2 -2
@@ -38,6 +38,7 @@ exports.hasValue = hasValue;
38
38
  exports.loadProfileClientConfig = loadProfileClientConfig;
39
39
  const env_js_1 = require("./env.cjs");
40
40
  const fsUtils = __importStar(require("./fs.cjs"));
41
+ const profile_lock_js_1 = require("./profile-lock.cjs");
41
42
  exports.DEFAULT_API_URL = "https://api.smith.langchain.com";
42
43
  const OAUTH_CLIENT_ID = "langsmith-cli";
43
44
  const TOKEN_REFRESH_LEEWAY_MS = 60_000;
@@ -227,17 +228,39 @@ class ProfileAuth {
227
228
  isProfileAuthorizationHeader(value) {
228
229
  return value === this.managedAuthorizationValue;
229
230
  }
231
+ reloadProfile() {
232
+ try {
233
+ const config = JSON.parse(fsUtils.readFileSync(this.state.configPath));
234
+ const profile = config.profiles?.[this.state.profileName];
235
+ if (!profile) {
236
+ return undefined;
237
+ }
238
+ this.state.config = config;
239
+ this.state.profile = profile;
240
+ return profile;
241
+ }
242
+ catch {
243
+ return undefined;
244
+ }
245
+ }
230
246
  async refreshOAuthToken(fetchImplementation) {
231
247
  const refreshToken = this.state.profile.oauth?.refresh_token;
232
248
  if (!refreshToken) {
233
249
  return;
234
250
  }
235
251
  const refreshApiUrl = trimConfigValue(this.state.profile.api_url) ?? exports.DEFAULT_API_URL;
252
+ const deadline = Date.now() + TOKEN_REFRESH_TIMEOUT_MS;
253
+ let lock;
236
254
  try {
255
+ lock = await (0, profile_lock_js_1.acquireOAuthRefreshLock)(this.state.configPath, deadline);
256
+ const fresh = this.reloadProfile();
257
+ if (fresh && !shouldRefreshProfileToken(this.state.profile)) {
258
+ return;
259
+ }
237
260
  const body = new URLSearchParams({
238
261
  grant_type: "refresh_token",
239
262
  client_id: OAUTH_CLIENT_ID,
240
- refresh_token: refreshToken,
263
+ refresh_token: this.state.profile.oauth?.refresh_token ?? refreshToken,
241
264
  });
242
265
  const response = await fetchImplementation(`${normalizeConfigUrl(refreshApiUrl)}/oauth/token`, {
243
266
  method: "POST",
@@ -245,7 +268,7 @@ class ProfileAuth {
245
268
  "Content-Type": "application/x-www-form-urlencoded",
246
269
  },
247
270
  body: body.toString(),
248
- signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS),
271
+ signal: AbortSignal.timeout(Math.max(0, deadline - Date.now())),
249
272
  });
250
273
  if (!response.ok) {
251
274
  return;
@@ -262,6 +285,9 @@ class ProfileAuth {
262
285
  catch {
263
286
  return;
264
287
  }
288
+ finally {
289
+ await lock?.release();
290
+ }
265
291
  }
266
292
  rememberProfileAuthHeader(header) {
267
293
  this.managedAuthorizationValue =
@@ -42,6 +42,7 @@ export declare class ProfileAuth {
42
42
  currentAuthHeader(): ProfileAuthHeader | undefined;
43
43
  getAuthHeader(fetchImplementation: typeof fetch, signal?: AbortSignal | null): Promise<ProfileAuthHeader | undefined>;
44
44
  isProfileAuthorizationHeader(value: string): boolean;
45
+ private reloadProfile;
45
46
  private refreshOAuthToken;
46
47
  private rememberProfileAuthHeader;
47
48
  }
@@ -1,5 +1,6 @@
1
1
  import { getEnv, getEnvironmentVariable } from "./env.js";
2
2
  import * as fsUtils from "./fs.js";
3
+ import { acquireOAuthRefreshLock } from "./profile-lock.js";
3
4
  export const DEFAULT_API_URL = "https://api.smith.langchain.com";
4
5
  const OAUTH_CLIENT_ID = "langsmith-cli";
5
6
  const TOKEN_REFRESH_LEEWAY_MS = 60_000;
@@ -189,17 +190,39 @@ export class ProfileAuth {
189
190
  isProfileAuthorizationHeader(value) {
190
191
  return value === this.managedAuthorizationValue;
191
192
  }
193
+ reloadProfile() {
194
+ try {
195
+ const config = JSON.parse(fsUtils.readFileSync(this.state.configPath));
196
+ const profile = config.profiles?.[this.state.profileName];
197
+ if (!profile) {
198
+ return undefined;
199
+ }
200
+ this.state.config = config;
201
+ this.state.profile = profile;
202
+ return profile;
203
+ }
204
+ catch {
205
+ return undefined;
206
+ }
207
+ }
192
208
  async refreshOAuthToken(fetchImplementation) {
193
209
  const refreshToken = this.state.profile.oauth?.refresh_token;
194
210
  if (!refreshToken) {
195
211
  return;
196
212
  }
197
213
  const refreshApiUrl = trimConfigValue(this.state.profile.api_url) ?? DEFAULT_API_URL;
214
+ const deadline = Date.now() + TOKEN_REFRESH_TIMEOUT_MS;
215
+ let lock;
198
216
  try {
217
+ lock = await acquireOAuthRefreshLock(this.state.configPath, deadline);
218
+ const fresh = this.reloadProfile();
219
+ if (fresh && !shouldRefreshProfileToken(this.state.profile)) {
220
+ return;
221
+ }
199
222
  const body = new URLSearchParams({
200
223
  grant_type: "refresh_token",
201
224
  client_id: OAUTH_CLIENT_ID,
202
- refresh_token: refreshToken,
225
+ refresh_token: this.state.profile.oauth?.refresh_token ?? refreshToken,
203
226
  });
204
227
  const response = await fetchImplementation(`${normalizeConfigUrl(refreshApiUrl)}/oauth/token`, {
205
228
  method: "POST",
@@ -207,7 +230,7 @@ export class ProfileAuth {
207
230
  "Content-Type": "application/x-www-form-urlencoded",
208
231
  },
209
232
  body: body.toString(),
210
- signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS),
233
+ signal: AbortSignal.timeout(Math.max(0, deadline - Date.now())),
211
234
  });
212
235
  if (!response.ok) {
213
236
  return;
@@ -224,6 +247,9 @@ export class ProfileAuth {
224
247
  catch {
225
248
  return;
226
249
  }
250
+ finally {
251
+ await lock?.release();
252
+ }
227
253
  }
228
254
  rememberProfileAuthHeader(header) {
229
255
  this.managedAuthorizationValue =
@@ -2,44 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.wrapGemini = wrapGemini;
4
4
  const traceable_js_1 = require("../traceable.cjs");
5
- const _createUsageMetadata = (usage) => {
6
- const usageMetadata = {
7
- input_tokens: usage.promptTokenCount || 0,
8
- output_tokens: (() => {
9
- if ("responseTokenCount" in usage) {
10
- return usage.responseTokenCount || 0;
11
- }
12
- if ("candidatesTokenCount" in usage) {
13
- return usage.candidatesTokenCount || 0;
14
- }
15
- return 0;
16
- })(),
17
- total_tokens: usage.totalTokenCount || 0,
18
- };
19
- // Add input token details if available
20
- usageMetadata.input_token_details = {
21
- ...(usage.cachedContentTokenCount && {
22
- cache_read_over_200k: Math.max(0, usage.cachedContentTokenCount - 200000),
23
- }),
24
- ...(usage.promptTokenCount && {
25
- over_200k: Math.max(0, usage.promptTokenCount - 200000),
26
- }),
27
- ...(usage.cachedContentTokenCount && {
28
- cache_read: usage.cachedContentTokenCount,
29
- }),
30
- };
31
- // Add output token details if available
32
- usageMetadata.output_token_details = {
33
- ...("candidatesTokenCount" in usage &&
34
- usage.candidatesTokenCount != null && {
35
- over_200k: Math.max(0, usage.candidatesTokenCount - 200000),
36
- }),
37
- ...(usage.thoughtsTokenCount && {
38
- reasoning: usage.thoughtsTokenCount,
39
- }),
40
- };
41
- return usageMetadata;
42
- };
5
+ const gemini_utils_js_1 = require("./gemini.utils.cjs");
43
6
  const chatAggregator = (input) => {
44
7
  const chunks = Array.isArray(input) &&
45
8
  input.every((item) => typeof item === "object" && item !== null)
@@ -161,7 +124,7 @@ const chatAggregator = (input) => {
161
124
  result.safety_ratings = safetyRatings;
162
125
  }
163
126
  if (usageMetadata) {
164
- result.usage_metadata = _createUsageMetadata(usageMetadata);
127
+ result.usage_metadata = (0, gemini_utils_js_1.createGeminiUsageMetadata)(usageMetadata);
165
128
  }
166
129
  return result;
167
130
  };
@@ -354,7 +317,7 @@ function processGeminiOutputs(outputs) {
354
317
  result.safety_ratings = safetyRatings;
355
318
  }
356
319
  if ("usageMetadata" in response && response.usageMetadata) {
357
- result.usage_metadata = _createUsageMetadata(response.usageMetadata);
320
+ result.usage_metadata = (0, gemini_utils_js_1.createGeminiUsageMetadata)(response.usageMetadata);
358
321
  }
359
322
  return result;
360
323
  }
@@ -1,42 +1,5 @@
1
1
  import { isTraceableFunction, traceable, } from "../traceable.js";
2
- const _createUsageMetadata = (usage) => {
3
- const usageMetadata = {
4
- input_tokens: usage.promptTokenCount || 0,
5
- output_tokens: (() => {
6
- if ("responseTokenCount" in usage) {
7
- return usage.responseTokenCount || 0;
8
- }
9
- if ("candidatesTokenCount" in usage) {
10
- return usage.candidatesTokenCount || 0;
11
- }
12
- return 0;
13
- })(),
14
- total_tokens: usage.totalTokenCount || 0,
15
- };
16
- // Add input token details if available
17
- usageMetadata.input_token_details = {
18
- ...(usage.cachedContentTokenCount && {
19
- cache_read_over_200k: Math.max(0, usage.cachedContentTokenCount - 200000),
20
- }),
21
- ...(usage.promptTokenCount && {
22
- over_200k: Math.max(0, usage.promptTokenCount - 200000),
23
- }),
24
- ...(usage.cachedContentTokenCount && {
25
- cache_read: usage.cachedContentTokenCount,
26
- }),
27
- };
28
- // Add output token details if available
29
- usageMetadata.output_token_details = {
30
- ...("candidatesTokenCount" in usage &&
31
- usage.candidatesTokenCount != null && {
32
- over_200k: Math.max(0, usage.candidatesTokenCount - 200000),
33
- }),
34
- ...(usage.thoughtsTokenCount && {
35
- reasoning: usage.thoughtsTokenCount,
36
- }),
37
- };
38
- return usageMetadata;
39
- };
2
+ import { createGeminiUsageMetadata } from "./gemini.utils.js";
40
3
  const chatAggregator = (input) => {
41
4
  const chunks = Array.isArray(input) &&
42
5
  input.every((item) => typeof item === "object" && item !== null)
@@ -158,7 +121,7 @@ const chatAggregator = (input) => {
158
121
  result.safety_ratings = safetyRatings;
159
122
  }
160
123
  if (usageMetadata) {
161
- result.usage_metadata = _createUsageMetadata(usageMetadata);
124
+ result.usage_metadata = createGeminiUsageMetadata(usageMetadata);
162
125
  }
163
126
  return result;
164
127
  };
@@ -351,7 +314,7 @@ function processGeminiOutputs(outputs) {
351
314
  result.safety_ratings = safetyRatings;
352
315
  }
353
316
  if ("usageMetadata" in response && response.usageMetadata) {
354
- result.usage_metadata = _createUsageMetadata(response.usageMetadata);
317
+ result.usage_metadata = createGeminiUsageMetadata(response.usageMetadata);
355
318
  }
356
319
  return result;
357
320
  }
@@ -0,0 +1,41 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createGeminiUsageMetadata = void 0;
4
+ const createGeminiUsageMetadata = (usage) => {
5
+ const usageMetadata = {
6
+ input_tokens: usage.promptTokenCount || 0,
7
+ output_tokens: (() => {
8
+ if ("responseTokenCount" in usage) {
9
+ return usage.responseTokenCount || 0;
10
+ }
11
+ if ("candidatesTokenCount" in usage) {
12
+ return usage.candidatesTokenCount || 0;
13
+ }
14
+ return 0;
15
+ })(),
16
+ total_tokens: usage.totalTokenCount || 0,
17
+ };
18
+ // Add input token details if available
19
+ usageMetadata.input_token_details = {
20
+ cache_read: usage.cachedContentTokenCount != null
21
+ ? Math.min(usage.cachedContentTokenCount, 200_000)
22
+ : undefined,
23
+ cache_read_over_200k: usage.cachedContentTokenCount != null
24
+ ? Math.max(0, usage.cachedContentTokenCount - 200_000)
25
+ : undefined,
26
+ over_200k: usage.promptTokenCount != null
27
+ ? Math.max(0, usage.promptTokenCount -
28
+ (usage.cachedContentTokenCount || 0) -
29
+ 200_000)
30
+ : undefined,
31
+ };
32
+ // Add output token details if available
33
+ usageMetadata.output_token_details = {
34
+ over_200k: "candidatesTokenCount" in usage && usage.candidatesTokenCount != null
35
+ ? Math.max(0, usage.candidatesTokenCount - 200_000)
36
+ : undefined,
37
+ reasoning: usage.thoughtsTokenCount ?? undefined,
38
+ };
39
+ return usageMetadata;
40
+ };
41
+ exports.createGeminiUsageMetadata = createGeminiUsageMetadata;
@@ -0,0 +1,3 @@
1
+ import type { GenerateContentResponseUsageMetadata, UsageMetadata } from "@google/genai";
2
+ import { KVMap } from "../schemas.js";
3
+ export declare const createGeminiUsageMetadata: (usage: UsageMetadata | GenerateContentResponseUsageMetadata) => KVMap;
@@ -0,0 +1,37 @@
1
+ export const createGeminiUsageMetadata = (usage) => {
2
+ const usageMetadata = {
3
+ input_tokens: usage.promptTokenCount || 0,
4
+ output_tokens: (() => {
5
+ if ("responseTokenCount" in usage) {
6
+ return usage.responseTokenCount || 0;
7
+ }
8
+ if ("candidatesTokenCount" in usage) {
9
+ return usage.candidatesTokenCount || 0;
10
+ }
11
+ return 0;
12
+ })(),
13
+ total_tokens: usage.totalTokenCount || 0,
14
+ };
15
+ // Add input token details if available
16
+ usageMetadata.input_token_details = {
17
+ cache_read: usage.cachedContentTokenCount != null
18
+ ? Math.min(usage.cachedContentTokenCount, 200_000)
19
+ : undefined,
20
+ cache_read_over_200k: usage.cachedContentTokenCount != null
21
+ ? Math.max(0, usage.cachedContentTokenCount - 200_000)
22
+ : undefined,
23
+ over_200k: usage.promptTokenCount != null
24
+ ? Math.max(0, usage.promptTokenCount -
25
+ (usage.cachedContentTokenCount || 0) -
26
+ 200_000)
27
+ : undefined,
28
+ };
29
+ // Add output token details if available
30
+ usageMetadata.output_token_details = {
31
+ over_200k: "candidatesTokenCount" in usage && usage.candidatesTokenCount != null
32
+ ? Math.max(0, usage.candidatesTokenCount - 200_000)
33
+ : undefined,
34
+ reasoning: usage.thoughtsTokenCount ?? undefined,
35
+ };
36
+ return usageMetadata;
37
+ };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "langsmith",
3
- "version": "0.7.4",
3
+ "version": "0.7.6",
4
4
  "description": "Client library to connect to the LangSmith Observability and Evaluation Platform.",
5
5
  "packageManager": "pnpm@10.33.0",
6
6
  "files": [
@@ -164,7 +164,7 @@
164
164
  "@babel/preset-env": "^7.22.4",
165
165
  "@faker-js/faker": "^8.4.1",
166
166
  "@google/genai": "^2.0.1",
167
- "@jest/globals": "^29.5.0",
167
+ "@jest/globals": "^30.4.1",
168
168
  "@jest/reporters": "^30.2.0",
169
169
  "@langchain/core": "^0.3.72",
170
170
  "@langchain/langgraph": "^0.3.6",