langsmith 0.7.8 → 0.7.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/README.md +161 -15
  2. package/dist/_openapi_client/client.cjs +814 -0
  3. package/dist/_openapi_client/client.d.ts +247 -0
  4. package/dist/_openapi_client/client.js +777 -0
  5. package/dist/_openapi_client/core/api-promise.cjs +100 -0
  6. package/dist/_openapi_client/core/api-promise.d.ts +45 -0
  7. package/dist/_openapi_client/core/api-promise.js +96 -0
  8. package/dist/_openapi_client/core/error.cjs +134 -0
  9. package/dist/_openapi_client/core/error.d.ts +45 -0
  10. package/dist/_openapi_client/core/error.js +118 -0
  11. package/dist/_openapi_client/core/pagination.cjs +374 -0
  12. package/dist/_openapi_client/core/pagination.d.ts +157 -0
  13. package/dist/_openapi_client/core/pagination.js +361 -0
  14. package/dist/_openapi_client/core/resource.cjs +17 -0
  15. package/dist/_openapi_client/core/resource.d.ts +5 -0
  16. package/dist/_openapi_client/core/resource.js +13 -0
  17. package/dist/_openapi_client/core/uploads.cjs +5 -0
  18. package/dist/_openapi_client/core/uploads.d.ts +2 -0
  19. package/dist/_openapi_client/core/uploads.js +1 -0
  20. package/dist/_openapi_client/index.cjs +29 -0
  21. package/dist/_openapi_client/index.d.ts +6 -0
  22. package/dist/_openapi_client/index.js +8 -0
  23. package/dist/_openapi_client/internal/builtin-types.cjs +4 -0
  24. package/dist/_openapi_client/internal/builtin-types.d.ts +72 -0
  25. package/dist/_openapi_client/internal/builtin-types.js +3 -0
  26. package/dist/_openapi_client/internal/detect-platform.cjs +162 -0
  27. package/dist/_openapi_client/internal/detect-platform.d.ts +14 -0
  28. package/dist/_openapi_client/internal/detect-platform.js +157 -0
  29. package/dist/_openapi_client/internal/errors.cjs +41 -0
  30. package/dist/_openapi_client/internal/errors.d.ts +2 -0
  31. package/dist/_openapi_client/internal/errors.js +36 -0
  32. package/dist/_openapi_client/internal/headers.cjs +79 -0
  33. package/dist/_openapi_client/internal/headers.d.ts +18 -0
  34. package/dist/_openapi_client/internal/headers.js +74 -0
  35. package/dist/_openapi_client/internal/parse.cjs +40 -0
  36. package/dist/_openapi_client/internal/parse.d.ts +11 -0
  37. package/dist/_openapi_client/internal/parse.js +37 -0
  38. package/dist/_openapi_client/internal/qs/formats.cjs +12 -0
  39. package/dist/_openapi_client/internal/qs/formats.d.ts +6 -0
  40. package/dist/_openapi_client/internal/qs/formats.js +8 -0
  41. package/dist/_openapi_client/internal/qs/stringify.cjs +277 -0
  42. package/dist/_openapi_client/internal/qs/stringify.d.ts +2 -0
  43. package/dist/_openapi_client/internal/qs/stringify.js +274 -0
  44. package/dist/_openapi_client/internal/qs/types.cjs +2 -0
  45. package/dist/_openapi_client/internal/qs/types.d.ts +56 -0
  46. package/dist/_openapi_client/internal/qs/types.js +1 -0
  47. package/dist/_openapi_client/internal/qs/utils.cjs +230 -0
  48. package/dist/_openapi_client/internal/qs/utils.d.ts +14 -0
  49. package/dist/_openapi_client/internal/qs/utils.js +217 -0
  50. package/dist/_openapi_client/internal/request-options.cjs +14 -0
  51. package/dist/_openapi_client/internal/request-options.d.ts +74 -0
  52. package/dist/_openapi_client/internal/request-options.js +10 -0
  53. package/dist/_openapi_client/internal/shim-types.cjs +4 -0
  54. package/dist/_openapi_client/internal/shim-types.d.ts +16 -0
  55. package/dist/_openapi_client/internal/shim-types.js +3 -0
  56. package/dist/_openapi_client/internal/shims.cjs +92 -0
  57. package/dist/_openapi_client/internal/shims.d.ts +25 -0
  58. package/dist/_openapi_client/internal/shims.js +85 -0
  59. package/dist/_openapi_client/internal/to-file.cjs +91 -0
  60. package/dist/_openapi_client/internal/to-file.d.ts +44 -0
  61. package/dist/_openapi_client/internal/to-file.js +88 -0
  62. package/dist/_openapi_client/internal/types.cjs +4 -0
  63. package/dist/_openapi_client/internal/types.d.ts +35 -0
  64. package/dist/_openapi_client/internal/types.js +3 -0
  65. package/dist/_openapi_client/internal/uploads.cjs +140 -0
  66. package/dist/_openapi_client/internal/uploads.d.ts +41 -0
  67. package/dist/_openapi_client/internal/uploads.js +130 -0
  68. package/dist/_openapi_client/internal/utils/env.cjs +22 -0
  69. package/dist/_openapi_client/internal/utils/env.d.ts +8 -0
  70. package/dist/_openapi_client/internal/utils/env.js +18 -0
  71. package/dist/_openapi_client/internal/utils/log.cjs +88 -0
  72. package/dist/_openapi_client/internal/utils/log.d.ts +36 -0
  73. package/dist/_openapi_client/internal/utils/log.js +82 -0
  74. package/dist/_openapi_client/internal/utils/path.cjs +79 -0
  75. package/dist/_openapi_client/internal/utils/path.d.ts +14 -0
  76. package/dist/_openapi_client/internal/utils/path.js +74 -0
  77. package/dist/_openapi_client/internal/utils/query.cjs +42 -0
  78. package/dist/_openapi_client/internal/utils/query.d.ts +1 -0
  79. package/dist/_openapi_client/internal/utils/query.js +6 -0
  80. package/dist/_openapi_client/internal/utils/sleep.cjs +7 -0
  81. package/dist/_openapi_client/internal/utils/sleep.d.ts +1 -0
  82. package/dist/_openapi_client/internal/utils/sleep.js +3 -0
  83. package/dist/_openapi_client/internal/utils/uuid.cjs +19 -0
  84. package/dist/_openapi_client/internal/utils/uuid.d.ts +4 -0
  85. package/dist/_openapi_client/internal/utils/uuid.js +15 -0
  86. package/dist/_openapi_client/internal/utils/values.cjs +112 -0
  87. package/dist/_openapi_client/internal/utils/values.d.ts +17 -0
  88. package/dist/_openapi_client/internal/utils/values.js +94 -0
  89. package/dist/_openapi_client/resources/annotation-queues/annotation-queues.cjs +138 -0
  90. package/dist/_openapi_client/resources/annotation-queues/annotation-queues.d.ts +367 -0
  91. package/dist/_openapi_client/resources/annotation-queues/annotation-queues.js +101 -0
  92. package/dist/_openapi_client/resources/annotation-queues/runs.cjs +46 -0
  93. package/dist/_openapi_client/resources/annotation-queues/runs.d.ts +128 -0
  94. package/dist/_openapi_client/resources/annotation-queues/runs.js +42 -0
  95. package/dist/_openapi_client/resources/commits.cjs +44 -0
  96. package/dist/_openapi_client/resources/commits.d.ts +204 -0
  97. package/dist/_openapi_client/resources/commits.js +40 -0
  98. package/dist/_openapi_client/resources/datasets/comparative.cjs +22 -0
  99. package/dist/_openapi_client/resources/datasets/comparative.d.ts +55 -0
  100. package/dist/_openapi_client/resources/datasets/comparative.js +18 -0
  101. package/dist/_openapi_client/resources/datasets/datasets.cjs +193 -0
  102. package/dist/_openapi_client/resources/datasets/datasets.d.ts +374 -0
  103. package/dist/_openapi_client/resources/datasets/datasets.js +156 -0
  104. package/dist/_openapi_client/resources/datasets/experiments.cjs +16 -0
  105. package/dist/_openapi_client/resources/datasets/experiments.d.ts +23 -0
  106. package/dist/_openapi_client/resources/datasets/experiments.js +12 -0
  107. package/dist/_openapi_client/resources/datasets/group.cjs +17 -0
  108. package/dist/_openapi_client/resources/datasets/group.d.ts +160 -0
  109. package/dist/_openapi_client/resources/datasets/group.js +13 -0
  110. package/dist/_openapi_client/resources/datasets/runs.cjs +29 -0
  111. package/dist/_openapi_client/resources/datasets/runs.d.ts +216 -0
  112. package/dist/_openapi_client/resources/datasets/runs.js +25 -0
  113. package/dist/_openapi_client/resources/datasets/share.cjs +32 -0
  114. package/dist/_openapi_client/resources/datasets/share.d.ts +28 -0
  115. package/dist/_openapi_client/resources/datasets/share.js +28 -0
  116. package/dist/_openapi_client/resources/datasets/splits.cjs +22 -0
  117. package/dist/_openapi_client/resources/datasets/splits.d.ts +30 -0
  118. package/dist/_openapi_client/resources/datasets/splits.js +18 -0
  119. package/dist/_openapi_client/resources/datasets/versions.cjs +23 -0
  120. package/dist/_openapi_client/resources/datasets/versions.d.ts +36 -0
  121. package/dist/_openapi_client/resources/datasets/versions.js +19 -0
  122. package/dist/_openapi_client/resources/evaluators.cjs +15 -0
  123. package/dist/_openapi_client/resources/evaluators.d.ts +125 -0
  124. package/dist/_openapi_client/resources/evaluators.js +11 -0
  125. package/dist/_openapi_client/resources/examples/bulk.cjs +24 -0
  126. package/dist/_openapi_client/resources/examples/bulk.d.ts +78 -0
  127. package/dist/_openapi_client/resources/examples/bulk.js +20 -0
  128. package/dist/_openapi_client/resources/examples/examples.cjs +124 -0
  129. package/dist/_openapi_client/resources/examples/examples.d.ts +182 -0
  130. package/dist/_openapi_client/resources/examples/examples.js +87 -0
  131. package/dist/_openapi_client/resources/examples/validate.cjs +21 -0
  132. package/dist/_openapi_client/resources/examples/validate.d.ts +38 -0
  133. package/dist/_openapi_client/resources/examples/validate.js +17 -0
  134. package/dist/_openapi_client/resources/feedback/configs.cjs +24 -0
  135. package/dist/_openapi_client/resources/feedback/configs.d.ts +18 -0
  136. package/dist/_openapi_client/resources/feedback/configs.js +20 -0
  137. package/dist/_openapi_client/resources/feedback/feedback.cjs +98 -0
  138. package/dist/_openapi_client/resources/feedback/feedback.d.ts +275 -0
  139. package/dist/_openapi_client/resources/feedback/feedback.js +61 -0
  140. package/dist/_openapi_client/resources/feedback/tokens.cjs +35 -0
  141. package/dist/_openapi_client/resources/feedback/tokens.d.ts +130 -0
  142. package/dist/_openapi_client/resources/feedback/tokens.js +31 -0
  143. package/dist/_openapi_client/resources/index.cjs +35 -0
  144. package/dist/_openapi_client/resources/index.d.ts +15 -0
  145. package/dist/_openapi_client/resources/index.js +17 -0
  146. package/dist/_openapi_client/resources/info.cjs +15 -0
  147. package/dist/_openapi_client/resources/info.d.ts +51 -0
  148. package/dist/_openapi_client/resources/info.js +11 -0
  149. package/dist/_openapi_client/resources/online-evaluators.cjs +70 -0
  150. package/dist/_openapi_client/resources/online-evaluators.d.ts +284 -0
  151. package/dist/_openapi_client/resources/online-evaluators.js +66 -0
  152. package/dist/_openapi_client/resources/public/datasets.cjs +47 -0
  153. package/dist/_openapi_client/resources/public/datasets.d.ts +152 -0
  154. package/dist/_openapi_client/resources/public/datasets.js +43 -0
  155. package/dist/_openapi_client/resources/public/public.cjs +62 -0
  156. package/dist/_openapi_client/resources/public/public.d.ts +32 -0
  157. package/dist/_openapi_client/resources/public/public.js +25 -0
  158. package/dist/_openapi_client/resources/repos/directories.cjs +40 -0
  159. package/dist/_openapi_client/resources/repos/directories.d.ts +72 -0
  160. package/dist/_openapi_client/resources/repos/directories.js +36 -0
  161. package/dist/_openapi_client/resources/repos/repos.cjs +93 -0
  162. package/dist/_openapi_client/resources/repos/repos.d.ts +188 -0
  163. package/dist/_openapi_client/resources/repos/repos.js +56 -0
  164. package/dist/_openapi_client/resources/runs/rules.cjs +9 -0
  165. package/dist/_openapi_client/resources/runs/rules.d.ts +3 -0
  166. package/dist/_openapi_client/resources/runs/rules.js +5 -0
  167. package/dist/_openapi_client/resources/runs/runs.cjs +102 -0
  168. package/dist/_openapi_client/resources/runs/runs.d.ts +542 -0
  169. package/dist/_openapi_client/resources/runs/runs.js +65 -0
  170. package/dist/_openapi_client/resources/sandboxes/boxes.cjs +86 -0
  171. package/dist/_openapi_client/resources/sandboxes/boxes.d.ts +1121 -0
  172. package/dist/_openapi_client/resources/sandboxes/boxes.js +82 -0
  173. package/dist/_openapi_client/resources/sandboxes/sandboxes.cjs +63 -0
  174. package/dist/_openapi_client/resources/sandboxes/sandboxes.d.ts +13 -0
  175. package/dist/_openapi_client/resources/sandboxes/sandboxes.js +26 -0
  176. package/dist/_openapi_client/resources/sandboxes/snapshots.cjs +39 -0
  177. package/dist/_openapi_client/resources/sandboxes/snapshots.d.ts +130 -0
  178. package/dist/_openapi_client/resources/sandboxes/snapshots.js +35 -0
  179. package/dist/_openapi_client/resources/sessions/insights.cjs +54 -0
  180. package/dist/_openapi_client/resources/sessions/insights.d.ts +246 -0
  181. package/dist/_openapi_client/resources/sessions/insights.js +50 -0
  182. package/dist/_openapi_client/resources/sessions/sessions.cjs +109 -0
  183. package/dist/_openapi_client/resources/sessions/sessions.d.ts +674 -0
  184. package/dist/_openapi_client/resources/sessions/sessions.js +72 -0
  185. package/dist/_openapi_client/resources/settings.cjs +15 -0
  186. package/dist/_openapi_client/resources/settings.d.ts +18 -0
  187. package/dist/_openapi_client/resources/settings.js +11 -0
  188. package/dist/_openapi_client/resources/workspaces.cjs +35 -0
  189. package/dist/_openapi_client/resources/workspaces.d.ts +84 -0
  190. package/dist/_openapi_client/resources/workspaces.js +31 -0
  191. package/dist/_openapi_client/version.cjs +5 -0
  192. package/dist/_openapi_client/version.d.ts +1 -0
  193. package/dist/_openapi_client/version.js +2 -0
  194. package/dist/client.cjs +48 -19
  195. package/dist/client.d.ts +5 -0
  196. package/dist/client.js +29 -0
  197. package/dist/env.cjs +3 -3
  198. package/dist/env.d.ts +1 -1
  199. package/dist/env.js +1 -1
  200. package/dist/experimental/otel/exporter.cjs +1 -1
  201. package/dist/experimental/otel/exporter.js +2 -2
  202. package/dist/experimental/vercel/telemetry.cjs +1 -1
  203. package/dist/experimental/vercel/telemetry.js +2 -2
  204. package/dist/index.cjs +4 -2
  205. package/dist/index.d.ts +2 -1
  206. package/dist/index.js +2 -1
  207. package/dist/run_trees.cjs +1 -1
  208. package/dist/run_trees.js +2 -2
  209. package/dist/sandbox/client.cjs +11 -3
  210. package/dist/sandbox/client.js +11 -3
  211. package/dist/sandbox/index.cjs +8 -2
  212. package/dist/sandbox/index.d.ts +3 -2
  213. package/dist/sandbox/index.js +2 -1
  214. package/dist/sandbox/mounts.cjs +114 -0
  215. package/dist/sandbox/mounts.d.ts +24 -0
  216. package/dist/sandbox/mounts.js +109 -0
  217. package/dist/sandbox/proxy_config.cjs +90 -5
  218. package/dist/sandbox/proxy_config.d.ts +19 -4
  219. package/dist/sandbox/proxy_config.js +86 -4
  220. package/dist/sandbox/types.d.ts +92 -3
  221. package/dist/traceable.cjs +1 -1
  222. package/dist/traceable.js +2 -2
  223. package/dist/utils/guard.cjs +13 -0
  224. package/dist/utils/guard.d.ts +6 -0
  225. package/dist/utils/guard.js +10 -0
  226. package/dist/utils/jestlike/globals.cjs +1 -1
  227. package/dist/utils/jestlike/globals.js +2 -2
  228. package/package.json +2 -2
@@ -0,0 +1,284 @@
1
+ import { APIResource } from '../core/resource.js';
2
+ import { APIPromise } from '../core/api-promise.js';
3
+ import { OffsetPaginationOnlineEvaluators, type OffsetPaginationOnlineEvaluatorsParams, PagePromise } from '../core/pagination.js';
4
+ import { RequestOptions } from '../internal/request-options.js';
5
+ export declare class OnlineEvaluators extends APIResource {
6
+ /**
7
+ * Create a new LLM or code evaluator for the current workspace.
8
+ */
9
+ create(body: OnlineEvaluatorCreateParams, options?: RequestOptions): APIPromise<CreateOnlineEvaluatorResponse>;
10
+ /**
11
+ * Retrieve a single evaluator by its ID.
12
+ */
13
+ retrieve(evaluatorID: string, options?: RequestOptions): APIPromise<OnlineEvaluator>;
14
+ /**
15
+ * Update an existing evaluator's name, LLM configuration, or code configuration.
16
+ */
17
+ update(evaluatorID: string, body: OnlineEvaluatorUpdateParams, options?: RequestOptions): APIPromise<UpdateOnlineEvaluatorResponse>;
18
+ /**
19
+ * List evaluators for the current workspace, with optional filtering by type,
20
+ * name, tag, feedback key, or resource ID.
21
+ */
22
+ list(query?: OnlineEvaluatorListParams | null | undefined, options?: RequestOptions): PagePromise<OnlineEvaluatorsOffsetPaginationOnlineEvaluators, OnlineEvaluator>;
23
+ /**
24
+ * Delete an evaluator. When delete_run_rules is true, all run rules referencing
25
+ * this evaluator are deleted first (same tenant). Associated llm_evaluators and
26
+ * code_evaluators rows are removed by foreign-key cascade when the evaluator row
27
+ * is deleted.
28
+ */
29
+ delete(evaluatorID: string, params?: OnlineEvaluatorDeleteParams | null | undefined, options?: RequestOptions): APIPromise<void>;
30
+ /**
31
+ * Delete multiple evaluators by their IDs. Returns per-item success/failure.
32
+ */
33
+ bulkDelete(params: OnlineEvaluatorBulkDeleteParams, options?: RequestOptions): APIPromise<BulkDeleteEvaluatorsResponse>;
34
+ /**
35
+ * Returns per-day LLM evaluator spend for the requested 7-day period, grouped by
36
+ * evaluator, resource, or run rule. Exactly one of group_by, evaluator_id,
37
+ * session_id, or dataset_id is required. resource_id, type, and feedback_key may
38
+ * be supplied with group_by to narrow listing aggregations.
39
+ */
40
+ spend(query: OnlineEvaluatorSpendParams, options?: RequestOptions): APIPromise<GetOnlineEvaluatorSpendResponse>;
41
+ }
42
+ export type OnlineEvaluatorsOffsetPaginationOnlineEvaluators = OffsetPaginationOnlineEvaluators<OnlineEvaluator>;
43
+ export interface BulkDeleteEvaluatorFailedItem {
44
+ id?: string;
45
+ error?: string;
46
+ }
47
+ export interface BulkDeleteEvaluatorsResponse {
48
+ failed?: Array<BulkDeleteEvaluatorFailedItem>;
49
+ succeeded?: Array<string>;
50
+ }
51
+ export interface CreateOnlineCodeEvaluatorRequest {
52
+ code?: string;
53
+ /**
54
+ * Default: "python"
55
+ */
56
+ language?: string;
57
+ }
58
+ export interface CreateOnlineEvaluatorRequest {
59
+ code_evaluator?: CreateOnlineCodeEvaluatorRequest;
60
+ llm_evaluator?: CreateOnlineLlmEvaluatorRequest;
61
+ name?: string;
62
+ type?: OnlineEvaluatorType;
63
+ }
64
+ export interface CreateOnlineEvaluatorResponse {
65
+ evaluator?: OnlineEvaluator;
66
+ }
67
+ export interface CreateOnlineLlmEvaluatorRequest {
68
+ commit_hash_or_tag?: string;
69
+ prompt_repo_handle?: string;
70
+ variable_mapping?: unknown;
71
+ }
72
+ export interface GetOnlineEvaluatorSpendResponse {
73
+ groups?: Array<OnlineEvaluatorSpendGroup>;
74
+ period_end?: string;
75
+ period_start?: string;
76
+ }
77
+ export interface OnlineCodeEvaluator {
78
+ code?: string;
79
+ evaluator_id?: string;
80
+ /**
81
+ * Default: "python"
82
+ */
83
+ language?: string;
84
+ }
85
+ export interface OnlineEvaluator {
86
+ id?: string;
87
+ code_evaluator?: OnlineCodeEvaluator;
88
+ created_at?: string;
89
+ created_by?: string;
90
+ feedback_keys?: Array<string>;
91
+ /**
92
+ * Embedded child evaluator (populated based on type)
93
+ */
94
+ llm_evaluator?: OnlineLlmEvaluator;
95
+ name?: string;
96
+ run_rules?: Array<OnlineEvaluatorRunRule>;
97
+ tenant_id?: string;
98
+ type?: OnlineEvaluatorType;
99
+ updated_at?: string;
100
+ }
101
+ export interface OnlineEvaluatorRunRule {
102
+ id?: string;
103
+ corrections_dataset_id?: string;
104
+ dataset_id?: string;
105
+ dataset_name?: string;
106
+ group_by?: string;
107
+ num_few_shot_examples?: number;
108
+ session_id?: string;
109
+ session_name?: string;
110
+ /**
111
+ * SpendLimit is the effective spend-cap limit for this rule (nil when
112
+ * unconfigured).
113
+ */
114
+ spend_limit?: OnlineSpendLimit;
115
+ /**
116
+ * Per-rule usage for the current ISO week (omitted when feature is disabled).
117
+ * LLM-evaluator rules are initialized to 0; code-evaluator rules include trace
118
+ * counts only.
119
+ */
120
+ spend_usd?: number;
121
+ trace_count?: number;
122
+ use_corrections_dataset?: boolean;
123
+ }
124
+ export interface OnlineEvaluatorSpendDay {
125
+ date?: string;
126
+ spend_usd?: number;
127
+ trace_count?: number;
128
+ }
129
+ export interface OnlineEvaluatorSpendGroup {
130
+ dataset_id?: string;
131
+ dataset_name?: string;
132
+ days?: Array<OnlineEvaluatorSpendDay>;
133
+ evaluator_id?: string;
134
+ evaluator_name?: string;
135
+ prev_total_spend_usd?: number;
136
+ prev_total_trace_count?: number;
137
+ run_rule_id?: string;
138
+ run_rule_name?: string;
139
+ session_id?: string;
140
+ session_name?: string;
141
+ spend_limit?: OnlineSpendLimit;
142
+ total_spend_usd?: number;
143
+ total_trace_count?: number;
144
+ }
145
+ export type OnlineEvaluatorType = 'llm' | 'code';
146
+ export interface OnlineLlmEvaluator {
147
+ annotation_queue_id?: string;
148
+ commit_hash_or_tag?: string;
149
+ corrections_dataset_id?: string;
150
+ evaluator_id?: string;
151
+ num_few_shot_examples?: number;
152
+ prompt_id?: string;
153
+ prompt_repo_handle?: string;
154
+ /**
155
+ * Derived from the evaluator's run rules — shared across all rules on this
156
+ * evaluator. Nil when the evaluator has no run rules.
157
+ */
158
+ use_corrections_dataset?: boolean;
159
+ /**
160
+ * JSONB
161
+ */
162
+ variable_mapping?: unknown;
163
+ }
164
+ export interface OnlineSpendLimit {
165
+ limit_usd?: number;
166
+ utilization_pct?: number;
167
+ window?: string;
168
+ }
169
+ export interface UpdateOnlineCodeEvaluatorRequest {
170
+ code?: string;
171
+ language?: string;
172
+ }
173
+ export interface UpdateOnlineEvaluatorRequest {
174
+ code_evaluator?: UpdateOnlineCodeEvaluatorRequest;
175
+ llm_evaluator?: UpdateOnlineLlmEvaluatorRequest;
176
+ name?: string;
177
+ }
178
+ export interface UpdateOnlineEvaluatorResponse {
179
+ evaluator?: OnlineEvaluator;
180
+ }
181
+ export interface UpdateOnlineLlmEvaluatorRequest {
182
+ commit_hash_or_tag?: string;
183
+ num_few_shot_examples?: number;
184
+ prompt_repo_handle?: string;
185
+ use_corrections_dataset?: boolean;
186
+ variable_mapping?: unknown;
187
+ }
188
+ export interface OnlineEvaluatorCreateParams {
189
+ code_evaluator?: CreateOnlineCodeEvaluatorRequest;
190
+ llm_evaluator?: CreateOnlineLlmEvaluatorRequest;
191
+ name?: string;
192
+ type?: OnlineEvaluatorType;
193
+ }
194
+ export interface OnlineEvaluatorUpdateParams {
195
+ code_evaluator?: UpdateOnlineCodeEvaluatorRequest;
196
+ llm_evaluator?: UpdateOnlineLlmEvaluatorRequest;
197
+ name?: string;
198
+ }
199
+ export interface OnlineEvaluatorListParams extends OffsetPaginationOnlineEvaluatorsParams {
200
+ /**
201
+ * Filter by feedback key
202
+ */
203
+ feedback_key?: string;
204
+ /**
205
+ * Filter by name substring (also searches creator names)
206
+ */
207
+ name_contains?: string;
208
+ /**
209
+ * Filter by resource IDs
210
+ */
211
+ resource_id?: Array<string>;
212
+ /**
213
+ * Field to sort by
214
+ */
215
+ sort_by?: string;
216
+ /**
217
+ * Sort in descending order
218
+ */
219
+ sort_by_desc?: boolean;
220
+ /**
221
+ * Filter by tag value IDs
222
+ */
223
+ tag_value_id?: Array<string>;
224
+ /**
225
+ * Filter by evaluator type
226
+ */
227
+ type?: string;
228
+ }
229
+ export interface OnlineEvaluatorDeleteParams {
230
+ /**
231
+ * When true, delete all run rules for this evaluator before deleting the evaluator
232
+ */
233
+ delete_run_rules?: boolean;
234
+ }
235
+ export interface OnlineEvaluatorBulkDeleteParams {
236
+ /**
237
+ * Evaluator IDs to delete
238
+ */
239
+ evaluator_ids: Array<string>;
240
+ /**
241
+ * When true, delete all run rules for this evaluator before deleting the evaluator
242
+ */
243
+ delete_run_rules?: boolean;
244
+ }
245
+ export interface OnlineEvaluatorSpendParams {
246
+ /**
247
+ * Start of the 7-day window (YYYY-MM-DD).
248
+ */
249
+ period_start: string;
250
+ /**
251
+ * Filter to a specific dataset (UUID). Mutually exclusive with group_by.
252
+ */
253
+ dataset_id?: string;
254
+ /**
255
+ * Filter to a specific evaluator (UUID). Mutually exclusive with group_by.
256
+ */
257
+ evaluator_id?: string;
258
+ /**
259
+ * Filter grouped results by evaluator feedback key. Only valid with group_by.
260
+ */
261
+ feedback_key?: string;
262
+ /**
263
+ * Aggregation mode: 'evaluator', 'resource', or 'run_rule'. Mutually exclusive
264
+ * with entity filters.
265
+ */
266
+ group_by?: string;
267
+ /**
268
+ * Filter grouped results to evaluators attached to all supplied project or dataset
269
+ * IDs. Only valid with group_by.
270
+ */
271
+ resource_id?: Array<string>;
272
+ /**
273
+ * Filter to a specific project (UUID). Mutually exclusive with group_by.
274
+ */
275
+ session_id?: string;
276
+ /**
277
+ * Filter grouped results by evaluator type: 'llm' or 'code'. Only valid with
278
+ * group_by.
279
+ */
280
+ type?: string;
281
+ }
282
+ export declare namespace OnlineEvaluators {
283
+ export { type BulkDeleteEvaluatorFailedItem as BulkDeleteEvaluatorFailedItem, type BulkDeleteEvaluatorsResponse as BulkDeleteEvaluatorsResponse, type CreateOnlineCodeEvaluatorRequest as CreateOnlineCodeEvaluatorRequest, type CreateOnlineEvaluatorRequest as CreateOnlineEvaluatorRequest, type CreateOnlineEvaluatorResponse as CreateOnlineEvaluatorResponse, type CreateOnlineLlmEvaluatorRequest as CreateOnlineLlmEvaluatorRequest, type GetOnlineEvaluatorSpendResponse as GetOnlineEvaluatorSpendResponse, type OnlineCodeEvaluator as OnlineCodeEvaluator, type OnlineEvaluator as OnlineEvaluator, type OnlineEvaluatorRunRule as OnlineEvaluatorRunRule, type OnlineEvaluatorSpendDay as OnlineEvaluatorSpendDay, type OnlineEvaluatorSpendGroup as OnlineEvaluatorSpendGroup, type OnlineEvaluatorType as OnlineEvaluatorType, type OnlineLlmEvaluator as OnlineLlmEvaluator, type OnlineSpendLimit as OnlineSpendLimit, type UpdateOnlineCodeEvaluatorRequest as UpdateOnlineCodeEvaluatorRequest, type UpdateOnlineEvaluatorRequest as UpdateOnlineEvaluatorRequest, type UpdateOnlineEvaluatorResponse as UpdateOnlineEvaluatorResponse, type UpdateOnlineLlmEvaluatorRequest as UpdateOnlineLlmEvaluatorRequest, type OnlineEvaluatorsOffsetPaginationOnlineEvaluators as OnlineEvaluatorsOffsetPaginationOnlineEvaluators, type OnlineEvaluatorCreateParams as OnlineEvaluatorCreateParams, type OnlineEvaluatorUpdateParams as OnlineEvaluatorUpdateParams, type OnlineEvaluatorListParams as OnlineEvaluatorListParams, type OnlineEvaluatorDeleteParams as OnlineEvaluatorDeleteParams, type OnlineEvaluatorBulkDeleteParams as OnlineEvaluatorBulkDeleteParams, type OnlineEvaluatorSpendParams as OnlineEvaluatorSpendParams, };
284
+ }
@@ -0,0 +1,66 @@
1
+ // @ts-nocheck
2
+ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
3
+ import { APIResource } from '../core/resource.js';
4
+ import { OffsetPaginationOnlineEvaluators, } from '../core/pagination.js';
5
+ import { buildHeaders } from '../internal/headers.js';
6
+ import { path } from '../internal/utils/path.js';
7
+ export class OnlineEvaluators extends APIResource {
8
+ /**
9
+ * Create a new LLM or code evaluator for the current workspace.
10
+ */
11
+ create(body, options) {
12
+ return this._client.post('/v1/platform/evaluators', { body, ...options });
13
+ }
14
+ /**
15
+ * Retrieve a single evaluator by its ID.
16
+ */
17
+ retrieve(evaluatorID, options) {
18
+ return this._client.get(path `/v1/platform/evaluators/${evaluatorID}`, options);
19
+ }
20
+ /**
21
+ * Update an existing evaluator's name, LLM configuration, or code configuration.
22
+ */
23
+ update(evaluatorID, body, options) {
24
+ return this._client.patch(path `/v1/platform/evaluators/${evaluatorID}`, { body, ...options });
25
+ }
26
+ /**
27
+ * List evaluators for the current workspace, with optional filtering by type,
28
+ * name, tag, feedback key, or resource ID.
29
+ */
30
+ list(query = {}, options) {
31
+ return this._client.getAPIList('/v1/platform/evaluators', (OffsetPaginationOnlineEvaluators), { query, ...options });
32
+ }
33
+ /**
34
+ * Delete an evaluator. When delete_run_rules is true, all run rules referencing
35
+ * this evaluator are deleted first (same tenant). Associated llm_evaluators and
36
+ * code_evaluators rows are removed by foreign-key cascade when the evaluator row
37
+ * is deleted.
38
+ */
39
+ delete(evaluatorID, params = {}, options) {
40
+ const { delete_run_rules } = params ?? {};
41
+ return this._client.delete(path `/v1/platform/evaluators/${evaluatorID}`, {
42
+ query: { delete_run_rules },
43
+ ...options,
44
+ headers: buildHeaders([{ Accept: '*/*' }, options?.headers]),
45
+ });
46
+ }
47
+ /**
48
+ * Delete multiple evaluators by their IDs. Returns per-item success/failure.
49
+ */
50
+ bulkDelete(params, options) {
51
+ const { evaluator_ids, delete_run_rules } = params;
52
+ return this._client.delete('/v1/platform/evaluators', {
53
+ query: { evaluator_ids, delete_run_rules },
54
+ ...options,
55
+ });
56
+ }
57
+ /**
58
+ * Returns per-day LLM evaluator spend for the requested 7-day period, grouped by
59
+ * evaluator, resource, or run rule. Exactly one of group_by, evaluator_id,
60
+ * session_id, or dataset_id is required. resource_id, type, and feedback_key may
61
+ * be supplied with group_by to narrow listing aggregations.
62
+ */
63
+ spend(query, options) {
64
+ return this._client.get('/v1/platform/evaluators/spend', { query, ...options });
65
+ }
66
+ }
@@ -0,0 +1,47 @@
1
+ "use strict";
2
+ // @ts-nocheck
3
+ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
4
+ Object.defineProperty(exports, "__esModule", { value: true });
5
+ exports.Datasets = void 0;
6
+ const resource_js_1 = require("../../core/resource.cjs");
7
+ const pagination_js_1 = require("../../core/pagination.cjs");
8
+ const headers_js_1 = require("../../internal/headers.cjs");
9
+ const path_js_1 = require("../../internal/utils/path.cjs");
10
+ class Datasets extends resource_js_1.APIResource {
11
+ /**
12
+ * Get dataset by ids or the shared dataset if not specifed.
13
+ */
14
+ list(shareToken, query = {}, options) {
15
+ return this._client.get((0, path_js_1.path) `/api/v1/public/${shareToken}/datasets`, { query, ...options });
16
+ }
17
+ /**
18
+ * Get all comparative experiments for a given dataset.
19
+ */
20
+ listComparative(shareToken, query = {}, options) {
21
+ return this._client.getAPIList((0, path_js_1.path) `/api/v1/public/${shareToken}/datasets/comparative`, (pagination_js_1.OffsetPaginationTopLevelArray), { query, ...options });
22
+ }
23
+ /**
24
+ * Get feedback for runs in projects run over a dataset that has been shared.
25
+ */
26
+ listFeedback(shareToken, query = {}, options) {
27
+ return this._client.getAPIList((0, path_js_1.path) `/api/v1/public/${shareToken}/datasets/feedback`, (pagination_js_1.OffsetPaginationTopLevelArray), { query, ...options });
28
+ }
29
+ /**
30
+ * Get projects run on a dataset that has been shared.
31
+ */
32
+ listSessions(shareToken, params = {}, options) {
33
+ const { accept, ...query } = params ?? {};
34
+ return this._client.getAPIList((0, path_js_1.path) `/api/v1/public/${shareToken}/datasets/sessions`, (pagination_js_1.OffsetPaginationTopLevelArray), {
35
+ query,
36
+ ...options,
37
+ headers: (0, headers_js_1.buildHeaders)([{ ...(accept != null ? { accept: accept } : undefined) }, options?.headers]),
38
+ });
39
+ }
40
+ /**
41
+ * Get sessions from multiple datasets using share tokens.
42
+ */
43
+ retrieveSessionsBulk(query, options) {
44
+ return this._client.get('/api/v1/public/datasets/sessions-bulk', { query, ...options });
45
+ }
46
+ }
47
+ exports.Datasets = Datasets;
@@ -0,0 +1,152 @@
1
+ import { APIResource } from '../../core/resource.js';
2
+ import * as ComparativeAPI from '../datasets/comparative.js';
3
+ import * as DatasetsAPI from '../datasets/datasets.js';
4
+ import * as FeedbackAPI from '../feedback/feedback.js';
5
+ import { FeedbackSchemasOffsetPaginationTopLevelArray } from '../feedback/feedback.js';
6
+ import * as SessionsAPI from '../sessions/sessions.js';
7
+ import { TracerSessionsOffsetPaginationTopLevelArray } from '../sessions/sessions.js';
8
+ import { APIPromise } from '../../core/api-promise.js';
9
+ import { OffsetPaginationTopLevelArray, type OffsetPaginationTopLevelArrayParams, PagePromise } from '../../core/pagination.js';
10
+ import { RequestOptions } from '../../internal/request-options.js';
11
+ export declare class Datasets extends APIResource {
12
+ /**
13
+ * Get dataset by ids or the shared dataset if not specifed.
14
+ */
15
+ list(shareToken: string, query?: DatasetListParams | null | undefined, options?: RequestOptions): APIPromise<DatasetListResponse>;
16
+ /**
17
+ * Get all comparative experiments for a given dataset.
18
+ */
19
+ listComparative(shareToken: string, query?: DatasetListComparativeParams | null | undefined, options?: RequestOptions): PagePromise<DatasetListComparativeResponsesOffsetPaginationTopLevelArray, DatasetListComparativeResponse>;
20
+ /**
21
+ * Get feedback for runs in projects run over a dataset that has been shared.
22
+ */
23
+ listFeedback(shareToken: string, query?: DatasetListFeedbackParams | null | undefined, options?: RequestOptions): PagePromise<FeedbackSchemasOffsetPaginationTopLevelArray, FeedbackAPI.FeedbackSchema>;
24
+ /**
25
+ * Get projects run on a dataset that has been shared.
26
+ */
27
+ listSessions(shareToken: string, params?: DatasetListSessionsParams | null | undefined, options?: RequestOptions): PagePromise<TracerSessionsOffsetPaginationTopLevelArray, SessionsAPI.TracerSession>;
28
+ /**
29
+ * Get sessions from multiple datasets using share tokens.
30
+ */
31
+ retrieveSessionsBulk(query: DatasetRetrieveSessionsBulkParams, options?: RequestOptions): APIPromise<DatasetRetrieveSessionsBulkResponse>;
32
+ }
33
+ export type DatasetListComparativeResponsesOffsetPaginationTopLevelArray = OffsetPaginationTopLevelArray<DatasetListComparativeResponse>;
34
+ /**
35
+ * Public schema for datasets.
36
+ *
37
+ * Doesn't currently include session counts/stats since public test project sharing
38
+ * is not yet shipped
39
+ */
40
+ export interface DatasetListResponse {
41
+ id: string;
42
+ example_count: number;
43
+ name: string;
44
+ created_at?: string;
45
+ /**
46
+ * Enum for dataset data types.
47
+ */
48
+ data_type?: DatasetsAPI.DataType | null;
49
+ description?: string | null;
50
+ externally_managed?: boolean | null;
51
+ inputs_schema_definition?: {
52
+ [key: string]: unknown;
53
+ } | null;
54
+ outputs_schema_definition?: {
55
+ [key: string]: unknown;
56
+ } | null;
57
+ transformations?: Array<DatasetsAPI.DatasetTransformation> | null;
58
+ }
59
+ /**
60
+ * Publicly-shared ComparativeExperiment schema.
61
+ */
62
+ export interface DatasetListComparativeResponse {
63
+ id: string;
64
+ created_at: string;
65
+ experiments_info: Array<ComparativeAPI.SimpleExperimentInfo>;
66
+ modified_at: string;
67
+ description?: string | null;
68
+ extra?: {
69
+ [key: string]: unknown;
70
+ } | null;
71
+ feedback_stats?: {
72
+ [key: string]: unknown;
73
+ } | null;
74
+ name?: string | null;
75
+ }
76
+ export type DatasetRetrieveSessionsBulkResponse = Array<SessionsAPI.TracerSession>;
77
+ export interface DatasetListParams {
78
+ limit?: number;
79
+ offset?: number;
80
+ /**
81
+ * Enum for available dataset columns to sort by.
82
+ */
83
+ sort_by?: DatasetsAPI.SortByDatasetColumn;
84
+ sort_by_desc?: boolean;
85
+ }
86
+ export interface DatasetListComparativeParams extends OffsetPaginationTopLevelArrayParams {
87
+ name?: string | null;
88
+ name_contains?: string | null;
89
+ /**
90
+ * Enum for available comparative experiment columns to sort by.
91
+ */
92
+ sort_by?: ComparativeAPI.SortByComparativeExperimentColumn;
93
+ sort_by_desc?: boolean;
94
+ }
95
+ export interface DatasetListFeedbackParams extends OffsetPaginationTopLevelArrayParams {
96
+ has_comment?: boolean | null;
97
+ has_score?: boolean | null;
98
+ key?: Array<string> | null;
99
+ /**
100
+ * Enum for feedback levels.
101
+ */
102
+ level?: FeedbackAPI.FeedbackLevel | null;
103
+ run?: Array<string> | null;
104
+ session?: Array<string> | null;
105
+ source?: Array<FeedbackAPI.SourceType> | null;
106
+ user?: Array<string> | null;
107
+ }
108
+ export interface DatasetListSessionsParams extends OffsetPaginationTopLevelArrayParams {
109
+ /**
110
+ * Query param
111
+ */
112
+ id?: Array<string> | null;
113
+ /**
114
+ * Query param
115
+ */
116
+ dataset_version?: string | null;
117
+ /**
118
+ * Query param
119
+ */
120
+ facets?: boolean;
121
+ /**
122
+ * Query param
123
+ */
124
+ name?: string | null;
125
+ /**
126
+ * Query param
127
+ */
128
+ name_contains?: string | null;
129
+ /**
130
+ * Query param
131
+ */
132
+ sort_by?: SessionsAPI.SessionSortableColumns;
133
+ /**
134
+ * Query param
135
+ */
136
+ sort_by_desc?: boolean;
137
+ /**
138
+ * Query param
139
+ */
140
+ sort_by_feedback_key?: string | null;
141
+ /**
142
+ * Header param
143
+ */
144
+ accept?: string;
145
+ }
146
+ export interface DatasetRetrieveSessionsBulkParams {
147
+ share_tokens: Array<string>;
148
+ }
149
+ export declare namespace Datasets {
150
+ export { type DatasetListResponse as DatasetListResponse, type DatasetListComparativeResponse as DatasetListComparativeResponse, type DatasetRetrieveSessionsBulkResponse as DatasetRetrieveSessionsBulkResponse, type DatasetListComparativeResponsesOffsetPaginationTopLevelArray as DatasetListComparativeResponsesOffsetPaginationTopLevelArray, type DatasetListParams as DatasetListParams, type DatasetListComparativeParams as DatasetListComparativeParams, type DatasetListFeedbackParams as DatasetListFeedbackParams, type DatasetListSessionsParams as DatasetListSessionsParams, type DatasetRetrieveSessionsBulkParams as DatasetRetrieveSessionsBulkParams, };
151
+ }
152
+ export { type FeedbackSchemasOffsetPaginationTopLevelArray, type TracerSessionsOffsetPaginationTopLevelArray, };
@@ -0,0 +1,43 @@
1
+ // @ts-nocheck
2
+ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
3
+ import { APIResource } from '../../core/resource.js';
4
+ import { OffsetPaginationTopLevelArray, } from '../../core/pagination.js';
5
+ import { buildHeaders } from '../../internal/headers.js';
6
+ import { path } from '../../internal/utils/path.js';
7
+ export class Datasets extends APIResource {
8
+ /**
9
+ * Get dataset by ids or the shared dataset if not specifed.
10
+ */
11
+ list(shareToken, query = {}, options) {
12
+ return this._client.get(path `/api/v1/public/${shareToken}/datasets`, { query, ...options });
13
+ }
14
+ /**
15
+ * Get all comparative experiments for a given dataset.
16
+ */
17
+ listComparative(shareToken, query = {}, options) {
18
+ return this._client.getAPIList(path `/api/v1/public/${shareToken}/datasets/comparative`, (OffsetPaginationTopLevelArray), { query, ...options });
19
+ }
20
+ /**
21
+ * Get feedback for runs in projects run over a dataset that has been shared.
22
+ */
23
+ listFeedback(shareToken, query = {}, options) {
24
+ return this._client.getAPIList(path `/api/v1/public/${shareToken}/datasets/feedback`, (OffsetPaginationTopLevelArray), { query, ...options });
25
+ }
26
+ /**
27
+ * Get projects run on a dataset that has been shared.
28
+ */
29
+ listSessions(shareToken, params = {}, options) {
30
+ const { accept, ...query } = params ?? {};
31
+ return this._client.getAPIList(path `/api/v1/public/${shareToken}/datasets/sessions`, (OffsetPaginationTopLevelArray), {
32
+ query,
33
+ ...options,
34
+ headers: buildHeaders([{ ...(accept != null ? { accept: accept } : undefined) }, options?.headers]),
35
+ });
36
+ }
37
+ /**
38
+ * Get sessions from multiple datasets using share tokens.
39
+ */
40
+ retrieveSessionsBulk(query, options) {
41
+ return this._client.get('/api/v1/public/datasets/sessions-bulk', { query, ...options });
42
+ }
43
+ }
@@ -0,0 +1,62 @@
1
+ "use strict";
2
+ // @ts-nocheck
3
+ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
4
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
5
+ if (k2 === undefined) k2 = k;
6
+ var desc = Object.getOwnPropertyDescriptor(m, k);
7
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
8
+ desc = { enumerable: true, get: function() { return m[k]; } };
9
+ }
10
+ Object.defineProperty(o, k2, desc);
11
+ }) : (function(o, m, k, k2) {
12
+ if (k2 === undefined) k2 = k;
13
+ o[k2] = m[k];
14
+ }));
15
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
16
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
17
+ }) : function(o, v) {
18
+ o["default"] = v;
19
+ });
20
+ var __importStar = (this && this.__importStar) || (function () {
21
+ var ownKeys = function(o) {
22
+ ownKeys = Object.getOwnPropertyNames || function (o) {
23
+ var ar = [];
24
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
25
+ return ar;
26
+ };
27
+ return ownKeys(o);
28
+ };
29
+ return function (mod) {
30
+ if (mod && mod.__esModule) return mod;
31
+ var result = {};
32
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
33
+ __setModuleDefault(result, mod);
34
+ return result;
35
+ };
36
+ })();
37
+ Object.defineProperty(exports, "__esModule", { value: true });
38
+ exports.Public = void 0;
39
+ const resource_js_1 = require("../../core/resource.cjs");
40
+ const DatasetsAPI = __importStar(require("./datasets.cjs"));
41
+ const datasets_js_1 = require("./datasets.cjs");
42
+ const pagination_js_1 = require("../../core/pagination.cjs");
43
+ const path_js_1 = require("../../internal/utils/path.cjs");
44
+ class Public extends resource_js_1.APIResource {
45
+ constructor() {
46
+ super(...arguments);
47
+ Object.defineProperty(this, "datasets", {
48
+ enumerable: true,
49
+ configurable: true,
50
+ writable: true,
51
+ value: new DatasetsAPI.Datasets(this._client)
52
+ });
53
+ }
54
+ /**
55
+ * Read Shared Feedbacks
56
+ */
57
+ retrieveFeedbacks(shareToken, query = {}, options) {
58
+ return this._client.getAPIList((0, path_js_1.path) `/api/v1/public/${shareToken}/feedbacks`, (pagination_js_1.OffsetPaginationTopLevelArray), { query, ...options });
59
+ }
60
+ }
61
+ exports.Public = Public;
62
+ Public.Datasets = datasets_js_1.Datasets;