notdiamond 2.0.0-rc2 → 2.0.0-rc20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +185 -0
- package/LICENSE +1 -1
- package/README.md +261 -168
- package/client.d.mts +26 -43
- package/client.d.mts.map +1 -1
- package/client.d.ts +26 -43
- package/client.d.ts.map +1 -1
- package/client.js +49 -68
- package/client.js.map +1 -1
- package/client.mjs +47 -66
- package/client.mjs.map +1 -1
- package/core/api-promise.d.mts +2 -2
- package/core/api-promise.d.ts +2 -2
- package/core/error.d.mts +2 -2
- package/core/error.d.ts +2 -2
- package/core/error.js +4 -4
- package/core/error.mjs +2 -2
- package/core/resource.d.mts +3 -3
- package/core/resource.d.ts +3 -3
- package/index.d.mts +3 -3
- package/index.d.ts +3 -3
- package/index.js +4 -4
- package/index.mjs +3 -3
- package/internal/parse.d.mts +2 -2
- package/internal/parse.d.ts +2 -2
- package/internal/shims.js +1 -1
- package/internal/shims.mjs +1 -1
- package/internal/uploads.d.mts +4 -4
- package/internal/uploads.d.ts +4 -4
- package/internal/utils/base64.js +2 -2
- package/internal/utils/base64.mjs +3 -3
- package/internal/utils/env.d.mts.map +1 -1
- package/internal/utils/env.d.ts.map +1 -1
- package/internal/utils/env.js +4 -2
- package/internal/utils/env.js.map +1 -1
- package/internal/utils/env.mjs +4 -2
- package/internal/utils/env.mjs.map +1 -1
- package/internal/utils/log.d.mts +3 -3
- package/internal/utils/log.d.ts +3 -3
- package/internal/utils/path.js +1 -1
- package/internal/utils/path.mjs +2 -2
- package/internal/utils/values.js +5 -5
- package/internal/utils/values.mjs +6 -6
- package/package.json +11 -12
- package/resources/custom-router.d.mts +145 -0
- package/resources/custom-router.d.mts.map +1 -0
- package/resources/custom-router.d.ts +145 -0
- package/resources/custom-router.d.ts.map +1 -0
- package/resources/custom-router.js +83 -0
- package/resources/custom-router.js.map +1 -0
- package/resources/custom-router.mjs +79 -0
- package/resources/custom-router.mjs.map +1 -0
- package/resources/index.d.mts +5 -6
- package/resources/index.d.mts.map +1 -1
- package/resources/index.d.ts +5 -6
- package/resources/index.d.ts.map +1 -1
- package/resources/index.js +5 -7
- package/resources/index.js.map +1 -1
- package/resources/index.mjs +2 -3
- package/resources/index.mjs.map +1 -1
- package/resources/model-router.d.mts +194 -0
- package/resources/model-router.d.mts.map +1 -0
- package/resources/model-router.d.ts +194 -0
- package/resources/model-router.d.ts.map +1 -0
- package/resources/model-router.js +68 -0
- package/resources/model-router.js.map +1 -0
- package/resources/model-router.mjs +64 -0
- package/resources/model-router.mjs.map +1 -0
- package/resources/models.d.mts +50 -25
- package/resources/models.d.mts.map +1 -1
- package/resources/models.d.ts +50 -25
- package/resources/models.d.ts.map +1 -1
- package/resources/models.js +5 -0
- package/resources/models.js.map +1 -1
- package/resources/models.mjs +5 -0
- package/resources/models.mjs.map +1 -1
- package/resources/preferences.d.mts +37 -54
- package/resources/preferences.d.mts.map +1 -1
- package/resources/preferences.d.ts +37 -54
- package/resources/preferences.d.ts.map +1 -1
- package/resources/preferences.js +17 -42
- package/resources/preferences.js.map +1 -1
- package/resources/preferences.mjs +17 -42
- package/resources/preferences.mjs.map +1 -1
- package/resources/prompt-adaptation.d.mts +340 -249
- package/resources/prompt-adaptation.d.mts.map +1 -1
- package/resources/prompt-adaptation.d.ts +340 -249
- package/resources/prompt-adaptation.d.ts.map +1 -1
- package/resources/prompt-adaptation.js +36 -79
- package/resources/prompt-adaptation.js.map +1 -1
- package/resources/prompt-adaptation.mjs +36 -79
- package/resources/prompt-adaptation.mjs.map +1 -1
- package/src/client.ts +72 -128
- package/src/core/api-promise.ts +4 -4
- package/src/core/error.ts +2 -2
- package/src/core/resource.ts +3 -3
- package/src/index.ts +3 -3
- package/src/internal/parse.ts +2 -2
- package/src/internal/shims.ts +1 -1
- package/src/internal/uploads.ts +5 -5
- package/src/internal/utils/base64.ts +3 -3
- package/src/internal/utils/env.ts +4 -2
- package/src/internal/utils/log.ts +3 -3
- package/src/internal/utils/path.ts +2 -2
- package/src/internal/utils/values.ts +6 -6
- package/src/resources/custom-router.ts +168 -0
- package/src/resources/index.ts +20 -32
- package/src/resources/model-router.ts +222 -0
- package/src/resources/models.ts +55 -32
- package/src/resources/preferences.ts +43 -83
- package/src/resources/prompt-adaptation.ts +358 -300
- package/src/version.ts +1 -1
- package/version.d.mts +1 -1
- package/version.d.mts.map +1 -1
- package/version.d.ts +1 -1
- package/version.d.ts.map +1 -1
- package/version.js +1 -1
- package/version.js.map +1 -1
- package/version.mjs +1 -1
- package/version.mjs.map +1 -1
- package/resources/admin.d.mts +0 -4
- package/resources/admin.d.mts.map +0 -1
- package/resources/admin.d.ts +0 -4
- package/resources/admin.d.ts.map +0 -1
- package/resources/admin.js +0 -9
- package/resources/admin.js.map +0 -1
- package/resources/admin.mjs +0 -5
- package/resources/admin.mjs.map +0 -1
- package/resources/report.d.mts +0 -245
- package/resources/report.d.mts.map +0 -1
- package/resources/report.d.ts +0 -245
- package/resources/report.d.ts.map +0 -1
- package/resources/report.js +0 -86
- package/resources/report.js.map +0 -1
- package/resources/report.mjs +0 -82
- package/resources/report.mjs.map +0 -1
- package/resources/routing.d.mts +0 -391
- package/resources/routing.d.mts.map +0 -1
- package/resources/routing.d.ts +0 -391
- package/resources/routing.d.ts.map +0 -1
- package/resources/routing.js +0 -163
- package/resources/routing.js.map +0 -1
- package/resources/routing.mjs +0 -159
- package/resources/routing.mjs.map +0 -1
- package/src/resources/admin.ts +0 -5
- package/src/resources/report.ts +0 -300
- package/src/resources/routing.ts +0 -476
|
@@ -7,14 +7,14 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
7
7
|
* Adapt your prompt from one LLM to work optimally across different target LLMs.
|
|
8
8
|
*
|
|
9
9
|
* This endpoint automatically optimizes your prompt (system prompt + user message
|
|
10
|
-
* template) to
|
|
11
|
-
*
|
|
12
|
-
*
|
|
10
|
+
* template) to improve accuracy on your use case across various models. Each model
|
|
11
|
+
* has unique characteristics, and what works well for GPT-5 might not work as well
|
|
12
|
+
* for Claude or Gemini.
|
|
13
13
|
*
|
|
14
14
|
* **How Prompt Adaptation Works:**
|
|
15
15
|
*
|
|
16
|
-
* 1. You provide your current prompt
|
|
17
|
-
* 2. You specify target models you want to adapt to
|
|
16
|
+
* 1. You provide your current prompt and optionally your current origin model
|
|
17
|
+
* 2. You specify the target models you want to adapt your prompt to
|
|
18
18
|
* 3. You provide evaluation examples (golden records) with expected answers
|
|
19
19
|
* 4. The system runs optimization to find the best prompt for each target model
|
|
20
20
|
* 5. You receive adapted prompts that perform well on your target models
|
|
@@ -22,14 +22,19 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
22
22
|
* **Evaluation Metrics:** Choose either a standard metric or provide custom
|
|
23
23
|
* evaluation:
|
|
24
24
|
*
|
|
25
|
-
* - **Standard metrics**: LLMaaJ:
|
|
26
|
-
* JSON_Match
|
|
25
|
+
* - **Standard metrics**: LLMaaJ:Sem_Sim_1 (semantic similarity), JSON_Match
|
|
27
26
|
* - **Custom evaluation**: Provide evaluation_config with your own LLM judge,
|
|
28
27
|
* prompt, and cutoff
|
|
29
28
|
*
|
|
30
29
|
* **Dataset Requirements:**
|
|
31
30
|
*
|
|
32
|
-
* - Minimum
|
|
31
|
+
* - Minimum 25 examples in train_goldens (more examples = better adaptation)
|
|
32
|
+
* - **Prototype mode**: Set `prototype_mode: true` to use as few as 3 examples for
|
|
33
|
+
* prototyping
|
|
34
|
+
* - Recommended when you don't have enough data yet to build a proof-of-concept
|
|
35
|
+
* - Note: Performance may be degraded compared to standard mode (25+ examples)
|
|
36
|
+
* - Trade-off: Faster iteration with less data vs. potentially less
|
|
37
|
+
* generalizability
|
|
33
38
|
* - Each example must have fields matching your template placeholders
|
|
34
39
|
* - Supervised evaluation requires 'answer' field in each golden record
|
|
35
40
|
* - Unsupervised evaluation can work without answers
|
|
@@ -40,21 +45,6 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
40
45
|
* - Time depends on: number of target models, dataset size, model availability
|
|
41
46
|
* - Use the returned adaptation_run_id to check status and retrieve results
|
|
42
47
|
*
|
|
43
|
-
* **Subscription Tiers:**
|
|
44
|
-
*
|
|
45
|
-
* - Free: 1 target model
|
|
46
|
-
* - Starter: 3 target models
|
|
47
|
-
* - Startup: 5 target models
|
|
48
|
-
* - Enterprise: 10 target models
|
|
49
|
-
*
|
|
50
|
-
* **Best Practices:**
|
|
51
|
-
*
|
|
52
|
-
* 1. Use diverse, representative examples from your production workload
|
|
53
|
-
* 2. Include 10-20 examples for best results (5 minimum)
|
|
54
|
-
* 3. Ensure consistent evaluation across all examples
|
|
55
|
-
* 4. Test both train_goldens and test_goldens split for validation
|
|
56
|
-
* 5. Use the same model versions you'll use in production
|
|
57
|
-
*
|
|
58
48
|
* **Example Workflow:**
|
|
59
49
|
*
|
|
60
50
|
* ```
|
|
@@ -75,7 +65,7 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
75
65
|
* system_prompt: 'You are a helpful assistant that answers questions accurately.',
|
|
76
66
|
* target_models: [
|
|
77
67
|
* { provider: 'anthropic', model: 'claude-sonnet-4-5-20250929' },
|
|
78
|
-
* { provider: 'google', model: 'gemini-
|
|
68
|
+
* { provider: 'google', model: 'gemini-2.5-flash' },
|
|
79
69
|
* ],
|
|
80
70
|
* template: 'Question: {question}\nAnswer:',
|
|
81
71
|
* evaluation_metric: 'LLMaaJ:Sem_Sim_3',
|
|
@@ -144,15 +134,6 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
144
134
|
* 3. Apply the optimized prompts when calling the respective target models
|
|
145
135
|
* 4. Compare pre/post optimization scores to see improvement
|
|
146
136
|
*
|
|
147
|
-
* **Evaluation Scores:**
|
|
148
|
-
*
|
|
149
|
-
* - Scores range from 0-10 (higher is better)
|
|
150
|
-
* - Compare origin_model score with target_models pre_optimization_score for
|
|
151
|
-
* baseline
|
|
152
|
-
* - Compare pre_optimization_score with post_optimization_score to see improvement
|
|
153
|
-
* from adaptation
|
|
154
|
-
* - Typical improvements range from 5-30% on evaluation metrics
|
|
155
|
-
*
|
|
156
137
|
* **Status Handling:**
|
|
157
138
|
*
|
|
158
139
|
* - If adaptation is still processing, target model results will have
|
|
@@ -176,38 +157,13 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
176
157
|
*
|
|
177
158
|
* @example
|
|
178
159
|
* ```ts
|
|
179
|
-
* const
|
|
160
|
+
* const response =
|
|
180
161
|
* await client.promptAdaptation.getAdaptResults(
|
|
181
162
|
* 'adaptation_run_id',
|
|
182
163
|
* );
|
|
183
164
|
* ```
|
|
184
165
|
*/
|
|
185
|
-
getAdaptResults(adaptationRunID: string, options?: RequestOptions): APIPromise<
|
|
186
|
-
/**
|
|
187
|
-
* Get Adapt Run Results
|
|
188
|
-
*
|
|
189
|
-
* @example
|
|
190
|
-
* ```ts
|
|
191
|
-
* const adaptationRunResults =
|
|
192
|
-
* await client.promptAdaptation.getAdaptRunResults(
|
|
193
|
-
* 'adaptation_run_id',
|
|
194
|
-
* { user_id: 'user_id', 'x-token': 'x-token' },
|
|
195
|
-
* );
|
|
196
|
-
* ```
|
|
197
|
-
*/
|
|
198
|
-
getAdaptRunResults(adaptationRunID: string, params: PromptAdaptationGetAdaptRunResultsParams, options?: RequestOptions): APIPromise<AdaptationRunResults>;
|
|
199
|
-
/**
|
|
200
|
-
* Get Adapt Runs
|
|
201
|
-
*
|
|
202
|
-
* @example
|
|
203
|
-
* ```ts
|
|
204
|
-
* const adaptationRunResults =
|
|
205
|
-
* await client.promptAdaptation.getAdaptRuns('user_id', {
|
|
206
|
-
* 'x-token': 'x-token',
|
|
207
|
-
* });
|
|
208
|
-
* ```
|
|
209
|
-
*/
|
|
210
|
-
getAdaptRuns(userID: string, params: PromptAdaptationGetAdaptRunsParams, options?: RequestOptions): APIPromise<PromptAdaptationGetAdaptRunsResponse>;
|
|
166
|
+
getAdaptResults(adaptationRunID: string, options?: RequestOptions): APIPromise<PromptAdaptationGetAdaptResultsResponse>;
|
|
211
167
|
/**
|
|
212
168
|
* Check the status of a prompt adaptation run.
|
|
213
169
|
*
|
|
@@ -250,22 +206,131 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
250
206
|
*/
|
|
251
207
|
getAdaptStatus(adaptationRunID: string, options?: RequestOptions): APIPromise<PromptAdaptationGetAdaptStatusResponse>;
|
|
252
208
|
/**
|
|
253
|
-
* Get LLM costs for a specific adaptation run
|
|
209
|
+
* Get LLM usage costs for a specific prompt adaptation run.
|
|
210
|
+
*
|
|
211
|
+
* This endpoint returns the total cost and detailed usage records for all LLM
|
|
212
|
+
* requests made during a prompt adaptation run. Use this to track costs associated
|
|
213
|
+
* with optimizing prompts for different target models.
|
|
214
|
+
*
|
|
215
|
+
* **Cost Breakdown:**
|
|
216
|
+
*
|
|
217
|
+
* - Total cost across all models used in the adaptation
|
|
218
|
+
* - Individual usage records with provider, model, tokens, and costs
|
|
219
|
+
* - Timestamps for each LLM request
|
|
220
|
+
*
|
|
221
|
+
* **Access Control:**
|
|
222
|
+
*
|
|
223
|
+
* - Only accessible by the user who created the adaptation run
|
|
224
|
+
* - Requires prompt adaptation access
|
|
254
225
|
*
|
|
255
226
|
* @example
|
|
256
227
|
* ```ts
|
|
257
|
-
* const response =
|
|
258
|
-
*
|
|
259
|
-
*
|
|
260
|
-
* );
|
|
228
|
+
* const response = await client.promptAdaptation.getCost(
|
|
229
|
+
* 'adaptation_run_id',
|
|
230
|
+
* );
|
|
261
231
|
* ```
|
|
262
232
|
*/
|
|
263
|
-
|
|
233
|
+
getCost(adaptationRunID: string, options?: RequestOptions): APIPromise<PromptAdaptationGetCostResponse>;
|
|
264
234
|
}
|
|
265
235
|
/**
|
|
266
|
-
*
|
|
236
|
+
* A training or test example for prompt adaptation.
|
|
267
237
|
*/
|
|
268
|
-
export interface
|
|
238
|
+
export interface GoldenRecord {
|
|
239
|
+
/**
|
|
240
|
+
* Dictionary mapping field names to their values. Keys must match the fields
|
|
241
|
+
* specified in the template
|
|
242
|
+
*/
|
|
243
|
+
fields: {
|
|
244
|
+
[key: string]: string;
|
|
245
|
+
};
|
|
246
|
+
/**
|
|
247
|
+
* Expected answer for supervised evaluation. Required for supervised metrics,
|
|
248
|
+
* optional for unsupervised
|
|
249
|
+
*/
|
|
250
|
+
answer?: string | null;
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Status enum for asynchronous jobs (prompt adaptation, custom router training,
|
|
254
|
+
* etc.).
|
|
255
|
+
*
|
|
256
|
+
* Represents the current state of a long-running operation:
|
|
257
|
+
*
|
|
258
|
+
* - **created**: Job has been initialized but not yet queued
|
|
259
|
+
* - **queued**: Job is waiting in the queue to be processed
|
|
260
|
+
* - **processing**: Job is currently being executed
|
|
261
|
+
* - **completed**: Job finished successfully and results are available
|
|
262
|
+
* - **failed**: Job encountered an error and did not complete
|
|
263
|
+
*/
|
|
264
|
+
export type JobStatus = 'created' | 'queued' | 'processing' | 'completed' | 'failed';
|
|
265
|
+
/**
|
|
266
|
+
* Model for specifying an LLM provider in API requests.
|
|
267
|
+
*/
|
|
268
|
+
export interface RequestProvider {
|
|
269
|
+
/**
|
|
270
|
+
* Model name (e.g., 'gpt-4o', 'claude-sonnet-4-5-20250929')
|
|
271
|
+
*/
|
|
272
|
+
model: string;
|
|
273
|
+
/**
|
|
274
|
+
* Provider name (e.g., 'openai', 'anthropic', 'google')
|
|
275
|
+
*/
|
|
276
|
+
provider: string;
|
|
277
|
+
/**
|
|
278
|
+
* Maximum context length for the model (required for custom models)
|
|
279
|
+
*/
|
|
280
|
+
context_length?: number | null;
|
|
281
|
+
/**
|
|
282
|
+
* Input token price per million tokens in USD (required for custom models)
|
|
283
|
+
*/
|
|
284
|
+
input_price?: number | null;
|
|
285
|
+
/**
|
|
286
|
+
* Whether this is a custom model not in Not Diamond's supported model list
|
|
287
|
+
*/
|
|
288
|
+
is_custom?: boolean;
|
|
289
|
+
/**
|
|
290
|
+
* Average latency in seconds (required for custom models)
|
|
291
|
+
*/
|
|
292
|
+
latency?: number | null;
|
|
293
|
+
/**
|
|
294
|
+
* Output token price per million tokens in USD (required for custom models)
|
|
295
|
+
*/
|
|
296
|
+
output_price?: number | null;
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Response model for POST /v2/prompt/adapt endpoint.
|
|
300
|
+
*
|
|
301
|
+
* Returned immediately after submitting a prompt adaptation request. The
|
|
302
|
+
* adaptation process runs asynchronously, so use the returned adaptation_run_id to
|
|
303
|
+
* track progress and retrieve results when complete.
|
|
304
|
+
*
|
|
305
|
+
* **Next steps:**
|
|
306
|
+
*
|
|
307
|
+
* 1. Store the adaptation_run_id
|
|
308
|
+
* 2. Poll GET /v2/prompt/adaptStatus/{adaptation_run_id} to check progress
|
|
309
|
+
* 3. When status is 'completed', retrieve optimized prompts from GET
|
|
310
|
+
* /v2/prompt/adaptResults/{adaptation_run_id}
|
|
311
|
+
* 4. Use the optimized prompts with your target models
|
|
312
|
+
*/
|
|
313
|
+
export interface PromptAdaptationAdaptResponse {
|
|
314
|
+
/**
|
|
315
|
+
* Unique identifier for this adaptation run. Use this to poll status and retrieve
|
|
316
|
+
* optimized prompts when complete
|
|
317
|
+
*/
|
|
318
|
+
adaptation_run_id: string;
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* Response model for GET /v2/prompt/adaptResults/{adaptation_run_id} endpoint.
|
|
322
|
+
*
|
|
323
|
+
* Contains the complete results of a prompt adaptation run, including optimized
|
|
324
|
+
* prompts and evaluation metrics for all target models. Use this to retrieve your
|
|
325
|
+
* adapted prompts after the adaptation status is 'completed'.
|
|
326
|
+
*
|
|
327
|
+
* The response includes:
|
|
328
|
+
*
|
|
329
|
+
* - Baseline performance of your original prompt on the origin model
|
|
330
|
+
* - Optimized prompts for each target model with pre/post optimization scores
|
|
331
|
+
* - Evaluation metrics and cost information for each model
|
|
332
|
+
*/
|
|
333
|
+
export interface PromptAdaptationGetAdaptResultsResponse {
|
|
269
334
|
/**
|
|
270
335
|
* Unique ID for this adaptation run
|
|
271
336
|
*/
|
|
@@ -275,13 +340,13 @@ export interface AdaptationRunResults {
|
|
|
275
340
|
*/
|
|
276
341
|
created_at: string;
|
|
277
342
|
/**
|
|
278
|
-
* Overall status of the adaptation run
|
|
343
|
+
* Overall status of the adaptation run (queued, running, completed, failed)
|
|
279
344
|
*/
|
|
280
345
|
job_status: JobStatus;
|
|
281
346
|
/**
|
|
282
|
-
* Results for each target model with optimized prompts
|
|
347
|
+
* Results for each target model with optimized prompts and improvement scores
|
|
283
348
|
*/
|
|
284
|
-
target_models: Array<
|
|
349
|
+
target_models: Array<PromptAdaptationGetAdaptResultsResponse.TargetModel>;
|
|
285
350
|
/**
|
|
286
351
|
* Timestamp of last update to this adaptation run
|
|
287
352
|
*/
|
|
@@ -289,19 +354,57 @@ export interface AdaptationRunResults {
|
|
|
289
354
|
evaluation_config?: string | null;
|
|
290
355
|
evaluation_metric?: string | null;
|
|
291
356
|
/**
|
|
292
|
-
* Metrics for the LLM requests made during the adaptation run
|
|
357
|
+
* Metrics for the LLM requests made during the adaptation run (e.g.,
|
|
358
|
+
* total_requests, avg_latency)
|
|
293
359
|
*/
|
|
294
360
|
llm_request_metrics?: {
|
|
295
361
|
[key: string]: number;
|
|
296
362
|
};
|
|
297
363
|
/**
|
|
298
|
-
*
|
|
364
|
+
* Baseline results for the origin model in prompt adaptation.
|
|
365
|
+
*
|
|
366
|
+
* Part of AdaptationRunResultsResponse. Contains the performance metrics and
|
|
367
|
+
* prompt configuration for your original prompt on the origin model. This serves
|
|
368
|
+
* as the baseline to compare against optimized prompts for target models.
|
|
369
|
+
*
|
|
370
|
+
* **Fields include:**
|
|
371
|
+
*
|
|
372
|
+
* - Original system prompt and user message template
|
|
373
|
+
* - Baseline performance score and evaluation metrics
|
|
374
|
+
* - Cost of running the baseline evaluation
|
|
375
|
+
* - Job status for the origin model evaluation
|
|
376
|
+
*/
|
|
377
|
+
origin_model?: PromptAdaptationGetAdaptResultsResponse.OriginModel | null;
|
|
378
|
+
/**
|
|
379
|
+
* Whether this adaptation run was created with prototype mode (3-24 training
|
|
380
|
+
* examples allowed). Prototype mode may have degraded performance compared to
|
|
381
|
+
* standard mode (25+ examples)
|
|
299
382
|
*/
|
|
300
|
-
|
|
383
|
+
prototype_mode?: boolean;
|
|
301
384
|
}
|
|
302
|
-
export declare namespace
|
|
385
|
+
export declare namespace PromptAdaptationGetAdaptResultsResponse {
|
|
303
386
|
/**
|
|
304
|
-
*
|
|
387
|
+
* Optimized prompt results for a single target model in prompt adaptation.
|
|
388
|
+
*
|
|
389
|
+
* Part of AdaptationRunResultsResponse. Contains the optimized system prompt and
|
|
390
|
+
* user message template for a specific target model, along with performance scores
|
|
391
|
+
* before and after optimization. Use these optimized prompts with the target model
|
|
392
|
+
* to achieve better performance than the original prompt.
|
|
393
|
+
*
|
|
394
|
+
* **Key metrics:**
|
|
395
|
+
*
|
|
396
|
+
* - **pre_optimization_score**: Performance with original prompt on this target
|
|
397
|
+
* model
|
|
398
|
+
* - **post_optimization_score**: Performance with optimized prompt on this target
|
|
399
|
+
* model
|
|
400
|
+
* - **Score improvement**: post - pre shows how much optimization helped
|
|
401
|
+
*
|
|
402
|
+
* **Usage:**
|
|
403
|
+
*
|
|
404
|
+
* 1. Extract the optimized system_prompt and user_message_template
|
|
405
|
+
* 2. Replace placeholders in user_message_template using fields from your data
|
|
406
|
+
* 3. Use these prompts when calling this target model
|
|
407
|
+
* 4. Compare pre/post scores to see improvement gained
|
|
305
408
|
*/
|
|
306
409
|
interface TargetModel {
|
|
307
410
|
cost: number | null;
|
|
@@ -314,26 +417,49 @@ export declare namespace AdaptationRunResults {
|
|
|
314
417
|
[key: string]: unknown;
|
|
315
418
|
} | null;
|
|
316
419
|
pre_optimization_score: number | null;
|
|
420
|
+
task_type: string | null;
|
|
317
421
|
/**
|
|
318
|
-
* Status
|
|
422
|
+
* Status enum for asynchronous jobs (prompt adaptation, custom router training,
|
|
423
|
+
* etc.).
|
|
424
|
+
*
|
|
425
|
+
* Represents the current state of a long-running operation:
|
|
426
|
+
*
|
|
427
|
+
* - **created**: Job has been initialized but not yet queued
|
|
428
|
+
* - **queued**: Job is waiting in the queue to be processed
|
|
429
|
+
* - **processing**: Job is currently being executed
|
|
430
|
+
* - **completed**: Job finished successfully and results are available
|
|
431
|
+
* - **failed**: Job encountered an error and did not complete
|
|
319
432
|
*/
|
|
320
|
-
result_status
|
|
433
|
+
result_status?: PromptAdaptationAPI.JobStatus | null;
|
|
321
434
|
/**
|
|
322
|
-
* Optimized system prompt for this target model
|
|
435
|
+
* Optimized system prompt for this target model. Use this as the system message in
|
|
436
|
+
* your LLM calls
|
|
323
437
|
*/
|
|
324
|
-
system_prompt
|
|
325
|
-
task_type: string | null;
|
|
438
|
+
system_prompt?: string | null;
|
|
326
439
|
/**
|
|
327
|
-
* Optimized user message template
|
|
440
|
+
* Optimized user message template with placeholders. Substitute fields using your
|
|
441
|
+
* data before calling the LLM
|
|
328
442
|
*/
|
|
329
|
-
user_message_template
|
|
443
|
+
user_message_template?: string | null;
|
|
330
444
|
/**
|
|
331
|
-
*
|
|
445
|
+
* List of field names to substitute in the template (e.g., ['question',
|
|
446
|
+
* 'context']). These match the curly-brace placeholders in user_message_template
|
|
332
447
|
*/
|
|
333
|
-
user_message_template_fields
|
|
448
|
+
user_message_template_fields?: Array<string> | null;
|
|
334
449
|
}
|
|
335
450
|
/**
|
|
336
|
-
*
|
|
451
|
+
* Baseline results for the origin model in prompt adaptation.
|
|
452
|
+
*
|
|
453
|
+
* Part of AdaptationRunResultsResponse. Contains the performance metrics and
|
|
454
|
+
* prompt configuration for your original prompt on the origin model. This serves
|
|
455
|
+
* as the baseline to compare against optimized prompts for target models.
|
|
456
|
+
*
|
|
457
|
+
* **Fields include:**
|
|
458
|
+
*
|
|
459
|
+
* - Original system prompt and user message template
|
|
460
|
+
* - Baseline performance score and evaluation metrics
|
|
461
|
+
* - Cost of running the baseline evaluation
|
|
462
|
+
* - Job status for the origin model evaluation
|
|
337
463
|
*/
|
|
338
464
|
interface OriginModel {
|
|
339
465
|
cost: number | null;
|
|
@@ -341,60 +467,148 @@ export declare namespace AdaptationRunResults {
|
|
|
341
467
|
[key: string]: unknown;
|
|
342
468
|
} | null;
|
|
343
469
|
model_name: string | null;
|
|
344
|
-
result_status: PromptAdaptationAPI.JobStatus | null;
|
|
345
470
|
score: number | null;
|
|
346
|
-
|
|
347
|
-
|
|
471
|
+
/**
|
|
472
|
+
* Status enum for asynchronous jobs (prompt adaptation, custom router training,
|
|
473
|
+
* etc.).
|
|
474
|
+
*
|
|
475
|
+
* Represents the current state of a long-running operation:
|
|
476
|
+
*
|
|
477
|
+
* - **created**: Job has been initialized but not yet queued
|
|
478
|
+
* - **queued**: Job is waiting in the queue to be processed
|
|
479
|
+
* - **processing**: Job is currently being executed
|
|
480
|
+
* - **completed**: Job finished successfully and results are available
|
|
481
|
+
* - **failed**: Job encountered an error and did not complete
|
|
482
|
+
*/
|
|
483
|
+
result_status?: PromptAdaptationAPI.JobStatus | null;
|
|
484
|
+
/**
|
|
485
|
+
* Original system prompt used for the origin model
|
|
486
|
+
*/
|
|
487
|
+
system_prompt?: string | null;
|
|
488
|
+
/**
|
|
489
|
+
* Original user message template used for the origin model
|
|
490
|
+
*/
|
|
491
|
+
user_message_template?: string | null;
|
|
348
492
|
}
|
|
349
493
|
}
|
|
350
|
-
export type JobStatus = 'created' | 'queued' | 'processing' | 'completed' | 'failed';
|
|
351
494
|
/**
|
|
352
|
-
* Response
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
*
|
|
495
|
+
* Response model for GET /v2/prompt/adaptStatus/{adaptation_run_id} endpoint.
|
|
496
|
+
*
|
|
497
|
+
* Returns the current status of an asynchronous prompt adaptation job. Poll this
|
|
498
|
+
* endpoint periodically to track progress. When status is 'completed', you can
|
|
499
|
+
* retrieve the optimized prompts using the /adaptResults endpoint.
|
|
500
|
+
*
|
|
501
|
+
* **Status values:**
|
|
502
|
+
*
|
|
503
|
+
* - **created**: Job has been initialized
|
|
504
|
+
* - **queued**: Waiting in queue (check queue_position for your place in line)
|
|
505
|
+
* - **processing**: Currently running optimization
|
|
506
|
+
* - **completed**: Finished successfully, results available via /adaptResults
|
|
507
|
+
* - **failed**: Encountered an error during processing
|
|
508
|
+
*
|
|
509
|
+
* **Polling recommendations:**
|
|
510
|
+
*
|
|
511
|
+
* - Poll every 30-60 seconds while status is incomplete
|
|
512
|
+
* - Stop polling once status is 'completed' or 'failed'
|
|
513
|
+
* - Adaptation typically takes 10-30 minutes total
|
|
363
514
|
*/
|
|
364
515
|
export interface PromptAdaptationGetAdaptStatusResponse {
|
|
365
516
|
/**
|
|
366
|
-
* Unique
|
|
517
|
+
* Unique identifier for this adaptation run. Use this to poll status and retrieve
|
|
518
|
+
* optimized prompts when complete
|
|
367
519
|
*/
|
|
368
520
|
adaptation_run_id: string;
|
|
369
521
|
/**
|
|
370
|
-
* Current status of the adaptation run
|
|
371
|
-
* failed)
|
|
522
|
+
* Current status of the adaptation run. Poll until this is 'completed' or 'failed'
|
|
372
523
|
*/
|
|
373
524
|
status: JobStatus;
|
|
374
525
|
/**
|
|
375
|
-
* Position in queue
|
|
526
|
+
* Position in queue when status is 'queued'. Lower numbers process sooner. Null
|
|
527
|
+
* when not queued
|
|
376
528
|
*/
|
|
377
529
|
queue_position?: number | null;
|
|
378
530
|
}
|
|
379
|
-
|
|
531
|
+
/**
|
|
532
|
+
* Response model for GET /v2/prompt/adapt/{adaptation_run_id}/costs endpoint.
|
|
533
|
+
*
|
|
534
|
+
* Contains the total LLM costs and detailed usage records for a prompt adaptation
|
|
535
|
+
* run. Use this to track costs associated with optimizing prompts for different
|
|
536
|
+
* target models.
|
|
537
|
+
*/
|
|
538
|
+
export interface PromptAdaptationGetCostResponse {
|
|
539
|
+
/**
|
|
540
|
+
* Unique identifier for the adaptation run
|
|
541
|
+
*/
|
|
380
542
|
adaptation_run_id: string;
|
|
543
|
+
/**
|
|
544
|
+
* Total cost in USD across all LLM requests in this adaptation run
|
|
545
|
+
*/
|
|
381
546
|
total_cost: number;
|
|
382
|
-
|
|
547
|
+
/**
|
|
548
|
+
* Detailed usage records for each LLM request made during the adaptation
|
|
549
|
+
*/
|
|
550
|
+
usage_records: Array<PromptAdaptationGetCostResponse.UsageRecord>;
|
|
383
551
|
}
|
|
384
|
-
export declare namespace
|
|
552
|
+
export declare namespace PromptAdaptationGetCostResponse {
|
|
553
|
+
/**
|
|
554
|
+
* Individual LLM usage record with token counts and cost breakdown.
|
|
555
|
+
*
|
|
556
|
+
* Returned by GET /llm-usage endpoint and included in AdaptationRunCostResponse.
|
|
557
|
+
* Each record represents a single LLM API call with detailed usage metrics.
|
|
558
|
+
*/
|
|
385
559
|
interface UsageRecord {
|
|
560
|
+
/**
|
|
561
|
+
* Unique identifier for this usage record
|
|
562
|
+
*/
|
|
386
563
|
id: string;
|
|
564
|
+
/**
|
|
565
|
+
* Adaptation run ID this usage is associated with
|
|
566
|
+
*/
|
|
387
567
|
adaptation_run_id: string;
|
|
568
|
+
/**
|
|
569
|
+
* Cost of input tokens in USD
|
|
570
|
+
*/
|
|
388
571
|
input_cost: number;
|
|
572
|
+
/**
|
|
573
|
+
* Number of input tokens consumed
|
|
574
|
+
*/
|
|
389
575
|
input_tokens: number;
|
|
576
|
+
/**
|
|
577
|
+
* Model name (e.g., 'gpt-4', 'claude-3-opus-20240229')
|
|
578
|
+
*/
|
|
390
579
|
model: string;
|
|
580
|
+
/**
|
|
581
|
+
* Organization ID associated with the request
|
|
582
|
+
*/
|
|
391
583
|
organization_id: string;
|
|
584
|
+
/**
|
|
585
|
+
* Cost of output tokens in USD
|
|
586
|
+
*/
|
|
392
587
|
output_cost: number;
|
|
588
|
+
/**
|
|
589
|
+
* Number of output tokens generated
|
|
590
|
+
*/
|
|
393
591
|
output_tokens: number;
|
|
592
|
+
/**
|
|
593
|
+
* LLM provider (e.g., 'openai', 'anthropic', 'google')
|
|
594
|
+
*/
|
|
394
595
|
provider: string;
|
|
596
|
+
/**
|
|
597
|
+
* Type of task: 'pre-optimization evaluation', 'optimization', or
|
|
598
|
+
* 'post-optimization evaluation'
|
|
599
|
+
*/
|
|
395
600
|
task_type: string;
|
|
601
|
+
/**
|
|
602
|
+
* Unix timestamp when the request was made
|
|
603
|
+
*/
|
|
396
604
|
timestamp: number;
|
|
605
|
+
/**
|
|
606
|
+
* Total cost (input + output) in USD
|
|
607
|
+
*/
|
|
397
608
|
total_cost: number;
|
|
609
|
+
/**
|
|
610
|
+
* User ID who made the request
|
|
611
|
+
*/
|
|
398
612
|
user_id: string;
|
|
399
613
|
}
|
|
400
614
|
}
|
|
@@ -411,9 +625,9 @@ export interface PromptAdaptationAdaptParams {
|
|
|
411
625
|
system_prompt: string;
|
|
412
626
|
/**
|
|
413
627
|
* List of models to adapt the prompt for. Maximum count depends on your
|
|
414
|
-
* subscription tier
|
|
628
|
+
* subscription tier (Free: 1, Starter: 3, Startup: 5, Enterprise: 10)
|
|
415
629
|
*/
|
|
416
|
-
target_models: Array<
|
|
630
|
+
target_models: Array<RequestProvider>;
|
|
417
631
|
/**
|
|
418
632
|
* User message template with placeholders for fields. Use curly braces for field
|
|
419
633
|
* substitution
|
|
@@ -423,159 +637,36 @@ export interface PromptAdaptationAdaptParams {
|
|
|
423
637
|
evaluation_metric?: string | null;
|
|
424
638
|
/**
|
|
425
639
|
* Training examples (legacy parameter). Use train_goldens and test_goldens for
|
|
426
|
-
* better control
|
|
640
|
+
* better control. Minimum 25 examples (or 3 with prototype_mode=true)
|
|
427
641
|
*/
|
|
428
|
-
goldens?: Array<
|
|
642
|
+
goldens?: Array<GoldenRecord> | null;
|
|
429
643
|
/**
|
|
430
644
|
* Model for specifying an LLM provider in API requests.
|
|
431
645
|
*/
|
|
432
|
-
origin_model?:
|
|
646
|
+
origin_model?: RequestProvider | null;
|
|
433
647
|
/**
|
|
434
|
-
* Optional baseline score for the origin model
|
|
648
|
+
* Optional baseline score for the origin model. If provided, can skip origin model
|
|
649
|
+
* evaluation
|
|
435
650
|
*/
|
|
436
651
|
origin_model_evaluation_score?: number | null;
|
|
437
652
|
/**
|
|
438
|
-
*
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
/**
|
|
442
|
-
* Training examples for prompt optimization. Minimum 5 examples required
|
|
443
|
-
*/
|
|
444
|
-
train_goldens?: Array<PromptAdaptationAdaptParams.TrainGolden> | null;
|
|
445
|
-
}
|
|
446
|
-
export declare namespace PromptAdaptationAdaptParams {
|
|
447
|
-
/**
|
|
448
|
-
* Model for specifying an LLM provider in API requests.
|
|
449
|
-
*/
|
|
450
|
-
interface TargetModel {
|
|
451
|
-
/**
|
|
452
|
-
* Model name (e.g., 'gpt-4o', 'claude-sonnet-4-5-20250929')
|
|
453
|
-
*/
|
|
454
|
-
model: string;
|
|
455
|
-
/**
|
|
456
|
-
* Provider name (e.g., 'openai', 'anthropic', 'google')
|
|
457
|
-
*/
|
|
458
|
-
provider: string;
|
|
459
|
-
/**
|
|
460
|
-
* Maximum context length for the model (required for custom models)
|
|
461
|
-
*/
|
|
462
|
-
context_length?: number | null;
|
|
463
|
-
/**
|
|
464
|
-
* Input token price per million tokens in USD (required for custom models)
|
|
465
|
-
*/
|
|
466
|
-
input_price?: number | null;
|
|
467
|
-
/**
|
|
468
|
-
* Whether this is a custom model not in Not Diamond's supported model list
|
|
469
|
-
*/
|
|
470
|
-
is_custom?: boolean;
|
|
471
|
-
/**
|
|
472
|
-
* Average latency in seconds (required for custom models)
|
|
473
|
-
*/
|
|
474
|
-
latency?: number | null;
|
|
475
|
-
/**
|
|
476
|
-
* Output token price per million tokens in USD (required for custom models)
|
|
477
|
-
*/
|
|
478
|
-
output_price?: number | null;
|
|
479
|
-
}
|
|
480
|
-
/**
|
|
481
|
-
* A training or test example for prompt adaptation.
|
|
482
|
-
*/
|
|
483
|
-
interface Golden {
|
|
484
|
-
/**
|
|
485
|
-
* Dictionary mapping field names to their values. Keys must match the fields
|
|
486
|
-
* specified in the template
|
|
487
|
-
*/
|
|
488
|
-
fields: {
|
|
489
|
-
[key: string]: string;
|
|
490
|
-
};
|
|
491
|
-
/**
|
|
492
|
-
* Expected answer for supervised evaluation. Required for supervised metrics,
|
|
493
|
-
* optional for unsupervised
|
|
494
|
-
*/
|
|
495
|
-
answer?: string | null;
|
|
496
|
-
}
|
|
497
|
-
/**
|
|
498
|
-
* Model for specifying an LLM provider in API requests.
|
|
499
|
-
*/
|
|
500
|
-
interface OriginModel {
|
|
501
|
-
/**
|
|
502
|
-
* Model name (e.g., 'gpt-4o', 'claude-sonnet-4-5-20250929')
|
|
503
|
-
*/
|
|
504
|
-
model: string;
|
|
505
|
-
/**
|
|
506
|
-
* Provider name (e.g., 'openai', 'anthropic', 'google')
|
|
507
|
-
*/
|
|
508
|
-
provider: string;
|
|
509
|
-
/**
|
|
510
|
-
* Maximum context length for the model (required for custom models)
|
|
511
|
-
*/
|
|
512
|
-
context_length?: number | null;
|
|
513
|
-
/**
|
|
514
|
-
* Input token price per million tokens in USD (required for custom models)
|
|
515
|
-
*/
|
|
516
|
-
input_price?: number | null;
|
|
517
|
-
/**
|
|
518
|
-
* Whether this is a custom model not in Not Diamond's supported model list
|
|
519
|
-
*/
|
|
520
|
-
is_custom?: boolean;
|
|
521
|
-
/**
|
|
522
|
-
* Average latency in seconds (required for custom models)
|
|
523
|
-
*/
|
|
524
|
-
latency?: number | null;
|
|
525
|
-
/**
|
|
526
|
-
* Output token price per million tokens in USD (required for custom models)
|
|
527
|
-
*/
|
|
528
|
-
output_price?: number | null;
|
|
529
|
-
}
|
|
530
|
-
/**
|
|
531
|
-
* A training or test example for prompt adaptation.
|
|
653
|
+
* Enable prototype mode to use as few as 3 training examples (instead of 25).
|
|
654
|
+
* Note: Performance may be degraded with fewer examples. Recommended for
|
|
655
|
+
* prototyping AI applications when you don't have enough data yet
|
|
532
656
|
*/
|
|
533
|
-
|
|
534
|
-
/**
|
|
535
|
-
* Dictionary mapping field names to their values. Keys must match the fields
|
|
536
|
-
* specified in the template
|
|
537
|
-
*/
|
|
538
|
-
fields: {
|
|
539
|
-
[key: string]: string;
|
|
540
|
-
};
|
|
541
|
-
/**
|
|
542
|
-
* Expected answer for supervised evaluation. Required for supervised metrics,
|
|
543
|
-
* optional for unsupervised
|
|
544
|
-
*/
|
|
545
|
-
answer?: string | null;
|
|
546
|
-
}
|
|
547
|
-
/**
|
|
548
|
-
* A training or test example for prompt adaptation.
|
|
549
|
-
*/
|
|
550
|
-
interface TrainGolden {
|
|
551
|
-
/**
|
|
552
|
-
* Dictionary mapping field names to their values. Keys must match the fields
|
|
553
|
-
* specified in the template
|
|
554
|
-
*/
|
|
555
|
-
fields: {
|
|
556
|
-
[key: string]: string;
|
|
557
|
-
};
|
|
558
|
-
/**
|
|
559
|
-
* Expected answer for supervised evaluation. Required for supervised metrics,
|
|
560
|
-
* optional for unsupervised
|
|
561
|
-
*/
|
|
562
|
-
answer?: string | null;
|
|
563
|
-
}
|
|
564
|
-
}
|
|
565
|
-
export interface PromptAdaptationGetAdaptRunResultsParams {
|
|
657
|
+
prototype_mode?: boolean;
|
|
566
658
|
/**
|
|
567
|
-
*
|
|
659
|
+
* Test examples for evaluation. Required if train_goldens is provided. Used to
|
|
660
|
+
* measure final performance on held-out data
|
|
568
661
|
*/
|
|
569
|
-
|
|
662
|
+
test_goldens?: Array<GoldenRecord> | null;
|
|
570
663
|
/**
|
|
571
|
-
*
|
|
664
|
+
* Training examples for prompt optimization. Minimum 25 examples required (or 3
|
|
665
|
+
* with prototype_mode=true). Cannot be used with 'goldens' parameter
|
|
572
666
|
*/
|
|
573
|
-
|
|
574
|
-
}
|
|
575
|
-
export interface PromptAdaptationGetAdaptRunsParams {
|
|
576
|
-
'x-token': string;
|
|
667
|
+
train_goldens?: Array<GoldenRecord> | null;
|
|
577
668
|
}
|
|
578
669
|
export declare namespace PromptAdaptation {
|
|
579
|
-
export { type
|
|
670
|
+
export { type GoldenRecord as GoldenRecord, type JobStatus as JobStatus, type RequestProvider as RequestProvider, type PromptAdaptationAdaptResponse as PromptAdaptationAdaptResponse, type PromptAdaptationGetAdaptResultsResponse as PromptAdaptationGetAdaptResultsResponse, type PromptAdaptationGetAdaptStatusResponse as PromptAdaptationGetAdaptStatusResponse, type PromptAdaptationGetCostResponse as PromptAdaptationGetCostResponse, type PromptAdaptationAdaptParams as PromptAdaptationAdaptParams, };
|
|
580
671
|
}
|
|
581
672
|
//# sourceMappingURL=prompt-adaptation.d.ts.map
|