notdiamond 2.0.0-rc1 → 2.0.0-rc10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +89 -0
- package/README.md +299 -97
- package/client.d.mts +19 -33
- package/client.d.mts.map +1 -1
- package/client.d.ts +19 -33
- package/client.d.ts.map +1 -1
- package/client.js +19 -35
- package/client.js.map +1 -1
- package/client.mjs +19 -35
- package/client.mjs.map +1 -1
- package/internal/tslib.js +17 -17
- package/internal/utils/env.d.mts.map +1 -1
- package/internal/utils/env.d.ts.map +1 -1
- package/internal/utils/env.js +4 -2
- package/internal/utils/env.js.map +1 -1
- package/internal/utils/env.mjs +4 -2
- package/internal/utils/env.mjs.map +1 -1
- package/package.json +10 -11
- package/resources/custom-router.d.mts +145 -0
- package/resources/custom-router.d.mts.map +1 -0
- package/resources/custom-router.d.ts +145 -0
- package/resources/custom-router.d.ts.map +1 -0
- package/resources/custom-router.js +83 -0
- package/resources/custom-router.js.map +1 -0
- package/resources/custom-router.mjs +79 -0
- package/resources/custom-router.mjs.map +1 -0
- package/resources/index.d.mts +6 -6
- package/resources/index.d.mts.map +1 -1
- package/resources/index.d.ts +6 -6
- package/resources/index.d.ts.map +1 -1
- package/resources/index.js +6 -6
- package/resources/index.js.map +1 -1
- package/resources/index.mjs +3 -3
- package/resources/index.mjs.map +1 -1
- package/resources/model-router.d.mts +196 -0
- package/resources/model-router.d.mts.map +1 -0
- package/resources/model-router.d.ts +196 -0
- package/resources/model-router.d.ts.map +1 -0
- package/resources/model-router.js +70 -0
- package/resources/model-router.js.map +1 -0
- package/resources/model-router.mjs +66 -0
- package/resources/model-router.mjs.map +1 -0
- package/resources/models.d.mts +50 -25
- package/resources/models.d.mts.map +1 -1
- package/resources/models.d.ts +50 -25
- package/resources/models.d.ts.map +1 -1
- package/resources/models.js +5 -0
- package/resources/models.js.map +1 -1
- package/resources/models.mjs +5 -0
- package/resources/models.mjs.map +1 -1
- package/resources/preferences.d.mts +34 -48
- package/resources/preferences.d.mts.map +1 -1
- package/resources/preferences.d.ts +34 -48
- package/resources/preferences.d.ts.map +1 -1
- package/resources/preferences.js +14 -36
- package/resources/preferences.js.map +1 -1
- package/resources/preferences.mjs +14 -36
- package/resources/preferences.mjs.map +1 -1
- package/resources/prompt-adaptation.d.mts +339 -234
- package/resources/prompt-adaptation.d.mts.map +1 -1
- package/resources/prompt-adaptation.d.ts +339 -234
- package/resources/prompt-adaptation.d.ts.map +1 -1
- package/resources/prompt-adaptation.js +30 -53
- package/resources/prompt-adaptation.js.map +1 -1
- package/resources/prompt-adaptation.mjs +30 -53
- package/resources/prompt-adaptation.mjs.map +1 -1
- package/resources/report/index.d.mts +3 -0
- package/resources/report/index.d.mts.map +1 -0
- package/resources/report/index.d.ts +3 -0
- package/resources/report/index.d.ts.map +1 -0
- package/resources/report/index.js +9 -0
- package/resources/report/index.js.map +1 -0
- package/resources/report/index.mjs +4 -0
- package/resources/report/index.mjs.map +1 -0
- package/resources/report/metrics.d.mts +87 -0
- package/resources/report/metrics.d.mts.map +1 -0
- package/resources/report/metrics.d.ts +87 -0
- package/resources/report/metrics.d.ts.map +1 -0
- package/resources/report/metrics.js +57 -0
- package/resources/report/metrics.js.map +1 -0
- package/resources/report/metrics.mjs +53 -0
- package/resources/report/metrics.mjs.map +1 -0
- package/resources/report/report.d.mts +10 -0
- package/resources/report/report.d.mts.map +1 -0
- package/resources/report/report.d.ts +10 -0
- package/resources/report/report.d.ts.map +1 -0
- package/resources/report/report.js +17 -0
- package/resources/report/report.js.map +1 -0
- package/resources/report/report.mjs +12 -0
- package/resources/report/report.mjs.map +1 -0
- package/resources/report.d.mts +1 -244
- package/resources/report.d.mts.map +1 -1
- package/resources/report.d.ts +1 -244
- package/resources/report.d.ts.map +1 -1
- package/resources/report.js +2 -82
- package/resources/report.js.map +1 -1
- package/resources/report.mjs +1 -80
- package/resources/report.mjs.map +1 -1
- package/src/client.ts +65 -116
- package/src/internal/utils/env.ts +4 -2
- package/src/resources/custom-router.ts +168 -0
- package/src/resources/index.ts +23 -34
- package/src/resources/model-router.ts +224 -0
- package/src/resources/models.ts +55 -32
- package/src/resources/preferences.ts +40 -77
- package/src/resources/prompt-adaptation.ts +361 -291
- package/src/resources/report/index.ts +4 -0
- package/src/resources/report/metrics.ts +99 -0
- package/src/resources/report/report.ts +19 -0
- package/src/resources/report.ts +1 -298
- package/src/version.ts +1 -1
- package/version.d.mts +1 -1
- package/version.d.mts.map +1 -1
- package/version.d.ts +1 -1
- package/version.d.ts.map +1 -1
- package/version.js +1 -1
- package/version.js.map +1 -1
- package/version.mjs +1 -1
- package/version.mjs.map +1 -1
- package/resources/admin.d.mts +0 -4
- package/resources/admin.d.mts.map +0 -1
- package/resources/admin.d.ts +0 -4
- package/resources/admin.d.ts.map +0 -1
- package/resources/admin.js +0 -9
- package/resources/admin.js.map +0 -1
- package/resources/admin.mjs +0 -5
- package/resources/admin.mjs.map +0 -1
- package/resources/routing.d.mts +0 -391
- package/resources/routing.d.mts.map +0 -1
- package/resources/routing.d.ts +0 -391
- package/resources/routing.d.ts.map +0 -1
- package/resources/routing.js +0 -163
- package/resources/routing.js.map +0 -1
- package/resources/routing.mjs +0 -159
- package/resources/routing.mjs.map +0 -1
- package/src/resources/admin.ts +0 -5
- package/src/resources/routing.ts +0 -476
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
import { APIResource } from '../core/resource';
|
|
4
4
|
import * as PromptAdaptationAPI from './prompt-adaptation';
|
|
5
5
|
import { APIPromise } from '../core/api-promise';
|
|
6
|
-
import { buildHeaders } from '../internal/headers';
|
|
7
6
|
import { RequestOptions } from '../internal/request-options';
|
|
8
7
|
import { path } from '../internal/utils/path';
|
|
9
8
|
|
|
@@ -34,7 +33,7 @@ export class PromptAdaptation extends APIResource {
|
|
|
34
33
|
*
|
|
35
34
|
* **Dataset Requirements:**
|
|
36
35
|
*
|
|
37
|
-
* - Minimum
|
|
36
|
+
* - Minimum 25 examples in train_goldens (more examples = better adaptation)
|
|
38
37
|
* - Each example must have fields matching your template placeholders
|
|
39
38
|
* - Supervised evaluation requires 'answer' field in each golden record
|
|
40
39
|
* - Unsupervised evaluation can work without answers
|
|
@@ -55,7 +54,7 @@ export class PromptAdaptation extends APIResource {
|
|
|
55
54
|
* **Best Practices:**
|
|
56
55
|
*
|
|
57
56
|
* 1. Use diverse, representative examples from your production workload
|
|
58
|
-
* 2. Include
|
|
57
|
+
* 2. Include examples for best results (25 minimum)
|
|
59
58
|
* 3. Ensure consistent evaluation across all examples
|
|
60
59
|
* 4. Test both train_goldens and test_goldens split for validation
|
|
61
60
|
* 5. Use the same model versions you'll use in production
|
|
@@ -75,16 +74,16 @@ export class PromptAdaptation extends APIResource {
|
|
|
75
74
|
*
|
|
76
75
|
* @example
|
|
77
76
|
* ```ts
|
|
78
|
-
* const
|
|
77
|
+
* const promptAdaptation = await client.promptAdaptation.create({
|
|
79
78
|
* fields: ['question'],
|
|
80
|
-
* origin_model: { provider: 'openai', model: 'gpt-4o' },
|
|
81
79
|
* system_prompt: 'You are a helpful assistant that answers questions accurately.',
|
|
82
80
|
* target_models: [
|
|
83
|
-
* { provider: 'anthropic', model: 'claude-
|
|
84
|
-
* { provider: 'google', model: 'gemini-
|
|
81
|
+
* { provider: 'anthropic', model: 'claude-sonnet-4-5-20250929' },
|
|
82
|
+
* { provider: 'google', model: 'gemini-2.5-flash' },
|
|
85
83
|
* ],
|
|
86
84
|
* template: 'Question: {question}\nAnswer:',
|
|
87
85
|
* evaluation_metric: 'LLMaaJ:Sem_Sim_3',
|
|
86
|
+
* origin_model: { provider: 'openai', model: 'gpt-4o' },
|
|
88
87
|
* test_goldens: [
|
|
89
88
|
* {
|
|
90
89
|
* fields: { ... },
|
|
@@ -120,10 +119,10 @@ export class PromptAdaptation extends APIResource {
|
|
|
120
119
|
* });
|
|
121
120
|
* ```
|
|
122
121
|
*/
|
|
123
|
-
|
|
124
|
-
body:
|
|
122
|
+
create(
|
|
123
|
+
body: PromptAdaptationCreateParams,
|
|
125
124
|
options?: RequestOptions,
|
|
126
|
-
): APIPromise<
|
|
125
|
+
): APIPromise<PromptAdaptationCreateResponse> {
|
|
127
126
|
return this._client.post('/v2/prompt/adapt', { body, ...options });
|
|
128
127
|
}
|
|
129
128
|
|
|
@@ -187,61 +186,17 @@ export class PromptAdaptation extends APIResource {
|
|
|
187
186
|
*
|
|
188
187
|
* @example
|
|
189
188
|
* ```ts
|
|
190
|
-
* const
|
|
189
|
+
* const response =
|
|
191
190
|
* await client.promptAdaptation.getAdaptResults(
|
|
192
191
|
* 'adaptation_run_id',
|
|
193
192
|
* );
|
|
194
193
|
* ```
|
|
195
194
|
*/
|
|
196
|
-
getAdaptResults(
|
|
197
|
-
return this._client.get(path`/v2/prompt/adaptResults/${adaptationRunID}`, options);
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
/**
|
|
201
|
-
* Get Adapt Run Results
|
|
202
|
-
*
|
|
203
|
-
* @example
|
|
204
|
-
* ```ts
|
|
205
|
-
* const adaptationRunResults =
|
|
206
|
-
* await client.promptAdaptation.getAdaptRunResults(
|
|
207
|
-
* 'adaptation_run_id',
|
|
208
|
-
* { user_id: 'user_id', 'x-token': 'x-token' },
|
|
209
|
-
* );
|
|
210
|
-
* ```
|
|
211
|
-
*/
|
|
212
|
-
getAdaptRunResults(
|
|
195
|
+
getAdaptResults(
|
|
213
196
|
adaptationRunID: string,
|
|
214
|
-
params: PromptAdaptationGetAdaptRunResultsParams,
|
|
215
197
|
options?: RequestOptions,
|
|
216
|
-
): APIPromise<
|
|
217
|
-
|
|
218
|
-
return this._client.get(path`/v2/prompt/frontendAdaptRunResults/${user_id}/${adaptationRunID}`, {
|
|
219
|
-
...options,
|
|
220
|
-
headers: buildHeaders([{ 'x-token': xToken }, options?.headers]),
|
|
221
|
-
});
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
/**
|
|
225
|
-
* Get Adapt Runs
|
|
226
|
-
*
|
|
227
|
-
* @example
|
|
228
|
-
* ```ts
|
|
229
|
-
* const adaptationRunResults =
|
|
230
|
-
* await client.promptAdaptation.getAdaptRuns('user_id', {
|
|
231
|
-
* 'x-token': 'x-token',
|
|
232
|
-
* });
|
|
233
|
-
* ```
|
|
234
|
-
*/
|
|
235
|
-
getAdaptRuns(
|
|
236
|
-
userID: string,
|
|
237
|
-
params: PromptAdaptationGetAdaptRunsParams,
|
|
238
|
-
options?: RequestOptions,
|
|
239
|
-
): APIPromise<PromptAdaptationGetAdaptRunsResponse> {
|
|
240
|
-
const { 'x-token': xToken } = params;
|
|
241
|
-
return this._client.get(path`/v2/prompt/frontendAdaptRuns/${userID}`, {
|
|
242
|
-
...options,
|
|
243
|
-
headers: buildHeaders([{ 'x-token': xToken }, options?.headers]),
|
|
244
|
-
});
|
|
198
|
+
): APIPromise<PromptAdaptationGetAdaptResultsResponse> {
|
|
199
|
+
return this._client.get(path`/v2/prompt/adaptResults/${adaptationRunID}`, options);
|
|
245
200
|
}
|
|
246
201
|
|
|
247
202
|
/**
|
|
@@ -292,85 +247,219 @@ export class PromptAdaptation extends APIResource {
|
|
|
292
247
|
}
|
|
293
248
|
|
|
294
249
|
/**
|
|
295
|
-
* Get LLM costs for a specific adaptation run
|
|
250
|
+
* Get LLM usage costs for a specific prompt adaptation run.
|
|
251
|
+
*
|
|
252
|
+
* This endpoint returns the total cost and detailed usage records for all LLM
|
|
253
|
+
* requests made during a prompt adaptation run. Use this to track costs associated
|
|
254
|
+
* with optimizing prompts for different target models.
|
|
255
|
+
*
|
|
256
|
+
* **Cost Breakdown:**
|
|
257
|
+
*
|
|
258
|
+
* - Total cost across all models used in the adaptation
|
|
259
|
+
* - Individual usage records with provider, model, tokens, and costs
|
|
260
|
+
* - Timestamps for each LLM request
|
|
261
|
+
* - Task type (e.g., optimization, evaluation)
|
|
262
|
+
*
|
|
263
|
+
* **Access Control:**
|
|
264
|
+
*
|
|
265
|
+
* - Only accessible by the user who created the adaptation run
|
|
266
|
+
* - Requires prompt adaptation access
|
|
296
267
|
*
|
|
297
268
|
* @example
|
|
298
269
|
* ```ts
|
|
299
|
-
* const response =
|
|
300
|
-
*
|
|
301
|
-
*
|
|
302
|
-
* );
|
|
270
|
+
* const response = await client.promptAdaptation.getCosts(
|
|
271
|
+
* 'adaptation_run_id',
|
|
272
|
+
* );
|
|
303
273
|
* ```
|
|
304
274
|
*/
|
|
305
|
-
|
|
306
|
-
adaptationRunID
|
|
307
|
-
options?: RequestOptions,
|
|
308
|
-
): APIPromise<PromptAdaptationRetrieveCostsResponse> {
|
|
309
|
-
return this._client.get(path`/v1/adaptation-runs/${adaptationRunID}/costs`, options);
|
|
275
|
+
getCosts(adaptationRunID: string, options?: RequestOptions): APIPromise<PromptAdaptationGetCostsResponse> {
|
|
276
|
+
return this._client.get(path`/v2/prompt/adapt/${adaptationRunID}/costs`, options);
|
|
310
277
|
}
|
|
311
278
|
}
|
|
312
279
|
|
|
313
280
|
/**
|
|
314
|
-
*
|
|
281
|
+
* A training or test example for prompt adaptation.
|
|
315
282
|
*/
|
|
316
|
-
export interface
|
|
283
|
+
export interface GoldenRecord {
|
|
317
284
|
/**
|
|
318
|
-
*
|
|
285
|
+
* Dictionary mapping field names to their values. Keys must match the fields
|
|
286
|
+
* specified in the template
|
|
319
287
|
*/
|
|
320
|
-
|
|
288
|
+
fields: { [key: string]: string };
|
|
321
289
|
|
|
322
290
|
/**
|
|
323
|
-
*
|
|
291
|
+
* Expected answer for supervised evaluation. Required for supervised metrics,
|
|
292
|
+
* optional for unsupervised
|
|
324
293
|
*/
|
|
325
|
-
|
|
294
|
+
answer?: string | null;
|
|
295
|
+
}
|
|
326
296
|
|
|
297
|
+
/**
|
|
298
|
+
* Status enum for asynchronous jobs (prompt adaptation, custom router training,
|
|
299
|
+
* etc.).
|
|
300
|
+
*
|
|
301
|
+
* Represents the current state of a long-running operation:
|
|
302
|
+
*
|
|
303
|
+
* - **created**: Job has been initialized but not yet queued
|
|
304
|
+
* - **queued**: Job is waiting in the queue to be processed
|
|
305
|
+
* - **processing**: Job is currently being executed
|
|
306
|
+
* - **completed**: Job finished successfully and results are available
|
|
307
|
+
* - **failed**: Job encountered an error and did not complete
|
|
308
|
+
*/
|
|
309
|
+
export type JobStatus = 'created' | 'queued' | 'processing' | 'completed' | 'failed';
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Model for specifying an LLM provider in API requests.
|
|
313
|
+
*/
|
|
314
|
+
export interface RequestProvider {
|
|
327
315
|
/**
|
|
328
|
-
*
|
|
316
|
+
* Model name (e.g., 'gpt-4o', 'claude-sonnet-4-5-20250929')
|
|
329
317
|
*/
|
|
330
|
-
|
|
318
|
+
model: string;
|
|
331
319
|
|
|
332
320
|
/**
|
|
333
|
-
*
|
|
321
|
+
* Provider name (e.g., 'openai', 'anthropic', 'google')
|
|
334
322
|
*/
|
|
335
|
-
|
|
323
|
+
provider: string;
|
|
336
324
|
|
|
337
325
|
/**
|
|
338
|
-
*
|
|
326
|
+
* Maximum context length for the model (required for custom models)
|
|
339
327
|
*/
|
|
340
|
-
|
|
328
|
+
context_length?: number | null;
|
|
341
329
|
|
|
342
330
|
/**
|
|
343
|
-
*
|
|
331
|
+
* Input token price per million tokens in USD (required for custom models)
|
|
344
332
|
*/
|
|
345
|
-
|
|
333
|
+
input_price?: number | null;
|
|
346
334
|
|
|
347
|
-
|
|
335
|
+
/**
|
|
336
|
+
* Whether this is a custom model not in Not Diamond's supported model list
|
|
337
|
+
*/
|
|
338
|
+
is_custom?: boolean;
|
|
348
339
|
|
|
349
|
-
|
|
340
|
+
/**
|
|
341
|
+
* Average latency in seconds (required for custom models)
|
|
342
|
+
*/
|
|
343
|
+
latency?: number | null;
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* Output token price per million tokens in USD (required for custom models)
|
|
347
|
+
*/
|
|
348
|
+
output_price?: number | null;
|
|
350
349
|
}
|
|
351
350
|
|
|
352
|
-
|
|
351
|
+
/**
|
|
352
|
+
* Response model for POST /v2/prompt/adapt endpoint.
|
|
353
|
+
*
|
|
354
|
+
* Returned immediately after submitting a prompt adaptation request. The
|
|
355
|
+
* adaptation process runs asynchronously, so use the returned adaptation_run_id to
|
|
356
|
+
* track progress and retrieve results when complete.
|
|
357
|
+
*
|
|
358
|
+
* **Next steps:**
|
|
359
|
+
*
|
|
360
|
+
* 1. Store the adaptation_run_id
|
|
361
|
+
* 2. Poll GET /v2/prompt/adaptStatus/{adaptation_run_id} to check progress
|
|
362
|
+
* 3. When status is 'completed', retrieve optimized prompts from GET
|
|
363
|
+
* /v2/prompt/adaptResults/{adaptation_run_id}
|
|
364
|
+
* 4. Use the optimized prompts with your target models
|
|
365
|
+
*/
|
|
366
|
+
export interface PromptAdaptationCreateResponse {
|
|
353
367
|
/**
|
|
354
|
-
*
|
|
368
|
+
* Unique identifier for this adaptation run. Use this to poll status and retrieve
|
|
369
|
+
* optimized prompts when complete
|
|
355
370
|
*/
|
|
356
|
-
|
|
357
|
-
|
|
371
|
+
adaptation_run_id: string;
|
|
372
|
+
}
|
|
358
373
|
|
|
359
|
-
|
|
374
|
+
/**
|
|
375
|
+
* Response model for GET /v2/prompt/adaptResults/{adaptation_run_id} endpoint.
|
|
376
|
+
*
|
|
377
|
+
* Contains the complete results of a prompt adaptation run, including optimized
|
|
378
|
+
* prompts and evaluation metrics for all target models. Use this to retrieve your
|
|
379
|
+
* adapted prompts after the adaptation status is 'completed'.
|
|
380
|
+
*
|
|
381
|
+
* The response includes:
|
|
382
|
+
*
|
|
383
|
+
* - Baseline performance of your original prompt on the origin model
|
|
384
|
+
* - Optimized prompts for each target model with pre/post optimization scores
|
|
385
|
+
* - Evaluation metrics and cost information for each model
|
|
386
|
+
*/
|
|
387
|
+
export interface PromptAdaptationGetAdaptResultsResponse {
|
|
388
|
+
/**
|
|
389
|
+
* Unique ID for this adaptation run
|
|
390
|
+
*/
|
|
391
|
+
id: string;
|
|
360
392
|
|
|
361
|
-
|
|
393
|
+
/**
|
|
394
|
+
* Timestamp when this adaptation run was created
|
|
395
|
+
*/
|
|
396
|
+
created_at: string;
|
|
362
397
|
|
|
363
|
-
|
|
398
|
+
/**
|
|
399
|
+
* Overall status of the adaptation run (queued, running, completed, failed)
|
|
400
|
+
*/
|
|
401
|
+
job_status: JobStatus;
|
|
364
402
|
|
|
365
|
-
|
|
403
|
+
/**
|
|
404
|
+
* Results for each target model with optimized prompts and improvement scores
|
|
405
|
+
*/
|
|
406
|
+
target_models: Array<PromptAdaptationGetAdaptResultsResponse.TargetModel>;
|
|
366
407
|
|
|
367
|
-
|
|
408
|
+
/**
|
|
409
|
+
* Timestamp of last update to this adaptation run
|
|
410
|
+
*/
|
|
411
|
+
updated_at: string | null;
|
|
368
412
|
|
|
369
|
-
|
|
370
|
-
|
|
413
|
+
evaluation_config?: string | null;
|
|
414
|
+
|
|
415
|
+
evaluation_metric?: string | null;
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* Metrics for the LLM requests made during the adaptation run (e.g.,
|
|
419
|
+
* total_requests, avg_latency)
|
|
420
|
+
*/
|
|
421
|
+
llm_request_metrics?: { [key: string]: number };
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Baseline results for the origin model in prompt adaptation.
|
|
425
|
+
*
|
|
426
|
+
* Part of AdaptationRunResultsResponse. Contains the performance metrics and
|
|
427
|
+
* prompt configuration for your original prompt on the origin model. This serves
|
|
428
|
+
* as the baseline to compare against optimized prompts for target models.
|
|
429
|
+
*
|
|
430
|
+
* **Fields include:**
|
|
431
|
+
*
|
|
432
|
+
* - Original system prompt and user message template
|
|
433
|
+
* - Baseline performance score and evaluation metrics
|
|
434
|
+
* - Cost of running the baseline evaluation
|
|
435
|
+
* - Job status for the origin model evaluation
|
|
436
|
+
*/
|
|
437
|
+
origin_model?: PromptAdaptationGetAdaptResultsResponse.OriginModel | null;
|
|
438
|
+
}
|
|
371
439
|
|
|
440
|
+
export namespace PromptAdaptationGetAdaptResultsResponse {
|
|
372
441
|
/**
|
|
373
|
-
*
|
|
442
|
+
* Optimized prompt results for a single target model in prompt adaptation.
|
|
443
|
+
*
|
|
444
|
+
* Part of AdaptationRunResultsResponse. Contains the optimized system prompt and
|
|
445
|
+
* user message template for a specific target model, along with performance scores
|
|
446
|
+
* before and after optimization. Use these optimized prompts with the target model
|
|
447
|
+
* to achieve better performance than the original prompt.
|
|
448
|
+
*
|
|
449
|
+
* **Key metrics:**
|
|
450
|
+
*
|
|
451
|
+
* - **pre_optimization_score**: Performance with original prompt on this target
|
|
452
|
+
* model
|
|
453
|
+
* - **post_optimization_score**: Performance with optimized prompt on this target
|
|
454
|
+
* model
|
|
455
|
+
* - **Score improvement**: post - pre shows how much optimization helped
|
|
456
|
+
*
|
|
457
|
+
* **Usage:**
|
|
458
|
+
*
|
|
459
|
+
* 1. Extract the optimized system_prompt and user_message_template
|
|
460
|
+
* 2. Replace placeholders in user_message_template using fields from your data
|
|
461
|
+
* 3. Use these prompts when calling this target model
|
|
462
|
+
* 4. Compare pre/post scores to see improvement gained
|
|
374
463
|
*/
|
|
375
464
|
export interface TargetModel {
|
|
376
465
|
cost: number | null;
|
|
@@ -385,113 +474,236 @@ export namespace AdaptationRunResults {
|
|
|
385
474
|
|
|
386
475
|
pre_optimization_score: number | null;
|
|
387
476
|
|
|
477
|
+
task_type: string | null;
|
|
478
|
+
|
|
388
479
|
/**
|
|
389
|
-
* Status
|
|
480
|
+
* Status enum for asynchronous jobs (prompt adaptation, custom router training,
|
|
481
|
+
* etc.).
|
|
482
|
+
*
|
|
483
|
+
* Represents the current state of a long-running operation:
|
|
484
|
+
*
|
|
485
|
+
* - **created**: Job has been initialized but not yet queued
|
|
486
|
+
* - **queued**: Job is waiting in the queue to be processed
|
|
487
|
+
* - **processing**: Job is currently being executed
|
|
488
|
+
* - **completed**: Job finished successfully and results are available
|
|
489
|
+
* - **failed**: Job encountered an error and did not complete
|
|
390
490
|
*/
|
|
391
|
-
result_status
|
|
491
|
+
result_status?: PromptAdaptationAPI.JobStatus | null;
|
|
392
492
|
|
|
393
493
|
/**
|
|
394
|
-
* Optimized system prompt for this target model
|
|
494
|
+
* Optimized system prompt for this target model. Use this as the system message in
|
|
495
|
+
* your LLM calls
|
|
395
496
|
*/
|
|
396
|
-
system_prompt
|
|
497
|
+
system_prompt?: string | null;
|
|
397
498
|
|
|
398
499
|
/**
|
|
399
|
-
* Optimized user message template
|
|
500
|
+
* Optimized user message template with placeholders. Substitute fields using your
|
|
501
|
+
* data before calling the LLM
|
|
400
502
|
*/
|
|
401
|
-
user_message_template
|
|
503
|
+
user_message_template?: string | null;
|
|
402
504
|
|
|
403
505
|
/**
|
|
404
|
-
*
|
|
506
|
+
* List of field names to substitute in the template (e.g., ['question',
|
|
507
|
+
* 'context']). These match the curly-brace placeholders in user_message_template
|
|
405
508
|
*/
|
|
406
|
-
user_message_template_fields
|
|
509
|
+
user_message_template_fields?: Array<string> | null;
|
|
407
510
|
}
|
|
408
|
-
}
|
|
409
511
|
|
|
410
|
-
export type JobStatus = 'created' | 'queued' | 'processing' | 'completed' | 'failed';
|
|
411
|
-
|
|
412
|
-
/**
|
|
413
|
-
* Response from prompt adaptation request.
|
|
414
|
-
*/
|
|
415
|
-
export interface PromptAdaptationAdaptResponse {
|
|
416
512
|
/**
|
|
417
|
-
*
|
|
513
|
+
* Baseline results for the origin model in prompt adaptation.
|
|
514
|
+
*
|
|
515
|
+
* Part of AdaptationRunResultsResponse. Contains the performance metrics and
|
|
516
|
+
* prompt configuration for your original prompt on the origin model. This serves
|
|
517
|
+
* as the baseline to compare against optimized prompts for target models.
|
|
518
|
+
*
|
|
519
|
+
* **Fields include:**
|
|
520
|
+
*
|
|
521
|
+
* - Original system prompt and user message template
|
|
522
|
+
* - Baseline performance score and evaluation metrics
|
|
523
|
+
* - Cost of running the baseline evaluation
|
|
524
|
+
* - Job status for the origin model evaluation
|
|
418
525
|
*/
|
|
419
|
-
|
|
420
|
-
|
|
526
|
+
export interface OriginModel {
|
|
527
|
+
cost: number | null;
|
|
528
|
+
|
|
529
|
+
evals: { [key: string]: unknown } | null;
|
|
530
|
+
|
|
531
|
+
model_name: string | null;
|
|
532
|
+
|
|
533
|
+
score: number | null;
|
|
421
534
|
|
|
422
|
-
|
|
535
|
+
/**
|
|
536
|
+
* Status enum for asynchronous jobs (prompt adaptation, custom router training,
|
|
537
|
+
* etc.).
|
|
538
|
+
*
|
|
539
|
+
* Represents the current state of a long-running operation:
|
|
540
|
+
*
|
|
541
|
+
* - **created**: Job has been initialized but not yet queued
|
|
542
|
+
* - **queued**: Job is waiting in the queue to be processed
|
|
543
|
+
* - **processing**: Job is currently being executed
|
|
544
|
+
* - **completed**: Job finished successfully and results are available
|
|
545
|
+
* - **failed**: Job encountered an error and did not complete
|
|
546
|
+
*/
|
|
547
|
+
result_status?: PromptAdaptationAPI.JobStatus | null;
|
|
548
|
+
|
|
549
|
+
/**
|
|
550
|
+
* Original system prompt used for the origin model
|
|
551
|
+
*/
|
|
552
|
+
system_prompt?: string | null;
|
|
553
|
+
|
|
554
|
+
/**
|
|
555
|
+
* Original user message template used for the origin model
|
|
556
|
+
*/
|
|
557
|
+
user_message_template?: string | null;
|
|
558
|
+
}
|
|
559
|
+
}
|
|
423
560
|
|
|
424
561
|
/**
|
|
425
|
-
*
|
|
562
|
+
* Response model for GET /v2/prompt/adaptStatus/{adaptation_run_id} endpoint.
|
|
563
|
+
*
|
|
564
|
+
* Returns the current status of an asynchronous prompt adaptation job. Poll this
|
|
565
|
+
* endpoint periodically to track progress. When status is 'completed', you can
|
|
566
|
+
* retrieve the optimized prompts using the /adaptResults endpoint.
|
|
567
|
+
*
|
|
568
|
+
* **Status values:**
|
|
569
|
+
*
|
|
570
|
+
* - **created**: Job has been initialized
|
|
571
|
+
* - **queued**: Waiting in queue (check queue_position for your place in line)
|
|
572
|
+
* - **processing**: Currently running optimization
|
|
573
|
+
* - **completed**: Finished successfully, results available via /adaptResults
|
|
574
|
+
* - **failed**: Encountered an error during processing
|
|
575
|
+
*
|
|
576
|
+
* **Polling recommendations:**
|
|
577
|
+
*
|
|
578
|
+
* - Poll every 30-60 seconds while status is incomplete
|
|
579
|
+
* - Stop polling once status is 'completed' or 'failed'
|
|
580
|
+
* - Adaptation typically takes 10-30 minutes total
|
|
426
581
|
*/
|
|
427
582
|
export interface PromptAdaptationGetAdaptStatusResponse {
|
|
428
583
|
/**
|
|
429
|
-
* Unique
|
|
584
|
+
* Unique identifier for this adaptation run. Use this to poll status and retrieve
|
|
585
|
+
* optimized prompts when complete
|
|
430
586
|
*/
|
|
431
587
|
adaptation_run_id: string;
|
|
432
588
|
|
|
433
589
|
/**
|
|
434
|
-
* Current status of the adaptation run
|
|
435
|
-
* failed)
|
|
590
|
+
* Current status of the adaptation run. Poll until this is 'completed' or 'failed'
|
|
436
591
|
*/
|
|
437
592
|
status: JobStatus;
|
|
438
593
|
|
|
439
594
|
/**
|
|
440
|
-
* Position in queue
|
|
595
|
+
* Position in queue when status is 'queued'. Lower numbers process sooner. Null
|
|
596
|
+
* when not queued
|
|
441
597
|
*/
|
|
442
598
|
queue_position?: number | null;
|
|
443
599
|
}
|
|
444
600
|
|
|
445
|
-
|
|
601
|
+
/**
|
|
602
|
+
* Response model for GET /v2/prompt/adapt/{adaptation_run_id}/costs endpoint.
|
|
603
|
+
*
|
|
604
|
+
* Contains the total LLM costs and detailed usage records for a prompt adaptation
|
|
605
|
+
* run. Use this to track costs associated with optimizing prompts for different
|
|
606
|
+
* target models.
|
|
607
|
+
*/
|
|
608
|
+
export interface PromptAdaptationGetCostsResponse {
|
|
609
|
+
/**
|
|
610
|
+
* Unique identifier for the adaptation run
|
|
611
|
+
*/
|
|
446
612
|
adaptation_run_id: string;
|
|
447
613
|
|
|
614
|
+
/**
|
|
615
|
+
* Total cost in USD across all LLM requests in this adaptation run
|
|
616
|
+
*/
|
|
448
617
|
total_cost: number;
|
|
449
618
|
|
|
450
|
-
|
|
619
|
+
/**
|
|
620
|
+
* Detailed usage records for each LLM request made during the adaptation
|
|
621
|
+
*/
|
|
622
|
+
usage_records: Array<PromptAdaptationGetCostsResponse.UsageRecord>;
|
|
451
623
|
}
|
|
452
624
|
|
|
453
|
-
export namespace
|
|
625
|
+
export namespace PromptAdaptationGetCostsResponse {
|
|
626
|
+
/**
|
|
627
|
+
* Individual LLM usage record with token counts and cost breakdown.
|
|
628
|
+
*
|
|
629
|
+
* Returned by GET /llm-usage endpoint and included in AdaptationRunCostResponse.
|
|
630
|
+
* Each record represents a single LLM API call with detailed usage metrics.
|
|
631
|
+
*/
|
|
454
632
|
export interface UsageRecord {
|
|
633
|
+
/**
|
|
634
|
+
* Unique identifier for this usage record
|
|
635
|
+
*/
|
|
455
636
|
id: string;
|
|
456
637
|
|
|
638
|
+
/**
|
|
639
|
+
* Adaptation run ID this usage is associated with
|
|
640
|
+
*/
|
|
457
641
|
adaptation_run_id: string;
|
|
458
642
|
|
|
643
|
+
/**
|
|
644
|
+
* Cost of input tokens in USD
|
|
645
|
+
*/
|
|
459
646
|
input_cost: number;
|
|
460
647
|
|
|
648
|
+
/**
|
|
649
|
+
* Number of input tokens consumed
|
|
650
|
+
*/
|
|
461
651
|
input_tokens: number;
|
|
462
652
|
|
|
653
|
+
/**
|
|
654
|
+
* Model name (e.g., 'gpt-4', 'claude-3-opus-20240229')
|
|
655
|
+
*/
|
|
463
656
|
model: string;
|
|
464
657
|
|
|
658
|
+
/**
|
|
659
|
+
* Organization ID associated with the request
|
|
660
|
+
*/
|
|
465
661
|
organization_id: string;
|
|
466
662
|
|
|
663
|
+
/**
|
|
664
|
+
* Cost of output tokens in USD
|
|
665
|
+
*/
|
|
467
666
|
output_cost: number;
|
|
468
667
|
|
|
668
|
+
/**
|
|
669
|
+
* Number of output tokens generated
|
|
670
|
+
*/
|
|
469
671
|
output_tokens: number;
|
|
470
672
|
|
|
673
|
+
/**
|
|
674
|
+
* LLM provider (e.g., 'openai', 'anthropic', 'google')
|
|
675
|
+
*/
|
|
471
676
|
provider: string;
|
|
472
677
|
|
|
678
|
+
/**
|
|
679
|
+
* Type of task (e.g., 'optimization', 'evaluation')
|
|
680
|
+
*/
|
|
473
681
|
task_type: string;
|
|
474
682
|
|
|
683
|
+
/**
|
|
684
|
+
* Unix timestamp when the request was made
|
|
685
|
+
*/
|
|
475
686
|
timestamp: number;
|
|
476
687
|
|
|
688
|
+
/**
|
|
689
|
+
* Total cost (input + output) in USD
|
|
690
|
+
*/
|
|
477
691
|
total_cost: number;
|
|
478
692
|
|
|
693
|
+
/**
|
|
694
|
+
* User ID who made the request
|
|
695
|
+
*/
|
|
479
696
|
user_id: string;
|
|
480
697
|
}
|
|
481
698
|
}
|
|
482
699
|
|
|
483
|
-
export interface
|
|
700
|
+
export interface PromptAdaptationCreateParams {
|
|
484
701
|
/**
|
|
485
702
|
* List of field names that will be substituted into the template. Must match keys
|
|
486
703
|
* in golden records
|
|
487
704
|
*/
|
|
488
705
|
fields: Array<string>;
|
|
489
706
|
|
|
490
|
-
/**
|
|
491
|
-
* The model your current prompt is optimized for
|
|
492
|
-
*/
|
|
493
|
-
origin_model: PromptAdaptationAdaptParams.OriginModel;
|
|
494
|
-
|
|
495
707
|
/**
|
|
496
708
|
* System prompt to use with the origin model. This sets the context and role for
|
|
497
709
|
* the LLM
|
|
@@ -500,9 +712,9 @@ export interface PromptAdaptationAdaptParams {
|
|
|
500
712
|
|
|
501
713
|
/**
|
|
502
714
|
* List of models to adapt the prompt for. Maximum count depends on your
|
|
503
|
-
* subscription tier
|
|
715
|
+
* subscription tier (Free: 1, Starter: 3, Startup: 5, Enterprise: 10)
|
|
504
716
|
*/
|
|
505
|
-
target_models: Array<
|
|
717
|
+
target_models: Array<RequestProvider>;
|
|
506
718
|
|
|
507
719
|
/**
|
|
508
720
|
* User message template with placeholders for fields. Use curly braces for field
|
|
@@ -516,185 +728,43 @@ export interface PromptAdaptationAdaptParams {
|
|
|
516
728
|
|
|
517
729
|
/**
|
|
518
730
|
* Training examples (legacy parameter). Use train_goldens and test_goldens for
|
|
519
|
-
* better control
|
|
731
|
+
* better control. Minimum 25 examples
|
|
520
732
|
*/
|
|
521
|
-
goldens?: Array<
|
|
522
|
-
|
|
523
|
-
/**
|
|
524
|
-
* Optional baseline score for the origin model
|
|
525
|
-
*/
|
|
526
|
-
origin_model_evaluation_score?: number | null;
|
|
527
|
-
|
|
528
|
-
/**
|
|
529
|
-
* Test examples for evaluation. Required if train_goldens is provided
|
|
530
|
-
*/
|
|
531
|
-
test_goldens?: Array<PromptAdaptationAdaptParams.TestGolden> | null;
|
|
532
|
-
|
|
533
|
-
/**
|
|
534
|
-
* Training examples for prompt optimization. Minimum 5 examples required
|
|
535
|
-
*/
|
|
536
|
-
train_goldens?: Array<PromptAdaptationAdaptParams.TrainGolden> | null;
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
export namespace PromptAdaptationAdaptParams {
|
|
540
|
-
/**
|
|
541
|
-
* The model your current prompt is optimized for
|
|
542
|
-
*/
|
|
543
|
-
export interface OriginModel {
|
|
544
|
-
/**
|
|
545
|
-
* Model name (e.g., 'gpt-4o', 'claude-3-5-sonnet-20241022')
|
|
546
|
-
*/
|
|
547
|
-
model: string;
|
|
548
|
-
|
|
549
|
-
/**
|
|
550
|
-
* Provider name (e.g., 'openai', 'anthropic', 'google')
|
|
551
|
-
*/
|
|
552
|
-
provider: string;
|
|
553
|
-
|
|
554
|
-
/**
|
|
555
|
-
* Maximum context length for the model (required for custom models)
|
|
556
|
-
*/
|
|
557
|
-
context_length?: number | null;
|
|
558
|
-
|
|
559
|
-
/**
|
|
560
|
-
* Input token price per million tokens in USD (required for custom models)
|
|
561
|
-
*/
|
|
562
|
-
input_price?: number | null;
|
|
563
|
-
|
|
564
|
-
/**
|
|
565
|
-
* Whether this is a custom model not in Not Diamond's supported model list
|
|
566
|
-
*/
|
|
567
|
-
is_custom?: boolean;
|
|
568
|
-
|
|
569
|
-
/**
|
|
570
|
-
* Average latency in seconds (required for custom models)
|
|
571
|
-
*/
|
|
572
|
-
latency?: number | null;
|
|
573
|
-
|
|
574
|
-
/**
|
|
575
|
-
* Output token price per million tokens in USD (required for custom models)
|
|
576
|
-
*/
|
|
577
|
-
output_price?: number | null;
|
|
578
|
-
}
|
|
733
|
+
goldens?: Array<GoldenRecord> | null;
|
|
579
734
|
|
|
580
735
|
/**
|
|
581
736
|
* Model for specifying an LLM provider in API requests.
|
|
582
737
|
*/
|
|
583
|
-
|
|
584
|
-
/**
|
|
585
|
-
* Model name (e.g., 'gpt-4o', 'claude-3-5-sonnet-20241022')
|
|
586
|
-
*/
|
|
587
|
-
model: string;
|
|
588
|
-
|
|
589
|
-
/**
|
|
590
|
-
* Provider name (e.g., 'openai', 'anthropic', 'google')
|
|
591
|
-
*/
|
|
592
|
-
provider: string;
|
|
593
|
-
|
|
594
|
-
/**
|
|
595
|
-
* Maximum context length for the model (required for custom models)
|
|
596
|
-
*/
|
|
597
|
-
context_length?: number | null;
|
|
598
|
-
|
|
599
|
-
/**
|
|
600
|
-
* Input token price per million tokens in USD (required for custom models)
|
|
601
|
-
*/
|
|
602
|
-
input_price?: number | null;
|
|
603
|
-
|
|
604
|
-
/**
|
|
605
|
-
* Whether this is a custom model not in Not Diamond's supported model list
|
|
606
|
-
*/
|
|
607
|
-
is_custom?: boolean;
|
|
608
|
-
|
|
609
|
-
/**
|
|
610
|
-
* Average latency in seconds (required for custom models)
|
|
611
|
-
*/
|
|
612
|
-
latency?: number | null;
|
|
613
|
-
|
|
614
|
-
/**
|
|
615
|
-
* Output token price per million tokens in USD (required for custom models)
|
|
616
|
-
*/
|
|
617
|
-
output_price?: number | null;
|
|
618
|
-
}
|
|
619
|
-
|
|
620
|
-
/**
|
|
621
|
-
* A training or test example for prompt adaptation.
|
|
622
|
-
*/
|
|
623
|
-
export interface Golden {
|
|
624
|
-
/**
|
|
625
|
-
* Dictionary mapping field names to their values. Keys must match the fields
|
|
626
|
-
* specified in the template
|
|
627
|
-
*/
|
|
628
|
-
fields: { [key: string]: string };
|
|
629
|
-
|
|
630
|
-
/**
|
|
631
|
-
* Expected answer for supervised evaluation. Required for supervised metrics,
|
|
632
|
-
* optional for unsupervised
|
|
633
|
-
*/
|
|
634
|
-
answer?: string | null;
|
|
635
|
-
}
|
|
738
|
+
origin_model?: RequestProvider | null;
|
|
636
739
|
|
|
637
740
|
/**
|
|
638
|
-
*
|
|
741
|
+
* Optional baseline score for the origin model. If provided, can skip origin model
|
|
742
|
+
* evaluation
|
|
639
743
|
*/
|
|
640
|
-
|
|
641
|
-
/**
|
|
642
|
-
* Dictionary mapping field names to their values. Keys must match the fields
|
|
643
|
-
* specified in the template
|
|
644
|
-
*/
|
|
645
|
-
fields: { [key: string]: string };
|
|
646
|
-
|
|
647
|
-
/**
|
|
648
|
-
* Expected answer for supervised evaluation. Required for supervised metrics,
|
|
649
|
-
* optional for unsupervised
|
|
650
|
-
*/
|
|
651
|
-
answer?: string | null;
|
|
652
|
-
}
|
|
653
|
-
|
|
654
|
-
/**
|
|
655
|
-
* A training or test example for prompt adaptation.
|
|
656
|
-
*/
|
|
657
|
-
export interface TrainGolden {
|
|
658
|
-
/**
|
|
659
|
-
* Dictionary mapping field names to their values. Keys must match the fields
|
|
660
|
-
* specified in the template
|
|
661
|
-
*/
|
|
662
|
-
fields: { [key: string]: string };
|
|
663
|
-
|
|
664
|
-
/**
|
|
665
|
-
* Expected answer for supervised evaluation. Required for supervised metrics,
|
|
666
|
-
* optional for unsupervised
|
|
667
|
-
*/
|
|
668
|
-
answer?: string | null;
|
|
669
|
-
}
|
|
670
|
-
}
|
|
744
|
+
origin_model_evaluation_score?: number | null;
|
|
671
745
|
|
|
672
|
-
export interface PromptAdaptationGetAdaptRunResultsParams {
|
|
673
746
|
/**
|
|
674
|
-
*
|
|
747
|
+
* Test examples for evaluation. Required if train_goldens is provided. Used to
|
|
748
|
+
* measure final performance on held-out data
|
|
675
749
|
*/
|
|
676
|
-
|
|
750
|
+
test_goldens?: Array<GoldenRecord> | null;
|
|
677
751
|
|
|
678
752
|
/**
|
|
679
|
-
*
|
|
753
|
+
* Training examples for prompt optimization. Minimum 25 examples required. Cannot
|
|
754
|
+
* be used with 'goldens' parameter
|
|
680
755
|
*/
|
|
681
|
-
|
|
682
|
-
}
|
|
683
|
-
|
|
684
|
-
export interface PromptAdaptationGetAdaptRunsParams {
|
|
685
|
-
'x-token': string;
|
|
756
|
+
train_goldens?: Array<GoldenRecord> | null;
|
|
686
757
|
}
|
|
687
758
|
|
|
688
759
|
export declare namespace PromptAdaptation {
|
|
689
760
|
export {
|
|
690
|
-
type
|
|
761
|
+
type GoldenRecord as GoldenRecord,
|
|
691
762
|
type JobStatus as JobStatus,
|
|
692
|
-
type
|
|
693
|
-
type
|
|
763
|
+
type RequestProvider as RequestProvider,
|
|
764
|
+
type PromptAdaptationCreateResponse as PromptAdaptationCreateResponse,
|
|
765
|
+
type PromptAdaptationGetAdaptResultsResponse as PromptAdaptationGetAdaptResultsResponse,
|
|
694
766
|
type PromptAdaptationGetAdaptStatusResponse as PromptAdaptationGetAdaptStatusResponse,
|
|
695
|
-
type
|
|
696
|
-
type
|
|
697
|
-
type PromptAdaptationGetAdaptRunResultsParams as PromptAdaptationGetAdaptRunResultsParams,
|
|
698
|
-
type PromptAdaptationGetAdaptRunsParams as PromptAdaptationGetAdaptRunsParams,
|
|
767
|
+
type PromptAdaptationGetCostsResponse as PromptAdaptationGetCostsResponse,
|
|
768
|
+
type PromptAdaptationCreateParams as PromptAdaptationCreateParams,
|
|
699
769
|
};
|
|
700
770
|
}
|