notdiamond 2.0.0-rc1 → 2.0.0-rc10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +89 -0
- package/README.md +299 -97
- package/client.d.mts +19 -33
- package/client.d.mts.map +1 -1
- package/client.d.ts +19 -33
- package/client.d.ts.map +1 -1
- package/client.js +19 -35
- package/client.js.map +1 -1
- package/client.mjs +19 -35
- package/client.mjs.map +1 -1
- package/internal/tslib.js +17 -17
- package/internal/utils/env.d.mts.map +1 -1
- package/internal/utils/env.d.ts.map +1 -1
- package/internal/utils/env.js +4 -2
- package/internal/utils/env.js.map +1 -1
- package/internal/utils/env.mjs +4 -2
- package/internal/utils/env.mjs.map +1 -1
- package/package.json +10 -11
- package/resources/custom-router.d.mts +145 -0
- package/resources/custom-router.d.mts.map +1 -0
- package/resources/custom-router.d.ts +145 -0
- package/resources/custom-router.d.ts.map +1 -0
- package/resources/custom-router.js +83 -0
- package/resources/custom-router.js.map +1 -0
- package/resources/custom-router.mjs +79 -0
- package/resources/custom-router.mjs.map +1 -0
- package/resources/index.d.mts +6 -6
- package/resources/index.d.mts.map +1 -1
- package/resources/index.d.ts +6 -6
- package/resources/index.d.ts.map +1 -1
- package/resources/index.js +6 -6
- package/resources/index.js.map +1 -1
- package/resources/index.mjs +3 -3
- package/resources/index.mjs.map +1 -1
- package/resources/model-router.d.mts +196 -0
- package/resources/model-router.d.mts.map +1 -0
- package/resources/model-router.d.ts +196 -0
- package/resources/model-router.d.ts.map +1 -0
- package/resources/model-router.js +70 -0
- package/resources/model-router.js.map +1 -0
- package/resources/model-router.mjs +66 -0
- package/resources/model-router.mjs.map +1 -0
- package/resources/models.d.mts +50 -25
- package/resources/models.d.mts.map +1 -1
- package/resources/models.d.ts +50 -25
- package/resources/models.d.ts.map +1 -1
- package/resources/models.js +5 -0
- package/resources/models.js.map +1 -1
- package/resources/models.mjs +5 -0
- package/resources/models.mjs.map +1 -1
- package/resources/preferences.d.mts +34 -48
- package/resources/preferences.d.mts.map +1 -1
- package/resources/preferences.d.ts +34 -48
- package/resources/preferences.d.ts.map +1 -1
- package/resources/preferences.js +14 -36
- package/resources/preferences.js.map +1 -1
- package/resources/preferences.mjs +14 -36
- package/resources/preferences.mjs.map +1 -1
- package/resources/prompt-adaptation.d.mts +339 -234
- package/resources/prompt-adaptation.d.mts.map +1 -1
- package/resources/prompt-adaptation.d.ts +339 -234
- package/resources/prompt-adaptation.d.ts.map +1 -1
- package/resources/prompt-adaptation.js +30 -53
- package/resources/prompt-adaptation.js.map +1 -1
- package/resources/prompt-adaptation.mjs +30 -53
- package/resources/prompt-adaptation.mjs.map +1 -1
- package/resources/report/index.d.mts +3 -0
- package/resources/report/index.d.mts.map +1 -0
- package/resources/report/index.d.ts +3 -0
- package/resources/report/index.d.ts.map +1 -0
- package/resources/report/index.js +9 -0
- package/resources/report/index.js.map +1 -0
- package/resources/report/index.mjs +4 -0
- package/resources/report/index.mjs.map +1 -0
- package/resources/report/metrics.d.mts +87 -0
- package/resources/report/metrics.d.mts.map +1 -0
- package/resources/report/metrics.d.ts +87 -0
- package/resources/report/metrics.d.ts.map +1 -0
- package/resources/report/metrics.js +57 -0
- package/resources/report/metrics.js.map +1 -0
- package/resources/report/metrics.mjs +53 -0
- package/resources/report/metrics.mjs.map +1 -0
- package/resources/report/report.d.mts +10 -0
- package/resources/report/report.d.mts.map +1 -0
- package/resources/report/report.d.ts +10 -0
- package/resources/report/report.d.ts.map +1 -0
- package/resources/report/report.js +17 -0
- package/resources/report/report.js.map +1 -0
- package/resources/report/report.mjs +12 -0
- package/resources/report/report.mjs.map +1 -0
- package/resources/report.d.mts +1 -244
- package/resources/report.d.mts.map +1 -1
- package/resources/report.d.ts +1 -244
- package/resources/report.d.ts.map +1 -1
- package/resources/report.js +2 -82
- package/resources/report.js.map +1 -1
- package/resources/report.mjs +1 -80
- package/resources/report.mjs.map +1 -1
- package/src/client.ts +65 -116
- package/src/internal/utils/env.ts +4 -2
- package/src/resources/custom-router.ts +168 -0
- package/src/resources/index.ts +23 -34
- package/src/resources/model-router.ts +224 -0
- package/src/resources/models.ts +55 -32
- package/src/resources/preferences.ts +40 -77
- package/src/resources/prompt-adaptation.ts +361 -291
- package/src/resources/report/index.ts +4 -0
- package/src/resources/report/metrics.ts +99 -0
- package/src/resources/report/report.ts +19 -0
- package/src/resources/report.ts +1 -298
- package/src/version.ts +1 -1
- package/version.d.mts +1 -1
- package/version.d.mts.map +1 -1
- package/version.d.ts +1 -1
- package/version.d.ts.map +1 -1
- package/version.js +1 -1
- package/version.js.map +1 -1
- package/version.mjs +1 -1
- package/version.mjs.map +1 -1
- package/resources/admin.d.mts +0 -4
- package/resources/admin.d.mts.map +0 -1
- package/resources/admin.d.ts +0 -4
- package/resources/admin.d.ts.map +0 -1
- package/resources/admin.js +0 -9
- package/resources/admin.js.map +0 -1
- package/resources/admin.mjs +0 -5
- package/resources/admin.mjs.map +0 -1
- package/resources/routing.d.mts +0 -391
- package/resources/routing.d.mts.map +0 -1
- package/resources/routing.d.ts +0 -391
- package/resources/routing.d.ts.map +0 -1
- package/resources/routing.js +0 -163
- package/resources/routing.js.map +0 -1
- package/resources/routing.mjs +0 -159
- package/resources/routing.mjs.map +0 -1
- package/src/resources/admin.ts +0 -5
- package/src/resources/routing.ts +0 -476
|
@@ -29,7 +29,7 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
29
29
|
*
|
|
30
30
|
* **Dataset Requirements:**
|
|
31
31
|
*
|
|
32
|
-
* - Minimum
|
|
32
|
+
* - Minimum 25 examples in train_goldens (more examples = better adaptation)
|
|
33
33
|
* - Each example must have fields matching your template placeholders
|
|
34
34
|
* - Supervised evaluation requires 'answer' field in each golden record
|
|
35
35
|
* - Unsupervised evaluation can work without answers
|
|
@@ -50,7 +50,7 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
50
50
|
* **Best Practices:**
|
|
51
51
|
*
|
|
52
52
|
* 1. Use diverse, representative examples from your production workload
|
|
53
|
-
* 2. Include
|
|
53
|
+
* 2. Include examples for best results (25 minimum)
|
|
54
54
|
* 3. Ensure consistent evaluation across all examples
|
|
55
55
|
* 4. Test both train_goldens and test_goldens split for validation
|
|
56
56
|
* 5. Use the same model versions you'll use in production
|
|
@@ -70,16 +70,16 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
70
70
|
*
|
|
71
71
|
* @example
|
|
72
72
|
* ```ts
|
|
73
|
-
* const
|
|
73
|
+
* const promptAdaptation = await client.promptAdaptation.create({
|
|
74
74
|
* fields: ['question'],
|
|
75
|
-
* origin_model: { provider: 'openai', model: 'gpt-4o' },
|
|
76
75
|
* system_prompt: 'You are a helpful assistant that answers questions accurately.',
|
|
77
76
|
* target_models: [
|
|
78
|
-
* { provider: 'anthropic', model: 'claude-
|
|
79
|
-
* { provider: 'google', model: 'gemini-
|
|
77
|
+
* { provider: 'anthropic', model: 'claude-sonnet-4-5-20250929' },
|
|
78
|
+
* { provider: 'google', model: 'gemini-2.5-flash' },
|
|
80
79
|
* ],
|
|
81
80
|
* template: 'Question: {question}\nAnswer:',
|
|
82
81
|
* evaluation_metric: 'LLMaaJ:Sem_Sim_3',
|
|
82
|
+
* origin_model: { provider: 'openai', model: 'gpt-4o' },
|
|
83
83
|
* test_goldens: [
|
|
84
84
|
* {
|
|
85
85
|
* fields: { ... },
|
|
@@ -115,7 +115,7 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
115
115
|
* });
|
|
116
116
|
* ```
|
|
117
117
|
*/
|
|
118
|
-
|
|
118
|
+
create(body: PromptAdaptationCreateParams, options?: RequestOptions): APIPromise<PromptAdaptationCreateResponse>;
|
|
119
119
|
/**
|
|
120
120
|
* Retrieve the complete results of a prompt adaptation run, including optimized
|
|
121
121
|
* prompts for all target models.
|
|
@@ -176,38 +176,13 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
176
176
|
*
|
|
177
177
|
* @example
|
|
178
178
|
* ```ts
|
|
179
|
-
* const
|
|
179
|
+
* const response =
|
|
180
180
|
* await client.promptAdaptation.getAdaptResults(
|
|
181
181
|
* 'adaptation_run_id',
|
|
182
182
|
* );
|
|
183
183
|
* ```
|
|
184
184
|
*/
|
|
185
|
-
getAdaptResults(adaptationRunID: string, options?: RequestOptions): APIPromise<
|
|
186
|
-
/**
|
|
187
|
-
* Get Adapt Run Results
|
|
188
|
-
*
|
|
189
|
-
* @example
|
|
190
|
-
* ```ts
|
|
191
|
-
* const adaptationRunResults =
|
|
192
|
-
* await client.promptAdaptation.getAdaptRunResults(
|
|
193
|
-
* 'adaptation_run_id',
|
|
194
|
-
* { user_id: 'user_id', 'x-token': 'x-token' },
|
|
195
|
-
* );
|
|
196
|
-
* ```
|
|
197
|
-
*/
|
|
198
|
-
getAdaptRunResults(adaptationRunID: string, params: PromptAdaptationGetAdaptRunResultsParams, options?: RequestOptions): APIPromise<AdaptationRunResults>;
|
|
199
|
-
/**
|
|
200
|
-
* Get Adapt Runs
|
|
201
|
-
*
|
|
202
|
-
* @example
|
|
203
|
-
* ```ts
|
|
204
|
-
* const adaptationRunResults =
|
|
205
|
-
* await client.promptAdaptation.getAdaptRuns('user_id', {
|
|
206
|
-
* 'x-token': 'x-token',
|
|
207
|
-
* });
|
|
208
|
-
* ```
|
|
209
|
-
*/
|
|
210
|
-
getAdaptRuns(userID: string, params: PromptAdaptationGetAdaptRunsParams, options?: RequestOptions): APIPromise<PromptAdaptationGetAdaptRunsResponse>;
|
|
185
|
+
getAdaptResults(adaptationRunID: string, options?: RequestOptions): APIPromise<PromptAdaptationGetAdaptResultsResponse>;
|
|
211
186
|
/**
|
|
212
187
|
* Check the status of a prompt adaptation run.
|
|
213
188
|
*
|
|
@@ -250,22 +225,132 @@ export declare class PromptAdaptation extends APIResource {
|
|
|
250
225
|
*/
|
|
251
226
|
getAdaptStatus(adaptationRunID: string, options?: RequestOptions): APIPromise<PromptAdaptationGetAdaptStatusResponse>;
|
|
252
227
|
/**
|
|
253
|
-
* Get LLM costs for a specific adaptation run
|
|
228
|
+
* Get LLM usage costs for a specific prompt adaptation run.
|
|
229
|
+
*
|
|
230
|
+
* This endpoint returns the total cost and detailed usage records for all LLM
|
|
231
|
+
* requests made during a prompt adaptation run. Use this to track costs associated
|
|
232
|
+
* with optimizing prompts for different target models.
|
|
233
|
+
*
|
|
234
|
+
* **Cost Breakdown:**
|
|
235
|
+
*
|
|
236
|
+
* - Total cost across all models used in the adaptation
|
|
237
|
+
* - Individual usage records with provider, model, tokens, and costs
|
|
238
|
+
* - Timestamps for each LLM request
|
|
239
|
+
* - Task type (e.g., optimization, evaluation)
|
|
240
|
+
*
|
|
241
|
+
* **Access Control:**
|
|
242
|
+
*
|
|
243
|
+
* - Only accessible by the user who created the adaptation run
|
|
244
|
+
* - Requires prompt adaptation access
|
|
254
245
|
*
|
|
255
246
|
* @example
|
|
256
247
|
* ```ts
|
|
257
|
-
* const response =
|
|
258
|
-
*
|
|
259
|
-
*
|
|
260
|
-
* );
|
|
248
|
+
* const response = await client.promptAdaptation.getCosts(
|
|
249
|
+
* 'adaptation_run_id',
|
|
250
|
+
* );
|
|
261
251
|
* ```
|
|
262
252
|
*/
|
|
263
|
-
|
|
253
|
+
getCosts(adaptationRunID: string, options?: RequestOptions): APIPromise<PromptAdaptationGetCostsResponse>;
|
|
264
254
|
}
|
|
265
255
|
/**
|
|
266
|
-
*
|
|
256
|
+
* A training or test example for prompt adaptation.
|
|
267
257
|
*/
|
|
268
|
-
export interface
|
|
258
|
+
export interface GoldenRecord {
|
|
259
|
+
/**
|
|
260
|
+
* Dictionary mapping field names to their values. Keys must match the fields
|
|
261
|
+
* specified in the template
|
|
262
|
+
*/
|
|
263
|
+
fields: {
|
|
264
|
+
[key: string]: string;
|
|
265
|
+
};
|
|
266
|
+
/**
|
|
267
|
+
* Expected answer for supervised evaluation. Required for supervised metrics,
|
|
268
|
+
* optional for unsupervised
|
|
269
|
+
*/
|
|
270
|
+
answer?: string | null;
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Status enum for asynchronous jobs (prompt adaptation, custom router training,
|
|
274
|
+
* etc.).
|
|
275
|
+
*
|
|
276
|
+
* Represents the current state of a long-running operation:
|
|
277
|
+
*
|
|
278
|
+
* - **created**: Job has been initialized but not yet queued
|
|
279
|
+
* - **queued**: Job is waiting in the queue to be processed
|
|
280
|
+
* - **processing**: Job is currently being executed
|
|
281
|
+
* - **completed**: Job finished successfully and results are available
|
|
282
|
+
* - **failed**: Job encountered an error and did not complete
|
|
283
|
+
*/
|
|
284
|
+
export type JobStatus = 'created' | 'queued' | 'processing' | 'completed' | 'failed';
|
|
285
|
+
/**
|
|
286
|
+
* Model for specifying an LLM provider in API requests.
|
|
287
|
+
*/
|
|
288
|
+
export interface RequestProvider {
|
|
289
|
+
/**
|
|
290
|
+
* Model name (e.g., 'gpt-4o', 'claude-sonnet-4-5-20250929')
|
|
291
|
+
*/
|
|
292
|
+
model: string;
|
|
293
|
+
/**
|
|
294
|
+
* Provider name (e.g., 'openai', 'anthropic', 'google')
|
|
295
|
+
*/
|
|
296
|
+
provider: string;
|
|
297
|
+
/**
|
|
298
|
+
* Maximum context length for the model (required for custom models)
|
|
299
|
+
*/
|
|
300
|
+
context_length?: number | null;
|
|
301
|
+
/**
|
|
302
|
+
* Input token price per million tokens in USD (required for custom models)
|
|
303
|
+
*/
|
|
304
|
+
input_price?: number | null;
|
|
305
|
+
/**
|
|
306
|
+
* Whether this is a custom model not in Not Diamond's supported model list
|
|
307
|
+
*/
|
|
308
|
+
is_custom?: boolean;
|
|
309
|
+
/**
|
|
310
|
+
* Average latency in seconds (required for custom models)
|
|
311
|
+
*/
|
|
312
|
+
latency?: number | null;
|
|
313
|
+
/**
|
|
314
|
+
* Output token price per million tokens in USD (required for custom models)
|
|
315
|
+
*/
|
|
316
|
+
output_price?: number | null;
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Response model for POST /v2/prompt/adapt endpoint.
|
|
320
|
+
*
|
|
321
|
+
* Returned immediately after submitting a prompt adaptation request. The
|
|
322
|
+
* adaptation process runs asynchronously, so use the returned adaptation_run_id to
|
|
323
|
+
* track progress and retrieve results when complete.
|
|
324
|
+
*
|
|
325
|
+
* **Next steps:**
|
|
326
|
+
*
|
|
327
|
+
* 1. Store the adaptation_run_id
|
|
328
|
+
* 2. Poll GET /v2/prompt/adaptStatus/{adaptation_run_id} to check progress
|
|
329
|
+
* 3. When status is 'completed', retrieve optimized prompts from GET
|
|
330
|
+
* /v2/prompt/adaptResults/{adaptation_run_id}
|
|
331
|
+
* 4. Use the optimized prompts with your target models
|
|
332
|
+
*/
|
|
333
|
+
export interface PromptAdaptationCreateResponse {
|
|
334
|
+
/**
|
|
335
|
+
* Unique identifier for this adaptation run. Use this to poll status and retrieve
|
|
336
|
+
* optimized prompts when complete
|
|
337
|
+
*/
|
|
338
|
+
adaptation_run_id: string;
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* Response model for GET /v2/prompt/adaptResults/{adaptation_run_id} endpoint.
|
|
342
|
+
*
|
|
343
|
+
* Contains the complete results of a prompt adaptation run, including optimized
|
|
344
|
+
* prompts and evaluation metrics for all target models. Use this to retrieve your
|
|
345
|
+
* adapted prompts after the adaptation status is 'completed'.
|
|
346
|
+
*
|
|
347
|
+
* The response includes:
|
|
348
|
+
*
|
|
349
|
+
* - Baseline performance of your original prompt on the origin model
|
|
350
|
+
* - Optimized prompts for each target model with pre/post optimization scores
|
|
351
|
+
* - Evaluation metrics and cost information for each model
|
|
352
|
+
*/
|
|
353
|
+
export interface PromptAdaptationGetAdaptResultsResponse {
|
|
269
354
|
/**
|
|
270
355
|
* Unique ID for this adaptation run
|
|
271
356
|
*/
|
|
@@ -275,41 +360,65 @@ export interface AdaptationRunResults {
|
|
|
275
360
|
*/
|
|
276
361
|
created_at: string;
|
|
277
362
|
/**
|
|
278
|
-
* Overall status of the adaptation run
|
|
363
|
+
* Overall status of the adaptation run (queued, running, completed, failed)
|
|
279
364
|
*/
|
|
280
365
|
job_status: JobStatus;
|
|
281
366
|
/**
|
|
282
|
-
* Results for
|
|
283
|
-
*/
|
|
284
|
-
origin_model: AdaptationRunResults.OriginModel;
|
|
285
|
-
/**
|
|
286
|
-
* Results for each target model with optimized prompts
|
|
367
|
+
* Results for each target model with optimized prompts and improvement scores
|
|
287
368
|
*/
|
|
288
|
-
target_models: Array<
|
|
369
|
+
target_models: Array<PromptAdaptationGetAdaptResultsResponse.TargetModel>;
|
|
289
370
|
/**
|
|
290
371
|
* Timestamp of last update to this adaptation run
|
|
291
372
|
*/
|
|
292
373
|
updated_at: string | null;
|
|
293
374
|
evaluation_config?: string | null;
|
|
294
375
|
evaluation_metric?: string | null;
|
|
295
|
-
}
|
|
296
|
-
export declare namespace AdaptationRunResults {
|
|
297
376
|
/**
|
|
298
|
-
*
|
|
377
|
+
* Metrics for the LLM requests made during the adaptation run (e.g.,
|
|
378
|
+
* total_requests, avg_latency)
|
|
299
379
|
*/
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
[key: string]: unknown;
|
|
304
|
-
} | null;
|
|
305
|
-
model_name: string;
|
|
306
|
-
result_status: PromptAdaptationAPI.JobStatus | null;
|
|
307
|
-
score: number | null;
|
|
308
|
-
system_prompt: string | null;
|
|
309
|
-
user_message_template: string | null;
|
|
310
|
-
}
|
|
380
|
+
llm_request_metrics?: {
|
|
381
|
+
[key: string]: number;
|
|
382
|
+
};
|
|
311
383
|
/**
|
|
312
|
-
*
|
|
384
|
+
* Baseline results for the origin model in prompt adaptation.
|
|
385
|
+
*
|
|
386
|
+
* Part of AdaptationRunResultsResponse. Contains the performance metrics and
|
|
387
|
+
* prompt configuration for your original prompt on the origin model. This serves
|
|
388
|
+
* as the baseline to compare against optimized prompts for target models.
|
|
389
|
+
*
|
|
390
|
+
* **Fields include:**
|
|
391
|
+
*
|
|
392
|
+
* - Original system prompt and user message template
|
|
393
|
+
* - Baseline performance score and evaluation metrics
|
|
394
|
+
* - Cost of running the baseline evaluation
|
|
395
|
+
* - Job status for the origin model evaluation
|
|
396
|
+
*/
|
|
397
|
+
origin_model?: PromptAdaptationGetAdaptResultsResponse.OriginModel | null;
|
|
398
|
+
}
|
|
399
|
+
export declare namespace PromptAdaptationGetAdaptResultsResponse {
|
|
400
|
+
/**
|
|
401
|
+
* Optimized prompt results for a single target model in prompt adaptation.
|
|
402
|
+
*
|
|
403
|
+
* Part of AdaptationRunResultsResponse. Contains the optimized system prompt and
|
|
404
|
+
* user message template for a specific target model, along with performance scores
|
|
405
|
+
* before and after optimization. Use these optimized prompts with the target model
|
|
406
|
+
* to achieve better performance than the original prompt.
|
|
407
|
+
*
|
|
408
|
+
* **Key metrics:**
|
|
409
|
+
*
|
|
410
|
+
* - **pre_optimization_score**: Performance with original prompt on this target
|
|
411
|
+
* model
|
|
412
|
+
* - **post_optimization_score**: Performance with optimized prompt on this target
|
|
413
|
+
* model
|
|
414
|
+
* - **Score improvement**: post - pre shows how much optimization helped
|
|
415
|
+
*
|
|
416
|
+
* **Usage:**
|
|
417
|
+
*
|
|
418
|
+
* 1. Extract the optimized system_prompt and user_message_template
|
|
419
|
+
* 2. Replace placeholders in user_message_template using fields from your data
|
|
420
|
+
* 3. Use these prompts when calling this target model
|
|
421
|
+
* 4. Compare pre/post scores to see improvement gained
|
|
313
422
|
*/
|
|
314
423
|
interface TargetModel {
|
|
315
424
|
cost: number | null;
|
|
@@ -322,85 +431,206 @@ export declare namespace AdaptationRunResults {
|
|
|
322
431
|
[key: string]: unknown;
|
|
323
432
|
} | null;
|
|
324
433
|
pre_optimization_score: number | null;
|
|
434
|
+
task_type: string | null;
|
|
325
435
|
/**
|
|
326
|
-
* Status
|
|
436
|
+
* Status enum for asynchronous jobs (prompt adaptation, custom router training,
|
|
437
|
+
* etc.).
|
|
438
|
+
*
|
|
439
|
+
* Represents the current state of a long-running operation:
|
|
440
|
+
*
|
|
441
|
+
* - **created**: Job has been initialized but not yet queued
|
|
442
|
+
* - **queued**: Job is waiting in the queue to be processed
|
|
443
|
+
* - **processing**: Job is currently being executed
|
|
444
|
+
* - **completed**: Job finished successfully and results are available
|
|
445
|
+
* - **failed**: Job encountered an error and did not complete
|
|
327
446
|
*/
|
|
328
|
-
result_status
|
|
447
|
+
result_status?: PromptAdaptationAPI.JobStatus | null;
|
|
329
448
|
/**
|
|
330
|
-
* Optimized system prompt for this target model
|
|
449
|
+
* Optimized system prompt for this target model. Use this as the system message in
|
|
450
|
+
* your LLM calls
|
|
331
451
|
*/
|
|
332
|
-
system_prompt
|
|
452
|
+
system_prompt?: string | null;
|
|
333
453
|
/**
|
|
334
|
-
* Optimized user message template
|
|
454
|
+
* Optimized user message template with placeholders. Substitute fields using your
|
|
455
|
+
* data before calling the LLM
|
|
335
456
|
*/
|
|
336
|
-
user_message_template
|
|
457
|
+
user_message_template?: string | null;
|
|
337
458
|
/**
|
|
338
|
-
*
|
|
459
|
+
* List of field names to substitute in the template (e.g., ['question',
|
|
460
|
+
* 'context']). These match the curly-brace placeholders in user_message_template
|
|
339
461
|
*/
|
|
340
|
-
user_message_template_fields
|
|
462
|
+
user_message_template_fields?: Array<string> | null;
|
|
341
463
|
}
|
|
342
|
-
}
|
|
343
|
-
export type JobStatus = 'created' | 'queued' | 'processing' | 'completed' | 'failed';
|
|
344
|
-
/**
|
|
345
|
-
* Response from prompt adaptation request.
|
|
346
|
-
*/
|
|
347
|
-
export interface PromptAdaptationAdaptResponse {
|
|
348
464
|
/**
|
|
349
|
-
*
|
|
465
|
+
* Baseline results for the origin model in prompt adaptation.
|
|
466
|
+
*
|
|
467
|
+
* Part of AdaptationRunResultsResponse. Contains the performance metrics and
|
|
468
|
+
* prompt configuration for your original prompt on the origin model. This serves
|
|
469
|
+
* as the baseline to compare against optimized prompts for target models.
|
|
470
|
+
*
|
|
471
|
+
* **Fields include:**
|
|
472
|
+
*
|
|
473
|
+
* - Original system prompt and user message template
|
|
474
|
+
* - Baseline performance score and evaluation metrics
|
|
475
|
+
* - Cost of running the baseline evaluation
|
|
476
|
+
* - Job status for the origin model evaluation
|
|
350
477
|
*/
|
|
351
|
-
|
|
478
|
+
interface OriginModel {
|
|
479
|
+
cost: number | null;
|
|
480
|
+
evals: {
|
|
481
|
+
[key: string]: unknown;
|
|
482
|
+
} | null;
|
|
483
|
+
model_name: string | null;
|
|
484
|
+
score: number | null;
|
|
485
|
+
/**
|
|
486
|
+
* Status enum for asynchronous jobs (prompt adaptation, custom router training,
|
|
487
|
+
* etc.).
|
|
488
|
+
*
|
|
489
|
+
* Represents the current state of a long-running operation:
|
|
490
|
+
*
|
|
491
|
+
* - **created**: Job has been initialized but not yet queued
|
|
492
|
+
* - **queued**: Job is waiting in the queue to be processed
|
|
493
|
+
* - **processing**: Job is currently being executed
|
|
494
|
+
* - **completed**: Job finished successfully and results are available
|
|
495
|
+
* - **failed**: Job encountered an error and did not complete
|
|
496
|
+
*/
|
|
497
|
+
result_status?: PromptAdaptationAPI.JobStatus | null;
|
|
498
|
+
/**
|
|
499
|
+
* Original system prompt used for the origin model
|
|
500
|
+
*/
|
|
501
|
+
system_prompt?: string | null;
|
|
502
|
+
/**
|
|
503
|
+
* Original user message template used for the origin model
|
|
504
|
+
*/
|
|
505
|
+
user_message_template?: string | null;
|
|
506
|
+
}
|
|
352
507
|
}
|
|
353
|
-
export type PromptAdaptationGetAdaptRunsResponse = Array<AdaptationRunResults>;
|
|
354
508
|
/**
|
|
355
|
-
*
|
|
509
|
+
* Response model for GET /v2/prompt/adaptStatus/{adaptation_run_id} endpoint.
|
|
510
|
+
*
|
|
511
|
+
* Returns the current status of an asynchronous prompt adaptation job. Poll this
|
|
512
|
+
* endpoint periodically to track progress. When status is 'completed', you can
|
|
513
|
+
* retrieve the optimized prompts using the /adaptResults endpoint.
|
|
514
|
+
*
|
|
515
|
+
* **Status values:**
|
|
516
|
+
*
|
|
517
|
+
* - **created**: Job has been initialized
|
|
518
|
+
* - **queued**: Waiting in queue (check queue_position for your place in line)
|
|
519
|
+
* - **processing**: Currently running optimization
|
|
520
|
+
* - **completed**: Finished successfully, results available via /adaptResults
|
|
521
|
+
* - **failed**: Encountered an error during processing
|
|
522
|
+
*
|
|
523
|
+
* **Polling recommendations:**
|
|
524
|
+
*
|
|
525
|
+
* - Poll every 30-60 seconds while status is incomplete
|
|
526
|
+
* - Stop polling once status is 'completed' or 'failed'
|
|
527
|
+
* - Adaptation typically takes 10-30 minutes total
|
|
356
528
|
*/
|
|
357
529
|
export interface PromptAdaptationGetAdaptStatusResponse {
|
|
358
530
|
/**
|
|
359
|
-
* Unique
|
|
531
|
+
* Unique identifier for this adaptation run. Use this to poll status and retrieve
|
|
532
|
+
* optimized prompts when complete
|
|
360
533
|
*/
|
|
361
534
|
adaptation_run_id: string;
|
|
362
535
|
/**
|
|
363
|
-
* Current status of the adaptation run
|
|
364
|
-
* failed)
|
|
536
|
+
* Current status of the adaptation run. Poll until this is 'completed' or 'failed'
|
|
365
537
|
*/
|
|
366
538
|
status: JobStatus;
|
|
367
539
|
/**
|
|
368
|
-
* Position in queue
|
|
540
|
+
* Position in queue when status is 'queued'. Lower numbers process sooner. Null
|
|
541
|
+
* when not queued
|
|
369
542
|
*/
|
|
370
543
|
queue_position?: number | null;
|
|
371
544
|
}
|
|
372
|
-
|
|
545
|
+
/**
|
|
546
|
+
* Response model for GET /v2/prompt/adapt/{adaptation_run_id}/costs endpoint.
|
|
547
|
+
*
|
|
548
|
+
* Contains the total LLM costs and detailed usage records for a prompt adaptation
|
|
549
|
+
* run. Use this to track costs associated with optimizing prompts for different
|
|
550
|
+
* target models.
|
|
551
|
+
*/
|
|
552
|
+
export interface PromptAdaptationGetCostsResponse {
|
|
553
|
+
/**
|
|
554
|
+
* Unique identifier for the adaptation run
|
|
555
|
+
*/
|
|
373
556
|
adaptation_run_id: string;
|
|
557
|
+
/**
|
|
558
|
+
* Total cost in USD across all LLM requests in this adaptation run
|
|
559
|
+
*/
|
|
374
560
|
total_cost: number;
|
|
375
|
-
|
|
561
|
+
/**
|
|
562
|
+
* Detailed usage records for each LLM request made during the adaptation
|
|
563
|
+
*/
|
|
564
|
+
usage_records: Array<PromptAdaptationGetCostsResponse.UsageRecord>;
|
|
376
565
|
}
|
|
377
|
-
export declare namespace
|
|
566
|
+
export declare namespace PromptAdaptationGetCostsResponse {
|
|
567
|
+
/**
|
|
568
|
+
* Individual LLM usage record with token counts and cost breakdown.
|
|
569
|
+
*
|
|
570
|
+
* Returned by GET /llm-usage endpoint and included in AdaptationRunCostResponse.
|
|
571
|
+
* Each record represents a single LLM API call with detailed usage metrics.
|
|
572
|
+
*/
|
|
378
573
|
interface UsageRecord {
|
|
574
|
+
/**
|
|
575
|
+
* Unique identifier for this usage record
|
|
576
|
+
*/
|
|
379
577
|
id: string;
|
|
578
|
+
/**
|
|
579
|
+
* Adaptation run ID this usage is associated with
|
|
580
|
+
*/
|
|
380
581
|
adaptation_run_id: string;
|
|
582
|
+
/**
|
|
583
|
+
* Cost of input tokens in USD
|
|
584
|
+
*/
|
|
381
585
|
input_cost: number;
|
|
586
|
+
/**
|
|
587
|
+
* Number of input tokens consumed
|
|
588
|
+
*/
|
|
382
589
|
input_tokens: number;
|
|
590
|
+
/**
|
|
591
|
+
* Model name (e.g., 'gpt-4', 'claude-3-opus-20240229')
|
|
592
|
+
*/
|
|
383
593
|
model: string;
|
|
594
|
+
/**
|
|
595
|
+
* Organization ID associated with the request
|
|
596
|
+
*/
|
|
384
597
|
organization_id: string;
|
|
598
|
+
/**
|
|
599
|
+
* Cost of output tokens in USD
|
|
600
|
+
*/
|
|
385
601
|
output_cost: number;
|
|
602
|
+
/**
|
|
603
|
+
* Number of output tokens generated
|
|
604
|
+
*/
|
|
386
605
|
output_tokens: number;
|
|
606
|
+
/**
|
|
607
|
+
* LLM provider (e.g., 'openai', 'anthropic', 'google')
|
|
608
|
+
*/
|
|
387
609
|
provider: string;
|
|
610
|
+
/**
|
|
611
|
+
* Type of task (e.g., 'optimization', 'evaluation')
|
|
612
|
+
*/
|
|
388
613
|
task_type: string;
|
|
614
|
+
/**
|
|
615
|
+
* Unix timestamp when the request was made
|
|
616
|
+
*/
|
|
389
617
|
timestamp: number;
|
|
618
|
+
/**
|
|
619
|
+
* Total cost (input + output) in USD
|
|
620
|
+
*/
|
|
390
621
|
total_cost: number;
|
|
622
|
+
/**
|
|
623
|
+
* User ID who made the request
|
|
624
|
+
*/
|
|
391
625
|
user_id: string;
|
|
392
626
|
}
|
|
393
627
|
}
|
|
394
|
-
export interface
|
|
628
|
+
export interface PromptAdaptationCreateParams {
|
|
395
629
|
/**
|
|
396
630
|
* List of field names that will be substituted into the template. Must match keys
|
|
397
631
|
* in golden records
|
|
398
632
|
*/
|
|
399
633
|
fields: Array<string>;
|
|
400
|
-
/**
|
|
401
|
-
* The model your current prompt is optimized for
|
|
402
|
-
*/
|
|
403
|
-
origin_model: PromptAdaptationAdaptParams.OriginModel;
|
|
404
634
|
/**
|
|
405
635
|
* System prompt to use with the origin model. This sets the context and role for
|
|
406
636
|
* the LLM
|
|
@@ -408,9 +638,9 @@ export interface PromptAdaptationAdaptParams {
|
|
|
408
638
|
system_prompt: string;
|
|
409
639
|
/**
|
|
410
640
|
* List of models to adapt the prompt for. Maximum count depends on your
|
|
411
|
-
* subscription tier
|
|
641
|
+
* subscription tier (Free: 1, Starter: 3, Startup: 5, Enterprise: 10)
|
|
412
642
|
*/
|
|
413
|
-
target_models: Array<
|
|
643
|
+
target_models: Array<RequestProvider>;
|
|
414
644
|
/**
|
|
415
645
|
* User message template with placeholders for fields. Use curly braces for field
|
|
416
646
|
* substitution
|
|
@@ -420,155 +650,30 @@ export interface PromptAdaptationAdaptParams {
|
|
|
420
650
|
evaluation_metric?: string | null;
|
|
421
651
|
/**
|
|
422
652
|
* Training examples (legacy parameter). Use train_goldens and test_goldens for
|
|
423
|
-
* better control
|
|
424
|
-
*/
|
|
425
|
-
goldens?: Array<PromptAdaptationAdaptParams.Golden> | null;
|
|
426
|
-
/**
|
|
427
|
-
* Optional baseline score for the origin model
|
|
428
|
-
*/
|
|
429
|
-
origin_model_evaluation_score?: number | null;
|
|
430
|
-
/**
|
|
431
|
-
* Test examples for evaluation. Required if train_goldens is provided
|
|
653
|
+
* better control. Minimum 25 examples
|
|
432
654
|
*/
|
|
433
|
-
|
|
434
|
-
/**
|
|
435
|
-
* Training examples for prompt optimization. Minimum 5 examples required
|
|
436
|
-
*/
|
|
437
|
-
train_goldens?: Array<PromptAdaptationAdaptParams.TrainGolden> | null;
|
|
438
|
-
}
|
|
439
|
-
export declare namespace PromptAdaptationAdaptParams {
|
|
440
|
-
/**
|
|
441
|
-
* The model your current prompt is optimized for
|
|
442
|
-
*/
|
|
443
|
-
interface OriginModel {
|
|
444
|
-
/**
|
|
445
|
-
* Model name (e.g., 'gpt-4o', 'claude-3-5-sonnet-20241022')
|
|
446
|
-
*/
|
|
447
|
-
model: string;
|
|
448
|
-
/**
|
|
449
|
-
* Provider name (e.g., 'openai', 'anthropic', 'google')
|
|
450
|
-
*/
|
|
451
|
-
provider: string;
|
|
452
|
-
/**
|
|
453
|
-
* Maximum context length for the model (required for custom models)
|
|
454
|
-
*/
|
|
455
|
-
context_length?: number | null;
|
|
456
|
-
/**
|
|
457
|
-
* Input token price per million tokens in USD (required for custom models)
|
|
458
|
-
*/
|
|
459
|
-
input_price?: number | null;
|
|
460
|
-
/**
|
|
461
|
-
* Whether this is a custom model not in Not Diamond's supported model list
|
|
462
|
-
*/
|
|
463
|
-
is_custom?: boolean;
|
|
464
|
-
/**
|
|
465
|
-
* Average latency in seconds (required for custom models)
|
|
466
|
-
*/
|
|
467
|
-
latency?: number | null;
|
|
468
|
-
/**
|
|
469
|
-
* Output token price per million tokens in USD (required for custom models)
|
|
470
|
-
*/
|
|
471
|
-
output_price?: number | null;
|
|
472
|
-
}
|
|
655
|
+
goldens?: Array<GoldenRecord> | null;
|
|
473
656
|
/**
|
|
474
657
|
* Model for specifying an LLM provider in API requests.
|
|
475
658
|
*/
|
|
476
|
-
|
|
477
|
-
/**
|
|
478
|
-
* Model name (e.g., 'gpt-4o', 'claude-3-5-sonnet-20241022')
|
|
479
|
-
*/
|
|
480
|
-
model: string;
|
|
481
|
-
/**
|
|
482
|
-
* Provider name (e.g., 'openai', 'anthropic', 'google')
|
|
483
|
-
*/
|
|
484
|
-
provider: string;
|
|
485
|
-
/**
|
|
486
|
-
* Maximum context length for the model (required for custom models)
|
|
487
|
-
*/
|
|
488
|
-
context_length?: number | null;
|
|
489
|
-
/**
|
|
490
|
-
* Input token price per million tokens in USD (required for custom models)
|
|
491
|
-
*/
|
|
492
|
-
input_price?: number | null;
|
|
493
|
-
/**
|
|
494
|
-
* Whether this is a custom model not in Not Diamond's supported model list
|
|
495
|
-
*/
|
|
496
|
-
is_custom?: boolean;
|
|
497
|
-
/**
|
|
498
|
-
* Average latency in seconds (required for custom models)
|
|
499
|
-
*/
|
|
500
|
-
latency?: number | null;
|
|
501
|
-
/**
|
|
502
|
-
* Output token price per million tokens in USD (required for custom models)
|
|
503
|
-
*/
|
|
504
|
-
output_price?: number | null;
|
|
505
|
-
}
|
|
506
|
-
/**
|
|
507
|
-
* A training or test example for prompt adaptation.
|
|
508
|
-
*/
|
|
509
|
-
interface Golden {
|
|
510
|
-
/**
|
|
511
|
-
* Dictionary mapping field names to their values. Keys must match the fields
|
|
512
|
-
* specified in the template
|
|
513
|
-
*/
|
|
514
|
-
fields: {
|
|
515
|
-
[key: string]: string;
|
|
516
|
-
};
|
|
517
|
-
/**
|
|
518
|
-
* Expected answer for supervised evaluation. Required for supervised metrics,
|
|
519
|
-
* optional for unsupervised
|
|
520
|
-
*/
|
|
521
|
-
answer?: string | null;
|
|
522
|
-
}
|
|
523
|
-
/**
|
|
524
|
-
* A training or test example for prompt adaptation.
|
|
525
|
-
*/
|
|
526
|
-
interface TestGolden {
|
|
527
|
-
/**
|
|
528
|
-
* Dictionary mapping field names to their values. Keys must match the fields
|
|
529
|
-
* specified in the template
|
|
530
|
-
*/
|
|
531
|
-
fields: {
|
|
532
|
-
[key: string]: string;
|
|
533
|
-
};
|
|
534
|
-
/**
|
|
535
|
-
* Expected answer for supervised evaluation. Required for supervised metrics,
|
|
536
|
-
* optional for unsupervised
|
|
537
|
-
*/
|
|
538
|
-
answer?: string | null;
|
|
539
|
-
}
|
|
659
|
+
origin_model?: RequestProvider | null;
|
|
540
660
|
/**
|
|
541
|
-
*
|
|
661
|
+
* Optional baseline score for the origin model. If provided, can skip origin model
|
|
662
|
+
* evaluation
|
|
542
663
|
*/
|
|
543
|
-
|
|
544
|
-
/**
|
|
545
|
-
* Dictionary mapping field names to their values. Keys must match the fields
|
|
546
|
-
* specified in the template
|
|
547
|
-
*/
|
|
548
|
-
fields: {
|
|
549
|
-
[key: string]: string;
|
|
550
|
-
};
|
|
551
|
-
/**
|
|
552
|
-
* Expected answer for supervised evaluation. Required for supervised metrics,
|
|
553
|
-
* optional for unsupervised
|
|
554
|
-
*/
|
|
555
|
-
answer?: string | null;
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
export interface PromptAdaptationGetAdaptRunResultsParams {
|
|
664
|
+
origin_model_evaluation_score?: number | null;
|
|
559
665
|
/**
|
|
560
|
-
*
|
|
666
|
+
* Test examples for evaluation. Required if train_goldens is provided. Used to
|
|
667
|
+
* measure final performance on held-out data
|
|
561
668
|
*/
|
|
562
|
-
|
|
669
|
+
test_goldens?: Array<GoldenRecord> | null;
|
|
563
670
|
/**
|
|
564
|
-
*
|
|
671
|
+
* Training examples for prompt optimization. Minimum 25 examples required. Cannot
|
|
672
|
+
* be used with 'goldens' parameter
|
|
565
673
|
*/
|
|
566
|
-
|
|
567
|
-
}
|
|
568
|
-
export interface PromptAdaptationGetAdaptRunsParams {
|
|
569
|
-
'x-token': string;
|
|
674
|
+
train_goldens?: Array<GoldenRecord> | null;
|
|
570
675
|
}
|
|
571
676
|
export declare namespace PromptAdaptation {
|
|
572
|
-
export { type
|
|
677
|
+
export { type GoldenRecord as GoldenRecord, type JobStatus as JobStatus, type RequestProvider as RequestProvider, type PromptAdaptationCreateResponse as PromptAdaptationCreateResponse, type PromptAdaptationGetAdaptResultsResponse as PromptAdaptationGetAdaptResultsResponse, type PromptAdaptationGetAdaptStatusResponse as PromptAdaptationGetAdaptStatusResponse, type PromptAdaptationGetCostsResponse as PromptAdaptationGetCostsResponse, type PromptAdaptationCreateParams as PromptAdaptationCreateParams, };
|
|
573
678
|
}
|
|
574
679
|
//# sourceMappingURL=prompt-adaptation.d.ts.map
|