notdiamond 2.0.0-rc2 → 2.0.0-rc5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/CHANGELOG.md +45 -0
  2. package/README.md +14 -16
  3. package/client.d.mts +20 -34
  4. package/client.d.mts.map +1 -1
  5. package/client.d.ts +20 -34
  6. package/client.d.ts.map +1 -1
  7. package/client.js +22 -38
  8. package/client.js.map +1 -1
  9. package/client.mjs +22 -38
  10. package/client.mjs.map +1 -1
  11. package/internal/tslib.js +17 -17
  12. package/internal/utils/env.d.mts.map +1 -1
  13. package/internal/utils/env.d.ts.map +1 -1
  14. package/internal/utils/env.js +4 -2
  15. package/internal/utils/env.js.map +1 -1
  16. package/internal/utils/env.mjs +4 -2
  17. package/internal/utils/env.mjs.map +1 -1
  18. package/package.json +10 -11
  19. package/resources/index.d.mts +6 -6
  20. package/resources/index.d.mts.map +1 -1
  21. package/resources/index.d.ts +6 -6
  22. package/resources/index.d.ts.map +1 -1
  23. package/resources/index.js +8 -8
  24. package/resources/index.js.map +1 -1
  25. package/resources/index.mjs +4 -4
  26. package/resources/index.mjs.map +1 -1
  27. package/resources/model-router.d.mts +311 -0
  28. package/resources/model-router.d.mts.map +1 -0
  29. package/resources/model-router.d.ts +311 -0
  30. package/resources/model-router.d.ts.map +1 -0
  31. package/resources/model-router.js +97 -0
  32. package/resources/model-router.js.map +1 -0
  33. package/resources/model-router.mjs +93 -0
  34. package/resources/model-router.mjs.map +1 -0
  35. package/resources/models.d.mts +50 -25
  36. package/resources/models.d.mts.map +1 -1
  37. package/resources/models.d.ts +50 -25
  38. package/resources/models.d.ts.map +1 -1
  39. package/resources/models.js +5 -0
  40. package/resources/models.js.map +1 -1
  41. package/resources/models.mjs +5 -0
  42. package/resources/models.mjs.map +1 -1
  43. package/resources/preferences.d.mts +49 -47
  44. package/resources/preferences.d.mts.map +1 -1
  45. package/resources/preferences.d.ts +49 -47
  46. package/resources/preferences.d.ts.map +1 -1
  47. package/resources/preferences.js +30 -33
  48. package/resources/preferences.js.map +1 -1
  49. package/resources/preferences.mjs +30 -33
  50. package/resources/preferences.mjs.map +1 -1
  51. package/resources/prompt/adapt.d.mts +352 -0
  52. package/resources/prompt/adapt.d.mts.map +1 -0
  53. package/resources/prompt/adapt.d.ts +352 -0
  54. package/resources/prompt/adapt.d.ts.map +1 -0
  55. package/resources/prompt/adapt.js +154 -0
  56. package/resources/prompt/adapt.js.map +1 -0
  57. package/resources/prompt/adapt.mjs +150 -0
  58. package/resources/prompt/adapt.mjs.map +1 -0
  59. package/resources/prompt/index.d.mts +3 -0
  60. package/resources/prompt/index.d.mts.map +1 -0
  61. package/resources/prompt/index.d.ts +3 -0
  62. package/resources/prompt/index.d.ts.map +1 -0
  63. package/resources/prompt/index.js +9 -0
  64. package/resources/prompt/index.js.map +1 -0
  65. package/resources/prompt/index.mjs +4 -0
  66. package/resources/prompt/index.mjs.map +1 -0
  67. package/resources/prompt/prompt.d.mts +338 -0
  68. package/resources/prompt/prompt.d.mts.map +1 -0
  69. package/resources/prompt/prompt.d.ts +338 -0
  70. package/resources/prompt/prompt.d.ts.map +1 -0
  71. package/resources/prompt/prompt.js +128 -0
  72. package/resources/prompt/prompt.js.map +1 -0
  73. package/resources/prompt/prompt.mjs +123 -0
  74. package/resources/prompt/prompt.mjs.map +1 -0
  75. package/resources/prompt.d.mts +2 -0
  76. package/resources/prompt.d.mts.map +1 -0
  77. package/resources/prompt.d.ts +2 -0
  78. package/resources/prompt.d.ts.map +1 -0
  79. package/resources/prompt.js +6 -0
  80. package/resources/prompt.js.map +1 -0
  81. package/resources/prompt.mjs +3 -0
  82. package/resources/prompt.mjs.map +1 -0
  83. package/resources/pzn.d.mts +225 -0
  84. package/resources/pzn.d.mts.map +1 -0
  85. package/resources/pzn.d.ts +225 -0
  86. package/resources/pzn.d.ts.map +1 -0
  87. package/resources/{routing.js → pzn.js} +26 -69
  88. package/resources/pzn.js.map +1 -0
  89. package/resources/{routing.mjs → pzn.mjs} +24 -67
  90. package/resources/pzn.mjs.map +1 -0
  91. package/resources/report/index.d.mts +3 -0
  92. package/resources/report/index.d.mts.map +1 -0
  93. package/resources/report/index.d.ts +3 -0
  94. package/resources/report/index.d.ts.map +1 -0
  95. package/resources/report/index.js +9 -0
  96. package/resources/report/index.js.map +1 -0
  97. package/resources/report/index.mjs +4 -0
  98. package/resources/report/index.mjs.map +1 -0
  99. package/resources/report/metrics.d.mts +87 -0
  100. package/resources/report/metrics.d.mts.map +1 -0
  101. package/resources/report/metrics.d.ts +87 -0
  102. package/resources/report/metrics.d.ts.map +1 -0
  103. package/resources/report/metrics.js +57 -0
  104. package/resources/report/metrics.js.map +1 -0
  105. package/resources/report/metrics.mjs +53 -0
  106. package/resources/report/metrics.mjs.map +1 -0
  107. package/resources/report/report.d.mts +10 -0
  108. package/resources/report/report.d.mts.map +1 -0
  109. package/resources/report/report.d.ts +10 -0
  110. package/resources/report/report.d.ts.map +1 -0
  111. package/resources/report/report.js +17 -0
  112. package/resources/report/report.js.map +1 -0
  113. package/resources/report/report.mjs +12 -0
  114. package/resources/report/report.mjs.map +1 -0
  115. package/resources/report.d.mts +1 -244
  116. package/resources/report.d.mts.map +1 -1
  117. package/resources/report.d.ts +1 -244
  118. package/resources/report.d.ts.map +1 -1
  119. package/resources/report.js +2 -82
  120. package/resources/report.js.map +1 -1
  121. package/resources/report.mjs +1 -80
  122. package/resources/report.mjs.map +1 -1
  123. package/src/client.ts +70 -119
  124. package/src/internal/utils/env.ts +4 -2
  125. package/src/resources/index.ts +24 -34
  126. package/src/resources/model-router.ts +356 -0
  127. package/src/resources/models.ts +55 -32
  128. package/src/resources/preferences.ts +61 -70
  129. package/src/resources/prompt/adapt.ts +402 -0
  130. package/src/resources/prompt/index.ts +16 -0
  131. package/src/resources/prompt/prompt.ts +398 -0
  132. package/src/resources/prompt.ts +3 -0
  133. package/src/resources/pzn.ts +273 -0
  134. package/src/resources/report/index.ts +4 -0
  135. package/src/resources/report/metrics.ts +99 -0
  136. package/src/resources/report/report.ts +19 -0
  137. package/src/resources/report.ts +1 -298
  138. package/src/version.ts +1 -1
  139. package/version.d.mts +1 -1
  140. package/version.d.ts +1 -1
  141. package/version.js +1 -1
  142. package/version.mjs +1 -1
  143. package/resources/admin.d.mts +0 -4
  144. package/resources/admin.d.mts.map +0 -1
  145. package/resources/admin.d.ts +0 -4
  146. package/resources/admin.d.ts.map +0 -1
  147. package/resources/admin.js +0 -9
  148. package/resources/admin.js.map +0 -1
  149. package/resources/admin.mjs +0 -5
  150. package/resources/admin.mjs.map +0 -1
  151. package/resources/prompt-adaptation.d.mts +0 -581
  152. package/resources/prompt-adaptation.d.mts.map +0 -1
  153. package/resources/prompt-adaptation.d.ts +0 -581
  154. package/resources/prompt-adaptation.d.ts.map +0 -1
  155. package/resources/prompt-adaptation.js +0 -289
  156. package/resources/prompt-adaptation.js.map +0 -1
  157. package/resources/prompt-adaptation.mjs +0 -285
  158. package/resources/prompt-adaptation.mjs.map +0 -1
  159. package/resources/routing.d.mts +0 -391
  160. package/resources/routing.d.mts.map +0 -1
  161. package/resources/routing.d.ts +0 -391
  162. package/resources/routing.d.ts.map +0 -1
  163. package/resources/routing.js.map +0 -1
  164. package/resources/routing.mjs.map +0 -1
  165. package/src/resources/admin.ts +0 -5
  166. package/src/resources/prompt-adaptation.ts +0 -707
  167. package/src/resources/routing.ts +0 -476
@@ -1,289 +0,0 @@
1
- "use strict";
2
- // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
3
- Object.defineProperty(exports, "__esModule", { value: true });
4
- exports.PromptAdaptation = void 0;
5
- const resource_1 = require("../core/resource.js");
6
- const headers_1 = require("../internal/headers.js");
7
- const path_1 = require("../internal/utils/path.js");
8
- class PromptAdaptation extends resource_1.APIResource {
9
- /**
10
- * Adapt your prompt from one LLM to work optimally across different target LLMs.
11
- *
12
- * This endpoint automatically optimizes your prompt (system prompt + user message
13
- * template) to achieve better performance when switching between different
14
- * language models. Each model has unique characteristics, and what works well for
15
- * GPT-4 might not work as well for Claude or Gemini.
16
- *
17
- * **How Prompt Adaptation Works:**
18
- *
19
- * 1. You provide your current prompt optimized for an origin model
20
- * 2. You specify target models you want to adapt to
21
- * 3. You provide evaluation examples (golden records) with expected answers
22
- * 4. The system runs optimization to find the best prompt for each target model
23
- * 5. You receive adapted prompts that perform well on your target models
24
- *
25
- * **Evaluation Metrics:** Choose either a standard metric or provide custom
26
- * evaluation:
27
- *
28
- * - **Standard metrics**: LLMaaJ:SQL, LLMaaJ:Sem_Sim_1/3/10 (semantic similarity),
29
- * JSON_Match
30
- * - **Custom evaluation**: Provide evaluation_config with your own LLM judge,
31
- * prompt, and cutoff
32
- *
33
- * **Dataset Requirements:**
34
- *
35
- * - Minimum 5 examples in train_goldens (more examples = better adaptation)
36
- * - Each example must have fields matching your template placeholders
37
- * - Supervised evaluation requires 'answer' field in each golden record
38
- * - Unsupervised evaluation can work without answers
39
- *
40
- * **Training Time:**
41
- *
42
- * - Processing is asynchronous and typically takes 10-30 minutes
43
- * - Time depends on: number of target models, dataset size, model availability
44
- * - Use the returned adaptation_run_id to check status and retrieve results
45
- *
46
- * **Subscription Tiers:**
47
- *
48
- * - Free: 1 target model
49
- * - Starter: 3 target models
50
- * - Startup: 5 target models
51
- * - Enterprise: 10 target models
52
- *
53
- * **Best Practices:**
54
- *
55
- * 1. Use diverse, representative examples from your production workload
56
- * 2. Include 10-20 examples for best results (5 minimum)
57
- * 3. Ensure consistent evaluation across all examples
58
- * 4. Test both train_goldens and test_goldens split for validation
59
- * 5. Use the same model versions you'll use in production
60
- *
61
- * **Example Workflow:**
62
- *
63
- * ```
64
- * 1. POST /v2/prompt/adapt - Submit adaptation request
65
- * 2. GET /v2/prompt/adaptStatus/{id} - Poll status until completed
66
- * 3. GET /v2/prompt/adaptResults/{id} - Retrieve optimized prompts
67
- * 4. Use optimized prompts in production with target models
68
- * ```
69
- *
70
- * **Related Documentation:** See
71
- * https://docs.notdiamond.ai/docs/adapting-prompts-to-new-models for detailed
72
- * guide.
73
- *
74
- * @example
75
- * ```ts
76
- * const response = await client.promptAdaptation.adapt({
77
- * fields: ['question'],
78
- * system_prompt: 'You are a helpful assistant that answers questions accurately.',
79
- * target_models: [
80
- * { provider: 'anthropic', model: 'claude-sonnet-4-5-20250929' },
81
- * { provider: 'google', model: 'gemini-1.5-pro' },
82
- * ],
83
- * template: 'Question: {question}\nAnswer:',
84
- * evaluation_metric: 'LLMaaJ:Sem_Sim_3',
85
- * origin_model: { provider: 'openai', model: 'gpt-4o' },
86
- * test_goldens: [
87
- * {
88
- * fields: { ... },
89
- * answer: '9',
90
- * },
91
- * {
92
- * fields: { ... },
93
- * answer: 'Pacific Ocean',
94
- * },
95
- * ],
96
- * train_goldens: [
97
- * {
98
- * fields: { ... },
99
- * answer: '4',
100
- * },
101
- * {
102
- * fields: { ... },
103
- * answer: 'Paris',
104
- * },
105
- * {
106
- * fields: { ... },
107
- * answer: 'William Shakespeare',
108
- * },
109
- * {
110
- * fields: { ... },
111
- * answer: 'Water',
112
- * },
113
- * {
114
- * fields: { ... },
115
- * answer: '7',
116
- * },
117
- * ],
118
- * });
119
- * ```
120
- */
121
- adapt(body, options) {
122
- return this._client.post('/v2/prompt/adapt', { body, ...options });
123
- }
124
- /**
125
- * Retrieve the complete results of a prompt adaptation run, including optimized
126
- * prompts for all target models.
127
- *
128
- * This endpoint returns the adapted prompts and evaluation metrics for each target
129
- * model in your adaptation request. Call this endpoint after the adaptation status
130
- * is 'completed' to get your optimized prompts.
131
- *
132
- * **Response Structure:**
133
- *
134
- * - **origin_model**: Baseline performance of your original prompt on the origin
135
- * model
136
- * - Includes: system_prompt, user_message_template, score, evaluation metrics,
137
- * cost
138
- * - **target_models**: Array of results for each target model
139
- * - Includes: optimized system_prompt, user_message_template, template_fields
140
- * - pre_optimization_score: Performance before adaptation
141
- * - post_optimization_score: Performance after adaptation
142
- * - Evaluation metrics and cost information
143
- *
144
- * **Using Adapted Prompts:**
145
- *
146
- * 1. Extract the `system_prompt` and `user_message_template` from each target
147
- * model result
148
- * 2. Use `user_message_template_fields` to know which fields to substitute
149
- * 3. Apply the optimized prompts when calling the respective target models
150
- * 4. Compare pre/post optimization scores to see improvement
151
- *
152
- * **Evaluation Scores:**
153
- *
154
- * - Scores range from 0-10 (higher is better)
155
- * - Compare origin_model score with target_models pre_optimization_score for
156
- * baseline
157
- * - Compare pre_optimization_score with post_optimization_score to see improvement
158
- * from adaptation
159
- * - Typical improvements range from 5-30% on evaluation metrics
160
- *
161
- * **Status Handling:**
162
- *
163
- * - If adaptation is still processing, target model results will have
164
- * `result_status: "processing"`
165
- * - Only completed target models will have system_prompt and template values
166
- * - Failed target models will have `result_status: "failed"` with null values
167
- *
168
- * **Cost Information:**
169
- *
170
- * - Each model result includes cost in USD for the adaptation process
171
- * - Costs vary based on model pricing and number of evaluation examples
172
- * - Typical range: $0.10 - $2.00 per target model
173
- *
174
- * **Best Practices:**
175
- *
176
- * 1. Wait for status 'completed' before calling this endpoint
177
- * 2. Check result_status for each target model
178
- * 3. Validate that post_optimization_score > pre_optimization_score
179
- * 4. Save optimized prompts for production use
180
- * 5. A/B test adapted prompts against originals in production
181
- *
182
- * @example
183
- * ```ts
184
- * const adaptationRunResults =
185
- * await client.promptAdaptation.getAdaptResults(
186
- * 'adaptation_run_id',
187
- * );
188
- * ```
189
- */
190
- getAdaptResults(adaptationRunID, options) {
191
- return this._client.get((0, path_1.path) `/v2/prompt/adaptResults/${adaptationRunID}`, options);
192
- }
193
- /**
194
- * Get Adapt Run Results
195
- *
196
- * @example
197
- * ```ts
198
- * const adaptationRunResults =
199
- * await client.promptAdaptation.getAdaptRunResults(
200
- * 'adaptation_run_id',
201
- * { user_id: 'user_id', 'x-token': 'x-token' },
202
- * );
203
- * ```
204
- */
205
- getAdaptRunResults(adaptationRunID, params, options) {
206
- const { user_id, 'x-token': xToken } = params;
207
- return this._client.get((0, path_1.path) `/v2/prompt/frontendAdaptRunResults/${user_id}/${adaptationRunID}`, {
208
- ...options,
209
- headers: (0, headers_1.buildHeaders)([{ 'x-token': xToken }, options?.headers]),
210
- });
211
- }
212
- /**
213
- * Get Adapt Runs
214
- *
215
- * @example
216
- * ```ts
217
- * const adaptationRunResults =
218
- * await client.promptAdaptation.getAdaptRuns('user_id', {
219
- * 'x-token': 'x-token',
220
- * });
221
- * ```
222
- */
223
- getAdaptRuns(userID, params, options) {
224
- const { 'x-token': xToken } = params;
225
- return this._client.get((0, path_1.path) `/v2/prompt/frontendAdaptRuns/${userID}`, {
226
- ...options,
227
- headers: (0, headers_1.buildHeaders)([{ 'x-token': xToken }, options?.headers]),
228
- });
229
- }
230
- /**
231
- * Check the status of a prompt adaptation run.
232
- *
233
- * Use this endpoint to poll the status of your adaptation request. Processing is
234
- * asynchronous, so you'll need to check periodically until the status indicates
235
- * completion.
236
- *
237
- * **Status Values:**
238
- *
239
- * - `created`: Initial state, not yet processing
240
- * - `queued`: Waiting for processing capacity (check queue_position)
241
- * - `processing`: Currently optimizing prompts
242
- * - `completed`: All target models have been processed successfully
243
- * - `failed`: One or more target models failed to process
244
- *
245
- * **Polling Recommendations:**
246
- *
247
- * - Poll every 30-60 seconds during processing
248
- * - Check queue_position if status is 'queued' to estimate wait time
249
- * - Stop polling once status is 'completed' or 'failed'
250
- * - Use GET /v2/prompt/adaptResults to retrieve results after completion
251
- *
252
- * **Queue Position:**
253
- *
254
- * - Only present when status is 'queued'
255
- * - Lower numbers mean earlier processing (position 1 is next)
256
- * - Typical wait time: 1-5 minutes per position
257
- *
258
- * **Note:** This endpoint only returns status information. To get the actual
259
- * adapted prompts and evaluation results, use GET /v2/prompt/adaptResults once
260
- * status is 'completed'.
261
- *
262
- * @example
263
- * ```ts
264
- * const response =
265
- * await client.promptAdaptation.getAdaptStatus(
266
- * 'adaptation_run_id',
267
- * );
268
- * ```
269
- */
270
- getAdaptStatus(adaptationRunID, options) {
271
- return this._client.get((0, path_1.path) `/v2/prompt/adaptStatus/${adaptationRunID}`, options);
272
- }
273
- /**
274
- * Get LLM costs for a specific adaptation run
275
- *
276
- * @example
277
- * ```ts
278
- * const response =
279
- * await client.promptAdaptation.retrieveCosts(
280
- * 'adaptation_run_id',
281
- * );
282
- * ```
283
- */
284
- retrieveCosts(adaptationRunID, options) {
285
- return this._client.get((0, path_1.path) `/v1/adaptation-runs/${adaptationRunID}/costs`, options);
286
- }
287
- }
288
- exports.PromptAdaptation = PromptAdaptation;
289
- //# sourceMappingURL=prompt-adaptation.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"prompt-adaptation.js","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kDAA+C;AAG/C,oDAAmD;AAEnD,oDAA8C;AAE9C,MAAa,gBAAiB,SAAQ,sBAAW;IAC/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+GG;IACH,KAAK,CACH,IAAiC,EACjC,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiEG;IACH,eAAe,CAAC,eAAuB,EAAE,OAAwB;QAC/D,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,2BAA2B,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IACrF,CAAC;IAED;;;;;;;;;;;OAWG;IACH,kBAAkB,CAChB,eAAuB,EACvB,MAAgD,EAChD,OAAwB;QAExB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;QAC9C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,sCAAsC,OAAO,IAAI,eAAe,EAAE,EAAE;YAC9F,GAAG,OAAO;YACV,OAAO,EAAE,IAAA,sBAAY,EAAC,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;SACjE,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,YAAY,CACV,MAAc,EACd,MAA0C,EAC1C,OAAwB;QAExB,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;QACrC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,gCAAgC,MAAM,EAAE,EAAE;YACpE,GAAG,OAAO;YACV,OAAO,EAAE,IAAA,sBAAY,EAAC,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;SACjE,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,cAAc,CACZ,eAAuB,EACvB,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,0BAA0B,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;IAED;;;;;;;;;;OAUG;IACH,aAAa,CACX,eAAuB,EACvB,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,uBAAuB,eAAe,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvF,CAAC;CACF;AA7SD,4CA6SC"}
@@ -1,285 +0,0 @@
1
- // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2
- import { APIResource } from "../core/resource.mjs";
3
- import { buildHeaders } from "../internal/headers.mjs";
4
- import { path } from "../internal/utils/path.mjs";
5
- export class PromptAdaptation extends APIResource {
6
- /**
7
- * Adapt your prompt from one LLM to work optimally across different target LLMs.
8
- *
9
- * This endpoint automatically optimizes your prompt (system prompt + user message
10
- * template) to achieve better performance when switching between different
11
- * language models. Each model has unique characteristics, and what works well for
12
- * GPT-4 might not work as well for Claude or Gemini.
13
- *
14
- * **How Prompt Adaptation Works:**
15
- *
16
- * 1. You provide your current prompt optimized for an origin model
17
- * 2. You specify target models you want to adapt to
18
- * 3. You provide evaluation examples (golden records) with expected answers
19
- * 4. The system runs optimization to find the best prompt for each target model
20
- * 5. You receive adapted prompts that perform well on your target models
21
- *
22
- * **Evaluation Metrics:** Choose either a standard metric or provide custom
23
- * evaluation:
24
- *
25
- * - **Standard metrics**: LLMaaJ:SQL, LLMaaJ:Sem_Sim_1/3/10 (semantic similarity),
26
- * JSON_Match
27
- * - **Custom evaluation**: Provide evaluation_config with your own LLM judge,
28
- * prompt, and cutoff
29
- *
30
- * **Dataset Requirements:**
31
- *
32
- * - Minimum 5 examples in train_goldens (more examples = better adaptation)
33
- * - Each example must have fields matching your template placeholders
34
- * - Supervised evaluation requires 'answer' field in each golden record
35
- * - Unsupervised evaluation can work without answers
36
- *
37
- * **Training Time:**
38
- *
39
- * - Processing is asynchronous and typically takes 10-30 minutes
40
- * - Time depends on: number of target models, dataset size, model availability
41
- * - Use the returned adaptation_run_id to check status and retrieve results
42
- *
43
- * **Subscription Tiers:**
44
- *
45
- * - Free: 1 target model
46
- * - Starter: 3 target models
47
- * - Startup: 5 target models
48
- * - Enterprise: 10 target models
49
- *
50
- * **Best Practices:**
51
- *
52
- * 1. Use diverse, representative examples from your production workload
53
- * 2. Include 10-20 examples for best results (5 minimum)
54
- * 3. Ensure consistent evaluation across all examples
55
- * 4. Test both train_goldens and test_goldens split for validation
56
- * 5. Use the same model versions you'll use in production
57
- *
58
- * **Example Workflow:**
59
- *
60
- * ```
61
- * 1. POST /v2/prompt/adapt - Submit adaptation request
62
- * 2. GET /v2/prompt/adaptStatus/{id} - Poll status until completed
63
- * 3. GET /v2/prompt/adaptResults/{id} - Retrieve optimized prompts
64
- * 4. Use optimized prompts in production with target models
65
- * ```
66
- *
67
- * **Related Documentation:** See
68
- * https://docs.notdiamond.ai/docs/adapting-prompts-to-new-models for detailed
69
- * guide.
70
- *
71
- * @example
72
- * ```ts
73
- * const response = await client.promptAdaptation.adapt({
74
- * fields: ['question'],
75
- * system_prompt: 'You are a helpful assistant that answers questions accurately.',
76
- * target_models: [
77
- * { provider: 'anthropic', model: 'claude-sonnet-4-5-20250929' },
78
- * { provider: 'google', model: 'gemini-1.5-pro' },
79
- * ],
80
- * template: 'Question: {question}\nAnswer:',
81
- * evaluation_metric: 'LLMaaJ:Sem_Sim_3',
82
- * origin_model: { provider: 'openai', model: 'gpt-4o' },
83
- * test_goldens: [
84
- * {
85
- * fields: { ... },
86
- * answer: '9',
87
- * },
88
- * {
89
- * fields: { ... },
90
- * answer: 'Pacific Ocean',
91
- * },
92
- * ],
93
- * train_goldens: [
94
- * {
95
- * fields: { ... },
96
- * answer: '4',
97
- * },
98
- * {
99
- * fields: { ... },
100
- * answer: 'Paris',
101
- * },
102
- * {
103
- * fields: { ... },
104
- * answer: 'William Shakespeare',
105
- * },
106
- * {
107
- * fields: { ... },
108
- * answer: 'Water',
109
- * },
110
- * {
111
- * fields: { ... },
112
- * answer: '7',
113
- * },
114
- * ],
115
- * });
116
- * ```
117
- */
118
- adapt(body, options) {
119
- return this._client.post('/v2/prompt/adapt', { body, ...options });
120
- }
121
- /**
122
- * Retrieve the complete results of a prompt adaptation run, including optimized
123
- * prompts for all target models.
124
- *
125
- * This endpoint returns the adapted prompts and evaluation metrics for each target
126
- * model in your adaptation request. Call this endpoint after the adaptation status
127
- * is 'completed' to get your optimized prompts.
128
- *
129
- * **Response Structure:**
130
- *
131
- * - **origin_model**: Baseline performance of your original prompt on the origin
132
- * model
133
- * - Includes: system_prompt, user_message_template, score, evaluation metrics,
134
- * cost
135
- * - **target_models**: Array of results for each target model
136
- * - Includes: optimized system_prompt, user_message_template, template_fields
137
- * - pre_optimization_score: Performance before adaptation
138
- * - post_optimization_score: Performance after adaptation
139
- * - Evaluation metrics and cost information
140
- *
141
- * **Using Adapted Prompts:**
142
- *
143
- * 1. Extract the `system_prompt` and `user_message_template` from each target
144
- * model result
145
- * 2. Use `user_message_template_fields` to know which fields to substitute
146
- * 3. Apply the optimized prompts when calling the respective target models
147
- * 4. Compare pre/post optimization scores to see improvement
148
- *
149
- * **Evaluation Scores:**
150
- *
151
- * - Scores range from 0-10 (higher is better)
152
- * - Compare origin_model score with target_models pre_optimization_score for
153
- * baseline
154
- * - Compare pre_optimization_score with post_optimization_score to see improvement
155
- * from adaptation
156
- * - Typical improvements range from 5-30% on evaluation metrics
157
- *
158
- * **Status Handling:**
159
- *
160
- * - If adaptation is still processing, target model results will have
161
- * `result_status: "processing"`
162
- * - Only completed target models will have system_prompt and template values
163
- * - Failed target models will have `result_status: "failed"` with null values
164
- *
165
- * **Cost Information:**
166
- *
167
- * - Each model result includes cost in USD for the adaptation process
168
- * - Costs vary based on model pricing and number of evaluation examples
169
- * - Typical range: $0.10 - $2.00 per target model
170
- *
171
- * **Best Practices:**
172
- *
173
- * 1. Wait for status 'completed' before calling this endpoint
174
- * 2. Check result_status for each target model
175
- * 3. Validate that post_optimization_score > pre_optimization_score
176
- * 4. Save optimized prompts for production use
177
- * 5. A/B test adapted prompts against originals in production
178
- *
179
- * @example
180
- * ```ts
181
- * const adaptationRunResults =
182
- * await client.promptAdaptation.getAdaptResults(
183
- * 'adaptation_run_id',
184
- * );
185
- * ```
186
- */
187
- getAdaptResults(adaptationRunID, options) {
188
- return this._client.get(path `/v2/prompt/adaptResults/${adaptationRunID}`, options);
189
- }
190
- /**
191
- * Get Adapt Run Results
192
- *
193
- * @example
194
- * ```ts
195
- * const adaptationRunResults =
196
- * await client.promptAdaptation.getAdaptRunResults(
197
- * 'adaptation_run_id',
198
- * { user_id: 'user_id', 'x-token': 'x-token' },
199
- * );
200
- * ```
201
- */
202
- getAdaptRunResults(adaptationRunID, params, options) {
203
- const { user_id, 'x-token': xToken } = params;
204
- return this._client.get(path `/v2/prompt/frontendAdaptRunResults/${user_id}/${adaptationRunID}`, {
205
- ...options,
206
- headers: buildHeaders([{ 'x-token': xToken }, options?.headers]),
207
- });
208
- }
209
- /**
210
- * Get Adapt Runs
211
- *
212
- * @example
213
- * ```ts
214
- * const adaptationRunResults =
215
- * await client.promptAdaptation.getAdaptRuns('user_id', {
216
- * 'x-token': 'x-token',
217
- * });
218
- * ```
219
- */
220
- getAdaptRuns(userID, params, options) {
221
- const { 'x-token': xToken } = params;
222
- return this._client.get(path `/v2/prompt/frontendAdaptRuns/${userID}`, {
223
- ...options,
224
- headers: buildHeaders([{ 'x-token': xToken }, options?.headers]),
225
- });
226
- }
227
- /**
228
- * Check the status of a prompt adaptation run.
229
- *
230
- * Use this endpoint to poll the status of your adaptation request. Processing is
231
- * asynchronous, so you'll need to check periodically until the status indicates
232
- * completion.
233
- *
234
- * **Status Values:**
235
- *
236
- * - `created`: Initial state, not yet processing
237
- * - `queued`: Waiting for processing capacity (check queue_position)
238
- * - `processing`: Currently optimizing prompts
239
- * - `completed`: All target models have been processed successfully
240
- * - `failed`: One or more target models failed to process
241
- *
242
- * **Polling Recommendations:**
243
- *
244
- * - Poll every 30-60 seconds during processing
245
- * - Check queue_position if status is 'queued' to estimate wait time
246
- * - Stop polling once status is 'completed' or 'failed'
247
- * - Use GET /v2/prompt/adaptResults to retrieve results after completion
248
- *
249
- * **Queue Position:**
250
- *
251
- * - Only present when status is 'queued'
252
- * - Lower numbers mean earlier processing (position 1 is next)
253
- * - Typical wait time: 1-5 minutes per position
254
- *
255
- * **Note:** This endpoint only returns status information. To get the actual
256
- * adapted prompts and evaluation results, use GET /v2/prompt/adaptResults once
257
- * status is 'completed'.
258
- *
259
- * @example
260
- * ```ts
261
- * const response =
262
- * await client.promptAdaptation.getAdaptStatus(
263
- * 'adaptation_run_id',
264
- * );
265
- * ```
266
- */
267
- getAdaptStatus(adaptationRunID, options) {
268
- return this._client.get(path `/v2/prompt/adaptStatus/${adaptationRunID}`, options);
269
- }
270
- /**
271
- * Get LLM costs for a specific adaptation run
272
- *
273
- * @example
274
- * ```ts
275
- * const response =
276
- * await client.promptAdaptation.retrieveCosts(
277
- * 'adaptation_run_id',
278
- * );
279
- * ```
280
- */
281
- retrieveCosts(adaptationRunID, options) {
282
- return this._client.get(path `/v1/adaptation-runs/${adaptationRunID}/costs`, options);
283
- }
284
- }
285
- //# sourceMappingURL=prompt-adaptation.mjs.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"prompt-adaptation.mjs","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAGf,EAAE,YAAY,EAAE;OAEhB,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,gBAAiB,SAAQ,WAAW;IAC/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+GG;IACH,KAAK,CACH,IAAiC,EACjC,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiEG;IACH,eAAe,CAAC,eAAuB,EAAE,OAAwB;QAC/D,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,2BAA2B,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IACrF,CAAC;IAED;;;;;;;;;;;OAWG;IACH,kBAAkB,CAChB,eAAuB,EACvB,MAAgD,EAChD,OAAwB;QAExB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;QAC9C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,sCAAsC,OAAO,IAAI,eAAe,EAAE,EAAE;YAC9F,GAAG,OAAO;YACV,OAAO,EAAE,YAAY,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;SACjE,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;OAUG;IACH,YAAY,CACV,MAAc,EACd,MAA0C,EAC1C,OAAwB;QAExB,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;QACrC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,gCAAgC,MAAM,EAAE,EAAE;YACpE,GAAG,OAAO;YACV,OAAO,EAAE,YAAY,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;SACjE,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,cAAc,CACZ,eAAuB,EACvB,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,0BAA0B,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;IAED;;;;;;;;;;OAUG;IACH,aAAa,CACX,eAAuB,EACvB,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,uBAAuB,eAAe,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvF,CAAC;CACF"}