notdiamond 2.0.0-rc1 → 2.0.0-rc10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +89 -0
- package/README.md +299 -97
- package/client.d.mts +19 -33
- package/client.d.mts.map +1 -1
- package/client.d.ts +19 -33
- package/client.d.ts.map +1 -1
- package/client.js +19 -35
- package/client.js.map +1 -1
- package/client.mjs +19 -35
- package/client.mjs.map +1 -1
- package/internal/tslib.js +17 -17
- package/internal/utils/env.d.mts.map +1 -1
- package/internal/utils/env.d.ts.map +1 -1
- package/internal/utils/env.js +4 -2
- package/internal/utils/env.js.map +1 -1
- package/internal/utils/env.mjs +4 -2
- package/internal/utils/env.mjs.map +1 -1
- package/package.json +10 -11
- package/resources/custom-router.d.mts +145 -0
- package/resources/custom-router.d.mts.map +1 -0
- package/resources/custom-router.d.ts +145 -0
- package/resources/custom-router.d.ts.map +1 -0
- package/resources/custom-router.js +83 -0
- package/resources/custom-router.js.map +1 -0
- package/resources/custom-router.mjs +79 -0
- package/resources/custom-router.mjs.map +1 -0
- package/resources/index.d.mts +6 -6
- package/resources/index.d.mts.map +1 -1
- package/resources/index.d.ts +6 -6
- package/resources/index.d.ts.map +1 -1
- package/resources/index.js +6 -6
- package/resources/index.js.map +1 -1
- package/resources/index.mjs +3 -3
- package/resources/index.mjs.map +1 -1
- package/resources/model-router.d.mts +196 -0
- package/resources/model-router.d.mts.map +1 -0
- package/resources/model-router.d.ts +196 -0
- package/resources/model-router.d.ts.map +1 -0
- package/resources/model-router.js +70 -0
- package/resources/model-router.js.map +1 -0
- package/resources/model-router.mjs +66 -0
- package/resources/model-router.mjs.map +1 -0
- package/resources/models.d.mts +50 -25
- package/resources/models.d.mts.map +1 -1
- package/resources/models.d.ts +50 -25
- package/resources/models.d.ts.map +1 -1
- package/resources/models.js +5 -0
- package/resources/models.js.map +1 -1
- package/resources/models.mjs +5 -0
- package/resources/models.mjs.map +1 -1
- package/resources/preferences.d.mts +34 -48
- package/resources/preferences.d.mts.map +1 -1
- package/resources/preferences.d.ts +34 -48
- package/resources/preferences.d.ts.map +1 -1
- package/resources/preferences.js +14 -36
- package/resources/preferences.js.map +1 -1
- package/resources/preferences.mjs +14 -36
- package/resources/preferences.mjs.map +1 -1
- package/resources/prompt-adaptation.d.mts +339 -234
- package/resources/prompt-adaptation.d.mts.map +1 -1
- package/resources/prompt-adaptation.d.ts +339 -234
- package/resources/prompt-adaptation.d.ts.map +1 -1
- package/resources/prompt-adaptation.js +30 -53
- package/resources/prompt-adaptation.js.map +1 -1
- package/resources/prompt-adaptation.mjs +30 -53
- package/resources/prompt-adaptation.mjs.map +1 -1
- package/resources/report/index.d.mts +3 -0
- package/resources/report/index.d.mts.map +1 -0
- package/resources/report/index.d.ts +3 -0
- package/resources/report/index.d.ts.map +1 -0
- package/resources/report/index.js +9 -0
- package/resources/report/index.js.map +1 -0
- package/resources/report/index.mjs +4 -0
- package/resources/report/index.mjs.map +1 -0
- package/resources/report/metrics.d.mts +87 -0
- package/resources/report/metrics.d.mts.map +1 -0
- package/resources/report/metrics.d.ts +87 -0
- package/resources/report/metrics.d.ts.map +1 -0
- package/resources/report/metrics.js +57 -0
- package/resources/report/metrics.js.map +1 -0
- package/resources/report/metrics.mjs +53 -0
- package/resources/report/metrics.mjs.map +1 -0
- package/resources/report/report.d.mts +10 -0
- package/resources/report/report.d.mts.map +1 -0
- package/resources/report/report.d.ts +10 -0
- package/resources/report/report.d.ts.map +1 -0
- package/resources/report/report.js +17 -0
- package/resources/report/report.js.map +1 -0
- package/resources/report/report.mjs +12 -0
- package/resources/report/report.mjs.map +1 -0
- package/resources/report.d.mts +1 -244
- package/resources/report.d.mts.map +1 -1
- package/resources/report.d.ts +1 -244
- package/resources/report.d.ts.map +1 -1
- package/resources/report.js +2 -82
- package/resources/report.js.map +1 -1
- package/resources/report.mjs +1 -80
- package/resources/report.mjs.map +1 -1
- package/src/client.ts +65 -116
- package/src/internal/utils/env.ts +4 -2
- package/src/resources/custom-router.ts +168 -0
- package/src/resources/index.ts +23 -34
- package/src/resources/model-router.ts +224 -0
- package/src/resources/models.ts +55 -32
- package/src/resources/preferences.ts +40 -77
- package/src/resources/prompt-adaptation.ts +361 -291
- package/src/resources/report/index.ts +4 -0
- package/src/resources/report/metrics.ts +99 -0
- package/src/resources/report/report.ts +19 -0
- package/src/resources/report.ts +1 -298
- package/src/version.ts +1 -1
- package/version.d.mts +1 -1
- package/version.d.mts.map +1 -1
- package/version.d.ts +1 -1
- package/version.d.ts.map +1 -1
- package/version.js +1 -1
- package/version.js.map +1 -1
- package/version.mjs +1 -1
- package/version.mjs.map +1 -1
- package/resources/admin.d.mts +0 -4
- package/resources/admin.d.mts.map +0 -1
- package/resources/admin.d.ts +0 -4
- package/resources/admin.d.ts.map +0 -1
- package/resources/admin.js +0 -9
- package/resources/admin.js.map +0 -1
- package/resources/admin.mjs +0 -5
- package/resources/admin.mjs.map +0 -1
- package/resources/routing.d.mts +0 -391
- package/resources/routing.d.mts.map +0 -1
- package/resources/routing.d.ts +0 -391
- package/resources/routing.d.ts.map +0 -1
- package/resources/routing.js +0 -163
- package/resources/routing.js.map +0 -1
- package/resources/routing.mjs +0 -159
- package/resources/routing.mjs.map +0 -1
- package/src/resources/admin.ts +0 -5
- package/src/resources/routing.ts +0 -476
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompt-adaptation.d.ts","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,KAAK,mBAAmB;OACxB,EAAE,UAAU,EAAE;
|
|
1
|
+
{"version":3,"file":"prompt-adaptation.d.ts","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,KAAK,mBAAmB;OACxB,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAGzB,qBAAa,gBAAiB,SAAQ,WAAW;IAC/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+GG;IACH,MAAM,CACJ,IAAI,EAAE,4BAA4B,EAClC,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,8BAA8B,CAAC;IAI7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiEG;IACH,eAAe,CACb,eAAe,EAAE,MAAM,EACvB,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,uCAAuC,CAAC;IAItD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,cAAc,CACZ,eAAe,EAAE,MAAM,EACvB,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,sCAAsC,CAAC;IAIrD;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,gCAAgC,CAAC;CAG1G;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,MAAM,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IAElC;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,YAAY,GAAG,WAAW,GAAG,QAAQ,CAAC;AAErF;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE/B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE5B;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAExB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,8BAA8B;IAC7C;;;OAGG;IACH,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,uCAAuC;IACtD;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,UAAU,EAAE,SAAS,CAAC;IAEtB;;OAEG;IACH,aAAa,EAAE,KAAK,CAAC,uCAAuC,CAAC,WAAW,CAAC,CAAC;IAE1E;;OAEG;IACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC;;;OAGG;IACH,mBAAmB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IAEhD;;;;;;;;;;;;;OAaG;IACH,YAAY,CAAC,EAAE,uCAAuC,CAAC,WAAW,GAAG,IAAI,CAAC;CAC3E;AAED,yBAAiB,uCAAuC,CAAC;IACvD;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,UAAiB,WAAW;QAC1B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAEpB,UAAU,EAAE,MAAM,CAAC;QAEnB,uBAAuB,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,GAAG,IAAI,CAAC;QAE3D,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAC;QAEvC,sBAAsB,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,GAAG,IAAI,CAAC;QAE1D,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;QAEtC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;QAEzB;;;;;;;;;;;WAWG;QACH,aAAa,CAAC,EAAE,mBAAmB,CAAC,SAAS,GAAG,IAAI,CAAC;QAErD;;;WAGG;QACH,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE9B;;;WAGG;QACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAEtC;;;WAGG;QACH,4BAA4B,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;KACrD;IAED;;;;;;;;;;;;;OAaG;IACH,UAAiB,WAAW;QAC1B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAEpB,KAAK,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,GAAG,IAAI,CAAC;QAEzC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAE1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAErB;;;;;;;;;;;WAWG;QACH,aAAa,CAAC,EAAE,mBAAmB,CAAC,SAAS,GAAG,IAAI,CAAC;QAErD;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE9B;;WAEG;QACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KACvC;CACF;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,sCAAsC;IACrD;;;OAGG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAE1B;;OAEG;IACH,MAAM,EAAE,SAAS,CAAC;IAElB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,gCAAgC;IAC/C;;OAEG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAE1B;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,aAAa,EAAE,KAAK,CAAC,gCAAgC,CAAC,WAAW,CAAC,CAAC;CACpE;AAED,yBAAiB,gCAAgC,CAAC;IAChD;;;;;OAKG;IACH,UAAiB,WAAW;QAC1B;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,iBAAiB,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,UAAU,EAAE,MAAM,CAAC;QAEnB;;WAEG;QACH,YAAY,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,eAAe,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,QAAQ,EAAE,MAAM,CAAC;QAEjB;;WAEG;QACH,SAAS,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,SAAS,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,UAAU,EAAE,MAAM,CAAC;QAEnB;;WAEG;QACH,OAAO,EAAE,MAAM,CAAC;KACjB;CACF;AAED,MAAM,WAAW,4BAA4B;IAC3C;;;OAGG;IACH,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEtB;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,aAAa,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;IAEtC;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC;;;OAGG;IACH,OAAO,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC;IAErC;;OAEG;IACH,YAAY,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IAEtC;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE9C;;;OAGG;IACH,YAAY,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC;IAE1C;;;OAGG;IACH,aAAa,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC;CAC5C;AAED,MAAM,CAAC,OAAO,WAAW,gBAAgB,CAAC;IACxC,OAAO,EACL,KAAK,YAAY,IAAI,YAAY,EACjC,KAAK,SAAS,IAAI,SAAS,EAC3B,KAAK,eAAe,IAAI,eAAe,EACvC,KAAK,8BAA8B,IAAI,8BAA8B,EACrE,KAAK,uCAAuC,IAAI,uCAAuC,EACvF,KAAK,sCAAsC,IAAI,sCAAsC,EACrF,KAAK,gCAAgC,IAAI,gCAAgC,EACzE,KAAK,4BAA4B,IAAI,4BAA4B,GAClE,CAAC;CACH"}
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
4
|
exports.PromptAdaptation = void 0;
|
|
5
5
|
const resource_1 = require("../core/resource.js");
|
|
6
|
-
const headers_1 = require("../internal/headers.js");
|
|
7
6
|
const path_1 = require("../internal/utils/path.js");
|
|
8
7
|
class PromptAdaptation extends resource_1.APIResource {
|
|
9
8
|
/**
|
|
@@ -32,7 +31,7 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
32
31
|
*
|
|
33
32
|
* **Dataset Requirements:**
|
|
34
33
|
*
|
|
35
|
-
* - Minimum
|
|
34
|
+
* - Minimum 25 examples in train_goldens (more examples = better adaptation)
|
|
36
35
|
* - Each example must have fields matching your template placeholders
|
|
37
36
|
* - Supervised evaluation requires 'answer' field in each golden record
|
|
38
37
|
* - Unsupervised evaluation can work without answers
|
|
@@ -53,7 +52,7 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
53
52
|
* **Best Practices:**
|
|
54
53
|
*
|
|
55
54
|
* 1. Use diverse, representative examples from your production workload
|
|
56
|
-
* 2. Include
|
|
55
|
+
* 2. Include examples for best results (25 minimum)
|
|
57
56
|
* 3. Ensure consistent evaluation across all examples
|
|
58
57
|
* 4. Test both train_goldens and test_goldens split for validation
|
|
59
58
|
* 5. Use the same model versions you'll use in production
|
|
@@ -73,16 +72,16 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
73
72
|
*
|
|
74
73
|
* @example
|
|
75
74
|
* ```ts
|
|
76
|
-
* const
|
|
75
|
+
* const promptAdaptation = await client.promptAdaptation.create({
|
|
77
76
|
* fields: ['question'],
|
|
78
|
-
* origin_model: { provider: 'openai', model: 'gpt-4o' },
|
|
79
77
|
* system_prompt: 'You are a helpful assistant that answers questions accurately.',
|
|
80
78
|
* target_models: [
|
|
81
|
-
* { provider: 'anthropic', model: 'claude-
|
|
82
|
-
* { provider: 'google', model: 'gemini-
|
|
79
|
+
* { provider: 'anthropic', model: 'claude-sonnet-4-5-20250929' },
|
|
80
|
+
* { provider: 'google', model: 'gemini-2.5-flash' },
|
|
83
81
|
* ],
|
|
84
82
|
* template: 'Question: {question}\nAnswer:',
|
|
85
83
|
* evaluation_metric: 'LLMaaJ:Sem_Sim_3',
|
|
84
|
+
* origin_model: { provider: 'openai', model: 'gpt-4o' },
|
|
86
85
|
* test_goldens: [
|
|
87
86
|
* {
|
|
88
87
|
* fields: { ... },
|
|
@@ -118,7 +117,7 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
118
117
|
* });
|
|
119
118
|
* ```
|
|
120
119
|
*/
|
|
121
|
-
|
|
120
|
+
create(body, options) {
|
|
122
121
|
return this._client.post('/v2/prompt/adapt', { body, ...options });
|
|
123
122
|
}
|
|
124
123
|
/**
|
|
@@ -181,7 +180,7 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
181
180
|
*
|
|
182
181
|
* @example
|
|
183
182
|
* ```ts
|
|
184
|
-
* const
|
|
183
|
+
* const response =
|
|
185
184
|
* await client.promptAdaptation.getAdaptResults(
|
|
186
185
|
* 'adaptation_run_id',
|
|
187
186
|
* );
|
|
@@ -190,43 +189,6 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
190
189
|
getAdaptResults(adaptationRunID, options) {
|
|
191
190
|
return this._client.get((0, path_1.path) `/v2/prompt/adaptResults/${adaptationRunID}`, options);
|
|
192
191
|
}
|
|
193
|
-
/**
|
|
194
|
-
* Get Adapt Run Results
|
|
195
|
-
*
|
|
196
|
-
* @example
|
|
197
|
-
* ```ts
|
|
198
|
-
* const adaptationRunResults =
|
|
199
|
-
* await client.promptAdaptation.getAdaptRunResults(
|
|
200
|
-
* 'adaptation_run_id',
|
|
201
|
-
* { user_id: 'user_id', 'x-token': 'x-token' },
|
|
202
|
-
* );
|
|
203
|
-
* ```
|
|
204
|
-
*/
|
|
205
|
-
getAdaptRunResults(adaptationRunID, params, options) {
|
|
206
|
-
const { user_id, 'x-token': xToken } = params;
|
|
207
|
-
return this._client.get((0, path_1.path) `/v2/prompt/frontendAdaptRunResults/${user_id}/${adaptationRunID}`, {
|
|
208
|
-
...options,
|
|
209
|
-
headers: (0, headers_1.buildHeaders)([{ 'x-token': xToken }, options?.headers]),
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
/**
|
|
213
|
-
* Get Adapt Runs
|
|
214
|
-
*
|
|
215
|
-
* @example
|
|
216
|
-
* ```ts
|
|
217
|
-
* const adaptationRunResults =
|
|
218
|
-
* await client.promptAdaptation.getAdaptRuns('user_id', {
|
|
219
|
-
* 'x-token': 'x-token',
|
|
220
|
-
* });
|
|
221
|
-
* ```
|
|
222
|
-
*/
|
|
223
|
-
getAdaptRuns(userID, params, options) {
|
|
224
|
-
const { 'x-token': xToken } = params;
|
|
225
|
-
return this._client.get((0, path_1.path) `/v2/prompt/frontendAdaptRuns/${userID}`, {
|
|
226
|
-
...options,
|
|
227
|
-
headers: (0, headers_1.buildHeaders)([{ 'x-token': xToken }, options?.headers]),
|
|
228
|
-
});
|
|
229
|
-
}
|
|
230
192
|
/**
|
|
231
193
|
* Check the status of a prompt adaptation run.
|
|
232
194
|
*
|
|
@@ -271,18 +233,33 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
271
233
|
return this._client.get((0, path_1.path) `/v2/prompt/adaptStatus/${adaptationRunID}`, options);
|
|
272
234
|
}
|
|
273
235
|
/**
|
|
274
|
-
* Get LLM costs for a specific adaptation run
|
|
236
|
+
* Get LLM usage costs for a specific prompt adaptation run.
|
|
237
|
+
*
|
|
238
|
+
* This endpoint returns the total cost and detailed usage records for all LLM
|
|
239
|
+
* requests made during a prompt adaptation run. Use this to track costs associated
|
|
240
|
+
* with optimizing prompts for different target models.
|
|
241
|
+
*
|
|
242
|
+
* **Cost Breakdown:**
|
|
243
|
+
*
|
|
244
|
+
* - Total cost across all models used in the adaptation
|
|
245
|
+
* - Individual usage records with provider, model, tokens, and costs
|
|
246
|
+
* - Timestamps for each LLM request
|
|
247
|
+
* - Task type (e.g., optimization, evaluation)
|
|
248
|
+
*
|
|
249
|
+
* **Access Control:**
|
|
250
|
+
*
|
|
251
|
+
* - Only accessible by the user who created the adaptation run
|
|
252
|
+
* - Requires prompt adaptation access
|
|
275
253
|
*
|
|
276
254
|
* @example
|
|
277
255
|
* ```ts
|
|
278
|
-
* const response =
|
|
279
|
-
*
|
|
280
|
-
*
|
|
281
|
-
* );
|
|
256
|
+
* const response = await client.promptAdaptation.getCosts(
|
|
257
|
+
* 'adaptation_run_id',
|
|
258
|
+
* );
|
|
282
259
|
* ```
|
|
283
260
|
*/
|
|
284
|
-
|
|
285
|
-
return this._client.get((0, path_1.path) `/
|
|
261
|
+
getCosts(adaptationRunID, options) {
|
|
262
|
+
return this._client.get((0, path_1.path) `/v2/prompt/adapt/${adaptationRunID}/costs`, options);
|
|
286
263
|
}
|
|
287
264
|
}
|
|
288
265
|
exports.PromptAdaptation = PromptAdaptation;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompt-adaptation.js","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kDAA+C;
|
|
1
|
+
{"version":3,"file":"prompt-adaptation.js","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kDAA+C;AAI/C,oDAA8C;AAE9C,MAAa,gBAAiB,SAAQ,sBAAW;IAC/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+GG;IACH,MAAM,CACJ,IAAkC,EAClC,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiEG;IACH,eAAe,CACb,eAAuB,EACvB,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,2BAA2B,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IACrF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,cAAc,CACZ,eAAuB,EACvB,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,0BAA0B,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,QAAQ,CAAC,eAAuB,EAAE,OAAwB;QACxD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,oBAAoB,eAAe,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;CACF;AA7QD,4CA6QC"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
2
2
|
import { APIResource } from "../core/resource.mjs";
|
|
3
|
-
import { buildHeaders } from "../internal/headers.mjs";
|
|
4
3
|
import { path } from "../internal/utils/path.mjs";
|
|
5
4
|
export class PromptAdaptation extends APIResource {
|
|
6
5
|
/**
|
|
@@ -29,7 +28,7 @@ export class PromptAdaptation extends APIResource {
|
|
|
29
28
|
*
|
|
30
29
|
* **Dataset Requirements:**
|
|
31
30
|
*
|
|
32
|
-
* - Minimum
|
|
31
|
+
* - Minimum 25 examples in train_goldens (more examples = better adaptation)
|
|
33
32
|
* - Each example must have fields matching your template placeholders
|
|
34
33
|
* - Supervised evaluation requires 'answer' field in each golden record
|
|
35
34
|
* - Unsupervised evaluation can work without answers
|
|
@@ -50,7 +49,7 @@ export class PromptAdaptation extends APIResource {
|
|
|
50
49
|
* **Best Practices:**
|
|
51
50
|
*
|
|
52
51
|
* 1. Use diverse, representative examples from your production workload
|
|
53
|
-
* 2. Include
|
|
52
|
+
* 2. Include examples for best results (25 minimum)
|
|
54
53
|
* 3. Ensure consistent evaluation across all examples
|
|
55
54
|
* 4. Test both train_goldens and test_goldens split for validation
|
|
56
55
|
* 5. Use the same model versions you'll use in production
|
|
@@ -70,16 +69,16 @@ export class PromptAdaptation extends APIResource {
|
|
|
70
69
|
*
|
|
71
70
|
* @example
|
|
72
71
|
* ```ts
|
|
73
|
-
* const
|
|
72
|
+
* const promptAdaptation = await client.promptAdaptation.create({
|
|
74
73
|
* fields: ['question'],
|
|
75
|
-
* origin_model: { provider: 'openai', model: 'gpt-4o' },
|
|
76
74
|
* system_prompt: 'You are a helpful assistant that answers questions accurately.',
|
|
77
75
|
* target_models: [
|
|
78
|
-
* { provider: 'anthropic', model: 'claude-
|
|
79
|
-
* { provider: 'google', model: 'gemini-
|
|
76
|
+
* { provider: 'anthropic', model: 'claude-sonnet-4-5-20250929' },
|
|
77
|
+
* { provider: 'google', model: 'gemini-2.5-flash' },
|
|
80
78
|
* ],
|
|
81
79
|
* template: 'Question: {question}\nAnswer:',
|
|
82
80
|
* evaluation_metric: 'LLMaaJ:Sem_Sim_3',
|
|
81
|
+
* origin_model: { provider: 'openai', model: 'gpt-4o' },
|
|
83
82
|
* test_goldens: [
|
|
84
83
|
* {
|
|
85
84
|
* fields: { ... },
|
|
@@ -115,7 +114,7 @@ export class PromptAdaptation extends APIResource {
|
|
|
115
114
|
* });
|
|
116
115
|
* ```
|
|
117
116
|
*/
|
|
118
|
-
|
|
117
|
+
create(body, options) {
|
|
119
118
|
return this._client.post('/v2/prompt/adapt', { body, ...options });
|
|
120
119
|
}
|
|
121
120
|
/**
|
|
@@ -178,7 +177,7 @@ export class PromptAdaptation extends APIResource {
|
|
|
178
177
|
*
|
|
179
178
|
* @example
|
|
180
179
|
* ```ts
|
|
181
|
-
* const
|
|
180
|
+
* const response =
|
|
182
181
|
* await client.promptAdaptation.getAdaptResults(
|
|
183
182
|
* 'adaptation_run_id',
|
|
184
183
|
* );
|
|
@@ -187,43 +186,6 @@ export class PromptAdaptation extends APIResource {
|
|
|
187
186
|
getAdaptResults(adaptationRunID, options) {
|
|
188
187
|
return this._client.get(path `/v2/prompt/adaptResults/${adaptationRunID}`, options);
|
|
189
188
|
}
|
|
190
|
-
/**
|
|
191
|
-
* Get Adapt Run Results
|
|
192
|
-
*
|
|
193
|
-
* @example
|
|
194
|
-
* ```ts
|
|
195
|
-
* const adaptationRunResults =
|
|
196
|
-
* await client.promptAdaptation.getAdaptRunResults(
|
|
197
|
-
* 'adaptation_run_id',
|
|
198
|
-
* { user_id: 'user_id', 'x-token': 'x-token' },
|
|
199
|
-
* );
|
|
200
|
-
* ```
|
|
201
|
-
*/
|
|
202
|
-
getAdaptRunResults(adaptationRunID, params, options) {
|
|
203
|
-
const { user_id, 'x-token': xToken } = params;
|
|
204
|
-
return this._client.get(path `/v2/prompt/frontendAdaptRunResults/${user_id}/${adaptationRunID}`, {
|
|
205
|
-
...options,
|
|
206
|
-
headers: buildHeaders([{ 'x-token': xToken }, options?.headers]),
|
|
207
|
-
});
|
|
208
|
-
}
|
|
209
|
-
/**
|
|
210
|
-
* Get Adapt Runs
|
|
211
|
-
*
|
|
212
|
-
* @example
|
|
213
|
-
* ```ts
|
|
214
|
-
* const adaptationRunResults =
|
|
215
|
-
* await client.promptAdaptation.getAdaptRuns('user_id', {
|
|
216
|
-
* 'x-token': 'x-token',
|
|
217
|
-
* });
|
|
218
|
-
* ```
|
|
219
|
-
*/
|
|
220
|
-
getAdaptRuns(userID, params, options) {
|
|
221
|
-
const { 'x-token': xToken } = params;
|
|
222
|
-
return this._client.get(path `/v2/prompt/frontendAdaptRuns/${userID}`, {
|
|
223
|
-
...options,
|
|
224
|
-
headers: buildHeaders([{ 'x-token': xToken }, options?.headers]),
|
|
225
|
-
});
|
|
226
|
-
}
|
|
227
189
|
/**
|
|
228
190
|
* Check the status of a prompt adaptation run.
|
|
229
191
|
*
|
|
@@ -268,18 +230,33 @@ export class PromptAdaptation extends APIResource {
|
|
|
268
230
|
return this._client.get(path `/v2/prompt/adaptStatus/${adaptationRunID}`, options);
|
|
269
231
|
}
|
|
270
232
|
/**
|
|
271
|
-
* Get LLM costs for a specific adaptation run
|
|
233
|
+
* Get LLM usage costs for a specific prompt adaptation run.
|
|
234
|
+
*
|
|
235
|
+
* This endpoint returns the total cost and detailed usage records for all LLM
|
|
236
|
+
* requests made during a prompt adaptation run. Use this to track costs associated
|
|
237
|
+
* with optimizing prompts for different target models.
|
|
238
|
+
*
|
|
239
|
+
* **Cost Breakdown:**
|
|
240
|
+
*
|
|
241
|
+
* - Total cost across all models used in the adaptation
|
|
242
|
+
* - Individual usage records with provider, model, tokens, and costs
|
|
243
|
+
* - Timestamps for each LLM request
|
|
244
|
+
* - Task type (e.g., optimization, evaluation)
|
|
245
|
+
*
|
|
246
|
+
* **Access Control:**
|
|
247
|
+
*
|
|
248
|
+
* - Only accessible by the user who created the adaptation run
|
|
249
|
+
* - Requires prompt adaptation access
|
|
272
250
|
*
|
|
273
251
|
* @example
|
|
274
252
|
* ```ts
|
|
275
|
-
* const response =
|
|
276
|
-
*
|
|
277
|
-
*
|
|
278
|
-
* );
|
|
253
|
+
* const response = await client.promptAdaptation.getCosts(
|
|
254
|
+
* 'adaptation_run_id',
|
|
255
|
+
* );
|
|
279
256
|
* ```
|
|
280
257
|
*/
|
|
281
|
-
|
|
282
|
-
return this._client.get(path `/
|
|
258
|
+
getCosts(adaptationRunID, options) {
|
|
259
|
+
return this._client.get(path `/v2/prompt/adapt/${adaptationRunID}/costs`, options);
|
|
283
260
|
}
|
|
284
261
|
}
|
|
285
262
|
//# sourceMappingURL=prompt-adaptation.mjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompt-adaptation.mjs","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;
|
|
1
|
+
{"version":3,"file":"prompt-adaptation.mjs","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAIf,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,gBAAiB,SAAQ,WAAW;IAC/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+GG;IACH,MAAM,CACJ,IAAkC,EAClC,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAiEG;IACH,eAAe,CACb,eAAuB,EACvB,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,2BAA2B,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IACrF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,cAAc,CACZ,eAAuB,EACvB,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,0BAA0B,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,QAAQ,CAAC,eAAuB,EAAE,OAAwB;QACxD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,oBAAoB,eAAe,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../../src/resources/report/index.ts"],"names":[],"mappings":"OAEO,EAAE,OAAO,EAAE,KAAK,4BAA4B,EAAE,KAAK,0BAA0B,EAAE;OAC/E,EAAE,MAAM,EAAE"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/resources/report/index.ts"],"names":[],"mappings":"OAEO,EAAE,OAAO,EAAE,KAAK,4BAA4B,EAAE,KAAK,0BAA0B,EAAE;OAC/E,EAAE,MAAM,EAAE"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.Report = exports.Metrics = void 0;
|
|
5
|
+
var metrics_1 = require("./metrics.js");
|
|
6
|
+
Object.defineProperty(exports, "Metrics", { enumerable: true, get: function () { return metrics_1.Metrics; } });
|
|
7
|
+
var report_1 = require("./report.js");
|
|
8
|
+
Object.defineProperty(exports, "Report", { enumerable: true, get: function () { return report_1.Report; } });
|
|
9
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/resources/report/index.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,wCAAwG;AAA/F,kGAAA,OAAO,OAAA;AAChB,sCAAkC;AAAzB,gGAAA,MAAM,OAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../../src/resources/report/index.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,OAAO,EAAsE;OAC/E,EAAE,MAAM,EAAE"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { APIResource } from "../../core/resource.mjs";
|
|
2
|
+
import * as PromptAdaptationAPI from "../prompt-adaptation.mjs";
|
|
3
|
+
import { APIPromise } from "../../core/api-promise.mjs";
|
|
4
|
+
import { RequestOptions } from "../../internal/request-options.mjs";
|
|
5
|
+
export declare class Metrics extends APIResource {
|
|
6
|
+
/**
|
|
7
|
+
* Submit feedback on a routing decision to improve future recommendations.
|
|
8
|
+
*
|
|
9
|
+
* This endpoint allows you to provide feedback on whether the router selected the
|
|
10
|
+
* right model for your query. Your feedback is used to:
|
|
11
|
+
*
|
|
12
|
+
* 1. Personalize routing decisions for your preference_id
|
|
13
|
+
* 2. Improve the overall routing quality
|
|
14
|
+
* 3. Train and refine custom routers
|
|
15
|
+
*
|
|
16
|
+
* **Feedback Format:**
|
|
17
|
+
*
|
|
18
|
+
* - `accuracy: 1` - Thumbs up (the model performed well)
|
|
19
|
+
* - `accuracy: 0` - Thumbs down (the model did not perform well)
|
|
20
|
+
*
|
|
21
|
+
* **Requirements:**
|
|
22
|
+
*
|
|
23
|
+
* - You must have used a preference_id in the original model_select() call
|
|
24
|
+
* - The session_id must be valid and belong to your account
|
|
25
|
+
* - The provider must match one of the providers returned by model_select()
|
|
26
|
+
*
|
|
27
|
+
* **How Feedback Works:** When you submit thumbs down, the router will:
|
|
28
|
+
*
|
|
29
|
+
* - Decrease the ranking of the selected model for similar queries
|
|
30
|
+
* - Consider alternative models more favorably
|
|
31
|
+
*
|
|
32
|
+
* When you submit thumbs up, the router will:
|
|
33
|
+
*
|
|
34
|
+
* - Increase the ranking of the selected model for similar queries
|
|
35
|
+
* - Prioritize this model for similar future requests
|
|
36
|
+
*
|
|
37
|
+
* **Note:** Feedback requires a valid preference_id. Create one via POST
|
|
38
|
+
* /v2/preferences/userPreferenceCreate
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* ```ts
|
|
42
|
+
* const response = await client.report.metrics.submitFeedback(
|
|
43
|
+
* {
|
|
44
|
+
* feedback: { accuracy: 1 },
|
|
45
|
+
* provider: { provider: 'openai', model: 'gpt-4o' },
|
|
46
|
+
* session_id: '550e8400-e29b-41d4-a716-446655440000',
|
|
47
|
+
* },
|
|
48
|
+
* );
|
|
49
|
+
* ```
|
|
50
|
+
*/
|
|
51
|
+
submitFeedback(body: MetricSubmitFeedbackParams, options?: RequestOptions): APIPromise<MetricSubmitFeedbackResponse>;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Response from feedback submission endpoint.
|
|
55
|
+
*/
|
|
56
|
+
export interface MetricSubmitFeedbackResponse {
|
|
57
|
+
/**
|
|
58
|
+
* The processed feedback
|
|
59
|
+
*/
|
|
60
|
+
feedback: {
|
|
61
|
+
[key: string]: unknown;
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* The session ID for which feedback was submitted
|
|
65
|
+
*/
|
|
66
|
+
session_id: string;
|
|
67
|
+
}
|
|
68
|
+
export interface MetricSubmitFeedbackParams {
|
|
69
|
+
/**
|
|
70
|
+
* Feedback dictionary with 'accuracy' key (0 for thumbs down, 1 for thumbs up)
|
|
71
|
+
*/
|
|
72
|
+
feedback: {
|
|
73
|
+
[key: string]: unknown;
|
|
74
|
+
};
|
|
75
|
+
/**
|
|
76
|
+
* The provider that was selected by the router
|
|
77
|
+
*/
|
|
78
|
+
provider: PromptAdaptationAPI.RequestProvider;
|
|
79
|
+
/**
|
|
80
|
+
* Session ID returned from POST /v2/modelRouter/modelSelect
|
|
81
|
+
*/
|
|
82
|
+
session_id: string;
|
|
83
|
+
}
|
|
84
|
+
export declare namespace Metrics {
|
|
85
|
+
export { type MetricSubmitFeedbackResponse as MetricSubmitFeedbackResponse, type MetricSubmitFeedbackParams as MetricSubmitFeedbackParams, };
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=metrics.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.mts","sourceRoot":"","sources":["../../src/resources/report/metrics.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,KAAK,mBAAmB;OACxB,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAEzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4CG;IACH,cAAc,CACZ,IAAI,EAAE,0BAA0B,EAChC,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,4BAA4B,CAAC;CAG5C;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;OAEG;IACH,QAAQ,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IAErC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,0BAA0B;IACzC;;OAEG;IACH,QAAQ,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IAErC;;OAEG;IACH,QAAQ,EAAE,mBAAmB,CAAC,eAAe,CAAC;IAE9C;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,4BAA4B,IAAI,4BAA4B,EACjE,KAAK,0BAA0B,IAAI,0BAA0B,GAC9D,CAAC;CACH"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { APIResource } from "../../core/resource.js";
|
|
2
|
+
import * as PromptAdaptationAPI from "../prompt-adaptation.js";
|
|
3
|
+
import { APIPromise } from "../../core/api-promise.js";
|
|
4
|
+
import { RequestOptions } from "../../internal/request-options.js";
|
|
5
|
+
export declare class Metrics extends APIResource {
|
|
6
|
+
/**
|
|
7
|
+
* Submit feedback on a routing decision to improve future recommendations.
|
|
8
|
+
*
|
|
9
|
+
* This endpoint allows you to provide feedback on whether the router selected the
|
|
10
|
+
* right model for your query. Your feedback is used to:
|
|
11
|
+
*
|
|
12
|
+
* 1. Personalize routing decisions for your preference_id
|
|
13
|
+
* 2. Improve the overall routing quality
|
|
14
|
+
* 3. Train and refine custom routers
|
|
15
|
+
*
|
|
16
|
+
* **Feedback Format:**
|
|
17
|
+
*
|
|
18
|
+
* - `accuracy: 1` - Thumbs up (the model performed well)
|
|
19
|
+
* - `accuracy: 0` - Thumbs down (the model did not perform well)
|
|
20
|
+
*
|
|
21
|
+
* **Requirements:**
|
|
22
|
+
*
|
|
23
|
+
* - You must have used a preference_id in the original model_select() call
|
|
24
|
+
* - The session_id must be valid and belong to your account
|
|
25
|
+
* - The provider must match one of the providers returned by model_select()
|
|
26
|
+
*
|
|
27
|
+
* **How Feedback Works:** When you submit thumbs down, the router will:
|
|
28
|
+
*
|
|
29
|
+
* - Decrease the ranking of the selected model for similar queries
|
|
30
|
+
* - Consider alternative models more favorably
|
|
31
|
+
*
|
|
32
|
+
* When you submit thumbs up, the router will:
|
|
33
|
+
*
|
|
34
|
+
* - Increase the ranking of the selected model for similar queries
|
|
35
|
+
* - Prioritize this model for similar future requests
|
|
36
|
+
*
|
|
37
|
+
* **Note:** Feedback requires a valid preference_id. Create one via POST
|
|
38
|
+
* /v2/preferences/userPreferenceCreate
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* ```ts
|
|
42
|
+
* const response = await client.report.metrics.submitFeedback(
|
|
43
|
+
* {
|
|
44
|
+
* feedback: { accuracy: 1 },
|
|
45
|
+
* provider: { provider: 'openai', model: 'gpt-4o' },
|
|
46
|
+
* session_id: '550e8400-e29b-41d4-a716-446655440000',
|
|
47
|
+
* },
|
|
48
|
+
* );
|
|
49
|
+
* ```
|
|
50
|
+
*/
|
|
51
|
+
submitFeedback(body: MetricSubmitFeedbackParams, options?: RequestOptions): APIPromise<MetricSubmitFeedbackResponse>;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Response from feedback submission endpoint.
|
|
55
|
+
*/
|
|
56
|
+
export interface MetricSubmitFeedbackResponse {
|
|
57
|
+
/**
|
|
58
|
+
* The processed feedback
|
|
59
|
+
*/
|
|
60
|
+
feedback: {
|
|
61
|
+
[key: string]: unknown;
|
|
62
|
+
};
|
|
63
|
+
/**
|
|
64
|
+
* The session ID for which feedback was submitted
|
|
65
|
+
*/
|
|
66
|
+
session_id: string;
|
|
67
|
+
}
|
|
68
|
+
export interface MetricSubmitFeedbackParams {
|
|
69
|
+
/**
|
|
70
|
+
* Feedback dictionary with 'accuracy' key (0 for thumbs down, 1 for thumbs up)
|
|
71
|
+
*/
|
|
72
|
+
feedback: {
|
|
73
|
+
[key: string]: unknown;
|
|
74
|
+
};
|
|
75
|
+
/**
|
|
76
|
+
* The provider that was selected by the router
|
|
77
|
+
*/
|
|
78
|
+
provider: PromptAdaptationAPI.RequestProvider;
|
|
79
|
+
/**
|
|
80
|
+
* Session ID returned from POST /v2/modelRouter/modelSelect
|
|
81
|
+
*/
|
|
82
|
+
session_id: string;
|
|
83
|
+
}
|
|
84
|
+
export declare namespace Metrics {
|
|
85
|
+
export { type MetricSubmitFeedbackResponse as MetricSubmitFeedbackResponse, type MetricSubmitFeedbackParams as MetricSubmitFeedbackParams, };
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=metrics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/resources/report/metrics.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,KAAK,mBAAmB;OACxB,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAEzB,qBAAa,OAAQ,SAAQ,WAAW;IACtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4CG;IACH,cAAc,CACZ,IAAI,EAAE,0BAA0B,EAChC,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,4BAA4B,CAAC;CAG5C;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC3C;;OAEG;IACH,QAAQ,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IAErC;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,0BAA0B;IACzC;;OAEG;IACH,QAAQ,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IAErC;;OAEG;IACH,QAAQ,EAAE,mBAAmB,CAAC,eAAe,CAAC;IAE9C;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,CAAC,OAAO,WAAW,OAAO,CAAC;IAC/B,OAAO,EACL,KAAK,4BAA4B,IAAI,4BAA4B,EACjE,KAAK,0BAA0B,IAAI,0BAA0B,GAC9D,CAAC;CACH"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.Metrics = void 0;
|
|
5
|
+
const resource_1 = require("../../core/resource.js");
|
|
6
|
+
class Metrics extends resource_1.APIResource {
|
|
7
|
+
/**
|
|
8
|
+
* Submit feedback on a routing decision to improve future recommendations.
|
|
9
|
+
*
|
|
10
|
+
* This endpoint allows you to provide feedback on whether the router selected the
|
|
11
|
+
* right model for your query. Your feedback is used to:
|
|
12
|
+
*
|
|
13
|
+
* 1. Personalize routing decisions for your preference_id
|
|
14
|
+
* 2. Improve the overall routing quality
|
|
15
|
+
* 3. Train and refine custom routers
|
|
16
|
+
*
|
|
17
|
+
* **Feedback Format:**
|
|
18
|
+
*
|
|
19
|
+
* - `accuracy: 1` - Thumbs up (the model performed well)
|
|
20
|
+
* - `accuracy: 0` - Thumbs down (the model did not perform well)
|
|
21
|
+
*
|
|
22
|
+
* **Requirements:**
|
|
23
|
+
*
|
|
24
|
+
* - You must have used a preference_id in the original model_select() call
|
|
25
|
+
* - The session_id must be valid and belong to your account
|
|
26
|
+
* - The provider must match one of the providers returned by model_select()
|
|
27
|
+
*
|
|
28
|
+
* **How Feedback Works:** When you submit thumbs down, the router will:
|
|
29
|
+
*
|
|
30
|
+
* - Decrease the ranking of the selected model for similar queries
|
|
31
|
+
* - Consider alternative models more favorably
|
|
32
|
+
*
|
|
33
|
+
* When you submit thumbs up, the router will:
|
|
34
|
+
*
|
|
35
|
+
* - Increase the ranking of the selected model for similar queries
|
|
36
|
+
* - Prioritize this model for similar future requests
|
|
37
|
+
*
|
|
38
|
+
* **Note:** Feedback requires a valid preference_id. Create one via POST
|
|
39
|
+
* /v2/preferences/userPreferenceCreate
|
|
40
|
+
*
|
|
41
|
+
* @example
|
|
42
|
+
* ```ts
|
|
43
|
+
* const response = await client.report.metrics.submitFeedback(
|
|
44
|
+
* {
|
|
45
|
+
* feedback: { accuracy: 1 },
|
|
46
|
+
* provider: { provider: 'openai', model: 'gpt-4o' },
|
|
47
|
+
* session_id: '550e8400-e29b-41d4-a716-446655440000',
|
|
48
|
+
* },
|
|
49
|
+
* );
|
|
50
|
+
* ```
|
|
51
|
+
*/
|
|
52
|
+
submitFeedback(body, options) {
|
|
53
|
+
return this._client.post('/v2/report/metrics/feedback', { body, ...options });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
exports.Metrics = Metrics;
|
|
57
|
+
//# sourceMappingURL=metrics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/resources/report/metrics.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,qDAAkD;AAKlD,MAAa,OAAQ,SAAQ,sBAAW;IACtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4CG;IACH,cAAc,CACZ,IAAgC,EAChC,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,6BAA6B,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IAChF,CAAC;CACF;AApDD,0BAoDC"}
|