notdiamond 2.0.0-rc2 → 2.0.0-rc20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +185 -0
- package/LICENSE +1 -1
- package/README.md +261 -168
- package/client.d.mts +26 -43
- package/client.d.mts.map +1 -1
- package/client.d.ts +26 -43
- package/client.d.ts.map +1 -1
- package/client.js +49 -68
- package/client.js.map +1 -1
- package/client.mjs +47 -66
- package/client.mjs.map +1 -1
- package/core/api-promise.d.mts +2 -2
- package/core/api-promise.d.ts +2 -2
- package/core/error.d.mts +2 -2
- package/core/error.d.ts +2 -2
- package/core/error.js +4 -4
- package/core/error.mjs +2 -2
- package/core/resource.d.mts +3 -3
- package/core/resource.d.ts +3 -3
- package/index.d.mts +3 -3
- package/index.d.ts +3 -3
- package/index.js +4 -4
- package/index.mjs +3 -3
- package/internal/parse.d.mts +2 -2
- package/internal/parse.d.ts +2 -2
- package/internal/shims.js +1 -1
- package/internal/shims.mjs +1 -1
- package/internal/uploads.d.mts +4 -4
- package/internal/uploads.d.ts +4 -4
- package/internal/utils/base64.js +2 -2
- package/internal/utils/base64.mjs +3 -3
- package/internal/utils/env.d.mts.map +1 -1
- package/internal/utils/env.d.ts.map +1 -1
- package/internal/utils/env.js +4 -2
- package/internal/utils/env.js.map +1 -1
- package/internal/utils/env.mjs +4 -2
- package/internal/utils/env.mjs.map +1 -1
- package/internal/utils/log.d.mts +3 -3
- package/internal/utils/log.d.ts +3 -3
- package/internal/utils/path.js +1 -1
- package/internal/utils/path.mjs +2 -2
- package/internal/utils/values.js +5 -5
- package/internal/utils/values.mjs +6 -6
- package/package.json +11 -12
- package/resources/custom-router.d.mts +145 -0
- package/resources/custom-router.d.mts.map +1 -0
- package/resources/custom-router.d.ts +145 -0
- package/resources/custom-router.d.ts.map +1 -0
- package/resources/custom-router.js +83 -0
- package/resources/custom-router.js.map +1 -0
- package/resources/custom-router.mjs +79 -0
- package/resources/custom-router.mjs.map +1 -0
- package/resources/index.d.mts +5 -6
- package/resources/index.d.mts.map +1 -1
- package/resources/index.d.ts +5 -6
- package/resources/index.d.ts.map +1 -1
- package/resources/index.js +5 -7
- package/resources/index.js.map +1 -1
- package/resources/index.mjs +2 -3
- package/resources/index.mjs.map +1 -1
- package/resources/model-router.d.mts +194 -0
- package/resources/model-router.d.mts.map +1 -0
- package/resources/model-router.d.ts +194 -0
- package/resources/model-router.d.ts.map +1 -0
- package/resources/model-router.js +68 -0
- package/resources/model-router.js.map +1 -0
- package/resources/model-router.mjs +64 -0
- package/resources/model-router.mjs.map +1 -0
- package/resources/models.d.mts +50 -25
- package/resources/models.d.mts.map +1 -1
- package/resources/models.d.ts +50 -25
- package/resources/models.d.ts.map +1 -1
- package/resources/models.js +5 -0
- package/resources/models.js.map +1 -1
- package/resources/models.mjs +5 -0
- package/resources/models.mjs.map +1 -1
- package/resources/preferences.d.mts +37 -54
- package/resources/preferences.d.mts.map +1 -1
- package/resources/preferences.d.ts +37 -54
- package/resources/preferences.d.ts.map +1 -1
- package/resources/preferences.js +17 -42
- package/resources/preferences.js.map +1 -1
- package/resources/preferences.mjs +17 -42
- package/resources/preferences.mjs.map +1 -1
- package/resources/prompt-adaptation.d.mts +340 -249
- package/resources/prompt-adaptation.d.mts.map +1 -1
- package/resources/prompt-adaptation.d.ts +340 -249
- package/resources/prompt-adaptation.d.ts.map +1 -1
- package/resources/prompt-adaptation.js +36 -79
- package/resources/prompt-adaptation.js.map +1 -1
- package/resources/prompt-adaptation.mjs +36 -79
- package/resources/prompt-adaptation.mjs.map +1 -1
- package/src/client.ts +72 -128
- package/src/core/api-promise.ts +4 -4
- package/src/core/error.ts +2 -2
- package/src/core/resource.ts +3 -3
- package/src/index.ts +3 -3
- package/src/internal/parse.ts +2 -2
- package/src/internal/shims.ts +1 -1
- package/src/internal/uploads.ts +5 -5
- package/src/internal/utils/base64.ts +3 -3
- package/src/internal/utils/env.ts +4 -2
- package/src/internal/utils/log.ts +3 -3
- package/src/internal/utils/path.ts +2 -2
- package/src/internal/utils/values.ts +6 -6
- package/src/resources/custom-router.ts +168 -0
- package/src/resources/index.ts +20 -32
- package/src/resources/model-router.ts +222 -0
- package/src/resources/models.ts +55 -32
- package/src/resources/preferences.ts +43 -83
- package/src/resources/prompt-adaptation.ts +358 -300
- package/src/version.ts +1 -1
- package/version.d.mts +1 -1
- package/version.d.mts.map +1 -1
- package/version.d.ts +1 -1
- package/version.d.ts.map +1 -1
- package/version.js +1 -1
- package/version.js.map +1 -1
- package/version.mjs +1 -1
- package/version.mjs.map +1 -1
- package/resources/admin.d.mts +0 -4
- package/resources/admin.d.mts.map +0 -1
- package/resources/admin.d.ts +0 -4
- package/resources/admin.d.ts.map +0 -1
- package/resources/admin.js +0 -9
- package/resources/admin.js.map +0 -1
- package/resources/admin.mjs +0 -5
- package/resources/admin.mjs.map +0 -1
- package/resources/report.d.mts +0 -245
- package/resources/report.d.mts.map +0 -1
- package/resources/report.d.ts +0 -245
- package/resources/report.d.ts.map +0 -1
- package/resources/report.js +0 -86
- package/resources/report.js.map +0 -1
- package/resources/report.mjs +0 -82
- package/resources/report.mjs.map +0 -1
- package/resources/routing.d.mts +0 -391
- package/resources/routing.d.mts.map +0 -1
- package/resources/routing.d.ts +0 -391
- package/resources/routing.d.ts.map +0 -1
- package/resources/routing.js +0 -163
- package/resources/routing.js.map +0 -1
- package/resources/routing.mjs +0 -159
- package/resources/routing.mjs.map +0 -1
- package/src/resources/admin.ts +0 -5
- package/src/resources/report.ts +0 -300
- package/src/resources/routing.ts +0 -476
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompt-adaptation.d.ts","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,KAAK,mBAAmB;OACxB,EAAE,UAAU,EAAE;
|
|
1
|
+
{"version":3,"file":"prompt-adaptation.d.ts","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":"OAEO,EAAE,WAAW,EAAE;OACf,KAAK,mBAAmB;OACxB,EAAE,UAAU,EAAE;OACd,EAAE,cAAc,EAAE;AAGzB,qBAAa,gBAAiB,SAAQ,WAAW;IAC/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqGG;IACH,KAAK,CACH,IAAI,EAAE,2BAA2B,EACjC,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,6BAA6B,CAAC;IAI5C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwDG;IACH,eAAe,CACb,eAAe,EAAE,MAAM,EACvB,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,uCAAuC,CAAC;IAItD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,cAAc,CACZ,eAAe,EAAE,MAAM,EACvB,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,sCAAsC,CAAC;IAIrD;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,OAAO,CAAC,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,UAAU,CAAC,+BAA+B,CAAC;CAGxG;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,MAAM,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IAElC;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,YAAY,GAAG,WAAW,GAAG,QAAQ,CAAC;AAErF;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE/B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE5B;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAExB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,6BAA6B;IAC5C;;;OAGG;IACH,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,uCAAuC;IACtD;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,UAAU,EAAE,SAAS,CAAC;IAEtB;;OAEG;IACH,aAAa,EAAE,KAAK,CAAC,uCAAuC,CAAC,WAAW,CAAC,CAAC;IAE1E;;OAEG;IACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAE1B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC;;;OAGG;IACH,mBAAmB,CAAC,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,CAAC;IAEhD;;;;;;;;;;;;;OAaG;IACH,YAAY,CAAC,EAAE,uCAAuC,CAAC,WAAW,GAAG,IAAI,CAAC;IAE1E;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,yBAAiB,uCAAuC,CAAC;IACvD;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,UAAiB,WAAW;QAC1B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAEpB,UAAU,EAAE,MAAM,CAAC;QAEnB,uBAAuB,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,GAAG,IAAI,CAAC;QAE3D,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAC;QAEvC,sBAAsB,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,GAAG,IAAI,CAAC;QAE1D,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;QAEtC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;QAEzB;;;;;;;;;;;WAWG;QACH,aAAa,CAAC,EAAE,mBAAmB,CAAC,SAAS,GAAG,IAAI,CAAC;QAErD;;;WAGG;QACH,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE9B;;;WAGG;QACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAEtC;;;WAGG;QACH,4BAA4B,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;KACrD;IAED;;;;;;;;;;;;;OAaG;IACH,UAAiB,WAAW;QAC1B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAEpB,KAAK,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,GAAG,IAAI,CAAC;QAEzC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAE1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QAErB;;;;;;;;;;;WAWG;QACH,aAAa,CAAC,EAAE,mBAAmB,CAAC,SAAS,GAAG,IAAI,CAAC;QAErD;;WAEG;QACH,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAE9B;;WAEG;QACH,qBAAqB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;KACvC;CACF;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,sCAAsC;IACrD;;;OAGG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAE1B;;OAEG;IACH,MAAM,EAAE,SAAS,CAAC;IAElB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,+BAA+B;IAC9C;;OAEG;IACH,iBAAiB,EAAE,MAAM,CAAC;IAE1B;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,aAAa,EAAE,KAAK,CAAC,+BAA+B,CAAC,WAAW,CAAC,CAAC;CACnE;AAED,yBAAiB,+BAA+B,CAAC;IAC/C;;;;;OAKG;IACH,UAAiB,WAAW;QAC1B;;WAEG;QACH,EAAE,EAAE,MAAM,CAAC;QAEX;;WAEG;QACH,iBAAiB,EAAE,MAAM,CAAC;QAE1B;;WAEG;QACH,UAAU,EAAE,MAAM,CAAC;QAEnB;;WAEG;QACH,YAAY,EAAE,MAAM,CAAC;QAErB;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,eAAe,EAAE,MAAM,CAAC;QAExB;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,QAAQ,EAAE,MAAM,CAAC;QAEjB;;;WAGG;QACH,SAAS,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,SAAS,EAAE,MAAM,CAAC;QAElB;;WAEG;QACH,UAAU,EAAE,MAAM,CAAC;QAEnB;;WAEG;QACH,OAAO,EAAE,MAAM,CAAC;KACjB;CACF;AAED,MAAM,WAAW,2BAA2B;IAC1C;;;OAGG;IACH,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAEtB;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,aAAa,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;IAEtC;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAElC;;;OAGG;IACH,OAAO,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC;IAErC;;OAEG;IACH,YAAY,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IAEtC;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAE9C;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,YAAY,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC;IAE1C;;;OAGG;IACH,aAAa,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC;CAC5C;AAED,MAAM,CAAC,OAAO,WAAW,gBAAgB,CAAC;IACxC,OAAO,EACL,KAAK,YAAY,IAAI,YAAY,EACjC,KAAK,SAAS,IAAI,SAAS,EAC3B,KAAK,eAAe,IAAI,eAAe,EACvC,KAAK,6BAA6B,IAAI,6BAA6B,EACnE,KAAK,uCAAuC,IAAI,uCAAuC,EACvF,KAAK,sCAAsC,IAAI,sCAAsC,EACrF,KAAK,+BAA+B,IAAI,+BAA+B,EACvE,KAAK,2BAA2B,IAAI,2BAA2B,GAChE,CAAC;CACH"}
|
|
@@ -3,21 +3,20 @@
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
4
|
exports.PromptAdaptation = void 0;
|
|
5
5
|
const resource_1 = require("../core/resource.js");
|
|
6
|
-
const headers_1 = require("../internal/headers.js");
|
|
7
6
|
const path_1 = require("../internal/utils/path.js");
|
|
8
7
|
class PromptAdaptation extends resource_1.APIResource {
|
|
9
8
|
/**
|
|
10
9
|
* Adapt your prompt from one LLM to work optimally across different target LLMs.
|
|
11
10
|
*
|
|
12
11
|
* This endpoint automatically optimizes your prompt (system prompt + user message
|
|
13
|
-
* template) to
|
|
14
|
-
*
|
|
15
|
-
*
|
|
12
|
+
* template) to improve accuracy on your use case across various models. Each model
|
|
13
|
+
* has unique characteristics, and what works well for GPT-5 might not work as well
|
|
14
|
+
* for Claude or Gemini.
|
|
16
15
|
*
|
|
17
16
|
* **How Prompt Adaptation Works:**
|
|
18
17
|
*
|
|
19
|
-
* 1. You provide your current prompt
|
|
20
|
-
* 2. You specify target models you want to adapt to
|
|
18
|
+
* 1. You provide your current prompt and optionally your current origin model
|
|
19
|
+
* 2. You specify the target models you want to adapt your prompt to
|
|
21
20
|
* 3. You provide evaluation examples (golden records) with expected answers
|
|
22
21
|
* 4. The system runs optimization to find the best prompt for each target model
|
|
23
22
|
* 5. You receive adapted prompts that perform well on your target models
|
|
@@ -25,14 +24,19 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
25
24
|
* **Evaluation Metrics:** Choose either a standard metric or provide custom
|
|
26
25
|
* evaluation:
|
|
27
26
|
*
|
|
28
|
-
* - **Standard metrics**: LLMaaJ:
|
|
29
|
-
* JSON_Match
|
|
27
|
+
* - **Standard metrics**: LLMaaJ:Sem_Sim_1 (semantic similarity), JSON_Match
|
|
30
28
|
* - **Custom evaluation**: Provide evaluation_config with your own LLM judge,
|
|
31
29
|
* prompt, and cutoff
|
|
32
30
|
*
|
|
33
31
|
* **Dataset Requirements:**
|
|
34
32
|
*
|
|
35
|
-
* - Minimum
|
|
33
|
+
* - Minimum 25 examples in train_goldens (more examples = better adaptation)
|
|
34
|
+
* - **Prototype mode**: Set `prototype_mode: true` to use as few as 3 examples for
|
|
35
|
+
* prototyping
|
|
36
|
+
* - Recommended when you don't have enough data yet to build a proof-of-concept
|
|
37
|
+
* - Note: Performance may be degraded compared to standard mode (25+ examples)
|
|
38
|
+
* - Trade-off: Faster iteration with less data vs. potentially less
|
|
39
|
+
* generalizability
|
|
36
40
|
* - Each example must have fields matching your template placeholders
|
|
37
41
|
* - Supervised evaluation requires 'answer' field in each golden record
|
|
38
42
|
* - Unsupervised evaluation can work without answers
|
|
@@ -43,21 +47,6 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
43
47
|
* - Time depends on: number of target models, dataset size, model availability
|
|
44
48
|
* - Use the returned adaptation_run_id to check status and retrieve results
|
|
45
49
|
*
|
|
46
|
-
* **Subscription Tiers:**
|
|
47
|
-
*
|
|
48
|
-
* - Free: 1 target model
|
|
49
|
-
* - Starter: 3 target models
|
|
50
|
-
* - Startup: 5 target models
|
|
51
|
-
* - Enterprise: 10 target models
|
|
52
|
-
*
|
|
53
|
-
* **Best Practices:**
|
|
54
|
-
*
|
|
55
|
-
* 1. Use diverse, representative examples from your production workload
|
|
56
|
-
* 2. Include 10-20 examples for best results (5 minimum)
|
|
57
|
-
* 3. Ensure consistent evaluation across all examples
|
|
58
|
-
* 4. Test both train_goldens and test_goldens split for validation
|
|
59
|
-
* 5. Use the same model versions you'll use in production
|
|
60
|
-
*
|
|
61
50
|
* **Example Workflow:**
|
|
62
51
|
*
|
|
63
52
|
* ```
|
|
@@ -78,7 +67,7 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
78
67
|
* system_prompt: 'You are a helpful assistant that answers questions accurately.',
|
|
79
68
|
* target_models: [
|
|
80
69
|
* { provider: 'anthropic', model: 'claude-sonnet-4-5-20250929' },
|
|
81
|
-
* { provider: 'google', model: 'gemini-
|
|
70
|
+
* { provider: 'google', model: 'gemini-2.5-flash' },
|
|
82
71
|
* ],
|
|
83
72
|
* template: 'Question: {question}\nAnswer:',
|
|
84
73
|
* evaluation_metric: 'LLMaaJ:Sem_Sim_3',
|
|
@@ -149,15 +138,6 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
149
138
|
* 3. Apply the optimized prompts when calling the respective target models
|
|
150
139
|
* 4. Compare pre/post optimization scores to see improvement
|
|
151
140
|
*
|
|
152
|
-
* **Evaluation Scores:**
|
|
153
|
-
*
|
|
154
|
-
* - Scores range from 0-10 (higher is better)
|
|
155
|
-
* - Compare origin_model score with target_models pre_optimization_score for
|
|
156
|
-
* baseline
|
|
157
|
-
* - Compare pre_optimization_score with post_optimization_score to see improvement
|
|
158
|
-
* from adaptation
|
|
159
|
-
* - Typical improvements range from 5-30% on evaluation metrics
|
|
160
|
-
*
|
|
161
141
|
* **Status Handling:**
|
|
162
142
|
*
|
|
163
143
|
* - If adaptation is still processing, target model results will have
|
|
@@ -181,7 +161,7 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
181
161
|
*
|
|
182
162
|
* @example
|
|
183
163
|
* ```ts
|
|
184
|
-
* const
|
|
164
|
+
* const response =
|
|
185
165
|
* await client.promptAdaptation.getAdaptResults(
|
|
186
166
|
* 'adaptation_run_id',
|
|
187
167
|
* );
|
|
@@ -190,43 +170,6 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
190
170
|
getAdaptResults(adaptationRunID, options) {
|
|
191
171
|
return this._client.get((0, path_1.path) `/v2/prompt/adaptResults/${adaptationRunID}`, options);
|
|
192
172
|
}
|
|
193
|
-
/**
|
|
194
|
-
* Get Adapt Run Results
|
|
195
|
-
*
|
|
196
|
-
* @example
|
|
197
|
-
* ```ts
|
|
198
|
-
* const adaptationRunResults =
|
|
199
|
-
* await client.promptAdaptation.getAdaptRunResults(
|
|
200
|
-
* 'adaptation_run_id',
|
|
201
|
-
* { user_id: 'user_id', 'x-token': 'x-token' },
|
|
202
|
-
* );
|
|
203
|
-
* ```
|
|
204
|
-
*/
|
|
205
|
-
getAdaptRunResults(adaptationRunID, params, options) {
|
|
206
|
-
const { user_id, 'x-token': xToken } = params;
|
|
207
|
-
return this._client.get((0, path_1.path) `/v2/prompt/frontendAdaptRunResults/${user_id}/${adaptationRunID}`, {
|
|
208
|
-
...options,
|
|
209
|
-
headers: (0, headers_1.buildHeaders)([{ 'x-token': xToken }, options?.headers]),
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
/**
|
|
213
|
-
* Get Adapt Runs
|
|
214
|
-
*
|
|
215
|
-
* @example
|
|
216
|
-
* ```ts
|
|
217
|
-
* const adaptationRunResults =
|
|
218
|
-
* await client.promptAdaptation.getAdaptRuns('user_id', {
|
|
219
|
-
* 'x-token': 'x-token',
|
|
220
|
-
* });
|
|
221
|
-
* ```
|
|
222
|
-
*/
|
|
223
|
-
getAdaptRuns(userID, params, options) {
|
|
224
|
-
const { 'x-token': xToken } = params;
|
|
225
|
-
return this._client.get((0, path_1.path) `/v2/prompt/frontendAdaptRuns/${userID}`, {
|
|
226
|
-
...options,
|
|
227
|
-
headers: (0, headers_1.buildHeaders)([{ 'x-token': xToken }, options?.headers]),
|
|
228
|
-
});
|
|
229
|
-
}
|
|
230
173
|
/**
|
|
231
174
|
* Check the status of a prompt adaptation run.
|
|
232
175
|
*
|
|
@@ -271,18 +214,32 @@ class PromptAdaptation extends resource_1.APIResource {
|
|
|
271
214
|
return this._client.get((0, path_1.path) `/v2/prompt/adaptStatus/${adaptationRunID}`, options);
|
|
272
215
|
}
|
|
273
216
|
/**
|
|
274
|
-
* Get LLM costs for a specific adaptation run
|
|
217
|
+
* Get LLM usage costs for a specific prompt adaptation run.
|
|
218
|
+
*
|
|
219
|
+
* This endpoint returns the total cost and detailed usage records for all LLM
|
|
220
|
+
* requests made during a prompt adaptation run. Use this to track costs associated
|
|
221
|
+
* with optimizing prompts for different target models.
|
|
222
|
+
*
|
|
223
|
+
* **Cost Breakdown:**
|
|
224
|
+
*
|
|
225
|
+
* - Total cost across all models used in the adaptation
|
|
226
|
+
* - Individual usage records with provider, model, tokens, and costs
|
|
227
|
+
* - Timestamps for each LLM request
|
|
228
|
+
*
|
|
229
|
+
* **Access Control:**
|
|
230
|
+
*
|
|
231
|
+
* - Only accessible by the user who created the adaptation run
|
|
232
|
+
* - Requires prompt adaptation access
|
|
275
233
|
*
|
|
276
234
|
* @example
|
|
277
235
|
* ```ts
|
|
278
|
-
* const response =
|
|
279
|
-
*
|
|
280
|
-
*
|
|
281
|
-
* );
|
|
236
|
+
* const response = await client.promptAdaptation.getCost(
|
|
237
|
+
* 'adaptation_run_id',
|
|
238
|
+
* );
|
|
282
239
|
* ```
|
|
283
240
|
*/
|
|
284
|
-
|
|
285
|
-
return this._client.get((0, path_1.path) `/
|
|
241
|
+
getCost(adaptationRunID, options) {
|
|
242
|
+
return this._client.get((0, path_1.path) `/v2/prompt/adapt/${adaptationRunID}/costs`, options);
|
|
286
243
|
}
|
|
287
244
|
}
|
|
288
245
|
exports.PromptAdaptation = PromptAdaptation;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompt-adaptation.js","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kDAA+C;
|
|
1
|
+
{"version":3,"file":"prompt-adaptation.js","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kDAA+C;AAI/C,oDAA8C;AAE9C,MAAa,gBAAiB,SAAQ,sBAAW;IAC/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqGG;IACH,KAAK,CACH,IAAiC,EACjC,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwDG;IACH,eAAe,CACb,eAAuB,EACvB,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,2BAA2B,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IACrF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,cAAc,CACZ,eAAuB,EACvB,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,0BAA0B,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,OAAO,CAAC,eAAuB,EAAE,OAAwB;QACvD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,oBAAoB,eAAe,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;CACF;AAzPD,4CAyPC"}
|
|
@@ -1,20 +1,19 @@
|
|
|
1
1
|
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
|
2
2
|
import { APIResource } from "../core/resource.mjs";
|
|
3
|
-
import { buildHeaders } from "../internal/headers.mjs";
|
|
4
3
|
import { path } from "../internal/utils/path.mjs";
|
|
5
4
|
export class PromptAdaptation extends APIResource {
|
|
6
5
|
/**
|
|
7
6
|
* Adapt your prompt from one LLM to work optimally across different target LLMs.
|
|
8
7
|
*
|
|
9
8
|
* This endpoint automatically optimizes your prompt (system prompt + user message
|
|
10
|
-
* template) to
|
|
11
|
-
*
|
|
12
|
-
*
|
|
9
|
+
* template) to improve accuracy on your use case across various models. Each model
|
|
10
|
+
* has unique characteristics, and what works well for GPT-5 might not work as well
|
|
11
|
+
* for Claude or Gemini.
|
|
13
12
|
*
|
|
14
13
|
* **How Prompt Adaptation Works:**
|
|
15
14
|
*
|
|
16
|
-
* 1. You provide your current prompt
|
|
17
|
-
* 2. You specify target models you want to adapt to
|
|
15
|
+
* 1. You provide your current prompt and optionally your current origin model
|
|
16
|
+
* 2. You specify the target models you want to adapt your prompt to
|
|
18
17
|
* 3. You provide evaluation examples (golden records) with expected answers
|
|
19
18
|
* 4. The system runs optimization to find the best prompt for each target model
|
|
20
19
|
* 5. You receive adapted prompts that perform well on your target models
|
|
@@ -22,14 +21,19 @@ export class PromptAdaptation extends APIResource {
|
|
|
22
21
|
* **Evaluation Metrics:** Choose either a standard metric or provide custom
|
|
23
22
|
* evaluation:
|
|
24
23
|
*
|
|
25
|
-
* - **Standard metrics**: LLMaaJ:
|
|
26
|
-
* JSON_Match
|
|
24
|
+
* - **Standard metrics**: LLMaaJ:Sem_Sim_1 (semantic similarity), JSON_Match
|
|
27
25
|
* - **Custom evaluation**: Provide evaluation_config with your own LLM judge,
|
|
28
26
|
* prompt, and cutoff
|
|
29
27
|
*
|
|
30
28
|
* **Dataset Requirements:**
|
|
31
29
|
*
|
|
32
|
-
* - Minimum
|
|
30
|
+
* - Minimum 25 examples in train_goldens (more examples = better adaptation)
|
|
31
|
+
* - **Prototype mode**: Set `prototype_mode: true` to use as few as 3 examples for
|
|
32
|
+
* prototyping
|
|
33
|
+
* - Recommended when you don't have enough data yet to build a proof-of-concept
|
|
34
|
+
* - Note: Performance may be degraded compared to standard mode (25+ examples)
|
|
35
|
+
* - Trade-off: Faster iteration with less data vs. potentially less
|
|
36
|
+
* generalizability
|
|
33
37
|
* - Each example must have fields matching your template placeholders
|
|
34
38
|
* - Supervised evaluation requires 'answer' field in each golden record
|
|
35
39
|
* - Unsupervised evaluation can work without answers
|
|
@@ -40,21 +44,6 @@ export class PromptAdaptation extends APIResource {
|
|
|
40
44
|
* - Time depends on: number of target models, dataset size, model availability
|
|
41
45
|
* - Use the returned adaptation_run_id to check status and retrieve results
|
|
42
46
|
*
|
|
43
|
-
* **Subscription Tiers:**
|
|
44
|
-
*
|
|
45
|
-
* - Free: 1 target model
|
|
46
|
-
* - Starter: 3 target models
|
|
47
|
-
* - Startup: 5 target models
|
|
48
|
-
* - Enterprise: 10 target models
|
|
49
|
-
*
|
|
50
|
-
* **Best Practices:**
|
|
51
|
-
*
|
|
52
|
-
* 1. Use diverse, representative examples from your production workload
|
|
53
|
-
* 2. Include 10-20 examples for best results (5 minimum)
|
|
54
|
-
* 3. Ensure consistent evaluation across all examples
|
|
55
|
-
* 4. Test both train_goldens and test_goldens split for validation
|
|
56
|
-
* 5. Use the same model versions you'll use in production
|
|
57
|
-
*
|
|
58
47
|
* **Example Workflow:**
|
|
59
48
|
*
|
|
60
49
|
* ```
|
|
@@ -75,7 +64,7 @@ export class PromptAdaptation extends APIResource {
|
|
|
75
64
|
* system_prompt: 'You are a helpful assistant that answers questions accurately.',
|
|
76
65
|
* target_models: [
|
|
77
66
|
* { provider: 'anthropic', model: 'claude-sonnet-4-5-20250929' },
|
|
78
|
-
* { provider: 'google', model: 'gemini-
|
|
67
|
+
* { provider: 'google', model: 'gemini-2.5-flash' },
|
|
79
68
|
* ],
|
|
80
69
|
* template: 'Question: {question}\nAnswer:',
|
|
81
70
|
* evaluation_metric: 'LLMaaJ:Sem_Sim_3',
|
|
@@ -146,15 +135,6 @@ export class PromptAdaptation extends APIResource {
|
|
|
146
135
|
* 3. Apply the optimized prompts when calling the respective target models
|
|
147
136
|
* 4. Compare pre/post optimization scores to see improvement
|
|
148
137
|
*
|
|
149
|
-
* **Evaluation Scores:**
|
|
150
|
-
*
|
|
151
|
-
* - Scores range from 0-10 (higher is better)
|
|
152
|
-
* - Compare origin_model score with target_models pre_optimization_score for
|
|
153
|
-
* baseline
|
|
154
|
-
* - Compare pre_optimization_score with post_optimization_score to see improvement
|
|
155
|
-
* from adaptation
|
|
156
|
-
* - Typical improvements range from 5-30% on evaluation metrics
|
|
157
|
-
*
|
|
158
138
|
* **Status Handling:**
|
|
159
139
|
*
|
|
160
140
|
* - If adaptation is still processing, target model results will have
|
|
@@ -178,7 +158,7 @@ export class PromptAdaptation extends APIResource {
|
|
|
178
158
|
*
|
|
179
159
|
* @example
|
|
180
160
|
* ```ts
|
|
181
|
-
* const
|
|
161
|
+
* const response =
|
|
182
162
|
* await client.promptAdaptation.getAdaptResults(
|
|
183
163
|
* 'adaptation_run_id',
|
|
184
164
|
* );
|
|
@@ -187,43 +167,6 @@ export class PromptAdaptation extends APIResource {
|
|
|
187
167
|
getAdaptResults(adaptationRunID, options) {
|
|
188
168
|
return this._client.get(path `/v2/prompt/adaptResults/${adaptationRunID}`, options);
|
|
189
169
|
}
|
|
190
|
-
/**
|
|
191
|
-
* Get Adapt Run Results
|
|
192
|
-
*
|
|
193
|
-
* @example
|
|
194
|
-
* ```ts
|
|
195
|
-
* const adaptationRunResults =
|
|
196
|
-
* await client.promptAdaptation.getAdaptRunResults(
|
|
197
|
-
* 'adaptation_run_id',
|
|
198
|
-
* { user_id: 'user_id', 'x-token': 'x-token' },
|
|
199
|
-
* );
|
|
200
|
-
* ```
|
|
201
|
-
*/
|
|
202
|
-
getAdaptRunResults(adaptationRunID, params, options) {
|
|
203
|
-
const { user_id, 'x-token': xToken } = params;
|
|
204
|
-
return this._client.get(path `/v2/prompt/frontendAdaptRunResults/${user_id}/${adaptationRunID}`, {
|
|
205
|
-
...options,
|
|
206
|
-
headers: buildHeaders([{ 'x-token': xToken }, options?.headers]),
|
|
207
|
-
});
|
|
208
|
-
}
|
|
209
|
-
/**
|
|
210
|
-
* Get Adapt Runs
|
|
211
|
-
*
|
|
212
|
-
* @example
|
|
213
|
-
* ```ts
|
|
214
|
-
* const adaptationRunResults =
|
|
215
|
-
* await client.promptAdaptation.getAdaptRuns('user_id', {
|
|
216
|
-
* 'x-token': 'x-token',
|
|
217
|
-
* });
|
|
218
|
-
* ```
|
|
219
|
-
*/
|
|
220
|
-
getAdaptRuns(userID, params, options) {
|
|
221
|
-
const { 'x-token': xToken } = params;
|
|
222
|
-
return this._client.get(path `/v2/prompt/frontendAdaptRuns/${userID}`, {
|
|
223
|
-
...options,
|
|
224
|
-
headers: buildHeaders([{ 'x-token': xToken }, options?.headers]),
|
|
225
|
-
});
|
|
226
|
-
}
|
|
227
170
|
/**
|
|
228
171
|
* Check the status of a prompt adaptation run.
|
|
229
172
|
*
|
|
@@ -268,18 +211,32 @@ export class PromptAdaptation extends APIResource {
|
|
|
268
211
|
return this._client.get(path `/v2/prompt/adaptStatus/${adaptationRunID}`, options);
|
|
269
212
|
}
|
|
270
213
|
/**
|
|
271
|
-
* Get LLM costs for a specific adaptation run
|
|
214
|
+
* Get LLM usage costs for a specific prompt adaptation run.
|
|
215
|
+
*
|
|
216
|
+
* This endpoint returns the total cost and detailed usage records for all LLM
|
|
217
|
+
* requests made during a prompt adaptation run. Use this to track costs associated
|
|
218
|
+
* with optimizing prompts for different target models.
|
|
219
|
+
*
|
|
220
|
+
* **Cost Breakdown:**
|
|
221
|
+
*
|
|
222
|
+
* - Total cost across all models used in the adaptation
|
|
223
|
+
* - Individual usage records with provider, model, tokens, and costs
|
|
224
|
+
* - Timestamps for each LLM request
|
|
225
|
+
*
|
|
226
|
+
* **Access Control:**
|
|
227
|
+
*
|
|
228
|
+
* - Only accessible by the user who created the adaptation run
|
|
229
|
+
* - Requires prompt adaptation access
|
|
272
230
|
*
|
|
273
231
|
* @example
|
|
274
232
|
* ```ts
|
|
275
|
-
* const response =
|
|
276
|
-
*
|
|
277
|
-
*
|
|
278
|
-
* );
|
|
233
|
+
* const response = await client.promptAdaptation.getCost(
|
|
234
|
+
* 'adaptation_run_id',
|
|
235
|
+
* );
|
|
279
236
|
* ```
|
|
280
237
|
*/
|
|
281
|
-
|
|
282
|
-
return this._client.get(path `/
|
|
238
|
+
getCost(adaptationRunID, options) {
|
|
239
|
+
return this._client.get(path `/v2/prompt/adapt/${adaptationRunID}/costs`, options);
|
|
283
240
|
}
|
|
284
241
|
}
|
|
285
242
|
//# sourceMappingURL=prompt-adaptation.mjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompt-adaptation.mjs","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;
|
|
1
|
+
{"version":3,"file":"prompt-adaptation.mjs","sourceRoot":"","sources":["../src/resources/prompt-adaptation.ts"],"names":[],"mappings":"AAAA,sFAAsF;OAE/E,EAAE,WAAW,EAAE;OAIf,EAAE,IAAI,EAAE;AAEf,MAAM,OAAO,gBAAiB,SAAQ,WAAW;IAC/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqGG;IACH,KAAK,CACH,IAAiC,EACjC,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwDG;IACH,eAAe,CACb,eAAuB,EACvB,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,2BAA2B,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IACrF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAuCG;IACH,cAAc,CACZ,eAAuB,EACvB,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,0BAA0B,eAAe,EAAE,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,OAAO,CAAC,eAAuB,EAAE,OAAwB;QACvD,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,oBAAoB,eAAe,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpF,CAAC;CACF"}
|