@prompt-helm/sdk 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -6
- package/dist/index.cjs +17 -6
- package/dist/index.d.cts +18 -1
- package/dist/index.d.ts +18 -1
- package/dist/index.js +17 -6
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -14,7 +14,9 @@ refresh, 60 seconds by default.
|
|
|
14
14
|
a refresh that finds nothing new is an empty `304`.
|
|
15
15
|
- **Your data stays with you.** Variables are filled in on your server. PromptHelm never sees
|
|
16
16
|
their values, your provider keys or your LLM traffic.
|
|
17
|
-
- **
|
|
17
|
+
- **Model settings travel with the text.** A version can carry its model, temperature and
|
|
18
|
+
output limit, so moving to a cheaper model is a publish you can roll back or A/B test.
|
|
19
|
+
- **No dependencies.** ESM and CommonJS, Node.js 18 and later.
|
|
18
20
|
|
|
19
21
|
## Install
|
|
20
22
|
|
|
@@ -108,6 +110,32 @@ Run it in CI before the build, so every deploy carries the text that was live th
|
|
|
108
110
|
`PROMPTHELM_API_KEY` (and `PROMPTHELM_BASE_URL`), and writes JSON for a `.json` path or with
|
|
109
111
|
`--format json`. A prompt appears in the file once it's published.
|
|
110
112
|
|
|
113
|
+
## Model settings
|
|
114
|
+
|
|
115
|
+
A version can carry the model to call it with, its temperature and its output limit, set next to
|
|
116
|
+
the text in the dashboard. Moving to a cheaper model is then a publish like any other: one click
|
|
117
|
+
rolls it back, and an A/B test can tell whether it does as well. `resolve` hands them over with
|
|
118
|
+
the text:
|
|
119
|
+
|
|
120
|
+
```ts
|
|
121
|
+
const r = await ph.resolve("support-reply", {
|
|
122
|
+
vars: { company: "Acme", customer_name: user.name },
|
|
123
|
+
fallback: SUPPORT_REPLY,
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
const reply = await openai.responses.create({
|
|
127
|
+
model: r.config.model ?? "gpt-5",
|
|
128
|
+
temperature: r.config.temperature,
|
|
129
|
+
max_output_tokens: r.config.maxTokens,
|
|
130
|
+
input: r.text,
|
|
131
|
+
});
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
`config` holds only the settings set in the dashboard (`model`, `temperature`, `maxTokens`), so
|
|
135
|
+
keep your defaults in the code; when the fallback is served it is empty. PromptHelm doesn't check
|
|
136
|
+
a model name with your provider, and a name your provider doesn't know fails your calls: try a new
|
|
137
|
+
model in an A/B test first. Apps on SDK versions before 1.4 ignore the settings.
|
|
138
|
+
|
|
111
139
|
## A/B tests
|
|
112
140
|
|
|
113
141
|
On the Pro plan, a prompt's page can run an experiment: the live version keeps most calls and a
|
|
@@ -123,14 +151,15 @@ const r = await ph.resolve("support-reply", {
|
|
|
123
151
|
unit: user.id,
|
|
124
152
|
});
|
|
125
153
|
|
|
126
|
-
const reply = await openai.responses.create({ model: "gpt-5", input: r.text });
|
|
154
|
+
const reply = await openai.responses.create({ model: r.config.model ?? "gpt-5", input: r.text });
|
|
127
155
|
r.usage(reply.usage); // tokens, and how long the call took since resolve
|
|
128
156
|
r.outcome(userKeptTheReply); // true or false, or a score from 0 to 1
|
|
129
157
|
```
|
|
130
158
|
|
|
131
159
|
The prompt's page then compares the two versions: how often each succeeded, with a range that
|
|
132
160
|
stays honest however often you look, and their tokens and latency per call. It says when there's
|
|
133
|
-
an answer; shipping the variant stays your call.
|
|
161
|
+
an answer; shipping the variant stays your call. Each version brings its own
|
|
162
|
+
[model settings](#model-settings), so a test can compare two models as well as two texts.
|
|
134
163
|
|
|
135
164
|
- **Outcomes that come later.** A purchase days later, in another service: keep `r.ticket` with
|
|
136
165
|
your own record, then call `ph.outcome(ticket, true)` from any process. The ticket names the
|
|
@@ -156,7 +185,7 @@ share only moves units from the live version to the variant, never back. Apps on
|
|
|
156
185
|
before 1.3 ignore experiments and serve the live version to every call.
|
|
157
186
|
|
|
158
187
|
To split calls the same way from another language: during an experiment the response carries
|
|
159
|
-
`experiment: { id, weight, variant: { version, content, variables } }`. A unit's bucket is the
|
|
188
|
+
`experiment: { id, weight, variant: { version, content, variables, config } }`. A unit's bucket is the
|
|
160
189
|
32-bit FNV-1a hash of the UTF-8 bytes of `<experiment id>:<unit>`, modulo 100; buckets below
|
|
161
190
|
`weight` get the variant.
|
|
162
191
|
|
|
@@ -169,8 +198,9 @@ curl https://prompthelm.app/v1/prompts/support-reply \
|
|
|
169
198
|
-H "Authorization: Bearer $PROMPTHELM_API_KEY"
|
|
170
199
|
```
|
|
171
200
|
|
|
172
|
-
|
|
173
|
-
`
|
|
201
|
+
The response carries the live version's `content`, its `variables` and its model settings as
|
|
202
|
+
`config` (only the ones set). `GET /v1/prompts` returns every published prompt of the key's
|
|
203
|
+
project at once, as `{ "prompts": [ … ] }`.
|
|
174
204
|
|
|
175
205
|
## Links
|
|
176
206
|
|
package/dist/index.cjs
CHANGED
|
@@ -151,7 +151,7 @@ var PromptHelm = class {
|
|
|
151
151
|
this.#remember(slug, failure.code);
|
|
152
152
|
if (fallback === void 0) throw failure;
|
|
153
153
|
this.#onError(failure, slug);
|
|
154
|
-
return this.#served(slug, render(fallback, vars), null);
|
|
154
|
+
return this.#served(slug, render(fallback, vars), {}, null);
|
|
155
155
|
}
|
|
156
156
|
}
|
|
157
157
|
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
@@ -210,9 +210,17 @@ var PromptHelm = class {
|
|
|
210
210
|
const test = experiment && !force ? { id: experiment.id, weight: experiment.weight } : void 0;
|
|
211
211
|
let skipped;
|
|
212
212
|
if (experiment && (force ? force === "variant" : bucket(experiment.id, unit) < experiment.weight)) {
|
|
213
|
-
const { version, content } = experiment.variant;
|
|
213
|
+
const { version, content, config = {} } = experiment.variant;
|
|
214
214
|
try {
|
|
215
|
-
return this.#served(
|
|
215
|
+
return this.#served(
|
|
216
|
+
slug,
|
|
217
|
+
render(content, vars),
|
|
218
|
+
config,
|
|
219
|
+
version,
|
|
220
|
+
experiment.id,
|
|
221
|
+
test,
|
|
222
|
+
true
|
|
223
|
+
);
|
|
216
224
|
} catch (error) {
|
|
217
225
|
skipped = toPromptHelmError(error);
|
|
218
226
|
}
|
|
@@ -222,10 +230,10 @@ var PromptHelm = class {
|
|
|
222
230
|
this.#onError(skipped, slug);
|
|
223
231
|
if (test) add(this.#counts(test, experiment.variant.version), SKIPPED, 1);
|
|
224
232
|
}
|
|
225
|
-
return this.#served(slug, text, entry.version, experiment?.id, test);
|
|
233
|
+
return this.#served(slug, text, entry.config, entry.version, experiment?.id, test);
|
|
226
234
|
}
|
|
227
|
-
/** Counts the call, tells `onResolve`, and hands the caller its text and what to report. */
|
|
228
|
-
#served(slug, text, version, experiment, test, variant = false) {
|
|
235
|
+
/** Counts the call, tells `onResolve`, and hands the caller its text, settings and what to report. */
|
|
236
|
+
#served(slug, text, config, version, experiment, test, variant = false) {
|
|
229
237
|
if (test && version !== null) add(this.#counts(test, version), SERVED, 1);
|
|
230
238
|
try {
|
|
231
239
|
this.#onResolve?.({ slug, version, experiment: experiment ?? null, variant });
|
|
@@ -235,6 +243,8 @@ var PromptHelm = class {
|
|
|
235
243
|
let reported = 0;
|
|
236
244
|
return {
|
|
237
245
|
text,
|
|
246
|
+
// A copy: the cached one serves every later call.
|
|
247
|
+
config: { ...config },
|
|
238
248
|
version,
|
|
239
249
|
ticket: test && version !== null ? `${test.id}.${version}.${test.weight}` : null,
|
|
240
250
|
outcome: (value) => {
|
|
@@ -344,6 +354,7 @@ var PromptHelm = class {
|
|
|
344
354
|
const entry = {
|
|
345
355
|
version: payload.version,
|
|
346
356
|
content: payload.content,
|
|
357
|
+
config: payload.config ?? {},
|
|
347
358
|
experiment: payload.experiment,
|
|
348
359
|
etag: response.headers.get("etag"),
|
|
349
360
|
fetchedAt: Date.now()
|
package/dist/index.d.cts
CHANGED
|
@@ -23,9 +23,20 @@ interface PromptPayload {
|
|
|
23
23
|
version: number;
|
|
24
24
|
content: string;
|
|
25
25
|
variables: string[];
|
|
26
|
+
/** The version's model settings; servers before SDK 1.4 send none. */
|
|
27
|
+
config?: PromptConfig;
|
|
26
28
|
/** An A/B test running on the prompt: the live version above is its control. */
|
|
27
29
|
experiment?: PromptExperiment;
|
|
28
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* Model settings published with a version, for your model call. Only the ones set in the
|
|
33
|
+
* dashboard are present; for the rest, your code's defaults apply.
|
|
34
|
+
*/
|
|
35
|
+
interface PromptConfig {
|
|
36
|
+
model?: string;
|
|
37
|
+
temperature?: number;
|
|
38
|
+
maxTokens?: number;
|
|
39
|
+
}
|
|
29
40
|
interface PromptExperiment {
|
|
30
41
|
id: string;
|
|
31
42
|
/** The variant's share of calls, in percent (1–99). */
|
|
@@ -34,6 +45,7 @@ interface PromptExperiment {
|
|
|
34
45
|
version: number;
|
|
35
46
|
content: string;
|
|
36
47
|
variables: string[];
|
|
48
|
+
config?: PromptConfig;
|
|
37
49
|
};
|
|
38
50
|
}
|
|
39
51
|
/**
|
|
@@ -126,6 +138,11 @@ interface ResolveEvent {
|
|
|
126
138
|
}
|
|
127
139
|
interface Resolved {
|
|
128
140
|
text: string;
|
|
141
|
+
/**
|
|
142
|
+
* The served version's model settings, such as `model`. Empty when the fallback was served,
|
|
143
|
+
* so your code's defaults apply.
|
|
144
|
+
*/
|
|
145
|
+
config: PromptConfig;
|
|
129
146
|
/** The version served, or null when the fallback was. */
|
|
130
147
|
version: number | null;
|
|
131
148
|
/**
|
|
@@ -165,4 +182,4 @@ declare const LATENCY_BUCKETS = 24;
|
|
|
165
182
|
declare function latencyBucket(ms: number): number;
|
|
166
183
|
|
|
167
184
|
export { ERROR_CODES, FALLBACK_HEADER, LATENCY_BUCKETS, PromptHelm, PromptHelmError, extractVariables, latencyBucket, render };
|
|
168
|
-
export type { GetOptions, PromptExperiment, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
|
|
185
|
+
export type { GetOptions, PromptConfig, PromptExperiment, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
|
package/dist/index.d.ts
CHANGED
|
@@ -23,9 +23,20 @@ interface PromptPayload {
|
|
|
23
23
|
version: number;
|
|
24
24
|
content: string;
|
|
25
25
|
variables: string[];
|
|
26
|
+
/** The version's model settings; servers before SDK 1.4 send none. */
|
|
27
|
+
config?: PromptConfig;
|
|
26
28
|
/** An A/B test running on the prompt: the live version above is its control. */
|
|
27
29
|
experiment?: PromptExperiment;
|
|
28
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* Model settings published with a version, for your model call. Only the ones set in the
|
|
33
|
+
* dashboard are present; for the rest, your code's defaults apply.
|
|
34
|
+
*/
|
|
35
|
+
interface PromptConfig {
|
|
36
|
+
model?: string;
|
|
37
|
+
temperature?: number;
|
|
38
|
+
maxTokens?: number;
|
|
39
|
+
}
|
|
29
40
|
interface PromptExperiment {
|
|
30
41
|
id: string;
|
|
31
42
|
/** The variant's share of calls, in percent (1–99). */
|
|
@@ -34,6 +45,7 @@ interface PromptExperiment {
|
|
|
34
45
|
version: number;
|
|
35
46
|
content: string;
|
|
36
47
|
variables: string[];
|
|
48
|
+
config?: PromptConfig;
|
|
37
49
|
};
|
|
38
50
|
}
|
|
39
51
|
/**
|
|
@@ -126,6 +138,11 @@ interface ResolveEvent {
|
|
|
126
138
|
}
|
|
127
139
|
interface Resolved {
|
|
128
140
|
text: string;
|
|
141
|
+
/**
|
|
142
|
+
* The served version's model settings, such as `model`. Empty when the fallback was served,
|
|
143
|
+
* so your code's defaults apply.
|
|
144
|
+
*/
|
|
145
|
+
config: PromptConfig;
|
|
129
146
|
/** The version served, or null when the fallback was. */
|
|
130
147
|
version: number | null;
|
|
131
148
|
/**
|
|
@@ -165,4 +182,4 @@ declare const LATENCY_BUCKETS = 24;
|
|
|
165
182
|
declare function latencyBucket(ms: number): number;
|
|
166
183
|
|
|
167
184
|
export { ERROR_CODES, FALLBACK_HEADER, LATENCY_BUCKETS, PromptHelm, PromptHelmError, extractVariables, latencyBucket, render };
|
|
168
|
-
export type { GetOptions, PromptExperiment, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
|
|
185
|
+
export type { GetOptions, PromptConfig, PromptExperiment, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
|
package/dist/index.js
CHANGED
|
@@ -118,7 +118,7 @@ var PromptHelm = class {
|
|
|
118
118
|
this.#remember(slug, failure.code);
|
|
119
119
|
if (fallback === void 0) throw failure;
|
|
120
120
|
this.#onError(failure, slug);
|
|
121
|
-
return this.#served(slug, render(fallback, vars), null);
|
|
121
|
+
return this.#served(slug, render(fallback, vars), {}, null);
|
|
122
122
|
}
|
|
123
123
|
}
|
|
124
124
|
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
@@ -177,9 +177,17 @@ var PromptHelm = class {
|
|
|
177
177
|
const test = experiment && !force ? { id: experiment.id, weight: experiment.weight } : void 0;
|
|
178
178
|
let skipped;
|
|
179
179
|
if (experiment && (force ? force === "variant" : bucket(experiment.id, unit) < experiment.weight)) {
|
|
180
|
-
const { version, content } = experiment.variant;
|
|
180
|
+
const { version, content, config = {} } = experiment.variant;
|
|
181
181
|
try {
|
|
182
|
-
return this.#served(
|
|
182
|
+
return this.#served(
|
|
183
|
+
slug,
|
|
184
|
+
render(content, vars),
|
|
185
|
+
config,
|
|
186
|
+
version,
|
|
187
|
+
experiment.id,
|
|
188
|
+
test,
|
|
189
|
+
true
|
|
190
|
+
);
|
|
183
191
|
} catch (error) {
|
|
184
192
|
skipped = toPromptHelmError(error);
|
|
185
193
|
}
|
|
@@ -189,10 +197,10 @@ var PromptHelm = class {
|
|
|
189
197
|
this.#onError(skipped, slug);
|
|
190
198
|
if (test) add(this.#counts(test, experiment.variant.version), SKIPPED, 1);
|
|
191
199
|
}
|
|
192
|
-
return this.#served(slug, text, entry.version, experiment?.id, test);
|
|
200
|
+
return this.#served(slug, text, entry.config, entry.version, experiment?.id, test);
|
|
193
201
|
}
|
|
194
|
-
/** Counts the call, tells `onResolve`, and hands the caller its text and what to report. */
|
|
195
|
-
#served(slug, text, version, experiment, test, variant = false) {
|
|
202
|
+
/** Counts the call, tells `onResolve`, and hands the caller its text, settings and what to report. */
|
|
203
|
+
#served(slug, text, config, version, experiment, test, variant = false) {
|
|
196
204
|
if (test && version !== null) add(this.#counts(test, version), SERVED, 1);
|
|
197
205
|
try {
|
|
198
206
|
this.#onResolve?.({ slug, version, experiment: experiment ?? null, variant });
|
|
@@ -202,6 +210,8 @@ var PromptHelm = class {
|
|
|
202
210
|
let reported = 0;
|
|
203
211
|
return {
|
|
204
212
|
text,
|
|
213
|
+
// A copy: the cached one serves every later call.
|
|
214
|
+
config: { ...config },
|
|
205
215
|
version,
|
|
206
216
|
ticket: test && version !== null ? `${test.id}.${version}.${test.weight}` : null,
|
|
207
217
|
outcome: (value) => {
|
|
@@ -311,6 +321,7 @@ var PromptHelm = class {
|
|
|
311
321
|
const entry = {
|
|
312
322
|
version: payload.version,
|
|
313
323
|
content: payload.content,
|
|
324
|
+
config: payload.config ?? {},
|
|
314
325
|
experiment: payload.experiment,
|
|
315
326
|
etag: response.headers.get("etag"),
|
|
316
327
|
fetchedAt: Date.now()
|
package/package.json
CHANGED