@prompt-helm/sdk 1.3.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -6
- package/dist/cli.js +5 -1
- package/dist/index.cjs +45 -14
- package/dist/index.d.cts +64 -5
- package/dist/index.d.ts +64 -5
- package/dist/index.js +45 -14
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -14,7 +14,11 @@ refresh, 60 seconds by default.
|
|
|
14
14
|
a refresh that finds nothing new is an empty `304`.
|
|
15
15
|
- **Your data stays with you.** Variables are filled in on your server. PromptHelm never sees
|
|
16
16
|
their values, your provider keys or your LLM traffic.
|
|
17
|
-
- **
|
|
17
|
+
- **Model settings travel with the text.** A version can carry its model, temperature and
|
|
18
|
+
output limit, so moving to a cheaper model is a publish you can roll back or A/B test.
|
|
19
|
+
- **System and user messages.** A prompt can hold the user message too, with its own variables,
|
|
20
|
+
and ask for an answer in JSON that matches a schema.
|
|
21
|
+
- **No dependencies.** ESM and CommonJS, Node.js 18 and later.
|
|
18
22
|
|
|
19
23
|
## Install
|
|
20
24
|
|
|
@@ -48,6 +52,30 @@ The stored prompt marks each variable in double braces, such as `{{customer_name
|
|
|
48
52
|
version uses a variable the call doesn't pass, `get` doesn't send a half-filled prompt: it
|
|
49
53
|
returns the fallback (or, without one, throws `missing_variables`).
|
|
50
54
|
|
|
55
|
+
## System and user messages
|
|
56
|
+
|
|
57
|
+
A prompt's text is its system message. It can also hold the user message, for a request your app
|
|
58
|
+
sends the same way each time with different values. `resolve` fills both in:
|
|
59
|
+
|
|
60
|
+
```ts
|
|
61
|
+
const r = await ph.resolve("classify-ticket", {
|
|
62
|
+
vars: { product: "Acme", ticket: ticket.text },
|
|
63
|
+
fallback: { system: CLASSIFY_TICKET, user: "Ticket: {{ticket}}" },
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
const reply = await openai.responses.create({
|
|
67
|
+
model: r.config.model ?? "gpt-5",
|
|
68
|
+
instructions: r.system,
|
|
69
|
+
input: r.user ?? ticket.text,
|
|
70
|
+
});
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
`user` is `null` when the live version has none: your app sends its own, such as what the person
|
|
74
|
+
typed. `get` returns the system message only and never needs the user message's variables, so an
|
|
75
|
+
app that sends its own user message keeps working when one is added in the dashboard. If the user
|
|
76
|
+
message uses a variable the call doesn't pass, `resolve` returns the fallback, as it does for the
|
|
77
|
+
system message. SDK versions before 1.5 receive the system message only.
|
|
78
|
+
|
|
51
79
|
## Options
|
|
52
80
|
|
|
53
81
|
All optional when the environment variables are set.
|
|
@@ -104,10 +132,66 @@ const system = await ph.get("support-reply", {
|
|
|
104
132
|
});
|
|
105
133
|
```
|
|
106
134
|
|
|
135
|
+
A prompt with a user message is written as `{ system, user }`, the shape `fallback` takes.
|
|
136
|
+
|
|
107
137
|
Run it in CI before the build, so every deploy carries the text that was live then. It reads
|
|
108
138
|
`PROMPTHELM_API_KEY` (and `PROMPTHELM_BASE_URL`), and writes JSON for a `.json` path or with
|
|
109
139
|
`--format json`. A prompt appears in the file once it's published.
|
|
110
140
|
|
|
141
|
+
## Model settings
|
|
142
|
+
|
|
143
|
+
A version can carry the model to call it with, its temperature and its output limit, set next to
|
|
144
|
+
the text in the dashboard. Moving to a cheaper model is then a publish like any other: one click
|
|
145
|
+
rolls it back, and an A/B test can tell whether it does as well. `resolve` hands them over with
|
|
146
|
+
the text:
|
|
147
|
+
|
|
148
|
+
```ts
|
|
149
|
+
const r = await ph.resolve("support-reply", {
|
|
150
|
+
vars: { company: "Acme", customer_name: user.name },
|
|
151
|
+
fallback: SUPPORT_REPLY,
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
const reply = await openai.responses.create({
|
|
155
|
+
model: r.config.model ?? "gpt-5",
|
|
156
|
+
temperature: r.config.temperature,
|
|
157
|
+
max_output_tokens: r.config.maxTokens,
|
|
158
|
+
input: r.text,
|
|
159
|
+
});
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
`config` holds only the settings set in the dashboard (`model`, `temperature`, `maxTokens`,
|
|
163
|
+
`response`), so keep your defaults in the code; when the fallback is served it is empty.
|
|
164
|
+
PromptHelm doesn't check a model name with your provider, and a name your provider doesn't know
|
|
165
|
+
fails your calls: try a new model in an A/B test first. Apps on SDK versions before 1.4 ignore the
|
|
166
|
+
settings.
|
|
167
|
+
|
|
168
|
+
### Answers in JSON
|
|
169
|
+
|
|
170
|
+
A version can ask for an answer in JSON with the fields set in the dashboard. `config.response` is
|
|
171
|
+
then `{ type: "json", name, schema }`: a JSON Schema kept inside what OpenAI, Anthropic, Gemini and
|
|
172
|
+
Amazon Bedrock all accept, so the same schema works with each. Hand it to your provider's
|
|
173
|
+
structured output:
|
|
174
|
+
|
|
175
|
+
```ts
|
|
176
|
+
const { response } = r.config;
|
|
177
|
+
const reply = await openai.responses.create({
|
|
178
|
+
model: r.config.model ?? "gpt-5",
|
|
179
|
+
instructions: r.system,
|
|
180
|
+
input: r.user ?? ticket.text,
|
|
181
|
+
...(response && {
|
|
182
|
+
text: {
|
|
183
|
+
format: { type: "json_schema", name: response.name, schema: response.schema, strict: true },
|
|
184
|
+
},
|
|
185
|
+
}),
|
|
186
|
+
});
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
On Anthropic's API the schema goes in `output_config: { format: { type: "json_schema", schema } }`,
|
|
190
|
+
on Gemini in `responseJsonSchema` with `responseMimeType: "application/json"`, and on Bedrock's
|
|
191
|
+
Converse in `outputConfig.textFormat` (as a string). PromptHelm never sees the answer, so check it
|
|
192
|
+
in your code. Without a response format the model answers in text; SDK versions before 1.5 ignore
|
|
193
|
+
it.
|
|
194
|
+
|
|
111
195
|
## A/B tests
|
|
112
196
|
|
|
113
197
|
On the Pro plan, a prompt's page can run an experiment: the live version keeps most calls and a
|
|
@@ -123,14 +207,15 @@ const r = await ph.resolve("support-reply", {
|
|
|
123
207
|
unit: user.id,
|
|
124
208
|
});
|
|
125
209
|
|
|
126
|
-
const reply = await openai.responses.create({ model: "gpt-5", input: r.text });
|
|
210
|
+
const reply = await openai.responses.create({ model: r.config.model ?? "gpt-5", input: r.text });
|
|
127
211
|
r.usage(reply.usage); // tokens, and how long the call took since resolve
|
|
128
212
|
r.outcome(userKeptTheReply); // true or false, or a score from 0 to 1
|
|
129
213
|
```
|
|
130
214
|
|
|
131
215
|
The prompt's page then compares the two versions: how often each succeeded, with a range that
|
|
132
216
|
stays honest however often you look, and their tokens and latency per call. It says when there's
|
|
133
|
-
an answer; shipping the variant stays your call.
|
|
217
|
+
an answer; shipping the variant stays your call. Each version brings its own
|
|
218
|
+
[model settings](#model-settings), so a test can compare two models as well as two texts.
|
|
134
219
|
|
|
135
220
|
- **Outcomes that come later.** A purchase days later, in another service: keep `r.ticket` with
|
|
136
221
|
your own record, then call `ph.outcome(ticket, true)` from any process. The ticket names the
|
|
@@ -156,7 +241,7 @@ share only moves units from the live version to the variant, never back. Apps on
|
|
|
156
241
|
before 1.3 ignore experiments and serve the live version to every call.
|
|
157
242
|
|
|
158
243
|
To split calls the same way from another language: during an experiment the response carries
|
|
159
|
-
`experiment: { id, weight, variant: { version, content, variables } }`. A unit's bucket is the
|
|
244
|
+
`experiment: { id, weight, variant: { version, content, variables, config } }`. A unit's bucket is the
|
|
160
245
|
32-bit FNV-1a hash of the UTF-8 bytes of `<experiment id>:<unit>`, modulo 100; buckets below
|
|
161
246
|
`weight` get the variant.
|
|
162
247
|
|
|
@@ -169,8 +254,9 @@ curl https://prompthelm.app/v1/prompts/support-reply \
|
|
|
169
254
|
-H "Authorization: Bearer $PROMPTHELM_API_KEY"
|
|
170
255
|
```
|
|
171
256
|
|
|
172
|
-
|
|
173
|
-
`
|
|
257
|
+
The response carries the live version's `content`, its `variables` and its model settings as
|
|
258
|
+
`config` (only the ones set). `GET /v1/prompts` returns every published prompt of the key's
|
|
259
|
+
project at once, as `{ "prompts": [ … ] }`.
|
|
174
260
|
|
|
175
261
|
## Links
|
|
176
262
|
|
package/dist/cli.js
CHANGED
|
@@ -94,7 +94,11 @@ async function fetchLivePrompts(options) {
|
|
|
94
94
|
}
|
|
95
95
|
function renderFallbacks(prompts, format) {
|
|
96
96
|
const sorted = [...prompts].sort((a, b) => a.slug < b.slug ? -1 : a.slug > b.slug ? 1 : 0);
|
|
97
|
-
const
|
|
97
|
+
const entries = sorted.map((p) => [
|
|
98
|
+
p.slug,
|
|
99
|
+
p.user ? { system: p.content, user: p.user.content } : p.content
|
|
100
|
+
]);
|
|
101
|
+
const json = JSON.stringify(Object.fromEntries(entries), null, 2);
|
|
98
102
|
return format === "json" ? `${json}
|
|
99
103
|
` : `${TS_HEADER}export const fallbacks = ${json} as const;
|
|
100
104
|
`;
|
package/dist/index.cjs
CHANGED
|
@@ -106,6 +106,13 @@ var TOKENS_IN = 6;
|
|
|
106
106
|
var TOKENS_OUT = 7;
|
|
107
107
|
var LATENCY = 8;
|
|
108
108
|
var BUCKETS = FIELDS.length;
|
|
109
|
+
function fill(system, user, vars, withUser) {
|
|
110
|
+
const text = typeof user === "string" ? user : user?.content;
|
|
111
|
+
return {
|
|
112
|
+
system: render(system, vars),
|
|
113
|
+
user: withUser && text !== void 0 ? render(text, vars) : null
|
|
114
|
+
};
|
|
115
|
+
}
|
|
109
116
|
var PromptHelm = class {
|
|
110
117
|
#apiKey;
|
|
111
118
|
#baseUrl;
|
|
@@ -137,21 +144,31 @@ var PromptHelm = class {
|
|
|
137
144
|
this.#reportFallbacks = options.reportFallbacks ?? true;
|
|
138
145
|
this.#reportResults = options.reportResults ?? this.#reportFallbacks;
|
|
139
146
|
}
|
|
140
|
-
/**
|
|
147
|
+
/**
|
|
148
|
+
* The live version's system message with its variables filled in. It never needs the user
|
|
149
|
+
* message's variables: an app that calls `get` sends its own.
|
|
150
|
+
*/
|
|
141
151
|
async get(slug, options = {}) {
|
|
142
|
-
return (await this
|
|
152
|
+
return (await this.#resolve(slug, options, false)).text;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Like `get`, with the user message too, the model settings, and which version was served, so
|
|
156
|
+
* you can tag your logs or traces with it.
|
|
157
|
+
*/
|
|
158
|
+
resolve(slug, options = {}) {
|
|
159
|
+
return this.#resolve(slug, options, true);
|
|
143
160
|
}
|
|
144
|
-
|
|
145
|
-
async resolve(slug, options = {}) {
|
|
161
|
+
async #resolve(slug, options, withUser) {
|
|
146
162
|
const { vars, fallback } = options;
|
|
147
163
|
try {
|
|
148
|
-
return this.#pick(slug, await this.#entry(slug), options);
|
|
164
|
+
return this.#pick(slug, await this.#entry(slug), options, withUser);
|
|
149
165
|
} catch (error) {
|
|
150
166
|
const failure = toPromptHelmError(error);
|
|
151
167
|
this.#remember(slug, failure.code);
|
|
152
168
|
if (fallback === void 0) throw failure;
|
|
153
169
|
this.#onError(failure, slug);
|
|
154
|
-
|
|
170
|
+
const own = typeof fallback === "string" ? { system: fallback } : fallback;
|
|
171
|
+
return this.#served(slug, fill(own.system, own.user, vars, withUser), {}, null);
|
|
155
172
|
}
|
|
156
173
|
}
|
|
157
174
|
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
@@ -204,28 +221,36 @@ var PromptHelm = class {
|
|
|
204
221
|
}
|
|
205
222
|
}
|
|
206
223
|
}
|
|
207
|
-
#pick(slug, entry, options) {
|
|
224
|
+
#pick(slug, entry, options, withUser) {
|
|
208
225
|
const { vars, unit, force } = options;
|
|
209
226
|
const { experiment } = entry;
|
|
210
227
|
const test = experiment && !force ? { id: experiment.id, weight: experiment.weight } : void 0;
|
|
211
228
|
let skipped;
|
|
212
229
|
if (experiment && (force ? force === "variant" : bucket(experiment.id, unit) < experiment.weight)) {
|
|
213
|
-
const { version, content } = experiment.variant;
|
|
230
|
+
const { version, content, user, config = {} } = experiment.variant;
|
|
214
231
|
try {
|
|
215
|
-
return this.#served(
|
|
232
|
+
return this.#served(
|
|
233
|
+
slug,
|
|
234
|
+
fill(content, user, vars, withUser),
|
|
235
|
+
config,
|
|
236
|
+
version,
|
|
237
|
+
experiment.id,
|
|
238
|
+
test,
|
|
239
|
+
true
|
|
240
|
+
);
|
|
216
241
|
} catch (error) {
|
|
217
242
|
skipped = toPromptHelmError(error);
|
|
218
243
|
}
|
|
219
244
|
}
|
|
220
|
-
const
|
|
245
|
+
const messages = fill(entry.content, entry.user, vars, withUser);
|
|
221
246
|
if (skipped && experiment) {
|
|
222
247
|
this.#onError(skipped, slug);
|
|
223
248
|
if (test) add(this.#counts(test, experiment.variant.version), SKIPPED, 1);
|
|
224
249
|
}
|
|
225
|
-
return this.#served(slug,
|
|
250
|
+
return this.#served(slug, messages, entry.config, entry.version, experiment?.id, test);
|
|
226
251
|
}
|
|
227
|
-
/** Counts the call, tells `onResolve`, and hands the caller its
|
|
228
|
-
#served(slug,
|
|
252
|
+
/** Counts the call, tells `onResolve`, and hands the caller its messages, settings and what to report. */
|
|
253
|
+
#served(slug, { system, user }, config, version, experiment, test, variant = false) {
|
|
229
254
|
if (test && version !== null) add(this.#counts(test, version), SERVED, 1);
|
|
230
255
|
try {
|
|
231
256
|
this.#onResolve?.({ slug, version, experiment: experiment ?? null, variant });
|
|
@@ -234,7 +259,11 @@ var PromptHelm = class {
|
|
|
234
259
|
const started = Date.now();
|
|
235
260
|
let reported = 0;
|
|
236
261
|
return {
|
|
237
|
-
text,
|
|
262
|
+
text: system,
|
|
263
|
+
system,
|
|
264
|
+
user,
|
|
265
|
+
// A copy, schema and all: the cached one serves every later call.
|
|
266
|
+
config: structuredClone(config),
|
|
238
267
|
version,
|
|
239
268
|
ticket: test && version !== null ? `${test.id}.${version}.${test.weight}` : null,
|
|
240
269
|
outcome: (value) => {
|
|
@@ -344,6 +373,8 @@ var PromptHelm = class {
|
|
|
344
373
|
const entry = {
|
|
345
374
|
version: payload.version,
|
|
346
375
|
content: payload.content,
|
|
376
|
+
user: payload.user,
|
|
377
|
+
config: payload.config ?? {},
|
|
347
378
|
experiment: payload.experiment,
|
|
348
379
|
etag: response.headers.get("etag"),
|
|
349
380
|
fetchedAt: Date.now()
|
package/dist/index.d.cts
CHANGED
|
@@ -21,11 +21,45 @@ declare function render(template: string, vars?: Vars): string;
|
|
|
21
21
|
interface PromptPayload {
|
|
22
22
|
slug: string;
|
|
23
23
|
version: number;
|
|
24
|
+
/** The system message: every SDK serves it, and SDKs before 1.5 serve nothing else. */
|
|
24
25
|
content: string;
|
|
26
|
+
/** The variables `content` uses. */
|
|
25
27
|
variables: string[];
|
|
28
|
+
/** The user message, when the version has one; servers before SDK 1.5 send none. */
|
|
29
|
+
user?: PromptMessage;
|
|
30
|
+
/** The version's model settings; servers before SDK 1.4 send none. */
|
|
31
|
+
config?: PromptConfig;
|
|
26
32
|
/** An A/B test running on the prompt: the live version above is its control. */
|
|
27
33
|
experiment?: PromptExperiment;
|
|
28
34
|
}
|
|
35
|
+
/** A message besides the system one, with the variables it uses. */
|
|
36
|
+
interface PromptMessage {
|
|
37
|
+
content: string;
|
|
38
|
+
variables: string[];
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Model settings published with a version, for your model call. Only the ones set in the
|
|
42
|
+
* dashboard are present; for the rest, your code's defaults apply.
|
|
43
|
+
*/
|
|
44
|
+
interface PromptConfig {
|
|
45
|
+
model?: string;
|
|
46
|
+
temperature?: number;
|
|
47
|
+
maxTokens?: number;
|
|
48
|
+
/** How the model must answer; absent when it answers in text. Servers before SDK 1.5 send none. */
|
|
49
|
+
response?: PromptResponse;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* An answer in JSON that matches `schema`: a JSON Schema (2020-12) kept inside what OpenAI,
|
|
53
|
+
* Anthropic, Gemini and Bedrock all accept. Hand it to your provider's structured output.
|
|
54
|
+
*/
|
|
55
|
+
interface PromptResponse {
|
|
56
|
+
type: "json";
|
|
57
|
+
/** A name for the schema, made from the prompt's slug; OpenAI and Bedrock ask for one. */
|
|
58
|
+
name: string;
|
|
59
|
+
schema: {
|
|
60
|
+
[key: string]: unknown;
|
|
61
|
+
};
|
|
62
|
+
}
|
|
29
63
|
interface PromptExperiment {
|
|
30
64
|
id: string;
|
|
31
65
|
/** The variant's share of calls, in percent (1–99). */
|
|
@@ -34,8 +68,16 @@ interface PromptExperiment {
|
|
|
34
68
|
version: number;
|
|
35
69
|
content: string;
|
|
36
70
|
variables: string[];
|
|
71
|
+
user?: PromptMessage;
|
|
72
|
+
config?: PromptConfig;
|
|
37
73
|
};
|
|
38
74
|
}
|
|
75
|
+
/** A fallback with both messages, for a prompt that has a user message. */
|
|
76
|
+
interface PromptFallback {
|
|
77
|
+
system: string;
|
|
78
|
+
/** Left out: `user` is null when the fallback is served. */
|
|
79
|
+
user?: string;
|
|
80
|
+
}
|
|
39
81
|
/**
|
|
40
82
|
* Sent with the next request for a slug after `get()` couldn't serve its live text: the error's
|
|
41
83
|
* code alone, never any text or variable value. The server records it for the prompt page.
|
|
@@ -104,9 +146,10 @@ interface GetOptions {
|
|
|
104
146
|
vars?: Vars;
|
|
105
147
|
/**
|
|
106
148
|
* Served when PromptHelm can't be reached, the prompt doesn't exist, or the live
|
|
107
|
-
* version uses a variable this call doesn't pass. Without it those cases throw.
|
|
149
|
+
* version uses a variable this call doesn't pass. Without it those cases throw. A string is
|
|
150
|
+
* the system message; `{ system, user }` gives both, filled in like the live version.
|
|
108
151
|
*/
|
|
109
|
-
fallback?: string;
|
|
152
|
+
fallback?: string | PromptFallback;
|
|
110
153
|
/**
|
|
111
154
|
* Who the call is for, such as a user or conversation id: during an A/B test the same unit
|
|
112
155
|
* always gets the same version. Never sent to PromptHelm.
|
|
@@ -125,7 +168,17 @@ interface ResolveEvent {
|
|
|
125
168
|
variant: boolean;
|
|
126
169
|
}
|
|
127
170
|
interface Resolved {
|
|
171
|
+
/** The system message, filled in: what `get` returns. */
|
|
128
172
|
text: string;
|
|
173
|
+
/** The same as `text`, named after the message it is. */
|
|
174
|
+
system: string;
|
|
175
|
+
/** The user message, filled in; null when the version has none, so your app sends its own. */
|
|
176
|
+
user: string | null;
|
|
177
|
+
/**
|
|
178
|
+
* The served version's model settings, such as `model`. Empty when the fallback was served,
|
|
179
|
+
* so your code's defaults apply.
|
|
180
|
+
*/
|
|
181
|
+
config: PromptConfig;
|
|
129
182
|
/** The version served, or null when the fallback was. */
|
|
130
183
|
version: number | null;
|
|
131
184
|
/**
|
|
@@ -144,9 +197,15 @@ interface Resolved {
|
|
|
144
197
|
declare class PromptHelm {
|
|
145
198
|
#private;
|
|
146
199
|
constructor(options?: PromptHelmOptions);
|
|
147
|
-
/**
|
|
200
|
+
/**
|
|
201
|
+
* The live version's system message with its variables filled in. It never needs the user
|
|
202
|
+
* message's variables: an app that calls `get` sends its own.
|
|
203
|
+
*/
|
|
148
204
|
get(slug: string, options?: GetOptions): Promise<string>;
|
|
149
|
-
/**
|
|
205
|
+
/**
|
|
206
|
+
* Like `get`, with the user message too, the model settings, and which version was served, so
|
|
207
|
+
* you can tag your logs or traces with it.
|
|
208
|
+
*/
|
|
150
209
|
resolve(slug: string, options?: GetOptions): Promise<Resolved>;
|
|
151
210
|
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
152
211
|
outcome(ticket: string, value: boolean | number): void;
|
|
@@ -165,4 +224,4 @@ declare const LATENCY_BUCKETS = 24;
|
|
|
165
224
|
declare function latencyBucket(ms: number): number;
|
|
166
225
|
|
|
167
226
|
export { ERROR_CODES, FALLBACK_HEADER, LATENCY_BUCKETS, PromptHelm, PromptHelmError, extractVariables, latencyBucket, render };
|
|
168
|
-
export type { GetOptions, PromptExperiment, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
|
|
227
|
+
export type { GetOptions, PromptConfig, PromptExperiment, PromptFallback, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptMessage, PromptPayload, PromptResponse, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
|
package/dist/index.d.ts
CHANGED
|
@@ -21,11 +21,45 @@ declare function render(template: string, vars?: Vars): string;
|
|
|
21
21
|
interface PromptPayload {
|
|
22
22
|
slug: string;
|
|
23
23
|
version: number;
|
|
24
|
+
/** The system message: every SDK serves it, and SDKs before 1.5 serve nothing else. */
|
|
24
25
|
content: string;
|
|
26
|
+
/** The variables `content` uses. */
|
|
25
27
|
variables: string[];
|
|
28
|
+
/** The user message, when the version has one; servers before SDK 1.5 send none. */
|
|
29
|
+
user?: PromptMessage;
|
|
30
|
+
/** The version's model settings; servers before SDK 1.4 send none. */
|
|
31
|
+
config?: PromptConfig;
|
|
26
32
|
/** An A/B test running on the prompt: the live version above is its control. */
|
|
27
33
|
experiment?: PromptExperiment;
|
|
28
34
|
}
|
|
35
|
+
/** A message besides the system one, with the variables it uses. */
|
|
36
|
+
interface PromptMessage {
|
|
37
|
+
content: string;
|
|
38
|
+
variables: string[];
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Model settings published with a version, for your model call. Only the ones set in the
|
|
42
|
+
* dashboard are present; for the rest, your code's defaults apply.
|
|
43
|
+
*/
|
|
44
|
+
interface PromptConfig {
|
|
45
|
+
model?: string;
|
|
46
|
+
temperature?: number;
|
|
47
|
+
maxTokens?: number;
|
|
48
|
+
/** How the model must answer; absent when it answers in text. Servers before SDK 1.5 send none. */
|
|
49
|
+
response?: PromptResponse;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* An answer in JSON that matches `schema`: a JSON Schema (2020-12) kept inside what OpenAI,
|
|
53
|
+
* Anthropic, Gemini and Bedrock all accept. Hand it to your provider's structured output.
|
|
54
|
+
*/
|
|
55
|
+
interface PromptResponse {
|
|
56
|
+
type: "json";
|
|
57
|
+
/** A name for the schema, made from the prompt's slug; OpenAI and Bedrock ask for one. */
|
|
58
|
+
name: string;
|
|
59
|
+
schema: {
|
|
60
|
+
[key: string]: unknown;
|
|
61
|
+
};
|
|
62
|
+
}
|
|
29
63
|
interface PromptExperiment {
|
|
30
64
|
id: string;
|
|
31
65
|
/** The variant's share of calls, in percent (1–99). */
|
|
@@ -34,8 +68,16 @@ interface PromptExperiment {
|
|
|
34
68
|
version: number;
|
|
35
69
|
content: string;
|
|
36
70
|
variables: string[];
|
|
71
|
+
user?: PromptMessage;
|
|
72
|
+
config?: PromptConfig;
|
|
37
73
|
};
|
|
38
74
|
}
|
|
75
|
+
/** A fallback with both messages, for a prompt that has a user message. */
|
|
76
|
+
interface PromptFallback {
|
|
77
|
+
system: string;
|
|
78
|
+
/** Left out: `user` is null when the fallback is served. */
|
|
79
|
+
user?: string;
|
|
80
|
+
}
|
|
39
81
|
/**
|
|
40
82
|
* Sent with the next request for a slug after `get()` couldn't serve its live text: the error's
|
|
41
83
|
* code alone, never any text or variable value. The server records it for the prompt page.
|
|
@@ -104,9 +146,10 @@ interface GetOptions {
|
|
|
104
146
|
vars?: Vars;
|
|
105
147
|
/**
|
|
106
148
|
* Served when PromptHelm can't be reached, the prompt doesn't exist, or the live
|
|
107
|
-
* version uses a variable this call doesn't pass. Without it those cases throw.
|
|
149
|
+
* version uses a variable this call doesn't pass. Without it those cases throw. A string is
|
|
150
|
+
* the system message; `{ system, user }` gives both, filled in like the live version.
|
|
108
151
|
*/
|
|
109
|
-
fallback?: string;
|
|
152
|
+
fallback?: string | PromptFallback;
|
|
110
153
|
/**
|
|
111
154
|
* Who the call is for, such as a user or conversation id: during an A/B test the same unit
|
|
112
155
|
* always gets the same version. Never sent to PromptHelm.
|
|
@@ -125,7 +168,17 @@ interface ResolveEvent {
|
|
|
125
168
|
variant: boolean;
|
|
126
169
|
}
|
|
127
170
|
interface Resolved {
|
|
171
|
+
/** The system message, filled in: what `get` returns. */
|
|
128
172
|
text: string;
|
|
173
|
+
/** The same as `text`, named after the message it is. */
|
|
174
|
+
system: string;
|
|
175
|
+
/** The user message, filled in; null when the version has none, so your app sends its own. */
|
|
176
|
+
user: string | null;
|
|
177
|
+
/**
|
|
178
|
+
* The served version's model settings, such as `model`. Empty when the fallback was served,
|
|
179
|
+
* so your code's defaults apply.
|
|
180
|
+
*/
|
|
181
|
+
config: PromptConfig;
|
|
129
182
|
/** The version served, or null when the fallback was. */
|
|
130
183
|
version: number | null;
|
|
131
184
|
/**
|
|
@@ -144,9 +197,15 @@ interface Resolved {
|
|
|
144
197
|
declare class PromptHelm {
|
|
145
198
|
#private;
|
|
146
199
|
constructor(options?: PromptHelmOptions);
|
|
147
|
-
/**
|
|
200
|
+
/**
|
|
201
|
+
* The live version's system message with its variables filled in. It never needs the user
|
|
202
|
+
* message's variables: an app that calls `get` sends its own.
|
|
203
|
+
*/
|
|
148
204
|
get(slug: string, options?: GetOptions): Promise<string>;
|
|
149
|
-
/**
|
|
205
|
+
/**
|
|
206
|
+
* Like `get`, with the user message too, the model settings, and which version was served, so
|
|
207
|
+
* you can tag your logs or traces with it.
|
|
208
|
+
*/
|
|
150
209
|
resolve(slug: string, options?: GetOptions): Promise<Resolved>;
|
|
151
210
|
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
152
211
|
outcome(ticket: string, value: boolean | number): void;
|
|
@@ -165,4 +224,4 @@ declare const LATENCY_BUCKETS = 24;
|
|
|
165
224
|
declare function latencyBucket(ms: number): number;
|
|
166
225
|
|
|
167
226
|
export { ERROR_CODES, FALLBACK_HEADER, LATENCY_BUCKETS, PromptHelm, PromptHelmError, extractVariables, latencyBucket, render };
|
|
168
|
-
export type { GetOptions, PromptExperiment, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
|
|
227
|
+
export type { GetOptions, PromptConfig, PromptExperiment, PromptFallback, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptMessage, PromptPayload, PromptResponse, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
|
package/dist/index.js
CHANGED
|
@@ -73,6 +73,13 @@ var TOKENS_IN = 6;
|
|
|
73
73
|
var TOKENS_OUT = 7;
|
|
74
74
|
var LATENCY = 8;
|
|
75
75
|
var BUCKETS = FIELDS.length;
|
|
76
|
+
function fill(system, user, vars, withUser) {
|
|
77
|
+
const text = typeof user === "string" ? user : user?.content;
|
|
78
|
+
return {
|
|
79
|
+
system: render(system, vars),
|
|
80
|
+
user: withUser && text !== void 0 ? render(text, vars) : null
|
|
81
|
+
};
|
|
82
|
+
}
|
|
76
83
|
var PromptHelm = class {
|
|
77
84
|
#apiKey;
|
|
78
85
|
#baseUrl;
|
|
@@ -104,21 +111,31 @@ var PromptHelm = class {
|
|
|
104
111
|
this.#reportFallbacks = options.reportFallbacks ?? true;
|
|
105
112
|
this.#reportResults = options.reportResults ?? this.#reportFallbacks;
|
|
106
113
|
}
|
|
107
|
-
/**
|
|
114
|
+
/**
|
|
115
|
+
* The live version's system message with its variables filled in. It never needs the user
|
|
116
|
+
* message's variables: an app that calls `get` sends its own.
|
|
117
|
+
*/
|
|
108
118
|
async get(slug, options = {}) {
|
|
109
|
-
return (await this
|
|
119
|
+
return (await this.#resolve(slug, options, false)).text;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Like `get`, with the user message too, the model settings, and which version was served, so
|
|
123
|
+
* you can tag your logs or traces with it.
|
|
124
|
+
*/
|
|
125
|
+
resolve(slug, options = {}) {
|
|
126
|
+
return this.#resolve(slug, options, true);
|
|
110
127
|
}
|
|
111
|
-
|
|
112
|
-
async resolve(slug, options = {}) {
|
|
128
|
+
async #resolve(slug, options, withUser) {
|
|
113
129
|
const { vars, fallback } = options;
|
|
114
130
|
try {
|
|
115
|
-
return this.#pick(slug, await this.#entry(slug), options);
|
|
131
|
+
return this.#pick(slug, await this.#entry(slug), options, withUser);
|
|
116
132
|
} catch (error) {
|
|
117
133
|
const failure = toPromptHelmError(error);
|
|
118
134
|
this.#remember(slug, failure.code);
|
|
119
135
|
if (fallback === void 0) throw failure;
|
|
120
136
|
this.#onError(failure, slug);
|
|
121
|
-
|
|
137
|
+
const own = typeof fallback === "string" ? { system: fallback } : fallback;
|
|
138
|
+
return this.#served(slug, fill(own.system, own.user, vars, withUser), {}, null);
|
|
122
139
|
}
|
|
123
140
|
}
|
|
124
141
|
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
@@ -171,28 +188,36 @@ var PromptHelm = class {
|
|
|
171
188
|
}
|
|
172
189
|
}
|
|
173
190
|
}
|
|
174
|
-
#pick(slug, entry, options) {
|
|
191
|
+
#pick(slug, entry, options, withUser) {
|
|
175
192
|
const { vars, unit, force } = options;
|
|
176
193
|
const { experiment } = entry;
|
|
177
194
|
const test = experiment && !force ? { id: experiment.id, weight: experiment.weight } : void 0;
|
|
178
195
|
let skipped;
|
|
179
196
|
if (experiment && (force ? force === "variant" : bucket(experiment.id, unit) < experiment.weight)) {
|
|
180
|
-
const { version, content } = experiment.variant;
|
|
197
|
+
const { version, content, user, config = {} } = experiment.variant;
|
|
181
198
|
try {
|
|
182
|
-
return this.#served(
|
|
199
|
+
return this.#served(
|
|
200
|
+
slug,
|
|
201
|
+
fill(content, user, vars, withUser),
|
|
202
|
+
config,
|
|
203
|
+
version,
|
|
204
|
+
experiment.id,
|
|
205
|
+
test,
|
|
206
|
+
true
|
|
207
|
+
);
|
|
183
208
|
} catch (error) {
|
|
184
209
|
skipped = toPromptHelmError(error);
|
|
185
210
|
}
|
|
186
211
|
}
|
|
187
|
-
const
|
|
212
|
+
const messages = fill(entry.content, entry.user, vars, withUser);
|
|
188
213
|
if (skipped && experiment) {
|
|
189
214
|
this.#onError(skipped, slug);
|
|
190
215
|
if (test) add(this.#counts(test, experiment.variant.version), SKIPPED, 1);
|
|
191
216
|
}
|
|
192
|
-
return this.#served(slug,
|
|
217
|
+
return this.#served(slug, messages, entry.config, entry.version, experiment?.id, test);
|
|
193
218
|
}
|
|
194
|
-
/** Counts the call, tells `onResolve`, and hands the caller its
|
|
195
|
-
#served(slug,
|
|
219
|
+
/** Counts the call, tells `onResolve`, and hands the caller its messages, settings and what to report. */
|
|
220
|
+
#served(slug, { system, user }, config, version, experiment, test, variant = false) {
|
|
196
221
|
if (test && version !== null) add(this.#counts(test, version), SERVED, 1);
|
|
197
222
|
try {
|
|
198
223
|
this.#onResolve?.({ slug, version, experiment: experiment ?? null, variant });
|
|
@@ -201,7 +226,11 @@ var PromptHelm = class {
|
|
|
201
226
|
const started = Date.now();
|
|
202
227
|
let reported = 0;
|
|
203
228
|
return {
|
|
204
|
-
text,
|
|
229
|
+
text: system,
|
|
230
|
+
system,
|
|
231
|
+
user,
|
|
232
|
+
// A copy, schema and all: the cached one serves every later call.
|
|
233
|
+
config: structuredClone(config),
|
|
205
234
|
version,
|
|
206
235
|
ticket: test && version !== null ? `${test.id}.${version}.${test.weight}` : null,
|
|
207
236
|
outcome: (value) => {
|
|
@@ -311,6 +340,8 @@ var PromptHelm = class {
|
|
|
311
340
|
const entry = {
|
|
312
341
|
version: payload.version,
|
|
313
342
|
content: payload.content,
|
|
343
|
+
user: payload.user,
|
|
344
|
+
config: payload.config ?? {},
|
|
314
345
|
experiment: payload.experiment,
|
|
315
346
|
etag: response.headers.get("etag"),
|
|
316
347
|
fetchedAt: Date.now()
|
package/package.json
CHANGED