@prompt-helm/sdk 1.2.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -3
- package/dist/cli.js +12 -0
- package/dist/index.cjs +208 -3
- package/dist/index.d.cts +117 -2
- package/dist/index.d.ts +117 -2
- package/dist/index.js +206 -3
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -14,7 +14,9 @@ refresh, 60 seconds by default.
|
|
|
14
14
|
a refresh that finds nothing new is an empty `304`.
|
|
15
15
|
- **Your data stays with you.** Variables are filled in on your server. PromptHelm never sees
|
|
16
16
|
their values, your provider keys or your LLM traffic.
|
|
17
|
-
- **
|
|
17
|
+
- **Model settings travel with the text.** A version can carry its model, temperature and
|
|
18
|
+
output limit, so moving to a cheaper model is a publish you can roll back or A/B test.
|
|
19
|
+
- **No dependencies.** ESM and CommonJS, Node.js 18 and later.
|
|
18
20
|
|
|
19
21
|
## Install
|
|
20
22
|
|
|
@@ -60,6 +62,8 @@ All optional when the environment variables are set.
|
|
|
60
62
|
| `timeout` | `3000` | How long, in ms, one request may take. |
|
|
61
63
|
| `onError` | `console.warn` | Called with the error and the slug when a fallback or a cached prompt was served instead of failing. |
|
|
62
64
|
| `reportFallbacks` | `true` | Tells PromptHelm, by error code only, when a fallback or a cached prompt was served, so the prompt's page shows which app fell back and why. |
|
|
65
|
+
| `reportResults` | `reportFallbacks` | During an [A/B test](#ab-tests), sends PromptHelm totals only: calls served per version, and the outcomes and usage you report. |
|
|
66
|
+
| `onResolve` | none | Called on every `get` and `resolve` with the version served, for your analytics or traces. |
|
|
63
67
|
| `fetch` | global `fetch` | Your own fetch, for tests or a proxy. |
|
|
64
68
|
|
|
65
69
|
## Errors
|
|
@@ -106,6 +110,85 @@ Run it in CI before the build, so every deploy carries the text that was live th
|
|
|
106
110
|
`PROMPTHELM_API_KEY` (and `PROMPTHELM_BASE_URL`), and writes JSON for a `.json` path or with
|
|
107
111
|
`--format json`. A prompt appears in the file once it's published.
|
|
108
112
|
|
|
113
|
+
## Model settings
|
|
114
|
+
|
|
115
|
+
A version can carry the model to call it with, its temperature and its output limit, set next to
|
|
116
|
+
the text in the dashboard. Moving to a cheaper model is then a publish like any other: one click
|
|
117
|
+
rolls it back, and an A/B test can tell whether it does as well. `resolve` hands them over with
|
|
118
|
+
the text:
|
|
119
|
+
|
|
120
|
+
```ts
|
|
121
|
+
const r = await ph.resolve("support-reply", {
|
|
122
|
+
vars: { company: "Acme", customer_name: user.name },
|
|
123
|
+
fallback: SUPPORT_REPLY,
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
const reply = await openai.responses.create({
|
|
127
|
+
model: r.config.model ?? "gpt-5",
|
|
128
|
+
temperature: r.config.temperature,
|
|
129
|
+
max_output_tokens: r.config.maxTokens,
|
|
130
|
+
input: r.text,
|
|
131
|
+
});
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
`config` holds only the settings set in the dashboard (`model`, `temperature`, `maxTokens`), so
|
|
135
|
+
keep your defaults in the code; when the fallback is served it is empty. PromptHelm doesn't check
|
|
136
|
+
a model name with your provider, and a name your provider doesn't know fails your calls: try a new
|
|
137
|
+
model in an A/B test first. Apps on SDK versions before 1.4 ignore the settings.
|
|
138
|
+
|
|
139
|
+
## A/B tests
|
|
140
|
+
|
|
141
|
+
On the Pro plan, a prompt's page can run an experiment: the live version keeps most calls and a
|
|
142
|
+
variant gets the share you pick. The SDK picks the version for each call on your server. Pass a
|
|
143
|
+
`unit`, such as a user or conversation id, and that unit gets the same version every time;
|
|
144
|
+
without one, each call is picked at random. Use `resolve` to learn which version a call got, and
|
|
145
|
+
tell PromptHelm how it went:
|
|
146
|
+
|
|
147
|
+
```ts
|
|
148
|
+
const r = await ph.resolve("support-reply", {
|
|
149
|
+
vars: { company: "Acme", customer_name: user.name },
|
|
150
|
+
fallback: SUPPORT_REPLY,
|
|
151
|
+
unit: user.id,
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
const reply = await openai.responses.create({ model: r.config.model ?? "gpt-5", input: r.text });
|
|
155
|
+
r.usage(reply.usage); // tokens, and how long the call took since resolve
|
|
156
|
+
r.outcome(userKeptTheReply); // true or false, or a score from 0 to 1
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
The prompt's page then compares the two versions: how often each succeeded, with a range that
|
|
160
|
+
stays honest however often you look, and their tokens and latency per call. It says when there's
|
|
161
|
+
an answer; shipping the variant stays your call. Each version brings its own
|
|
162
|
+
[model settings](#model-settings), so a test can compare two models as well as two texts.
|
|
163
|
+
|
|
164
|
+
- **Outcomes that come later.** A purchase days later, in another service: keep `r.ticket` with
|
|
165
|
+
your own record, then call `ph.outcome(ticket, true)` from any process. The ticket names the
|
|
166
|
+
test and the version, not the user.
|
|
167
|
+
- **Your own analytics.** `onResolve` is called on every `get` and `resolve` with the slug, the
|
|
168
|
+
version, the test and whether the call got the variant, so your events or traces can carry it:
|
|
169
|
+
`new PromptHelm({ onResolve: (e) => posthog.capture("prompt_served", e) })`.
|
|
170
|
+
- **Your team first.** `force: "variant"` (or `"control"`) serves that side to a call, such as
|
|
171
|
+
to your own staff. Forced calls are never counted.
|
|
172
|
+
- **Serverless.** Counts are sent within a ttl. A function that freezes after its response
|
|
173
|
+
should `await ph.flush()` first (with `waitUntil`, where the platform has it).
|
|
174
|
+
- **Usage objects** from OpenAI (Responses and Chat Completions), Anthropic, the Vercel AI SDK
|
|
175
|
+
and Gemini are read as they are. `r.usage(null, ms)` reports a duration alone.
|
|
176
|
+
|
|
177
|
+
Only totals reach PromptHelm: how many calls each version served, the outcomes, tokens and
|
|
178
|
+
durations you report. Never the text, a variable's value or the unit. Counts a process holds
|
|
179
|
+
when it exits are lost, so read them as a sample. `reportResults: false` sends none; unless you
|
|
180
|
+
set it, it follows `reportFallbacks`.
|
|
181
|
+
|
|
182
|
+
If the variant uses a variable the call doesn't pass, that call gets the live version and
|
|
183
|
+
`onError` receives `missing_variables`; it isn't reported as a fallback. Growing the variant's
|
|
184
|
+
share only moves units from the live version to the variant, never back. Apps on SDK versions
|
|
185
|
+
before 1.3 ignore experiments and serve the live version to every call.
|
|
186
|
+
|
|
187
|
+
To split calls the same way from another language: during an experiment the response carries
|
|
188
|
+
`experiment: { id, weight, variant: { version, content, variables, config } }`. A unit's bucket is the
|
|
189
|
+
32-bit FNV-1a hash of the UTF-8 bytes of `<experiment id>:<unit>`, modulo 100; buckets below
|
|
190
|
+
`weight` get the variant.
|
|
191
|
+
|
|
109
192
|
## Other languages
|
|
110
193
|
|
|
111
194
|
Any language can fetch the live version with one request:
|
|
@@ -115,8 +198,9 @@ curl https://prompthelm.app/v1/prompts/support-reply \
|
|
|
115
198
|
-H "Authorization: Bearer $PROMPTHELM_API_KEY"
|
|
116
199
|
```
|
|
117
200
|
|
|
118
|
-
|
|
119
|
-
`
|
|
201
|
+
The response carries the live version's `content`, its `variables` and its model settings as
|
|
202
|
+
`config` (only the ones set). `GET /v1/prompts` returns every published prompt of the key's
|
|
203
|
+
project at once, as `{ "prompts": [ … ] }`.
|
|
120
204
|
|
|
121
205
|
## Links
|
|
122
206
|
|
package/dist/cli.js
CHANGED
|
@@ -18,6 +18,18 @@ var PromptHelmError = class extends Error {
|
|
|
18
18
|
|
|
19
19
|
// src/client.ts
|
|
20
20
|
var DEFAULT_BASE_URL = "https://prompthelm.app";
|
|
21
|
+
var FIELDS = [
|
|
22
|
+
"served",
|
|
23
|
+
"skipped",
|
|
24
|
+
"outcomes",
|
|
25
|
+
"scoreSum",
|
|
26
|
+
"scoreSqSum",
|
|
27
|
+
"usage",
|
|
28
|
+
"tokensIn",
|
|
29
|
+
"tokensOut",
|
|
30
|
+
"latencyMs"
|
|
31
|
+
];
|
|
32
|
+
var BUCKETS = FIELDS.length;
|
|
21
33
|
async function errorFromResponse(response) {
|
|
22
34
|
const body = await response.json().catch(() => null);
|
|
23
35
|
const message = body?.error?.message ?? `HTTP ${response.status}`;
|
package/dist/index.cjs
CHANGED
|
@@ -22,9 +22,11 @@ var index_exports = {};
|
|
|
22
22
|
__export(index_exports, {
|
|
23
23
|
ERROR_CODES: () => ERROR_CODES,
|
|
24
24
|
FALLBACK_HEADER: () => FALLBACK_HEADER,
|
|
25
|
+
LATENCY_BUCKETS: () => LATENCY_BUCKETS,
|
|
25
26
|
PromptHelm: () => PromptHelm,
|
|
26
27
|
PromptHelmError: () => PromptHelmError,
|
|
27
28
|
extractVariables: () => extractVariables,
|
|
29
|
+
latencyBucket: () => latencyBucket,
|
|
28
30
|
render: () => render
|
|
29
31
|
});
|
|
30
32
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -49,6 +51,20 @@ var PromptHelmError = class extends Error {
|
|
|
49
51
|
name = "PromptHelmError";
|
|
50
52
|
};
|
|
51
53
|
|
|
54
|
+
// src/experiment.ts
|
|
55
|
+
function fnv1a32(text) {
|
|
56
|
+
let hash = 2166136261;
|
|
57
|
+
for (const byte of new TextEncoder().encode(text)) hash = Math.imul(hash ^ byte, 16777619);
|
|
58
|
+
return hash >>> 0;
|
|
59
|
+
}
|
|
60
|
+
function bucket(experimentId, unit) {
|
|
61
|
+
return unit === void 0 ? Math.floor(Math.random() * 100) : fnv1a32(`${experimentId}:${unit}`) % 100;
|
|
62
|
+
}
|
|
63
|
+
var LATENCY_BUCKETS = 24;
|
|
64
|
+
function latencyBucket(ms) {
|
|
65
|
+
return ms < 50 ? 0 : Math.min(LATENCY_BUCKETS - 1, 1 + Math.floor(2 * Math.log2(ms / 50)));
|
|
66
|
+
}
|
|
67
|
+
|
|
52
68
|
// src/template.ts
|
|
53
69
|
var VARIABLE = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
|
|
54
70
|
function extractVariables(template) {
|
|
@@ -69,6 +85,27 @@ function render(template, vars = {}) {
|
|
|
69
85
|
// src/client.ts
|
|
70
86
|
var DEFAULT_BASE_URL = "https://prompthelm.app";
|
|
71
87
|
var FALLBACK_HEADER = "x-prompthelm-fallback";
|
|
88
|
+
var FIELDS = [
|
|
89
|
+
"served",
|
|
90
|
+
"skipped",
|
|
91
|
+
"outcomes",
|
|
92
|
+
"scoreSum",
|
|
93
|
+
"scoreSqSum",
|
|
94
|
+
"usage",
|
|
95
|
+
"tokensIn",
|
|
96
|
+
"tokensOut",
|
|
97
|
+
"latencyMs"
|
|
98
|
+
];
|
|
99
|
+
var SERVED = 0;
|
|
100
|
+
var SKIPPED = 1;
|
|
101
|
+
var OUTCOMES = 2;
|
|
102
|
+
var SCORE = 3;
|
|
103
|
+
var SCORE_SQ = 4;
|
|
104
|
+
var USAGE = 5;
|
|
105
|
+
var TOKENS_IN = 6;
|
|
106
|
+
var TOKENS_OUT = 7;
|
|
107
|
+
var LATENCY = 8;
|
|
108
|
+
var BUCKETS = FIELDS.length;
|
|
72
109
|
var PromptHelm = class {
|
|
73
110
|
#apiKey;
|
|
74
111
|
#baseUrl;
|
|
@@ -76,12 +113,16 @@ var PromptHelm = class {
|
|
|
76
113
|
#timeout;
|
|
77
114
|
#fetch;
|
|
78
115
|
#onError;
|
|
116
|
+
#onResolve;
|
|
79
117
|
#cache = /* @__PURE__ */ new Map();
|
|
80
118
|
#inflight = /* @__PURE__ */ new Map();
|
|
81
119
|
#reportFallbacks;
|
|
120
|
+
#reportResults;
|
|
82
121
|
// Per slug, why the live text last couldn't be served, until a request carries it. An object,
|
|
83
122
|
// so a report made while that request is in flight isn't cleared with the one it carried.
|
|
84
123
|
#unreported = /* @__PURE__ */ new Map();
|
|
124
|
+
#pending = /* @__PURE__ */ new Map();
|
|
125
|
+
#timer;
|
|
85
126
|
constructor(options = {}) {
|
|
86
127
|
const apiKey = options.apiKey ?? env("PROMPTHELM_API_KEY");
|
|
87
128
|
if (!apiKey) throw new Error("PromptHelm: pass apiKey or set PROMPTHELM_API_KEY");
|
|
@@ -92,22 +133,169 @@ var PromptHelm = class {
|
|
|
92
133
|
this.#timeout = options.timeout ?? 3e3;
|
|
93
134
|
this.#fetch = options.fetch ?? ((input, init) => globalThis.fetch(input, init));
|
|
94
135
|
this.#onError = options.onError ?? ((error, slug) => console.warn(`[PromptHelm] ${slug}: ${error.message}`));
|
|
136
|
+
this.#onResolve = options.onResolve;
|
|
95
137
|
this.#reportFallbacks = options.reportFallbacks ?? true;
|
|
138
|
+
this.#reportResults = options.reportResults ?? this.#reportFallbacks;
|
|
96
139
|
}
|
|
97
140
|
/** The live version of a prompt with its variables filled in. */
|
|
98
141
|
async get(slug, options = {}) {
|
|
142
|
+
return (await this.resolve(slug, options)).text;
|
|
143
|
+
}
|
|
144
|
+
/** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
|
|
145
|
+
async resolve(slug, options = {}) {
|
|
99
146
|
const { vars, fallback } = options;
|
|
100
147
|
try {
|
|
101
|
-
|
|
102
|
-
return render(entry.content, vars);
|
|
148
|
+
return this.#pick(slug, await this.#entry(slug), options);
|
|
103
149
|
} catch (error) {
|
|
104
150
|
const failure = toPromptHelmError(error);
|
|
105
151
|
this.#remember(slug, failure.code);
|
|
106
152
|
if (fallback === void 0) throw failure;
|
|
107
153
|
this.#onError(failure, slug);
|
|
108
|
-
return render(fallback, vars);
|
|
154
|
+
return this.#served(slug, render(fallback, vars), {}, null);
|
|
109
155
|
}
|
|
110
156
|
}
|
|
157
|
+
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
158
|
+
outcome(ticket, value) {
|
|
159
|
+
const [id = "", version, weight] = ticket.split(".");
|
|
160
|
+
const test = { id, weight: Number(weight) };
|
|
161
|
+
if (id && test.weight > 0) this.#outcome(test, Number(version), value);
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Sends what was counted for A/B tests now, instead of within a ttl: call it before a
|
|
165
|
+
* serverless function freezes. Never throws; what can't be sent is kept for the next try.
|
|
166
|
+
*/
|
|
167
|
+
async flush() {
|
|
168
|
+
clearTimeout(this.#timer);
|
|
169
|
+
this.#timer = void 0;
|
|
170
|
+
const sent = this.#pending;
|
|
171
|
+
if (sent.size === 0) return;
|
|
172
|
+
this.#pending = /* @__PURE__ */ new Map();
|
|
173
|
+
const body = {
|
|
174
|
+
experiments: [...sent.values()].map(({ id, weight, versions }) => ({
|
|
175
|
+
id,
|
|
176
|
+
weight,
|
|
177
|
+
versions: Object.fromEntries(
|
|
178
|
+
[...versions].map(([version, counts]) => [
|
|
179
|
+
version,
|
|
180
|
+
{
|
|
181
|
+
...Object.fromEntries(FIELDS.map((field, i) => [field, counts[i]])),
|
|
182
|
+
latency: counts.slice(BUCKETS)
|
|
183
|
+
}
|
|
184
|
+
])
|
|
185
|
+
)
|
|
186
|
+
}))
|
|
187
|
+
};
|
|
188
|
+
try {
|
|
189
|
+
const response = await this.#fetch(`${this.#baseUrl}/v1/reports`, {
|
|
190
|
+
method: "POST",
|
|
191
|
+
headers: { authorization: `Bearer ${this.#apiKey}`, "content-type": "application/json" },
|
|
192
|
+
body: JSON.stringify(body),
|
|
193
|
+
signal: AbortSignal.timeout(this.#timeout)
|
|
194
|
+
});
|
|
195
|
+
if (response.status < 500) return;
|
|
196
|
+
} catch {
|
|
197
|
+
}
|
|
198
|
+
for (const { id, weight, versions } of sent.values()) {
|
|
199
|
+
for (const [version, counts] of versions) {
|
|
200
|
+
const total = this.#counts({ id, weight }, version);
|
|
201
|
+
counts.forEach((n, i) => {
|
|
202
|
+
add(total, i, n);
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
#pick(slug, entry, options) {
|
|
208
|
+
const { vars, unit, force } = options;
|
|
209
|
+
const { experiment } = entry;
|
|
210
|
+
const test = experiment && !force ? { id: experiment.id, weight: experiment.weight } : void 0;
|
|
211
|
+
let skipped;
|
|
212
|
+
if (experiment && (force ? force === "variant" : bucket(experiment.id, unit) < experiment.weight)) {
|
|
213
|
+
const { version, content, config = {} } = experiment.variant;
|
|
214
|
+
try {
|
|
215
|
+
return this.#served(
|
|
216
|
+
slug,
|
|
217
|
+
render(content, vars),
|
|
218
|
+
config,
|
|
219
|
+
version,
|
|
220
|
+
experiment.id,
|
|
221
|
+
test,
|
|
222
|
+
true
|
|
223
|
+
);
|
|
224
|
+
} catch (error) {
|
|
225
|
+
skipped = toPromptHelmError(error);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
const text = render(entry.content, vars);
|
|
229
|
+
if (skipped && experiment) {
|
|
230
|
+
this.#onError(skipped, slug);
|
|
231
|
+
if (test) add(this.#counts(test, experiment.variant.version), SKIPPED, 1);
|
|
232
|
+
}
|
|
233
|
+
return this.#served(slug, text, entry.config, entry.version, experiment?.id, test);
|
|
234
|
+
}
|
|
235
|
+
/** Counts the call, tells `onResolve`, and hands the caller its text, settings and what to report. */
|
|
236
|
+
#served(slug, text, config, version, experiment, test, variant = false) {
|
|
237
|
+
if (test && version !== null) add(this.#counts(test, version), SERVED, 1);
|
|
238
|
+
try {
|
|
239
|
+
this.#onResolve?.({ slug, version, experiment: experiment ?? null, variant });
|
|
240
|
+
} catch {
|
|
241
|
+
}
|
|
242
|
+
const started = Date.now();
|
|
243
|
+
let reported = 0;
|
|
244
|
+
return {
|
|
245
|
+
text,
|
|
246
|
+
// A copy: the cached one serves every later call.
|
|
247
|
+
config: { ...config },
|
|
248
|
+
version,
|
|
249
|
+
ticket: test && version !== null ? `${test.id}.${version}.${test.weight}` : null,
|
|
250
|
+
outcome: (value) => {
|
|
251
|
+
if (!test || version === null || reported & 1) return;
|
|
252
|
+
reported |= 1;
|
|
253
|
+
this.#outcome(test, version, value);
|
|
254
|
+
},
|
|
255
|
+
usage: (usage, ms = Date.now() - started) => {
|
|
256
|
+
if (!test || version === null || reported & 2) return;
|
|
257
|
+
reported |= 2;
|
|
258
|
+
const counts = this.#counts(test, version);
|
|
259
|
+
const fields = usage ?? {};
|
|
260
|
+
add(counts, USAGE, 1);
|
|
261
|
+
add(counts, TOKENS_IN, tokens(fields, IN));
|
|
262
|
+
add(counts, TOKENS_OUT, tokens(fields, OUT));
|
|
263
|
+
add(counts, LATENCY, Math.max(0, ms));
|
|
264
|
+
add(counts, BUCKETS + latencyBucket(ms), 1);
|
|
265
|
+
}
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
#outcome(test, version, value) {
|
|
269
|
+
const score = value === true ? 1 : value === false ? 0 : value;
|
|
270
|
+
if (!Number.isInteger(version) || !(score >= 0 && score <= 1)) return;
|
|
271
|
+
const counts = this.#counts(test, version);
|
|
272
|
+
add(counts, OUTCOMES, 1);
|
|
273
|
+
add(counts, SCORE, score);
|
|
274
|
+
add(counts, SCORE_SQ, score * score);
|
|
275
|
+
}
|
|
276
|
+
/** The counts to add a call to, kept until the next report; none when reporting is off. */
|
|
277
|
+
#counts(test, version) {
|
|
278
|
+
if (!this.#reportResults) return;
|
|
279
|
+
const key = `${test.id}.${test.weight}`;
|
|
280
|
+
let pending = this.#pending.get(key);
|
|
281
|
+
if (!pending) {
|
|
282
|
+
pending = { ...test, versions: /* @__PURE__ */ new Map() };
|
|
283
|
+
this.#pending.set(key, pending);
|
|
284
|
+
}
|
|
285
|
+
let counts = pending.versions.get(version);
|
|
286
|
+
if (!counts) {
|
|
287
|
+
counts = new Array(BUCKETS + LATENCY_BUCKETS).fill(0);
|
|
288
|
+
pending.versions.set(version, counts);
|
|
289
|
+
}
|
|
290
|
+
this.#schedule();
|
|
291
|
+
return counts;
|
|
292
|
+
}
|
|
293
|
+
/** Reports within a ttl of the first count, without keeping the process alive for it. */
|
|
294
|
+
#schedule() {
|
|
295
|
+
if (this.#timer) return;
|
|
296
|
+
this.#timer = setTimeout(() => this.flush(), this.#ttl);
|
|
297
|
+
this.#timer.unref?.();
|
|
298
|
+
}
|
|
111
299
|
#entry(slug) {
|
|
112
300
|
const cached = this.#cache.get(slug);
|
|
113
301
|
if (!cached) return this.#load(slug);
|
|
@@ -164,7 +352,10 @@ var PromptHelm = class {
|
|
|
164
352
|
if (!response.ok) throw await errorFromResponse(response);
|
|
165
353
|
const payload = await response.json();
|
|
166
354
|
const entry = {
|
|
355
|
+
version: payload.version,
|
|
167
356
|
content: payload.content,
|
|
357
|
+
config: payload.config ?? {},
|
|
358
|
+
experiment: payload.experiment,
|
|
168
359
|
etag: response.headers.get("etag"),
|
|
169
360
|
fetchedAt: Date.now()
|
|
170
361
|
};
|
|
@@ -172,6 +363,18 @@ var PromptHelm = class {
|
|
|
172
363
|
return entry;
|
|
173
364
|
}
|
|
174
365
|
};
|
|
366
|
+
var IN = ["input_tokens", "prompt_tokens", "inputTokens", "promptTokenCount"];
|
|
367
|
+
var OUT = ["output_tokens", "completion_tokens", "outputTokens", "candidatesTokenCount"];
|
|
368
|
+
function tokens(usage, keys) {
|
|
369
|
+
for (const key of keys) {
|
|
370
|
+
const value = usage[key];
|
|
371
|
+
if (typeof value === "number" && value >= 0 && Number.isFinite(value)) return value;
|
|
372
|
+
}
|
|
373
|
+
return 0;
|
|
374
|
+
}
|
|
375
|
+
function add(counts, index, amount) {
|
|
376
|
+
if (counts) counts[index] = (counts[index] ?? 0) + amount;
|
|
377
|
+
}
|
|
175
378
|
async function errorFromResponse(response) {
|
|
176
379
|
const body = await response.json().catch(() => null);
|
|
177
380
|
const message = body?.error?.message ?? `HTTP ${response.status}`;
|
|
@@ -189,8 +392,10 @@ function toPromptHelmError(error) {
|
|
|
189
392
|
0 && (module.exports = {
|
|
190
393
|
ERROR_CODES,
|
|
191
394
|
FALLBACK_HEADER,
|
|
395
|
+
LATENCY_BUCKETS,
|
|
192
396
|
PromptHelm,
|
|
193
397
|
PromptHelmError,
|
|
194
398
|
extractVariables,
|
|
399
|
+
latencyBucket,
|
|
195
400
|
render
|
|
196
401
|
});
|
package/dist/index.d.cts
CHANGED
|
@@ -23,6 +23,30 @@ interface PromptPayload {
|
|
|
23
23
|
version: number;
|
|
24
24
|
content: string;
|
|
25
25
|
variables: string[];
|
|
26
|
+
/** The version's model settings; servers before SDK 1.4 send none. */
|
|
27
|
+
config?: PromptConfig;
|
|
28
|
+
/** An A/B test running on the prompt: the live version above is its control. */
|
|
29
|
+
experiment?: PromptExperiment;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Model settings published with a version, for your model call. Only the ones set in the
|
|
33
|
+
* dashboard are present; for the rest, your code's defaults apply.
|
|
34
|
+
*/
|
|
35
|
+
interface PromptConfig {
|
|
36
|
+
model?: string;
|
|
37
|
+
temperature?: number;
|
|
38
|
+
maxTokens?: number;
|
|
39
|
+
}
|
|
40
|
+
interface PromptExperiment {
|
|
41
|
+
id: string;
|
|
42
|
+
/** The variant's share of calls, in percent (1–99). */
|
|
43
|
+
weight: number;
|
|
44
|
+
variant: {
|
|
45
|
+
version: number;
|
|
46
|
+
content: string;
|
|
47
|
+
variables: string[];
|
|
48
|
+
config?: PromptConfig;
|
|
49
|
+
};
|
|
26
50
|
}
|
|
27
51
|
/**
|
|
28
52
|
* Sent with the next request for a slug after `get()` couldn't serve its live text: the error's
|
|
@@ -33,6 +57,35 @@ declare const FALLBACK_HEADER = "x-prompthelm-fallback";
|
|
|
33
57
|
interface PromptListPayload {
|
|
34
58
|
prompts: PromptPayload[];
|
|
35
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* Body of `POST /v1/reports`: what an app counted under running A/B tests since its last report.
|
|
62
|
+
* Numbers only, never any text, variable value or unit.
|
|
63
|
+
*/
|
|
64
|
+
interface ReportPayload {
|
|
65
|
+
experiments: {
|
|
66
|
+
id: string;
|
|
67
|
+
/** The variant's share when these calls were assigned. */
|
|
68
|
+
weight: number;
|
|
69
|
+
/** By version number. */
|
|
70
|
+
versions: Record<string, ReportCounts>;
|
|
71
|
+
}[];
|
|
72
|
+
}
|
|
73
|
+
interface ReportCounts {
|
|
74
|
+
served: number;
|
|
75
|
+
/** Picked for the variant, served the live version because a variable wasn't passed. */
|
|
76
|
+
skipped: number;
|
|
77
|
+
outcomes: number;
|
|
78
|
+
/** The outcomes' scores (true = 1, false = 0), summed, and their squares summed. */
|
|
79
|
+
scoreSum: number;
|
|
80
|
+
scoreSqSum: number;
|
|
81
|
+
/** Calls whose model usage was reported, their tokens and their durations in ms, summed. */
|
|
82
|
+
usage: number;
|
|
83
|
+
tokensIn: number;
|
|
84
|
+
tokensOut: number;
|
|
85
|
+
latencyMs: number;
|
|
86
|
+
/** Those calls per latency bucket (`latencyBucket`). */
|
|
87
|
+
latency: number[];
|
|
88
|
+
}
|
|
36
89
|
interface PromptHelmOptions {
|
|
37
90
|
/** Defaults to the PROMPTHELM_API_KEY environment variable. */
|
|
38
91
|
apiKey?: string;
|
|
@@ -50,6 +103,14 @@ interface PromptHelmOptions {
|
|
|
50
103
|
* prompt's page shows it. Defaults to true.
|
|
51
104
|
*/
|
|
52
105
|
reportFallbacks?: boolean;
|
|
106
|
+
/**
|
|
107
|
+
* Whether to send PromptHelm, as totals only, what happened under an A/B test: how often each
|
|
108
|
+
* version was served, the outcomes and the model usage you report. Defaults to
|
|
109
|
+
* `reportFallbacks`.
|
|
110
|
+
*/
|
|
111
|
+
reportResults?: boolean;
|
|
112
|
+
/** Called on every `get` and `resolve`, so your analytics or traces can carry the version. */
|
|
113
|
+
onResolve?: (event: ResolveEvent) => void;
|
|
53
114
|
}
|
|
54
115
|
interface GetOptions {
|
|
55
116
|
vars?: Vars;
|
|
@@ -58,13 +119,67 @@ interface GetOptions {
|
|
|
58
119
|
* version uses a variable this call doesn't pass. Without it those cases throw.
|
|
59
120
|
*/
|
|
60
121
|
fallback?: string;
|
|
122
|
+
/**
|
|
123
|
+
* Who the call is for, such as a user or conversation id: during an A/B test the same unit
|
|
124
|
+
* always gets the same version. Never sent to PromptHelm.
|
|
125
|
+
*/
|
|
126
|
+
unit?: string | number;
|
|
127
|
+
/** During an A/B test, serves this side, such as the variant to your own team. Never counted. */
|
|
128
|
+
force?: "control" | "variant";
|
|
129
|
+
}
|
|
130
|
+
interface ResolveEvent {
|
|
131
|
+
slug: string;
|
|
132
|
+
/** The version served, or null when the fallback was. */
|
|
133
|
+
version: number | null;
|
|
134
|
+
/** The A/B test running on the prompt, if any. */
|
|
135
|
+
experiment: string | null;
|
|
136
|
+
/** Whether the call got the test's variant. */
|
|
137
|
+
variant: boolean;
|
|
138
|
+
}
|
|
139
|
+
interface Resolved {
|
|
140
|
+
text: string;
|
|
141
|
+
/**
|
|
142
|
+
* The served version's model settings, such as `model`. Empty when the fallback was served,
|
|
143
|
+
* so your code's defaults apply.
|
|
144
|
+
*/
|
|
145
|
+
config: PromptConfig;
|
|
146
|
+
/** The version served, or null when the fallback was. */
|
|
147
|
+
version: number | null;
|
|
148
|
+
/**
|
|
149
|
+
* During an A/B test, names the test and the version this call got, so its outcome can be
|
|
150
|
+
* reported later from any process: `ph.outcome(ticket, true)`. No user in it. Null otherwise.
|
|
151
|
+
*/
|
|
152
|
+
ticket: string | null;
|
|
153
|
+
/** How the call went: true or false, or a score from 0 to 1. The first report counts. */
|
|
154
|
+
outcome(value: boolean | number): void;
|
|
155
|
+
/**
|
|
156
|
+
* The model call's tokens, from your provider's `usage` object, and how long it took: `ms`,
|
|
157
|
+
* else the time since `resolve` returned. The first report counts.
|
|
158
|
+
*/
|
|
159
|
+
usage(usage: object | null | undefined, ms?: number): void;
|
|
61
160
|
}
|
|
62
161
|
declare class PromptHelm {
|
|
63
162
|
#private;
|
|
64
163
|
constructor(options?: PromptHelmOptions);
|
|
65
164
|
/** The live version of a prompt with its variables filled in. */
|
|
66
165
|
get(slug: string, options?: GetOptions): Promise<string>;
|
|
166
|
+
/** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
|
|
167
|
+
resolve(slug: string, options?: GetOptions): Promise<Resolved>;
|
|
168
|
+
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
169
|
+
outcome(ticket: string, value: boolean | number): void;
|
|
170
|
+
/**
|
|
171
|
+
* Sends what was counted for A/B tests now, instead of within a ttl: call it before a
|
|
172
|
+
* serverless function freezes. Never throws; what can't be sent is kept for the next try.
|
|
173
|
+
*/
|
|
174
|
+
flush(): Promise<void>;
|
|
67
175
|
}
|
|
68
176
|
|
|
69
|
-
|
|
70
|
-
|
|
177
|
+
declare const LATENCY_BUCKETS = 24;
|
|
178
|
+
/**
|
|
179
|
+
* Which of 24 latency buckets a model call's duration falls in, for its percentiles: bucket i
|
|
180
|
+
* holds calls up to 50·2^(i/2) ms (bucket 0 under 50 ms), and the last one everything slower.
|
|
181
|
+
*/
|
|
182
|
+
declare function latencyBucket(ms: number): number;
|
|
183
|
+
|
|
184
|
+
export { ERROR_CODES, FALLBACK_HEADER, LATENCY_BUCKETS, PromptHelm, PromptHelmError, extractVariables, latencyBucket, render };
|
|
185
|
+
export type { GetOptions, PromptConfig, PromptExperiment, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
|
package/dist/index.d.ts
CHANGED
|
@@ -23,6 +23,30 @@ interface PromptPayload {
|
|
|
23
23
|
version: number;
|
|
24
24
|
content: string;
|
|
25
25
|
variables: string[];
|
|
26
|
+
/** The version's model settings; servers before SDK 1.4 send none. */
|
|
27
|
+
config?: PromptConfig;
|
|
28
|
+
/** An A/B test running on the prompt: the live version above is its control. */
|
|
29
|
+
experiment?: PromptExperiment;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Model settings published with a version, for your model call. Only the ones set in the
|
|
33
|
+
* dashboard are present; for the rest, your code's defaults apply.
|
|
34
|
+
*/
|
|
35
|
+
interface PromptConfig {
|
|
36
|
+
model?: string;
|
|
37
|
+
temperature?: number;
|
|
38
|
+
maxTokens?: number;
|
|
39
|
+
}
|
|
40
|
+
interface PromptExperiment {
|
|
41
|
+
id: string;
|
|
42
|
+
/** The variant's share of calls, in percent (1–99). */
|
|
43
|
+
weight: number;
|
|
44
|
+
variant: {
|
|
45
|
+
version: number;
|
|
46
|
+
content: string;
|
|
47
|
+
variables: string[];
|
|
48
|
+
config?: PromptConfig;
|
|
49
|
+
};
|
|
26
50
|
}
|
|
27
51
|
/**
|
|
28
52
|
* Sent with the next request for a slug after `get()` couldn't serve its live text: the error's
|
|
@@ -33,6 +57,35 @@ declare const FALLBACK_HEADER = "x-prompthelm-fallback";
|
|
|
33
57
|
interface PromptListPayload {
|
|
34
58
|
prompts: PromptPayload[];
|
|
35
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* Body of `POST /v1/reports`: what an app counted under running A/B tests since its last report.
|
|
62
|
+
* Numbers only, never any text, variable value or unit.
|
|
63
|
+
*/
|
|
64
|
+
interface ReportPayload {
|
|
65
|
+
experiments: {
|
|
66
|
+
id: string;
|
|
67
|
+
/** The variant's share when these calls were assigned. */
|
|
68
|
+
weight: number;
|
|
69
|
+
/** By version number. */
|
|
70
|
+
versions: Record<string, ReportCounts>;
|
|
71
|
+
}[];
|
|
72
|
+
}
|
|
73
|
+
interface ReportCounts {
|
|
74
|
+
served: number;
|
|
75
|
+
/** Picked for the variant, served the live version because a variable wasn't passed. */
|
|
76
|
+
skipped: number;
|
|
77
|
+
outcomes: number;
|
|
78
|
+
/** The outcomes' scores (true = 1, false = 0), summed, and their squares summed. */
|
|
79
|
+
scoreSum: number;
|
|
80
|
+
scoreSqSum: number;
|
|
81
|
+
/** Calls whose model usage was reported, their tokens and their durations in ms, summed. */
|
|
82
|
+
usage: number;
|
|
83
|
+
tokensIn: number;
|
|
84
|
+
tokensOut: number;
|
|
85
|
+
latencyMs: number;
|
|
86
|
+
/** Those calls per latency bucket (`latencyBucket`). */
|
|
87
|
+
latency: number[];
|
|
88
|
+
}
|
|
36
89
|
interface PromptHelmOptions {
|
|
37
90
|
/** Defaults to the PROMPTHELM_API_KEY environment variable. */
|
|
38
91
|
apiKey?: string;
|
|
@@ -50,6 +103,14 @@ interface PromptHelmOptions {
|
|
|
50
103
|
* prompt's page shows it. Defaults to true.
|
|
51
104
|
*/
|
|
52
105
|
reportFallbacks?: boolean;
|
|
106
|
+
/**
|
|
107
|
+
* Whether to send PromptHelm, as totals only, what happened under an A/B test: how often each
|
|
108
|
+
* version was served, the outcomes and the model usage you report. Defaults to
|
|
109
|
+
* `reportFallbacks`.
|
|
110
|
+
*/
|
|
111
|
+
reportResults?: boolean;
|
|
112
|
+
/** Called on every `get` and `resolve`, so your analytics or traces can carry the version. */
|
|
113
|
+
onResolve?: (event: ResolveEvent) => void;
|
|
53
114
|
}
|
|
54
115
|
interface GetOptions {
|
|
55
116
|
vars?: Vars;
|
|
@@ -58,13 +119,67 @@ interface GetOptions {
|
|
|
58
119
|
* version uses a variable this call doesn't pass. Without it those cases throw.
|
|
59
120
|
*/
|
|
60
121
|
fallback?: string;
|
|
122
|
+
/**
|
|
123
|
+
* Who the call is for, such as a user or conversation id: during an A/B test the same unit
|
|
124
|
+
* always gets the same version. Never sent to PromptHelm.
|
|
125
|
+
*/
|
|
126
|
+
unit?: string | number;
|
|
127
|
+
/** During an A/B test, serves this side, such as the variant to your own team. Never counted. */
|
|
128
|
+
force?: "control" | "variant";
|
|
129
|
+
}
|
|
130
|
+
interface ResolveEvent {
|
|
131
|
+
slug: string;
|
|
132
|
+
/** The version served, or null when the fallback was. */
|
|
133
|
+
version: number | null;
|
|
134
|
+
/** The A/B test running on the prompt, if any. */
|
|
135
|
+
experiment: string | null;
|
|
136
|
+
/** Whether the call got the test's variant. */
|
|
137
|
+
variant: boolean;
|
|
138
|
+
}
|
|
139
|
+
interface Resolved {
|
|
140
|
+
text: string;
|
|
141
|
+
/**
|
|
142
|
+
* The served version's model settings, such as `model`. Empty when the fallback was served,
|
|
143
|
+
* so your code's defaults apply.
|
|
144
|
+
*/
|
|
145
|
+
config: PromptConfig;
|
|
146
|
+
/** The version served, or null when the fallback was. */
|
|
147
|
+
version: number | null;
|
|
148
|
+
/**
|
|
149
|
+
* During an A/B test, names the test and the version this call got, so its outcome can be
|
|
150
|
+
* reported later from any process: `ph.outcome(ticket, true)`. No user in it. Null otherwise.
|
|
151
|
+
*/
|
|
152
|
+
ticket: string | null;
|
|
153
|
+
/** How the call went: true or false, or a score from 0 to 1. The first report counts. */
|
|
154
|
+
outcome(value: boolean | number): void;
|
|
155
|
+
/**
|
|
156
|
+
* The model call's tokens, from your provider's `usage` object, and how long it took: `ms`,
|
|
157
|
+
* else the time since `resolve` returned. The first report counts.
|
|
158
|
+
*/
|
|
159
|
+
usage(usage: object | null | undefined, ms?: number): void;
|
|
61
160
|
}
|
|
62
161
|
declare class PromptHelm {
|
|
63
162
|
#private;
|
|
64
163
|
constructor(options?: PromptHelmOptions);
|
|
65
164
|
/** The live version of a prompt with its variables filled in. */
|
|
66
165
|
get(slug: string, options?: GetOptions): Promise<string>;
|
|
166
|
+
/** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
|
|
167
|
+
resolve(slug: string, options?: GetOptions): Promise<Resolved>;
|
|
168
|
+
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
169
|
+
outcome(ticket: string, value: boolean | number): void;
|
|
170
|
+
/**
|
|
171
|
+
* Sends what was counted for A/B tests now, instead of within a ttl: call it before a
|
|
172
|
+
* serverless function freezes. Never throws; what can't be sent is kept for the next try.
|
|
173
|
+
*/
|
|
174
|
+
flush(): Promise<void>;
|
|
67
175
|
}
|
|
68
176
|
|
|
69
|
-
|
|
70
|
-
|
|
177
|
+
declare const LATENCY_BUCKETS = 24;
|
|
178
|
+
/**
|
|
179
|
+
* Which of 24 latency buckets a model call's duration falls in, for its percentiles: bucket i
|
|
180
|
+
* holds calls up to 50·2^(i/2) ms (bucket 0 under 50 ms), and the last one everything slower.
|
|
181
|
+
*/
|
|
182
|
+
declare function latencyBucket(ms: number): number;
|
|
183
|
+
|
|
184
|
+
export { ERROR_CODES, FALLBACK_HEADER, LATENCY_BUCKETS, PromptHelm, PromptHelmError, extractVariables, latencyBucket, render };
|
|
185
|
+
export type { GetOptions, PromptConfig, PromptExperiment, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
|
package/dist/index.js
CHANGED
|
@@ -18,6 +18,20 @@ var PromptHelmError = class extends Error {
|
|
|
18
18
|
name = "PromptHelmError";
|
|
19
19
|
};
|
|
20
20
|
|
|
21
|
+
// src/experiment.ts
|
|
22
|
+
function fnv1a32(text) {
|
|
23
|
+
let hash = 2166136261;
|
|
24
|
+
for (const byte of new TextEncoder().encode(text)) hash = Math.imul(hash ^ byte, 16777619);
|
|
25
|
+
return hash >>> 0;
|
|
26
|
+
}
|
|
27
|
+
function bucket(experimentId, unit) {
|
|
28
|
+
return unit === void 0 ? Math.floor(Math.random() * 100) : fnv1a32(`${experimentId}:${unit}`) % 100;
|
|
29
|
+
}
|
|
30
|
+
var LATENCY_BUCKETS = 24;
|
|
31
|
+
function latencyBucket(ms) {
|
|
32
|
+
return ms < 50 ? 0 : Math.min(LATENCY_BUCKETS - 1, 1 + Math.floor(2 * Math.log2(ms / 50)));
|
|
33
|
+
}
|
|
34
|
+
|
|
21
35
|
// src/template.ts
|
|
22
36
|
var VARIABLE = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
|
|
23
37
|
function extractVariables(template) {
|
|
@@ -38,6 +52,27 @@ function render(template, vars = {}) {
|
|
|
38
52
|
// src/client.ts
|
|
39
53
|
var DEFAULT_BASE_URL = "https://prompthelm.app";
|
|
40
54
|
var FALLBACK_HEADER = "x-prompthelm-fallback";
|
|
55
|
+
var FIELDS = [
|
|
56
|
+
"served",
|
|
57
|
+
"skipped",
|
|
58
|
+
"outcomes",
|
|
59
|
+
"scoreSum",
|
|
60
|
+
"scoreSqSum",
|
|
61
|
+
"usage",
|
|
62
|
+
"tokensIn",
|
|
63
|
+
"tokensOut",
|
|
64
|
+
"latencyMs"
|
|
65
|
+
];
|
|
66
|
+
var SERVED = 0;
|
|
67
|
+
var SKIPPED = 1;
|
|
68
|
+
var OUTCOMES = 2;
|
|
69
|
+
var SCORE = 3;
|
|
70
|
+
var SCORE_SQ = 4;
|
|
71
|
+
var USAGE = 5;
|
|
72
|
+
var TOKENS_IN = 6;
|
|
73
|
+
var TOKENS_OUT = 7;
|
|
74
|
+
var LATENCY = 8;
|
|
75
|
+
var BUCKETS = FIELDS.length;
|
|
41
76
|
var PromptHelm = class {
|
|
42
77
|
#apiKey;
|
|
43
78
|
#baseUrl;
|
|
@@ -45,12 +80,16 @@ var PromptHelm = class {
|
|
|
45
80
|
#timeout;
|
|
46
81
|
#fetch;
|
|
47
82
|
#onError;
|
|
83
|
+
#onResolve;
|
|
48
84
|
#cache = /* @__PURE__ */ new Map();
|
|
49
85
|
#inflight = /* @__PURE__ */ new Map();
|
|
50
86
|
#reportFallbacks;
|
|
87
|
+
#reportResults;
|
|
51
88
|
// Per slug, why the live text last couldn't be served, until a request carries it. An object,
|
|
52
89
|
// so a report made while that request is in flight isn't cleared with the one it carried.
|
|
53
90
|
#unreported = /* @__PURE__ */ new Map();
|
|
91
|
+
#pending = /* @__PURE__ */ new Map();
|
|
92
|
+
#timer;
|
|
54
93
|
constructor(options = {}) {
|
|
55
94
|
const apiKey = options.apiKey ?? env("PROMPTHELM_API_KEY");
|
|
56
95
|
if (!apiKey) throw new Error("PromptHelm: pass apiKey or set PROMPTHELM_API_KEY");
|
|
@@ -61,22 +100,169 @@ var PromptHelm = class {
|
|
|
61
100
|
this.#timeout = options.timeout ?? 3e3;
|
|
62
101
|
this.#fetch = options.fetch ?? ((input, init) => globalThis.fetch(input, init));
|
|
63
102
|
this.#onError = options.onError ?? ((error, slug) => console.warn(`[PromptHelm] ${slug}: ${error.message}`));
|
|
103
|
+
this.#onResolve = options.onResolve;
|
|
64
104
|
this.#reportFallbacks = options.reportFallbacks ?? true;
|
|
105
|
+
this.#reportResults = options.reportResults ?? this.#reportFallbacks;
|
|
65
106
|
}
|
|
66
107
|
/** The live version of a prompt with its variables filled in. */
|
|
67
108
|
async get(slug, options = {}) {
|
|
109
|
+
return (await this.resolve(slug, options)).text;
|
|
110
|
+
}
|
|
111
|
+
/** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
|
|
112
|
+
async resolve(slug, options = {}) {
|
|
68
113
|
const { vars, fallback } = options;
|
|
69
114
|
try {
|
|
70
|
-
|
|
71
|
-
return render(entry.content, vars);
|
|
115
|
+
return this.#pick(slug, await this.#entry(slug), options);
|
|
72
116
|
} catch (error) {
|
|
73
117
|
const failure = toPromptHelmError(error);
|
|
74
118
|
this.#remember(slug, failure.code);
|
|
75
119
|
if (fallback === void 0) throw failure;
|
|
76
120
|
this.#onError(failure, slug);
|
|
77
|
-
return render(fallback, vars);
|
|
121
|
+
return this.#served(slug, render(fallback, vars), {}, null);
|
|
78
122
|
}
|
|
79
123
|
}
|
|
124
|
+
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
125
|
+
outcome(ticket, value) {
|
|
126
|
+
const [id = "", version, weight] = ticket.split(".");
|
|
127
|
+
const test = { id, weight: Number(weight) };
|
|
128
|
+
if (id && test.weight > 0) this.#outcome(test, Number(version), value);
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Sends what was counted for A/B tests now, instead of within a ttl: call it before a
|
|
132
|
+
* serverless function freezes. Never throws; what can't be sent is kept for the next try.
|
|
133
|
+
*/
|
|
134
|
+
async flush() {
|
|
135
|
+
clearTimeout(this.#timer);
|
|
136
|
+
this.#timer = void 0;
|
|
137
|
+
const sent = this.#pending;
|
|
138
|
+
if (sent.size === 0) return;
|
|
139
|
+
this.#pending = /* @__PURE__ */ new Map();
|
|
140
|
+
const body = {
|
|
141
|
+
experiments: [...sent.values()].map(({ id, weight, versions }) => ({
|
|
142
|
+
id,
|
|
143
|
+
weight,
|
|
144
|
+
versions: Object.fromEntries(
|
|
145
|
+
[...versions].map(([version, counts]) => [
|
|
146
|
+
version,
|
|
147
|
+
{
|
|
148
|
+
...Object.fromEntries(FIELDS.map((field, i) => [field, counts[i]])),
|
|
149
|
+
latency: counts.slice(BUCKETS)
|
|
150
|
+
}
|
|
151
|
+
])
|
|
152
|
+
)
|
|
153
|
+
}))
|
|
154
|
+
};
|
|
155
|
+
try {
|
|
156
|
+
const response = await this.#fetch(`${this.#baseUrl}/v1/reports`, {
|
|
157
|
+
method: "POST",
|
|
158
|
+
headers: { authorization: `Bearer ${this.#apiKey}`, "content-type": "application/json" },
|
|
159
|
+
body: JSON.stringify(body),
|
|
160
|
+
signal: AbortSignal.timeout(this.#timeout)
|
|
161
|
+
});
|
|
162
|
+
if (response.status < 500) return;
|
|
163
|
+
} catch {
|
|
164
|
+
}
|
|
165
|
+
for (const { id, weight, versions } of sent.values()) {
|
|
166
|
+
for (const [version, counts] of versions) {
|
|
167
|
+
const total = this.#counts({ id, weight }, version);
|
|
168
|
+
counts.forEach((n, i) => {
|
|
169
|
+
add(total, i, n);
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
#pick(slug, entry, options) {
|
|
175
|
+
const { vars, unit, force } = options;
|
|
176
|
+
const { experiment } = entry;
|
|
177
|
+
const test = experiment && !force ? { id: experiment.id, weight: experiment.weight } : void 0;
|
|
178
|
+
let skipped;
|
|
179
|
+
if (experiment && (force ? force === "variant" : bucket(experiment.id, unit) < experiment.weight)) {
|
|
180
|
+
const { version, content, config = {} } = experiment.variant;
|
|
181
|
+
try {
|
|
182
|
+
return this.#served(
|
|
183
|
+
slug,
|
|
184
|
+
render(content, vars),
|
|
185
|
+
config,
|
|
186
|
+
version,
|
|
187
|
+
experiment.id,
|
|
188
|
+
test,
|
|
189
|
+
true
|
|
190
|
+
);
|
|
191
|
+
} catch (error) {
|
|
192
|
+
skipped = toPromptHelmError(error);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
const text = render(entry.content, vars);
|
|
196
|
+
if (skipped && experiment) {
|
|
197
|
+
this.#onError(skipped, slug);
|
|
198
|
+
if (test) add(this.#counts(test, experiment.variant.version), SKIPPED, 1);
|
|
199
|
+
}
|
|
200
|
+
return this.#served(slug, text, entry.config, entry.version, experiment?.id, test);
|
|
201
|
+
}
|
|
202
|
+
/** Counts the call, tells `onResolve`, and hands the caller its text, settings and what to report. */
|
|
203
|
+
#served(slug, text, config, version, experiment, test, variant = false) {
|
|
204
|
+
if (test && version !== null) add(this.#counts(test, version), SERVED, 1);
|
|
205
|
+
try {
|
|
206
|
+
this.#onResolve?.({ slug, version, experiment: experiment ?? null, variant });
|
|
207
|
+
} catch {
|
|
208
|
+
}
|
|
209
|
+
const started = Date.now();
|
|
210
|
+
let reported = 0;
|
|
211
|
+
return {
|
|
212
|
+
text,
|
|
213
|
+
// A copy: the cached one serves every later call.
|
|
214
|
+
config: { ...config },
|
|
215
|
+
version,
|
|
216
|
+
ticket: test && version !== null ? `${test.id}.${version}.${test.weight}` : null,
|
|
217
|
+
outcome: (value) => {
|
|
218
|
+
if (!test || version === null || reported & 1) return;
|
|
219
|
+
reported |= 1;
|
|
220
|
+
this.#outcome(test, version, value);
|
|
221
|
+
},
|
|
222
|
+
usage: (usage, ms = Date.now() - started) => {
|
|
223
|
+
if (!test || version === null || reported & 2) return;
|
|
224
|
+
reported |= 2;
|
|
225
|
+
const counts = this.#counts(test, version);
|
|
226
|
+
const fields = usage ?? {};
|
|
227
|
+
add(counts, USAGE, 1);
|
|
228
|
+
add(counts, TOKENS_IN, tokens(fields, IN));
|
|
229
|
+
add(counts, TOKENS_OUT, tokens(fields, OUT));
|
|
230
|
+
add(counts, LATENCY, Math.max(0, ms));
|
|
231
|
+
add(counts, BUCKETS + latencyBucket(ms), 1);
|
|
232
|
+
}
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
#outcome(test, version, value) {
|
|
236
|
+
const score = value === true ? 1 : value === false ? 0 : value;
|
|
237
|
+
if (!Number.isInteger(version) || !(score >= 0 && score <= 1)) return;
|
|
238
|
+
const counts = this.#counts(test, version);
|
|
239
|
+
add(counts, OUTCOMES, 1);
|
|
240
|
+
add(counts, SCORE, score);
|
|
241
|
+
add(counts, SCORE_SQ, score * score);
|
|
242
|
+
}
|
|
243
|
+
/** The counts to add a call to, kept until the next report; none when reporting is off. */
|
|
244
|
+
#counts(test, version) {
|
|
245
|
+
if (!this.#reportResults) return;
|
|
246
|
+
const key = `${test.id}.${test.weight}`;
|
|
247
|
+
let pending = this.#pending.get(key);
|
|
248
|
+
if (!pending) {
|
|
249
|
+
pending = { ...test, versions: /* @__PURE__ */ new Map() };
|
|
250
|
+
this.#pending.set(key, pending);
|
|
251
|
+
}
|
|
252
|
+
let counts = pending.versions.get(version);
|
|
253
|
+
if (!counts) {
|
|
254
|
+
counts = new Array(BUCKETS + LATENCY_BUCKETS).fill(0);
|
|
255
|
+
pending.versions.set(version, counts);
|
|
256
|
+
}
|
|
257
|
+
this.#schedule();
|
|
258
|
+
return counts;
|
|
259
|
+
}
|
|
260
|
+
/** Reports within a ttl of the first count, without keeping the process alive for it. */
|
|
261
|
+
#schedule() {
|
|
262
|
+
if (this.#timer) return;
|
|
263
|
+
this.#timer = setTimeout(() => this.flush(), this.#ttl);
|
|
264
|
+
this.#timer.unref?.();
|
|
265
|
+
}
|
|
80
266
|
#entry(slug) {
|
|
81
267
|
const cached = this.#cache.get(slug);
|
|
82
268
|
if (!cached) return this.#load(slug);
|
|
@@ -133,7 +319,10 @@ var PromptHelm = class {
|
|
|
133
319
|
if (!response.ok) throw await errorFromResponse(response);
|
|
134
320
|
const payload = await response.json();
|
|
135
321
|
const entry = {
|
|
322
|
+
version: payload.version,
|
|
136
323
|
content: payload.content,
|
|
324
|
+
config: payload.config ?? {},
|
|
325
|
+
experiment: payload.experiment,
|
|
137
326
|
etag: response.headers.get("etag"),
|
|
138
327
|
fetchedAt: Date.now()
|
|
139
328
|
};
|
|
@@ -141,6 +330,18 @@ var PromptHelm = class {
|
|
|
141
330
|
return entry;
|
|
142
331
|
}
|
|
143
332
|
};
|
|
333
|
+
var IN = ["input_tokens", "prompt_tokens", "inputTokens", "promptTokenCount"];
|
|
334
|
+
var OUT = ["output_tokens", "completion_tokens", "outputTokens", "candidatesTokenCount"];
|
|
335
|
+
function tokens(usage, keys) {
|
|
336
|
+
for (const key of keys) {
|
|
337
|
+
const value = usage[key];
|
|
338
|
+
if (typeof value === "number" && value >= 0 && Number.isFinite(value)) return value;
|
|
339
|
+
}
|
|
340
|
+
return 0;
|
|
341
|
+
}
|
|
342
|
+
function add(counts, index, amount) {
|
|
343
|
+
if (counts) counts[index] = (counts[index] ?? 0) + amount;
|
|
344
|
+
}
|
|
144
345
|
async function errorFromResponse(response) {
|
|
145
346
|
const body = await response.json().catch(() => null);
|
|
146
347
|
const message = body?.error?.message ?? `HTTP ${response.status}`;
|
|
@@ -157,8 +358,10 @@ function toPromptHelmError(error) {
|
|
|
157
358
|
export {
|
|
158
359
|
ERROR_CODES,
|
|
159
360
|
FALLBACK_HEADER,
|
|
361
|
+
LATENCY_BUCKETS,
|
|
160
362
|
PromptHelm,
|
|
161
363
|
PromptHelmError,
|
|
162
364
|
extractVariables,
|
|
365
|
+
latencyBucket,
|
|
163
366
|
render
|
|
164
367
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@prompt-helm/sdk",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.4.0",
|
|
4
4
|
"description": "Fetch your PromptHelm prompts at runtime, so changing a prompt needs no redeploy.",
|
|
5
5
|
"homepage": "https://prompthelm.app/quickstart",
|
|
6
6
|
"repository": {
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
}
|
|
38
38
|
},
|
|
39
39
|
"bin": {
|
|
40
|
-
"prompthelm": "
|
|
40
|
+
"prompthelm": "dist/cli.js"
|
|
41
41
|
},
|
|
42
42
|
"files": [
|
|
43
43
|
"dist"
|