@prompt-helm/sdk 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -1
- package/dist/cli.js +12 -0
- package/dist/index.cjs +197 -3
- package/dist/index.d.cts +100 -2
- package/dist/index.d.ts +100 -2
- package/dist/index.js +195 -3
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -14,7 +14,7 @@ refresh, 60 seconds by default.
|
|
|
14
14
|
a refresh that finds nothing new is an empty `304`.
|
|
15
15
|
- **Your data stays with you.** Variables are filled in on your server. PromptHelm never sees
|
|
16
16
|
their values, your provider keys or your LLM traffic.
|
|
17
|
-
- **Small.** No dependencies, under
|
|
17
|
+
- **Small.** No dependencies, under 3 kB gzipped, ESM and CommonJS, Node.js 18 and later.
|
|
18
18
|
|
|
19
19
|
## Install
|
|
20
20
|
|
|
@@ -60,6 +60,8 @@ All optional when the environment variables are set.
|
|
|
60
60
|
| `timeout` | `3000` | How long, in ms, one request may take. |
|
|
61
61
|
| `onError` | `console.warn` | Called with the error and the slug when a fallback or a cached prompt was served instead of failing. |
|
|
62
62
|
| `reportFallbacks` | `true` | Tells PromptHelm, by error code only, when a fallback or a cached prompt was served, so the prompt's page shows which app fell back and why. |
|
|
63
|
+
| `reportResults` | `reportFallbacks` | During an [A/B test](#ab-tests), sends PromptHelm totals only: calls served per version, and the outcomes and usage you report. |
|
|
64
|
+
| `onResolve` | none | Called on every `get` and `resolve` with the version served, for your analytics or traces. |
|
|
63
65
|
| `fetch` | global `fetch` | Your own fetch, for tests or a proxy. |
|
|
64
66
|
|
|
65
67
|
## Errors
|
|
@@ -106,6 +108,58 @@ Run it in CI before the build, so every deploy carries the text that was live th
|
|
|
106
108
|
`PROMPTHELM_API_KEY` (and `PROMPTHELM_BASE_URL`), and writes JSON for a `.json` path or with
|
|
107
109
|
`--format json`. A prompt appears in the file once it's published.
|
|
108
110
|
|
|
111
|
+
## A/B tests
|
|
112
|
+
|
|
113
|
+
On the Pro plan, a prompt's page can run an experiment: the live version keeps most calls and a
|
|
114
|
+
variant gets the share you pick. The SDK picks the version for each call on your server. Pass a
|
|
115
|
+
`unit`, such as a user or conversation id, and that unit gets the same version every time;
|
|
116
|
+
without one, each call is picked at random. Use `resolve` to learn which version a call got, and
|
|
117
|
+
tell PromptHelm how it went:
|
|
118
|
+
|
|
119
|
+
```ts
|
|
120
|
+
const r = await ph.resolve("support-reply", {
|
|
121
|
+
vars: { company: "Acme", customer_name: user.name },
|
|
122
|
+
fallback: SUPPORT_REPLY,
|
|
123
|
+
unit: user.id,
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
const reply = await openai.responses.create({ model: "gpt-5", input: r.text });
|
|
127
|
+
r.usage(reply.usage); // tokens, and how long the call took since resolve
|
|
128
|
+
r.outcome(userKeptTheReply); // true or false, or a score from 0 to 1
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
The prompt's page then compares the two versions: how often each succeeded, with a range that
|
|
132
|
+
stays honest however often you look, and their tokens and latency per call. It says when there's
|
|
133
|
+
an answer; shipping the variant stays your call.
|
|
134
|
+
|
|
135
|
+
- **Outcomes that come later.** A purchase days later, in another service: keep `r.ticket` with
|
|
136
|
+
your own record, then call `ph.outcome(ticket, true)` from any process. The ticket names the
|
|
137
|
+
test and the version, not the user.
|
|
138
|
+
- **Your own analytics.** `onResolve` is called on every `get` and `resolve` with the slug, the
|
|
139
|
+
version, the test and whether the call got the variant, so your events or traces can carry it:
|
|
140
|
+
`new PromptHelm({ onResolve: (e) => posthog.capture("prompt_served", e) })`.
|
|
141
|
+
- **Your team first.** `force: "variant"` (or `"control"`) serves that side to a call, such as
|
|
142
|
+
to your own staff. Forced calls are never counted.
|
|
143
|
+
- **Serverless.** Counts are sent within a ttl. A function that freezes after its response
|
|
144
|
+
should `await ph.flush()` first (with `waitUntil`, where the platform has it).
|
|
145
|
+
- **Usage objects** from OpenAI (Responses and Chat Completions), Anthropic, the Vercel AI SDK
|
|
146
|
+
and Gemini are read as they are. `r.usage(null, ms)` reports a duration alone.
|
|
147
|
+
|
|
148
|
+
Only totals reach PromptHelm: how many calls each version served, the outcomes, tokens and
|
|
149
|
+
durations you report. Never the text, a variable's value or the unit. Counts a process holds
|
|
150
|
+
when it exits are lost, so read them as a sample. `reportResults: false` sends none; unless you
|
|
151
|
+
set it, it follows `reportFallbacks`.
|
|
152
|
+
|
|
153
|
+
If the variant uses a variable the call doesn't pass, that call gets the live version and
|
|
154
|
+
`onError` receives `missing_variables`; it isn't reported as a fallback. Growing the variant's
|
|
155
|
+
share only moves units from the live version to the variant, never back. Apps on SDK versions
|
|
156
|
+
before 1.3 ignore experiments and serve the live version to every call.
|
|
157
|
+
|
|
158
|
+
To split calls the same way from another language: during an experiment the response carries
|
|
159
|
+
`experiment: { id, weight, variant: { version, content, variables } }`. A unit's bucket is the
|
|
160
|
+
32-bit FNV-1a hash of the UTF-8 bytes of `<experiment id>:<unit>`, modulo 100; buckets below
|
|
161
|
+
`weight` get the variant.
|
|
162
|
+
|
|
109
163
|
## Other languages
|
|
110
164
|
|
|
111
165
|
Any language can fetch the live version with one request:
|
package/dist/cli.js
CHANGED
|
@@ -18,6 +18,18 @@ var PromptHelmError = class extends Error {
|
|
|
18
18
|
|
|
19
19
|
// src/client.ts
|
|
20
20
|
var DEFAULT_BASE_URL = "https://prompthelm.app";
|
|
21
|
+
var FIELDS = [
|
|
22
|
+
"served",
|
|
23
|
+
"skipped",
|
|
24
|
+
"outcomes",
|
|
25
|
+
"scoreSum",
|
|
26
|
+
"scoreSqSum",
|
|
27
|
+
"usage",
|
|
28
|
+
"tokensIn",
|
|
29
|
+
"tokensOut",
|
|
30
|
+
"latencyMs"
|
|
31
|
+
];
|
|
32
|
+
var BUCKETS = FIELDS.length;
|
|
21
33
|
async function errorFromResponse(response) {
|
|
22
34
|
const body = await response.json().catch(() => null);
|
|
23
35
|
const message = body?.error?.message ?? `HTTP ${response.status}`;
|
package/dist/index.cjs
CHANGED
|
@@ -22,9 +22,11 @@ var index_exports = {};
|
|
|
22
22
|
__export(index_exports, {
|
|
23
23
|
ERROR_CODES: () => ERROR_CODES,
|
|
24
24
|
FALLBACK_HEADER: () => FALLBACK_HEADER,
|
|
25
|
+
LATENCY_BUCKETS: () => LATENCY_BUCKETS,
|
|
25
26
|
PromptHelm: () => PromptHelm,
|
|
26
27
|
PromptHelmError: () => PromptHelmError,
|
|
27
28
|
extractVariables: () => extractVariables,
|
|
29
|
+
latencyBucket: () => latencyBucket,
|
|
28
30
|
render: () => render
|
|
29
31
|
});
|
|
30
32
|
module.exports = __toCommonJS(index_exports);
|
|
@@ -49,6 +51,20 @@ var PromptHelmError = class extends Error {
|
|
|
49
51
|
name = "PromptHelmError";
|
|
50
52
|
};
|
|
51
53
|
|
|
54
|
+
// src/experiment.ts
|
|
55
|
+
function fnv1a32(text) {
|
|
56
|
+
let hash = 2166136261;
|
|
57
|
+
for (const byte of new TextEncoder().encode(text)) hash = Math.imul(hash ^ byte, 16777619);
|
|
58
|
+
return hash >>> 0;
|
|
59
|
+
}
|
|
60
|
+
function bucket(experimentId, unit) {
|
|
61
|
+
return unit === void 0 ? Math.floor(Math.random() * 100) : fnv1a32(`${experimentId}:${unit}`) % 100;
|
|
62
|
+
}
|
|
63
|
+
var LATENCY_BUCKETS = 24;
|
|
64
|
+
function latencyBucket(ms) {
|
|
65
|
+
return ms < 50 ? 0 : Math.min(LATENCY_BUCKETS - 1, 1 + Math.floor(2 * Math.log2(ms / 50)));
|
|
66
|
+
}
|
|
67
|
+
|
|
52
68
|
// src/template.ts
|
|
53
69
|
var VARIABLE = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
|
|
54
70
|
function extractVariables(template) {
|
|
@@ -69,6 +85,27 @@ function render(template, vars = {}) {
|
|
|
69
85
|
// src/client.ts
|
|
70
86
|
var DEFAULT_BASE_URL = "https://prompthelm.app";
|
|
71
87
|
var FALLBACK_HEADER = "x-prompthelm-fallback";
|
|
88
|
+
var FIELDS = [
|
|
89
|
+
"served",
|
|
90
|
+
"skipped",
|
|
91
|
+
"outcomes",
|
|
92
|
+
"scoreSum",
|
|
93
|
+
"scoreSqSum",
|
|
94
|
+
"usage",
|
|
95
|
+
"tokensIn",
|
|
96
|
+
"tokensOut",
|
|
97
|
+
"latencyMs"
|
|
98
|
+
];
|
|
99
|
+
var SERVED = 0;
|
|
100
|
+
var SKIPPED = 1;
|
|
101
|
+
var OUTCOMES = 2;
|
|
102
|
+
var SCORE = 3;
|
|
103
|
+
var SCORE_SQ = 4;
|
|
104
|
+
var USAGE = 5;
|
|
105
|
+
var TOKENS_IN = 6;
|
|
106
|
+
var TOKENS_OUT = 7;
|
|
107
|
+
var LATENCY = 8;
|
|
108
|
+
var BUCKETS = FIELDS.length;
|
|
72
109
|
var PromptHelm = class {
|
|
73
110
|
#apiKey;
|
|
74
111
|
#baseUrl;
|
|
@@ -76,12 +113,16 @@ var PromptHelm = class {
|
|
|
76
113
|
#timeout;
|
|
77
114
|
#fetch;
|
|
78
115
|
#onError;
|
|
116
|
+
#onResolve;
|
|
79
117
|
#cache = /* @__PURE__ */ new Map();
|
|
80
118
|
#inflight = /* @__PURE__ */ new Map();
|
|
81
119
|
#reportFallbacks;
|
|
120
|
+
#reportResults;
|
|
82
121
|
// Per slug, why the live text last couldn't be served, until a request carries it. An object,
|
|
83
122
|
// so a report made while that request is in flight isn't cleared with the one it carried.
|
|
84
123
|
#unreported = /* @__PURE__ */ new Map();
|
|
124
|
+
#pending = /* @__PURE__ */ new Map();
|
|
125
|
+
#timer;
|
|
85
126
|
constructor(options = {}) {
|
|
86
127
|
const apiKey = options.apiKey ?? env("PROMPTHELM_API_KEY");
|
|
87
128
|
if (!apiKey) throw new Error("PromptHelm: pass apiKey or set PROMPTHELM_API_KEY");
|
|
@@ -92,22 +133,159 @@ var PromptHelm = class {
|
|
|
92
133
|
this.#timeout = options.timeout ?? 3e3;
|
|
93
134
|
this.#fetch = options.fetch ?? ((input, init) => globalThis.fetch(input, init));
|
|
94
135
|
this.#onError = options.onError ?? ((error, slug) => console.warn(`[PromptHelm] ${slug}: ${error.message}`));
|
|
136
|
+
this.#onResolve = options.onResolve;
|
|
95
137
|
this.#reportFallbacks = options.reportFallbacks ?? true;
|
|
138
|
+
this.#reportResults = options.reportResults ?? this.#reportFallbacks;
|
|
96
139
|
}
|
|
97
140
|
/** The live version of a prompt with its variables filled in. */
|
|
98
141
|
async get(slug, options = {}) {
|
|
142
|
+
return (await this.resolve(slug, options)).text;
|
|
143
|
+
}
|
|
144
|
+
/** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
|
|
145
|
+
async resolve(slug, options = {}) {
|
|
99
146
|
const { vars, fallback } = options;
|
|
100
147
|
try {
|
|
101
|
-
|
|
102
|
-
return render(entry.content, vars);
|
|
148
|
+
return this.#pick(slug, await this.#entry(slug), options);
|
|
103
149
|
} catch (error) {
|
|
104
150
|
const failure = toPromptHelmError(error);
|
|
105
151
|
this.#remember(slug, failure.code);
|
|
106
152
|
if (fallback === void 0) throw failure;
|
|
107
153
|
this.#onError(failure, slug);
|
|
108
|
-
return render(fallback, vars);
|
|
154
|
+
return this.#served(slug, render(fallback, vars), null);
|
|
109
155
|
}
|
|
110
156
|
}
|
|
157
|
+
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
158
|
+
outcome(ticket, value) {
|
|
159
|
+
const [id = "", version, weight] = ticket.split(".");
|
|
160
|
+
const test = { id, weight: Number(weight) };
|
|
161
|
+
if (id && test.weight > 0) this.#outcome(test, Number(version), value);
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Sends what was counted for A/B tests now, instead of within a ttl: call it before a
|
|
165
|
+
* serverless function freezes. Never throws; what can't be sent is kept for the next try.
|
|
166
|
+
*/
|
|
167
|
+
async flush() {
|
|
168
|
+
clearTimeout(this.#timer);
|
|
169
|
+
this.#timer = void 0;
|
|
170
|
+
const sent = this.#pending;
|
|
171
|
+
if (sent.size === 0) return;
|
|
172
|
+
this.#pending = /* @__PURE__ */ new Map();
|
|
173
|
+
const body = {
|
|
174
|
+
experiments: [...sent.values()].map(({ id, weight, versions }) => ({
|
|
175
|
+
id,
|
|
176
|
+
weight,
|
|
177
|
+
versions: Object.fromEntries(
|
|
178
|
+
[...versions].map(([version, counts]) => [
|
|
179
|
+
version,
|
|
180
|
+
{
|
|
181
|
+
...Object.fromEntries(FIELDS.map((field, i) => [field, counts[i]])),
|
|
182
|
+
latency: counts.slice(BUCKETS)
|
|
183
|
+
}
|
|
184
|
+
])
|
|
185
|
+
)
|
|
186
|
+
}))
|
|
187
|
+
};
|
|
188
|
+
try {
|
|
189
|
+
const response = await this.#fetch(`${this.#baseUrl}/v1/reports`, {
|
|
190
|
+
method: "POST",
|
|
191
|
+
headers: { authorization: `Bearer ${this.#apiKey}`, "content-type": "application/json" },
|
|
192
|
+
body: JSON.stringify(body),
|
|
193
|
+
signal: AbortSignal.timeout(this.#timeout)
|
|
194
|
+
});
|
|
195
|
+
if (response.status < 500) return;
|
|
196
|
+
} catch {
|
|
197
|
+
}
|
|
198
|
+
for (const { id, weight, versions } of sent.values()) {
|
|
199
|
+
for (const [version, counts] of versions) {
|
|
200
|
+
const total = this.#counts({ id, weight }, version);
|
|
201
|
+
counts.forEach((n, i) => {
|
|
202
|
+
add(total, i, n);
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
#pick(slug, entry, options) {
|
|
208
|
+
const { vars, unit, force } = options;
|
|
209
|
+
const { experiment } = entry;
|
|
210
|
+
const test = experiment && !force ? { id: experiment.id, weight: experiment.weight } : void 0;
|
|
211
|
+
let skipped;
|
|
212
|
+
if (experiment && (force ? force === "variant" : bucket(experiment.id, unit) < experiment.weight)) {
|
|
213
|
+
const { version, content } = experiment.variant;
|
|
214
|
+
try {
|
|
215
|
+
return this.#served(slug, render(content, vars), version, experiment.id, test, true);
|
|
216
|
+
} catch (error) {
|
|
217
|
+
skipped = toPromptHelmError(error);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
const text = render(entry.content, vars);
|
|
221
|
+
if (skipped && experiment) {
|
|
222
|
+
this.#onError(skipped, slug);
|
|
223
|
+
if (test) add(this.#counts(test, experiment.variant.version), SKIPPED, 1);
|
|
224
|
+
}
|
|
225
|
+
return this.#served(slug, text, entry.version, experiment?.id, test);
|
|
226
|
+
}
|
|
227
|
+
/** Counts the call, tells `onResolve`, and hands the caller its text and what to report. */
|
|
228
|
+
#served(slug, text, version, experiment, test, variant = false) {
|
|
229
|
+
if (test && version !== null) add(this.#counts(test, version), SERVED, 1);
|
|
230
|
+
try {
|
|
231
|
+
this.#onResolve?.({ slug, version, experiment: experiment ?? null, variant });
|
|
232
|
+
} catch {
|
|
233
|
+
}
|
|
234
|
+
const started = Date.now();
|
|
235
|
+
let reported = 0;
|
|
236
|
+
return {
|
|
237
|
+
text,
|
|
238
|
+
version,
|
|
239
|
+
ticket: test && version !== null ? `${test.id}.${version}.${test.weight}` : null,
|
|
240
|
+
outcome: (value) => {
|
|
241
|
+
if (!test || version === null || reported & 1) return;
|
|
242
|
+
reported |= 1;
|
|
243
|
+
this.#outcome(test, version, value);
|
|
244
|
+
},
|
|
245
|
+
usage: (usage, ms = Date.now() - started) => {
|
|
246
|
+
if (!test || version === null || reported & 2) return;
|
|
247
|
+
reported |= 2;
|
|
248
|
+
const counts = this.#counts(test, version);
|
|
249
|
+
const fields = usage ?? {};
|
|
250
|
+
add(counts, USAGE, 1);
|
|
251
|
+
add(counts, TOKENS_IN, tokens(fields, IN));
|
|
252
|
+
add(counts, TOKENS_OUT, tokens(fields, OUT));
|
|
253
|
+
add(counts, LATENCY, Math.max(0, ms));
|
|
254
|
+
add(counts, BUCKETS + latencyBucket(ms), 1);
|
|
255
|
+
}
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
#outcome(test, version, value) {
|
|
259
|
+
const score = value === true ? 1 : value === false ? 0 : value;
|
|
260
|
+
if (!Number.isInteger(version) || !(score >= 0 && score <= 1)) return;
|
|
261
|
+
const counts = this.#counts(test, version);
|
|
262
|
+
add(counts, OUTCOMES, 1);
|
|
263
|
+
add(counts, SCORE, score);
|
|
264
|
+
add(counts, SCORE_SQ, score * score);
|
|
265
|
+
}
|
|
266
|
+
/** The counts to add a call to, kept until the next report; none when reporting is off. */
|
|
267
|
+
#counts(test, version) {
|
|
268
|
+
if (!this.#reportResults) return;
|
|
269
|
+
const key = `${test.id}.${test.weight}`;
|
|
270
|
+
let pending = this.#pending.get(key);
|
|
271
|
+
if (!pending) {
|
|
272
|
+
pending = { ...test, versions: /* @__PURE__ */ new Map() };
|
|
273
|
+
this.#pending.set(key, pending);
|
|
274
|
+
}
|
|
275
|
+
let counts = pending.versions.get(version);
|
|
276
|
+
if (!counts) {
|
|
277
|
+
counts = new Array(BUCKETS + LATENCY_BUCKETS).fill(0);
|
|
278
|
+
pending.versions.set(version, counts);
|
|
279
|
+
}
|
|
280
|
+
this.#schedule();
|
|
281
|
+
return counts;
|
|
282
|
+
}
|
|
283
|
+
/** Reports within a ttl of the first count, without keeping the process alive for it. */
|
|
284
|
+
#schedule() {
|
|
285
|
+
if (this.#timer) return;
|
|
286
|
+
this.#timer = setTimeout(() => this.flush(), this.#ttl);
|
|
287
|
+
this.#timer.unref?.();
|
|
288
|
+
}
|
|
111
289
|
#entry(slug) {
|
|
112
290
|
const cached = this.#cache.get(slug);
|
|
113
291
|
if (!cached) return this.#load(slug);
|
|
@@ -164,7 +342,9 @@ var PromptHelm = class {
|
|
|
164
342
|
if (!response.ok) throw await errorFromResponse(response);
|
|
165
343
|
const payload = await response.json();
|
|
166
344
|
const entry = {
|
|
345
|
+
version: payload.version,
|
|
167
346
|
content: payload.content,
|
|
347
|
+
experiment: payload.experiment,
|
|
168
348
|
etag: response.headers.get("etag"),
|
|
169
349
|
fetchedAt: Date.now()
|
|
170
350
|
};
|
|
@@ -172,6 +352,18 @@ var PromptHelm = class {
|
|
|
172
352
|
return entry;
|
|
173
353
|
}
|
|
174
354
|
};
|
|
355
|
+
var IN = ["input_tokens", "prompt_tokens", "inputTokens", "promptTokenCount"];
|
|
356
|
+
var OUT = ["output_tokens", "completion_tokens", "outputTokens", "candidatesTokenCount"];
|
|
357
|
+
function tokens(usage, keys) {
|
|
358
|
+
for (const key of keys) {
|
|
359
|
+
const value = usage[key];
|
|
360
|
+
if (typeof value === "number" && value >= 0 && Number.isFinite(value)) return value;
|
|
361
|
+
}
|
|
362
|
+
return 0;
|
|
363
|
+
}
|
|
364
|
+
function add(counts, index, amount) {
|
|
365
|
+
if (counts) counts[index] = (counts[index] ?? 0) + amount;
|
|
366
|
+
}
|
|
175
367
|
async function errorFromResponse(response) {
|
|
176
368
|
const body = await response.json().catch(() => null);
|
|
177
369
|
const message = body?.error?.message ?? `HTTP ${response.status}`;
|
|
@@ -189,8 +381,10 @@ function toPromptHelmError(error) {
|
|
|
189
381
|
0 && (module.exports = {
|
|
190
382
|
ERROR_CODES,
|
|
191
383
|
FALLBACK_HEADER,
|
|
384
|
+
LATENCY_BUCKETS,
|
|
192
385
|
PromptHelm,
|
|
193
386
|
PromptHelmError,
|
|
194
387
|
extractVariables,
|
|
388
|
+
latencyBucket,
|
|
195
389
|
render
|
|
196
390
|
});
|
package/dist/index.d.cts
CHANGED
|
@@ -23,6 +23,18 @@ interface PromptPayload {
|
|
|
23
23
|
version: number;
|
|
24
24
|
content: string;
|
|
25
25
|
variables: string[];
|
|
26
|
+
/** An A/B test running on the prompt: the live version above is its control. */
|
|
27
|
+
experiment?: PromptExperiment;
|
|
28
|
+
}
|
|
29
|
+
interface PromptExperiment {
|
|
30
|
+
id: string;
|
|
31
|
+
/** The variant's share of calls, in percent (1–99). */
|
|
32
|
+
weight: number;
|
|
33
|
+
variant: {
|
|
34
|
+
version: number;
|
|
35
|
+
content: string;
|
|
36
|
+
variables: string[];
|
|
37
|
+
};
|
|
26
38
|
}
|
|
27
39
|
/**
|
|
28
40
|
* Sent with the next request for a slug after `get()` couldn't serve its live text: the error's
|
|
@@ -33,6 +45,35 @@ declare const FALLBACK_HEADER = "x-prompthelm-fallback";
|
|
|
33
45
|
interface PromptListPayload {
|
|
34
46
|
prompts: PromptPayload[];
|
|
35
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Body of `POST /v1/reports`: what an app counted under running A/B tests since its last report.
|
|
50
|
+
* Numbers only, never any text, variable value or unit.
|
|
51
|
+
*/
|
|
52
|
+
interface ReportPayload {
|
|
53
|
+
experiments: {
|
|
54
|
+
id: string;
|
|
55
|
+
/** The variant's share when these calls were assigned. */
|
|
56
|
+
weight: number;
|
|
57
|
+
/** By version number. */
|
|
58
|
+
versions: Record<string, ReportCounts>;
|
|
59
|
+
}[];
|
|
60
|
+
}
|
|
61
|
+
interface ReportCounts {
|
|
62
|
+
served: number;
|
|
63
|
+
/** Picked for the variant, served the live version because a variable wasn't passed. */
|
|
64
|
+
skipped: number;
|
|
65
|
+
outcomes: number;
|
|
66
|
+
/** The outcomes' scores (true = 1, false = 0), summed, and their squares summed. */
|
|
67
|
+
scoreSum: number;
|
|
68
|
+
scoreSqSum: number;
|
|
69
|
+
/** Calls whose model usage was reported, their tokens and their durations in ms, summed. */
|
|
70
|
+
usage: number;
|
|
71
|
+
tokensIn: number;
|
|
72
|
+
tokensOut: number;
|
|
73
|
+
latencyMs: number;
|
|
74
|
+
/** Those calls per latency bucket (`latencyBucket`). */
|
|
75
|
+
latency: number[];
|
|
76
|
+
}
|
|
36
77
|
interface PromptHelmOptions {
|
|
37
78
|
/** Defaults to the PROMPTHELM_API_KEY environment variable. */
|
|
38
79
|
apiKey?: string;
|
|
@@ -50,6 +91,14 @@ interface PromptHelmOptions {
|
|
|
50
91
|
* prompt's page shows it. Defaults to true.
|
|
51
92
|
*/
|
|
52
93
|
reportFallbacks?: boolean;
|
|
94
|
+
/**
|
|
95
|
+
* Whether to send PromptHelm, as totals only, what happened under an A/B test: how often each
|
|
96
|
+
* version was served, the outcomes and the model usage you report. Defaults to
|
|
97
|
+
* `reportFallbacks`.
|
|
98
|
+
*/
|
|
99
|
+
reportResults?: boolean;
|
|
100
|
+
/** Called on every `get` and `resolve`, so your analytics or traces can carry the version. */
|
|
101
|
+
onResolve?: (event: ResolveEvent) => void;
|
|
53
102
|
}
|
|
54
103
|
interface GetOptions {
|
|
55
104
|
vars?: Vars;
|
|
@@ -58,13 +107,62 @@ interface GetOptions {
|
|
|
58
107
|
* version uses a variable this call doesn't pass. Without it those cases throw.
|
|
59
108
|
*/
|
|
60
109
|
fallback?: string;
|
|
110
|
+
/**
|
|
111
|
+
* Who the call is for, such as a user or conversation id: during an A/B test the same unit
|
|
112
|
+
* always gets the same version. Never sent to PromptHelm.
|
|
113
|
+
*/
|
|
114
|
+
unit?: string | number;
|
|
115
|
+
/** During an A/B test, serves this side, such as the variant to your own team. Never counted. */
|
|
116
|
+
force?: "control" | "variant";
|
|
117
|
+
}
|
|
118
|
+
interface ResolveEvent {
|
|
119
|
+
slug: string;
|
|
120
|
+
/** The version served, or null when the fallback was. */
|
|
121
|
+
version: number | null;
|
|
122
|
+
/** The A/B test running on the prompt, if any. */
|
|
123
|
+
experiment: string | null;
|
|
124
|
+
/** Whether the call got the test's variant. */
|
|
125
|
+
variant: boolean;
|
|
126
|
+
}
|
|
127
|
+
interface Resolved {
|
|
128
|
+
text: string;
|
|
129
|
+
/** The version served, or null when the fallback was. */
|
|
130
|
+
version: number | null;
|
|
131
|
+
/**
|
|
132
|
+
* During an A/B test, names the test and the version this call got, so its outcome can be
|
|
133
|
+
* reported later from any process: `ph.outcome(ticket, true)`. No user in it. Null otherwise.
|
|
134
|
+
*/
|
|
135
|
+
ticket: string | null;
|
|
136
|
+
/** How the call went: true or false, or a score from 0 to 1. The first report counts. */
|
|
137
|
+
outcome(value: boolean | number): void;
|
|
138
|
+
/**
|
|
139
|
+
* The model call's tokens, from your provider's `usage` object, and how long it took: `ms`,
|
|
140
|
+
* else the time since `resolve` returned. The first report counts.
|
|
141
|
+
*/
|
|
142
|
+
usage(usage: object | null | undefined, ms?: number): void;
|
|
61
143
|
}
|
|
62
144
|
declare class PromptHelm {
|
|
63
145
|
#private;
|
|
64
146
|
constructor(options?: PromptHelmOptions);
|
|
65
147
|
/** The live version of a prompt with its variables filled in. */
|
|
66
148
|
get(slug: string, options?: GetOptions): Promise<string>;
|
|
149
|
+
/** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
|
|
150
|
+
resolve(slug: string, options?: GetOptions): Promise<Resolved>;
|
|
151
|
+
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
152
|
+
outcome(ticket: string, value: boolean | number): void;
|
|
153
|
+
/**
|
|
154
|
+
* Sends what was counted for A/B tests now, instead of within a ttl: call it before a
|
|
155
|
+
* serverless function freezes. Never throws; what can't be sent is kept for the next try.
|
|
156
|
+
*/
|
|
157
|
+
flush(): Promise<void>;
|
|
67
158
|
}
|
|
68
159
|
|
|
69
|
-
|
|
70
|
-
|
|
160
|
+
declare const LATENCY_BUCKETS = 24;
|
|
161
|
+
/**
|
|
162
|
+
* Which of 24 latency buckets a model call's duration falls in, for its percentiles: bucket i
|
|
163
|
+
* holds calls up to 50·2^(i/2) ms (bucket 0 under 50 ms), and the last one everything slower.
|
|
164
|
+
*/
|
|
165
|
+
declare function latencyBucket(ms: number): number;
|
|
166
|
+
|
|
167
|
+
export { ERROR_CODES, FALLBACK_HEADER, LATENCY_BUCKETS, PromptHelm, PromptHelmError, extractVariables, latencyBucket, render };
|
|
168
|
+
export type { GetOptions, PromptExperiment, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
|
package/dist/index.d.ts
CHANGED
|
@@ -23,6 +23,18 @@ interface PromptPayload {
|
|
|
23
23
|
version: number;
|
|
24
24
|
content: string;
|
|
25
25
|
variables: string[];
|
|
26
|
+
/** An A/B test running on the prompt: the live version above is its control. */
|
|
27
|
+
experiment?: PromptExperiment;
|
|
28
|
+
}
|
|
29
|
+
interface PromptExperiment {
|
|
30
|
+
id: string;
|
|
31
|
+
/** The variant's share of calls, in percent (1–99). */
|
|
32
|
+
weight: number;
|
|
33
|
+
variant: {
|
|
34
|
+
version: number;
|
|
35
|
+
content: string;
|
|
36
|
+
variables: string[];
|
|
37
|
+
};
|
|
26
38
|
}
|
|
27
39
|
/**
|
|
28
40
|
* Sent with the next request for a slug after `get()` couldn't serve its live text: the error's
|
|
@@ -33,6 +45,35 @@ declare const FALLBACK_HEADER = "x-prompthelm-fallback";
|
|
|
33
45
|
interface PromptListPayload {
|
|
34
46
|
prompts: PromptPayload[];
|
|
35
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* Body of `POST /v1/reports`: what an app counted under running A/B tests since its last report.
|
|
50
|
+
* Numbers only, never any text, variable value or unit.
|
|
51
|
+
*/
|
|
52
|
+
interface ReportPayload {
|
|
53
|
+
experiments: {
|
|
54
|
+
id: string;
|
|
55
|
+
/** The variant's share when these calls were assigned. */
|
|
56
|
+
weight: number;
|
|
57
|
+
/** By version number. */
|
|
58
|
+
versions: Record<string, ReportCounts>;
|
|
59
|
+
}[];
|
|
60
|
+
}
|
|
61
|
+
interface ReportCounts {
|
|
62
|
+
served: number;
|
|
63
|
+
/** Picked for the variant, served the live version because a variable wasn't passed. */
|
|
64
|
+
skipped: number;
|
|
65
|
+
outcomes: number;
|
|
66
|
+
/** The outcomes' scores (true = 1, false = 0), summed, and their squares summed. */
|
|
67
|
+
scoreSum: number;
|
|
68
|
+
scoreSqSum: number;
|
|
69
|
+
/** Calls whose model usage was reported, their tokens and their durations in ms, summed. */
|
|
70
|
+
usage: number;
|
|
71
|
+
tokensIn: number;
|
|
72
|
+
tokensOut: number;
|
|
73
|
+
latencyMs: number;
|
|
74
|
+
/** Those calls per latency bucket (`latencyBucket`). */
|
|
75
|
+
latency: number[];
|
|
76
|
+
}
|
|
36
77
|
interface PromptHelmOptions {
|
|
37
78
|
/** Defaults to the PROMPTHELM_API_KEY environment variable. */
|
|
38
79
|
apiKey?: string;
|
|
@@ -50,6 +91,14 @@ interface PromptHelmOptions {
|
|
|
50
91
|
* prompt's page shows it. Defaults to true.
|
|
51
92
|
*/
|
|
52
93
|
reportFallbacks?: boolean;
|
|
94
|
+
/**
|
|
95
|
+
* Whether to send PromptHelm, as totals only, what happened under an A/B test: how often each
|
|
96
|
+
* version was served, the outcomes and the model usage you report. Defaults to
|
|
97
|
+
* `reportFallbacks`.
|
|
98
|
+
*/
|
|
99
|
+
reportResults?: boolean;
|
|
100
|
+
/** Called on every `get` and `resolve`, so your analytics or traces can carry the version. */
|
|
101
|
+
onResolve?: (event: ResolveEvent) => void;
|
|
53
102
|
}
|
|
54
103
|
interface GetOptions {
|
|
55
104
|
vars?: Vars;
|
|
@@ -58,13 +107,62 @@ interface GetOptions {
|
|
|
58
107
|
* version uses a variable this call doesn't pass. Without it those cases throw.
|
|
59
108
|
*/
|
|
60
109
|
fallback?: string;
|
|
110
|
+
/**
|
|
111
|
+
* Who the call is for, such as a user or conversation id: during an A/B test the same unit
|
|
112
|
+
* always gets the same version. Never sent to PromptHelm.
|
|
113
|
+
*/
|
|
114
|
+
unit?: string | number;
|
|
115
|
+
/** During an A/B test, serves this side, such as the variant to your own team. Never counted. */
|
|
116
|
+
force?: "control" | "variant";
|
|
117
|
+
}
|
|
118
|
+
interface ResolveEvent {
|
|
119
|
+
slug: string;
|
|
120
|
+
/** The version served, or null when the fallback was. */
|
|
121
|
+
version: number | null;
|
|
122
|
+
/** The A/B test running on the prompt, if any. */
|
|
123
|
+
experiment: string | null;
|
|
124
|
+
/** Whether the call got the test's variant. */
|
|
125
|
+
variant: boolean;
|
|
126
|
+
}
|
|
127
|
+
interface Resolved {
|
|
128
|
+
text: string;
|
|
129
|
+
/** The version served, or null when the fallback was. */
|
|
130
|
+
version: number | null;
|
|
131
|
+
/**
|
|
132
|
+
* During an A/B test, names the test and the version this call got, so its outcome can be
|
|
133
|
+
* reported later from any process: `ph.outcome(ticket, true)`. No user in it. Null otherwise.
|
|
134
|
+
*/
|
|
135
|
+
ticket: string | null;
|
|
136
|
+
/** How the call went: true or false, or a score from 0 to 1. The first report counts. */
|
|
137
|
+
outcome(value: boolean | number): void;
|
|
138
|
+
/**
|
|
139
|
+
* The model call's tokens, from your provider's `usage` object, and how long it took: `ms`,
|
|
140
|
+
* else the time since `resolve` returned. The first report counts.
|
|
141
|
+
*/
|
|
142
|
+
usage(usage: object | null | undefined, ms?: number): void;
|
|
61
143
|
}
|
|
62
144
|
declare class PromptHelm {
|
|
63
145
|
#private;
|
|
64
146
|
constructor(options?: PromptHelmOptions);
|
|
65
147
|
/** The live version of a prompt with its variables filled in. */
|
|
66
148
|
get(slug: string, options?: GetOptions): Promise<string>;
|
|
149
|
+
/** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
|
|
150
|
+
resolve(slug: string, options?: GetOptions): Promise<Resolved>;
|
|
151
|
+
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
152
|
+
outcome(ticket: string, value: boolean | number): void;
|
|
153
|
+
/**
|
|
154
|
+
* Sends what was counted for A/B tests now, instead of within a ttl: call it before a
|
|
155
|
+
* serverless function freezes. Never throws; what can't be sent is kept for the next try.
|
|
156
|
+
*/
|
|
157
|
+
flush(): Promise<void>;
|
|
67
158
|
}
|
|
68
159
|
|
|
69
|
-
|
|
70
|
-
|
|
160
|
+
declare const LATENCY_BUCKETS = 24;
|
|
161
|
+
/**
|
|
162
|
+
* Which of 24 latency buckets a model call's duration falls in, for its percentiles: bucket i
|
|
163
|
+
* holds calls up to 50·2^(i/2) ms (bucket 0 under 50 ms), and the last one everything slower.
|
|
164
|
+
*/
|
|
165
|
+
declare function latencyBucket(ms: number): number;
|
|
166
|
+
|
|
167
|
+
export { ERROR_CODES, FALLBACK_HEADER, LATENCY_BUCKETS, PromptHelm, PromptHelmError, extractVariables, latencyBucket, render };
|
|
168
|
+
export type { GetOptions, PromptExperiment, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
|
package/dist/index.js
CHANGED
|
@@ -18,6 +18,20 @@ var PromptHelmError = class extends Error {
|
|
|
18
18
|
name = "PromptHelmError";
|
|
19
19
|
};
|
|
20
20
|
|
|
21
|
+
// src/experiment.ts
|
|
22
|
+
function fnv1a32(text) {
|
|
23
|
+
let hash = 2166136261;
|
|
24
|
+
for (const byte of new TextEncoder().encode(text)) hash = Math.imul(hash ^ byte, 16777619);
|
|
25
|
+
return hash >>> 0;
|
|
26
|
+
}
|
|
27
|
+
function bucket(experimentId, unit) {
|
|
28
|
+
return unit === void 0 ? Math.floor(Math.random() * 100) : fnv1a32(`${experimentId}:${unit}`) % 100;
|
|
29
|
+
}
|
|
30
|
+
var LATENCY_BUCKETS = 24;
|
|
31
|
+
function latencyBucket(ms) {
|
|
32
|
+
return ms < 50 ? 0 : Math.min(LATENCY_BUCKETS - 1, 1 + Math.floor(2 * Math.log2(ms / 50)));
|
|
33
|
+
}
|
|
34
|
+
|
|
21
35
|
// src/template.ts
|
|
22
36
|
var VARIABLE = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
|
|
23
37
|
function extractVariables(template) {
|
|
@@ -38,6 +52,27 @@ function render(template, vars = {}) {
|
|
|
38
52
|
// src/client.ts
|
|
39
53
|
var DEFAULT_BASE_URL = "https://prompthelm.app";
|
|
40
54
|
var FALLBACK_HEADER = "x-prompthelm-fallback";
|
|
55
|
+
var FIELDS = [
|
|
56
|
+
"served",
|
|
57
|
+
"skipped",
|
|
58
|
+
"outcomes",
|
|
59
|
+
"scoreSum",
|
|
60
|
+
"scoreSqSum",
|
|
61
|
+
"usage",
|
|
62
|
+
"tokensIn",
|
|
63
|
+
"tokensOut",
|
|
64
|
+
"latencyMs"
|
|
65
|
+
];
|
|
66
|
+
var SERVED = 0;
|
|
67
|
+
var SKIPPED = 1;
|
|
68
|
+
var OUTCOMES = 2;
|
|
69
|
+
var SCORE = 3;
|
|
70
|
+
var SCORE_SQ = 4;
|
|
71
|
+
var USAGE = 5;
|
|
72
|
+
var TOKENS_IN = 6;
|
|
73
|
+
var TOKENS_OUT = 7;
|
|
74
|
+
var LATENCY = 8;
|
|
75
|
+
var BUCKETS = FIELDS.length;
|
|
41
76
|
var PromptHelm = class {
|
|
42
77
|
#apiKey;
|
|
43
78
|
#baseUrl;
|
|
@@ -45,12 +80,16 @@ var PromptHelm = class {
|
|
|
45
80
|
#timeout;
|
|
46
81
|
#fetch;
|
|
47
82
|
#onError;
|
|
83
|
+
#onResolve;
|
|
48
84
|
#cache = /* @__PURE__ */ new Map();
|
|
49
85
|
#inflight = /* @__PURE__ */ new Map();
|
|
50
86
|
#reportFallbacks;
|
|
87
|
+
#reportResults;
|
|
51
88
|
// Per slug, why the live text last couldn't be served, until a request carries it. An object,
|
|
52
89
|
// so a report made while that request is in flight isn't cleared with the one it carried.
|
|
53
90
|
#unreported = /* @__PURE__ */ new Map();
|
|
91
|
+
#pending = /* @__PURE__ */ new Map();
|
|
92
|
+
#timer;
|
|
54
93
|
constructor(options = {}) {
|
|
55
94
|
const apiKey = options.apiKey ?? env("PROMPTHELM_API_KEY");
|
|
56
95
|
if (!apiKey) throw new Error("PromptHelm: pass apiKey or set PROMPTHELM_API_KEY");
|
|
@@ -61,22 +100,159 @@ var PromptHelm = class {
|
|
|
61
100
|
this.#timeout = options.timeout ?? 3e3;
|
|
62
101
|
this.#fetch = options.fetch ?? ((input, init) => globalThis.fetch(input, init));
|
|
63
102
|
this.#onError = options.onError ?? ((error, slug) => console.warn(`[PromptHelm] ${slug}: ${error.message}`));
|
|
103
|
+
this.#onResolve = options.onResolve;
|
|
64
104
|
this.#reportFallbacks = options.reportFallbacks ?? true;
|
|
105
|
+
this.#reportResults = options.reportResults ?? this.#reportFallbacks;
|
|
65
106
|
}
|
|
66
107
|
/** The live version of a prompt with its variables filled in. */
|
|
67
108
|
async get(slug, options = {}) {
|
|
109
|
+
return (await this.resolve(slug, options)).text;
|
|
110
|
+
}
|
|
111
|
+
/** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
|
|
112
|
+
async resolve(slug, options = {}) {
|
|
68
113
|
const { vars, fallback } = options;
|
|
69
114
|
try {
|
|
70
|
-
|
|
71
|
-
return render(entry.content, vars);
|
|
115
|
+
return this.#pick(slug, await this.#entry(slug), options);
|
|
72
116
|
} catch (error) {
|
|
73
117
|
const failure = toPromptHelmError(error);
|
|
74
118
|
this.#remember(slug, failure.code);
|
|
75
119
|
if (fallback === void 0) throw failure;
|
|
76
120
|
this.#onError(failure, slug);
|
|
77
|
-
return render(fallback, vars);
|
|
121
|
+
return this.#served(slug, render(fallback, vars), null);
|
|
78
122
|
}
|
|
79
123
|
}
|
|
124
|
+
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
125
|
+
outcome(ticket, value) {
|
|
126
|
+
const [id = "", version, weight] = ticket.split(".");
|
|
127
|
+
const test = { id, weight: Number(weight) };
|
|
128
|
+
if (id && test.weight > 0) this.#outcome(test, Number(version), value);
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Sends what was counted for A/B tests now, instead of within a ttl: call it before a
|
|
132
|
+
* serverless function freezes. Never throws; what can't be sent is kept for the next try.
|
|
133
|
+
*/
|
|
134
|
+
async flush() {
|
|
135
|
+
clearTimeout(this.#timer);
|
|
136
|
+
this.#timer = void 0;
|
|
137
|
+
const sent = this.#pending;
|
|
138
|
+
if (sent.size === 0) return;
|
|
139
|
+
this.#pending = /* @__PURE__ */ new Map();
|
|
140
|
+
const body = {
|
|
141
|
+
experiments: [...sent.values()].map(({ id, weight, versions }) => ({
|
|
142
|
+
id,
|
|
143
|
+
weight,
|
|
144
|
+
versions: Object.fromEntries(
|
|
145
|
+
[...versions].map(([version, counts]) => [
|
|
146
|
+
version,
|
|
147
|
+
{
|
|
148
|
+
...Object.fromEntries(FIELDS.map((field, i) => [field, counts[i]])),
|
|
149
|
+
latency: counts.slice(BUCKETS)
|
|
150
|
+
}
|
|
151
|
+
])
|
|
152
|
+
)
|
|
153
|
+
}))
|
|
154
|
+
};
|
|
155
|
+
try {
|
|
156
|
+
const response = await this.#fetch(`${this.#baseUrl}/v1/reports`, {
|
|
157
|
+
method: "POST",
|
|
158
|
+
headers: { authorization: `Bearer ${this.#apiKey}`, "content-type": "application/json" },
|
|
159
|
+
body: JSON.stringify(body),
|
|
160
|
+
signal: AbortSignal.timeout(this.#timeout)
|
|
161
|
+
});
|
|
162
|
+
if (response.status < 500) return;
|
|
163
|
+
} catch {
|
|
164
|
+
}
|
|
165
|
+
for (const { id, weight, versions } of sent.values()) {
|
|
166
|
+
for (const [version, counts] of versions) {
|
|
167
|
+
const total = this.#counts({ id, weight }, version);
|
|
168
|
+
counts.forEach((n, i) => {
|
|
169
|
+
add(total, i, n);
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
#pick(slug, entry, options) {
|
|
175
|
+
const { vars, unit, force } = options;
|
|
176
|
+
const { experiment } = entry;
|
|
177
|
+
const test = experiment && !force ? { id: experiment.id, weight: experiment.weight } : void 0;
|
|
178
|
+
let skipped;
|
|
179
|
+
if (experiment && (force ? force === "variant" : bucket(experiment.id, unit) < experiment.weight)) {
|
|
180
|
+
const { version, content } = experiment.variant;
|
|
181
|
+
try {
|
|
182
|
+
return this.#served(slug, render(content, vars), version, experiment.id, test, true);
|
|
183
|
+
} catch (error) {
|
|
184
|
+
skipped = toPromptHelmError(error);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
const text = render(entry.content, vars);
|
|
188
|
+
if (skipped && experiment) {
|
|
189
|
+
this.#onError(skipped, slug);
|
|
190
|
+
if (test) add(this.#counts(test, experiment.variant.version), SKIPPED, 1);
|
|
191
|
+
}
|
|
192
|
+
return this.#served(slug, text, entry.version, experiment?.id, test);
|
|
193
|
+
}
|
|
194
|
+
/** Counts the call, tells `onResolve`, and hands the caller its text and what to report. */
|
|
195
|
+
#served(slug, text, version, experiment, test, variant = false) {
|
|
196
|
+
if (test && version !== null) add(this.#counts(test, version), SERVED, 1);
|
|
197
|
+
try {
|
|
198
|
+
this.#onResolve?.({ slug, version, experiment: experiment ?? null, variant });
|
|
199
|
+
} catch {
|
|
200
|
+
}
|
|
201
|
+
const started = Date.now();
|
|
202
|
+
let reported = 0;
|
|
203
|
+
return {
|
|
204
|
+
text,
|
|
205
|
+
version,
|
|
206
|
+
ticket: test && version !== null ? `${test.id}.${version}.${test.weight}` : null,
|
|
207
|
+
outcome: (value) => {
|
|
208
|
+
if (!test || version === null || reported & 1) return;
|
|
209
|
+
reported |= 1;
|
|
210
|
+
this.#outcome(test, version, value);
|
|
211
|
+
},
|
|
212
|
+
usage: (usage, ms = Date.now() - started) => {
|
|
213
|
+
if (!test || version === null || reported & 2) return;
|
|
214
|
+
reported |= 2;
|
|
215
|
+
const counts = this.#counts(test, version);
|
|
216
|
+
const fields = usage ?? {};
|
|
217
|
+
add(counts, USAGE, 1);
|
|
218
|
+
add(counts, TOKENS_IN, tokens(fields, IN));
|
|
219
|
+
add(counts, TOKENS_OUT, tokens(fields, OUT));
|
|
220
|
+
add(counts, LATENCY, Math.max(0, ms));
|
|
221
|
+
add(counts, BUCKETS + latencyBucket(ms), 1);
|
|
222
|
+
}
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
#outcome(test, version, value) {
|
|
226
|
+
const score = value === true ? 1 : value === false ? 0 : value;
|
|
227
|
+
if (!Number.isInteger(version) || !(score >= 0 && score <= 1)) return;
|
|
228
|
+
const counts = this.#counts(test, version);
|
|
229
|
+
add(counts, OUTCOMES, 1);
|
|
230
|
+
add(counts, SCORE, score);
|
|
231
|
+
add(counts, SCORE_SQ, score * score);
|
|
232
|
+
}
|
|
233
|
+
/** The counts to add a call to, kept until the next report; none when reporting is off. */
|
|
234
|
+
#counts(test, version) {
|
|
235
|
+
if (!this.#reportResults) return;
|
|
236
|
+
const key = `${test.id}.${test.weight}`;
|
|
237
|
+
let pending = this.#pending.get(key);
|
|
238
|
+
if (!pending) {
|
|
239
|
+
pending = { ...test, versions: /* @__PURE__ */ new Map() };
|
|
240
|
+
this.#pending.set(key, pending);
|
|
241
|
+
}
|
|
242
|
+
let counts = pending.versions.get(version);
|
|
243
|
+
if (!counts) {
|
|
244
|
+
counts = new Array(BUCKETS + LATENCY_BUCKETS).fill(0);
|
|
245
|
+
pending.versions.set(version, counts);
|
|
246
|
+
}
|
|
247
|
+
this.#schedule();
|
|
248
|
+
return counts;
|
|
249
|
+
}
|
|
250
|
+
/** Reports within a ttl of the first count, without keeping the process alive for it. */
|
|
251
|
+
#schedule() {
|
|
252
|
+
if (this.#timer) return;
|
|
253
|
+
this.#timer = setTimeout(() => this.flush(), this.#ttl);
|
|
254
|
+
this.#timer.unref?.();
|
|
255
|
+
}
|
|
80
256
|
#entry(slug) {
|
|
81
257
|
const cached = this.#cache.get(slug);
|
|
82
258
|
if (!cached) return this.#load(slug);
|
|
@@ -133,7 +309,9 @@ var PromptHelm = class {
|
|
|
133
309
|
if (!response.ok) throw await errorFromResponse(response);
|
|
134
310
|
const payload = await response.json();
|
|
135
311
|
const entry = {
|
|
312
|
+
version: payload.version,
|
|
136
313
|
content: payload.content,
|
|
314
|
+
experiment: payload.experiment,
|
|
137
315
|
etag: response.headers.get("etag"),
|
|
138
316
|
fetchedAt: Date.now()
|
|
139
317
|
};
|
|
@@ -141,6 +319,18 @@ var PromptHelm = class {
|
|
|
141
319
|
return entry;
|
|
142
320
|
}
|
|
143
321
|
};
|
|
322
|
+
var IN = ["input_tokens", "prompt_tokens", "inputTokens", "promptTokenCount"];
|
|
323
|
+
var OUT = ["output_tokens", "completion_tokens", "outputTokens", "candidatesTokenCount"];
|
|
324
|
+
function tokens(usage, keys) {
|
|
325
|
+
for (const key of keys) {
|
|
326
|
+
const value = usage[key];
|
|
327
|
+
if (typeof value === "number" && value >= 0 && Number.isFinite(value)) return value;
|
|
328
|
+
}
|
|
329
|
+
return 0;
|
|
330
|
+
}
|
|
331
|
+
function add(counts, index, amount) {
|
|
332
|
+
if (counts) counts[index] = (counts[index] ?? 0) + amount;
|
|
333
|
+
}
|
|
144
334
|
async function errorFromResponse(response) {
|
|
145
335
|
const body = await response.json().catch(() => null);
|
|
146
336
|
const message = body?.error?.message ?? `HTTP ${response.status}`;
|
|
@@ -157,8 +347,10 @@ function toPromptHelmError(error) {
|
|
|
157
347
|
export {
|
|
158
348
|
ERROR_CODES,
|
|
159
349
|
FALLBACK_HEADER,
|
|
350
|
+
LATENCY_BUCKETS,
|
|
160
351
|
PromptHelm,
|
|
161
352
|
PromptHelmError,
|
|
162
353
|
extractVariables,
|
|
354
|
+
latencyBucket,
|
|
163
355
|
render
|
|
164
356
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@prompt-helm/sdk",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "Fetch your PromptHelm prompts at runtime, so changing a prompt needs no redeploy.",
|
|
5
5
|
"homepage": "https://prompthelm.app/quickstart",
|
|
6
6
|
"repository": {
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
}
|
|
38
38
|
},
|
|
39
39
|
"bin": {
|
|
40
|
-
"prompthelm": "
|
|
40
|
+
"prompthelm": "dist/cli.js"
|
|
41
41
|
},
|
|
42
42
|
"files": [
|
|
43
43
|
"dist"
|