@prompt-helm/sdk 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -14,7 +14,7 @@ refresh, 60 seconds by default.
14
14
  a refresh that finds nothing new is an empty `304`.
15
15
  - **Your data stays with you.** Variables are filled in on your server. PromptHelm never sees
16
16
  their values, your provider keys or your LLM traffic.
17
- - **Small.** No dependencies, under 2 kB gzipped, ESM and CommonJS, Node.js 18 and later.
17
+ - **Small.** No dependencies, under 3 kB gzipped, ESM and CommonJS, Node.js 18 and later.
18
18
 
19
19
  ## Install
20
20
 
@@ -60,6 +60,8 @@ All optional when the environment variables are set.
60
60
  | `timeout` | `3000` | How long, in ms, one request may take. |
61
61
  | `onError` | `console.warn` | Called with the error and the slug when a fallback or a cached prompt was served instead of failing. |
62
62
  | `reportFallbacks` | `true` | Tells PromptHelm, by error code only, when a fallback or a cached prompt was served, so the prompt's page shows which app fell back and why. |
63
+ | `reportResults` | `reportFallbacks` | During an [A/B test](#ab-tests), sends PromptHelm totals only: calls served per version, and the outcomes and usage you report. |
64
+ | `onResolve` | none | Called on every `get` and `resolve` with the version served, for your analytics or traces. |
63
65
  | `fetch` | global `fetch` | Your own fetch, for tests or a proxy. |
64
66
 
65
67
  ## Errors
@@ -106,6 +108,58 @@ Run it in CI before the build, so every deploy carries the text that was live th
106
108
  `PROMPTHELM_API_KEY` (and `PROMPTHELM_BASE_URL`), and writes JSON for a `.json` path or with
107
109
  `--format json`. A prompt appears in the file once it's published.
108
110
 
111
+ ## A/B tests
112
+
113
+ On the Pro plan, a prompt's page can run an experiment: the live version keeps most calls and a
114
+ variant gets the share you pick. The SDK picks the version for each call on your server. Pass a
115
+ `unit`, such as a user or conversation id, and that unit gets the same version every time;
116
+ without one, each call is picked at random. Use `resolve` to learn which version a call got, and
117
+ tell PromptHelm how it went:
118
+
119
+ ```ts
120
+ const r = await ph.resolve("support-reply", {
121
+ vars: { company: "Acme", customer_name: user.name },
122
+ fallback: SUPPORT_REPLY,
123
+ unit: user.id,
124
+ });
125
+
126
+ const reply = await openai.responses.create({ model: "gpt-5", input: r.text });
127
+ r.usage(reply.usage); // tokens, and how long the call took since resolve
128
+ r.outcome(userKeptTheReply); // true or false, or a score from 0 to 1
129
+ ```
130
+
131
+ The prompt's page then compares the two versions: how often each succeeded, with a range that
132
+ stays honest however often you look, and their tokens and latency per call. It says when there's
133
+ an answer; shipping the variant stays your call.
134
+
135
+ - **Outcomes that come later.** A purchase days later, in another service: keep `r.ticket` with
136
+ your own record, then call `ph.outcome(ticket, true)` from any process. The ticket names the
137
+ test and the version, not the user.
138
+ - **Your own analytics.** `onResolve` is called on every `get` and `resolve` with the slug, the
139
+ version, the test and whether the call got the variant, so your events or traces can carry it:
140
+ `new PromptHelm({ onResolve: (e) => posthog.capture("prompt_served", e) })`.
141
+ - **Your team first.** `force: "variant"` (or `"control"`) serves that side to a call, such as
142
+ to your own staff. Forced calls are never counted.
143
+ - **Serverless.** Counts are sent within a ttl. A function that freezes after its response
144
+ should `await ph.flush()` first (with `waitUntil`, where the platform has it).
145
+ - **Usage objects** from OpenAI (Responses and Chat Completions), Anthropic, the Vercel AI SDK
146
+ and Gemini are read as they are. `r.usage(null, ms)` reports a duration alone.
147
+
148
+ Only totals reach PromptHelm: how many calls each version served, the outcomes, tokens and
149
+ durations you report. Never the text, a variable's value or the unit. Counts a process holds
150
+ when it exits are lost, so read them as a sample. `reportResults: false` sends none; unless you
151
+ set it, it follows `reportFallbacks`.
152
+
153
+ If the variant uses a variable the call doesn't pass, that call gets the live version and
154
+ `onError` receives `missing_variables`; it isn't reported as a fallback. Growing the variant's
155
+ share only moves units from the live version to the variant, never back. Apps on SDK versions
156
+ before 1.3 ignore experiments and serve the live version to every call.
157
+
158
+ To split calls the same way from another language: during an experiment the response carries
159
+ `experiment: { id, weight, variant: { version, content, variables } }`. A unit's bucket is the
160
+ 32-bit FNV-1a hash of the UTF-8 bytes of `<experiment id>:<unit>`, modulo 100; buckets below
161
+ `weight` get the variant.
162
+
109
163
  ## Other languages
110
164
 
111
165
  Any language can fetch the live version with one request:
package/dist/cli.js CHANGED
@@ -18,6 +18,18 @@ var PromptHelmError = class extends Error {
18
18
 
19
19
  // src/client.ts
20
20
  var DEFAULT_BASE_URL = "https://prompthelm.app";
21
+ var FIELDS = [
22
+ "served",
23
+ "skipped",
24
+ "outcomes",
25
+ "scoreSum",
26
+ "scoreSqSum",
27
+ "usage",
28
+ "tokensIn",
29
+ "tokensOut",
30
+ "latencyMs"
31
+ ];
32
+ var BUCKETS = FIELDS.length;
21
33
  async function errorFromResponse(response) {
22
34
  const body = await response.json().catch(() => null);
23
35
  const message = body?.error?.message ?? `HTTP ${response.status}`;
package/dist/index.cjs CHANGED
@@ -22,9 +22,11 @@ var index_exports = {};
22
22
  __export(index_exports, {
23
23
  ERROR_CODES: () => ERROR_CODES,
24
24
  FALLBACK_HEADER: () => FALLBACK_HEADER,
25
+ LATENCY_BUCKETS: () => LATENCY_BUCKETS,
25
26
  PromptHelm: () => PromptHelm,
26
27
  PromptHelmError: () => PromptHelmError,
27
28
  extractVariables: () => extractVariables,
29
+ latencyBucket: () => latencyBucket,
28
30
  render: () => render
29
31
  });
30
32
  module.exports = __toCommonJS(index_exports);
@@ -49,6 +51,20 @@ var PromptHelmError = class extends Error {
49
51
  name = "PromptHelmError";
50
52
  };
51
53
 
54
+ // src/experiment.ts
55
+ function fnv1a32(text) {
56
+ let hash = 2166136261;
57
+ for (const byte of new TextEncoder().encode(text)) hash = Math.imul(hash ^ byte, 16777619);
58
+ return hash >>> 0;
59
+ }
60
+ function bucket(experimentId, unit) {
61
+ return unit === void 0 ? Math.floor(Math.random() * 100) : fnv1a32(`${experimentId}:${unit}`) % 100;
62
+ }
63
+ var LATENCY_BUCKETS = 24;
64
+ function latencyBucket(ms) {
65
+ return ms < 50 ? 0 : Math.min(LATENCY_BUCKETS - 1, 1 + Math.floor(2 * Math.log2(ms / 50)));
66
+ }
67
+
52
68
  // src/template.ts
53
69
  var VARIABLE = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
54
70
  function extractVariables(template) {
@@ -69,6 +85,27 @@ function render(template, vars = {}) {
69
85
  // src/client.ts
70
86
  var DEFAULT_BASE_URL = "https://prompthelm.app";
71
87
  var FALLBACK_HEADER = "x-prompthelm-fallback";
88
+ var FIELDS = [
89
+ "served",
90
+ "skipped",
91
+ "outcomes",
92
+ "scoreSum",
93
+ "scoreSqSum",
94
+ "usage",
95
+ "tokensIn",
96
+ "tokensOut",
97
+ "latencyMs"
98
+ ];
99
+ var SERVED = 0;
100
+ var SKIPPED = 1;
101
+ var OUTCOMES = 2;
102
+ var SCORE = 3;
103
+ var SCORE_SQ = 4;
104
+ var USAGE = 5;
105
+ var TOKENS_IN = 6;
106
+ var TOKENS_OUT = 7;
107
+ var LATENCY = 8;
108
+ var BUCKETS = FIELDS.length;
72
109
  var PromptHelm = class {
73
110
  #apiKey;
74
111
  #baseUrl;
@@ -76,12 +113,16 @@ var PromptHelm = class {
76
113
  #timeout;
77
114
  #fetch;
78
115
  #onError;
116
+ #onResolve;
79
117
  #cache = /* @__PURE__ */ new Map();
80
118
  #inflight = /* @__PURE__ */ new Map();
81
119
  #reportFallbacks;
120
+ #reportResults;
82
121
  // Per slug, why the live text last couldn't be served, until a request carries it. An object,
83
122
  // so a report made while that request is in flight isn't cleared with the one it carried.
84
123
  #unreported = /* @__PURE__ */ new Map();
124
+ #pending = /* @__PURE__ */ new Map();
125
+ #timer;
85
126
  constructor(options = {}) {
86
127
  const apiKey = options.apiKey ?? env("PROMPTHELM_API_KEY");
87
128
  if (!apiKey) throw new Error("PromptHelm: pass apiKey or set PROMPTHELM_API_KEY");
@@ -92,22 +133,159 @@ var PromptHelm = class {
92
133
  this.#timeout = options.timeout ?? 3e3;
93
134
  this.#fetch = options.fetch ?? ((input, init) => globalThis.fetch(input, init));
94
135
  this.#onError = options.onError ?? ((error, slug) => console.warn(`[PromptHelm] ${slug}: ${error.message}`));
136
+ this.#onResolve = options.onResolve;
95
137
  this.#reportFallbacks = options.reportFallbacks ?? true;
138
+ this.#reportResults = options.reportResults ?? this.#reportFallbacks;
96
139
  }
97
140
  /** The live version of a prompt with its variables filled in. */
98
141
  async get(slug, options = {}) {
142
+ return (await this.resolve(slug, options)).text;
143
+ }
144
+ /** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
145
+ async resolve(slug, options = {}) {
99
146
  const { vars, fallback } = options;
100
147
  try {
101
- const entry = await this.#entry(slug);
102
- return render(entry.content, vars);
148
+ return this.#pick(slug, await this.#entry(slug), options);
103
149
  } catch (error) {
104
150
  const failure = toPromptHelmError(error);
105
151
  this.#remember(slug, failure.code);
106
152
  if (fallback === void 0) throw failure;
107
153
  this.#onError(failure, slug);
108
- return render(fallback, vars);
154
+ return this.#served(slug, render(fallback, vars), null);
109
155
  }
110
156
  }
157
+ /** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
158
+ outcome(ticket, value) {
159
+ const [id = "", version, weight] = ticket.split(".");
160
+ const test = { id, weight: Number(weight) };
161
+ if (id && test.weight > 0) this.#outcome(test, Number(version), value);
162
+ }
163
+ /**
164
+ * Sends what was counted for A/B tests now, instead of within a ttl: call it before a
165
+ * serverless function freezes. Never throws; what can't be sent is kept for the next try.
166
+ */
167
+ async flush() {
168
+ clearTimeout(this.#timer);
169
+ this.#timer = void 0;
170
+ const sent = this.#pending;
171
+ if (sent.size === 0) return;
172
+ this.#pending = /* @__PURE__ */ new Map();
173
+ const body = {
174
+ experiments: [...sent.values()].map(({ id, weight, versions }) => ({
175
+ id,
176
+ weight,
177
+ versions: Object.fromEntries(
178
+ [...versions].map(([version, counts]) => [
179
+ version,
180
+ {
181
+ ...Object.fromEntries(FIELDS.map((field, i) => [field, counts[i]])),
182
+ latency: counts.slice(BUCKETS)
183
+ }
184
+ ])
185
+ )
186
+ }))
187
+ };
188
+ try {
189
+ const response = await this.#fetch(`${this.#baseUrl}/v1/reports`, {
190
+ method: "POST",
191
+ headers: { authorization: `Bearer ${this.#apiKey}`, "content-type": "application/json" },
192
+ body: JSON.stringify(body),
193
+ signal: AbortSignal.timeout(this.#timeout)
194
+ });
195
+ if (response.status < 500) return;
196
+ } catch {
197
+ }
198
+ for (const { id, weight, versions } of sent.values()) {
199
+ for (const [version, counts] of versions) {
200
+ const total = this.#counts({ id, weight }, version);
201
+ counts.forEach((n, i) => {
202
+ add(total, i, n);
203
+ });
204
+ }
205
+ }
206
+ }
207
+ #pick(slug, entry, options) {
208
+ const { vars, unit, force } = options;
209
+ const { experiment } = entry;
210
+ const test = experiment && !force ? { id: experiment.id, weight: experiment.weight } : void 0;
211
+ let skipped;
212
+ if (experiment && (force ? force === "variant" : bucket(experiment.id, unit) < experiment.weight)) {
213
+ const { version, content } = experiment.variant;
214
+ try {
215
+ return this.#served(slug, render(content, vars), version, experiment.id, test, true);
216
+ } catch (error) {
217
+ skipped = toPromptHelmError(error);
218
+ }
219
+ }
220
+ const text = render(entry.content, vars);
221
+ if (skipped && experiment) {
222
+ this.#onError(skipped, slug);
223
+ if (test) add(this.#counts(test, experiment.variant.version), SKIPPED, 1);
224
+ }
225
+ return this.#served(slug, text, entry.version, experiment?.id, test);
226
+ }
227
+ /** Counts the call, tells `onResolve`, and hands the caller its text and what to report. */
228
+ #served(slug, text, version, experiment, test, variant = false) {
229
+ if (test && version !== null) add(this.#counts(test, version), SERVED, 1);
230
+ try {
231
+ this.#onResolve?.({ slug, version, experiment: experiment ?? null, variant });
232
+ } catch {
233
+ }
234
+ const started = Date.now();
235
+ let reported = 0;
236
+ return {
237
+ text,
238
+ version,
239
+ ticket: test && version !== null ? `${test.id}.${version}.${test.weight}` : null,
240
+ outcome: (value) => {
241
+ if (!test || version === null || reported & 1) return;
242
+ reported |= 1;
243
+ this.#outcome(test, version, value);
244
+ },
245
+ usage: (usage, ms = Date.now() - started) => {
246
+ if (!test || version === null || reported & 2) return;
247
+ reported |= 2;
248
+ const counts = this.#counts(test, version);
249
+ const fields = usage ?? {};
250
+ add(counts, USAGE, 1);
251
+ add(counts, TOKENS_IN, tokens(fields, IN));
252
+ add(counts, TOKENS_OUT, tokens(fields, OUT));
253
+ add(counts, LATENCY, Math.max(0, ms));
254
+ add(counts, BUCKETS + latencyBucket(ms), 1);
255
+ }
256
+ };
257
+ }
258
+ #outcome(test, version, value) {
259
+ const score = value === true ? 1 : value === false ? 0 : value;
260
+ if (!Number.isInteger(version) || !(score >= 0 && score <= 1)) return;
261
+ const counts = this.#counts(test, version);
262
+ add(counts, OUTCOMES, 1);
263
+ add(counts, SCORE, score);
264
+ add(counts, SCORE_SQ, score * score);
265
+ }
266
+ /** The counts to add a call to, kept until the next report; none when reporting is off. */
267
+ #counts(test, version) {
268
+ if (!this.#reportResults) return;
269
+ const key = `${test.id}.${test.weight}`;
270
+ let pending = this.#pending.get(key);
271
+ if (!pending) {
272
+ pending = { ...test, versions: /* @__PURE__ */ new Map() };
273
+ this.#pending.set(key, pending);
274
+ }
275
+ let counts = pending.versions.get(version);
276
+ if (!counts) {
277
+ counts = new Array(BUCKETS + LATENCY_BUCKETS).fill(0);
278
+ pending.versions.set(version, counts);
279
+ }
280
+ this.#schedule();
281
+ return counts;
282
+ }
283
+ /** Reports within a ttl of the first count, without keeping the process alive for it. */
284
+ #schedule() {
285
+ if (this.#timer) return;
286
+ this.#timer = setTimeout(() => this.flush(), this.#ttl);
287
+ this.#timer.unref?.();
288
+ }
111
289
  #entry(slug) {
112
290
  const cached = this.#cache.get(slug);
113
291
  if (!cached) return this.#load(slug);
@@ -164,7 +342,9 @@ var PromptHelm = class {
164
342
  if (!response.ok) throw await errorFromResponse(response);
165
343
  const payload = await response.json();
166
344
  const entry = {
345
+ version: payload.version,
167
346
  content: payload.content,
347
+ experiment: payload.experiment,
168
348
  etag: response.headers.get("etag"),
169
349
  fetchedAt: Date.now()
170
350
  };
@@ -172,6 +352,18 @@ var PromptHelm = class {
172
352
  return entry;
173
353
  }
174
354
  };
355
+ var IN = ["input_tokens", "prompt_tokens", "inputTokens", "promptTokenCount"];
356
+ var OUT = ["output_tokens", "completion_tokens", "outputTokens", "candidatesTokenCount"];
357
+ function tokens(usage, keys) {
358
+ for (const key of keys) {
359
+ const value = usage[key];
360
+ if (typeof value === "number" && value >= 0 && Number.isFinite(value)) return value;
361
+ }
362
+ return 0;
363
+ }
364
+ function add(counts, index, amount) {
365
+ if (counts) counts[index] = (counts[index] ?? 0) + amount;
366
+ }
175
367
  async function errorFromResponse(response) {
176
368
  const body = await response.json().catch(() => null);
177
369
  const message = body?.error?.message ?? `HTTP ${response.status}`;
@@ -189,8 +381,10 @@ function toPromptHelmError(error) {
189
381
  0 && (module.exports = {
190
382
  ERROR_CODES,
191
383
  FALLBACK_HEADER,
384
+ LATENCY_BUCKETS,
192
385
  PromptHelm,
193
386
  PromptHelmError,
194
387
  extractVariables,
388
+ latencyBucket,
195
389
  render
196
390
  });
package/dist/index.d.cts CHANGED
@@ -23,6 +23,18 @@ interface PromptPayload {
23
23
  version: number;
24
24
  content: string;
25
25
  variables: string[];
26
+ /** An A/B test running on the prompt: the live version above is its control. */
27
+ experiment?: PromptExperiment;
28
+ }
29
+ interface PromptExperiment {
30
+ id: string;
31
+ /** The variant's share of calls, in percent (1–99). */
32
+ weight: number;
33
+ variant: {
34
+ version: number;
35
+ content: string;
36
+ variables: string[];
37
+ };
26
38
  }
27
39
  /**
28
40
  * Sent with the next request for a slug after `get()` couldn't serve its live text: the error's
@@ -33,6 +45,35 @@ declare const FALLBACK_HEADER = "x-prompthelm-fallback";
33
45
  interface PromptListPayload {
34
46
  prompts: PromptPayload[];
35
47
  }
48
+ /**
49
+ * Body of `POST /v1/reports`: what an app counted under running A/B tests since its last report.
50
+ * Numbers only, never any text, variable value or unit.
51
+ */
52
+ interface ReportPayload {
53
+ experiments: {
54
+ id: string;
55
+ /** The variant's share when these calls were assigned. */
56
+ weight: number;
57
+ /** By version number. */
58
+ versions: Record<string, ReportCounts>;
59
+ }[];
60
+ }
61
+ interface ReportCounts {
62
+ served: number;
63
+ /** Picked for the variant, served the live version because a variable wasn't passed. */
64
+ skipped: number;
65
+ outcomes: number;
66
+ /** The outcomes' scores (true = 1, false = 0), summed, and their squares summed. */
67
+ scoreSum: number;
68
+ scoreSqSum: number;
69
+ /** Calls whose model usage was reported, their tokens and their durations in ms, summed. */
70
+ usage: number;
71
+ tokensIn: number;
72
+ tokensOut: number;
73
+ latencyMs: number;
74
+ /** Those calls per latency bucket (`latencyBucket`). */
75
+ latency: number[];
76
+ }
36
77
  interface PromptHelmOptions {
37
78
  /** Defaults to the PROMPTHELM_API_KEY environment variable. */
38
79
  apiKey?: string;
@@ -50,6 +91,14 @@ interface PromptHelmOptions {
50
91
  * prompt's page shows it. Defaults to true.
51
92
  */
52
93
  reportFallbacks?: boolean;
94
+ /**
95
+ * Whether to send PromptHelm, as totals only, what happened under an A/B test: how often each
96
+ * version was served, the outcomes and the model usage you report. Defaults to
97
+ * `reportFallbacks`.
98
+ */
99
+ reportResults?: boolean;
100
+ /** Called on every `get` and `resolve`, so your analytics or traces can carry the version. */
101
+ onResolve?: (event: ResolveEvent) => void;
53
102
  }
54
103
  interface GetOptions {
55
104
  vars?: Vars;
@@ -58,13 +107,62 @@ interface GetOptions {
58
107
  * version uses a variable this call doesn't pass. Without it those cases throw.
59
108
  */
60
109
  fallback?: string;
110
+ /**
111
+ * Who the call is for, such as a user or conversation id: during an A/B test the same unit
112
+ * always gets the same version. Never sent to PromptHelm.
113
+ */
114
+ unit?: string | number;
115
+ /** During an A/B test, serves this side, such as the variant to your own team. Never counted. */
116
+ force?: "control" | "variant";
117
+ }
118
+ interface ResolveEvent {
119
+ slug: string;
120
+ /** The version served, or null when the fallback was. */
121
+ version: number | null;
122
+ /** The A/B test running on the prompt, if any. */
123
+ experiment: string | null;
124
+ /** Whether the call got the test's variant. */
125
+ variant: boolean;
126
+ }
127
+ interface Resolved {
128
+ text: string;
129
+ /** The version served, or null when the fallback was. */
130
+ version: number | null;
131
+ /**
132
+ * During an A/B test, names the test and the version this call got, so its outcome can be
133
+ * reported later from any process: `ph.outcome(ticket, true)`. No user in it. Null otherwise.
134
+ */
135
+ ticket: string | null;
136
+ /** How the call went: true or false, or a score from 0 to 1. The first report counts. */
137
+ outcome(value: boolean | number): void;
138
+ /**
139
+ * The model call's tokens, from your provider's `usage` object, and how long it took: `ms`,
140
+ * else the time since `resolve` returned. The first report counts.
141
+ */
142
+ usage(usage: object | null | undefined, ms?: number): void;
61
143
  }
62
144
  declare class PromptHelm {
63
145
  #private;
64
146
  constructor(options?: PromptHelmOptions);
65
147
  /** The live version of a prompt with its variables filled in. */
66
148
  get(slug: string, options?: GetOptions): Promise<string>;
149
+ /** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
150
+ resolve(slug: string, options?: GetOptions): Promise<Resolved>;
151
+ /** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
152
+ outcome(ticket: string, value: boolean | number): void;
153
+ /**
154
+ * Sends what was counted for A/B tests now, instead of within a ttl: call it before a
155
+ * serverless function freezes. Never throws; what can't be sent is kept for the next try.
156
+ */
157
+ flush(): Promise<void>;
67
158
  }
68
159
 
69
- export { ERROR_CODES, FALLBACK_HEADER, PromptHelm, PromptHelmError, extractVariables, render };
70
- export type { GetOptions, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, Vars };
160
+ declare const LATENCY_BUCKETS = 24;
161
+ /**
162
+ * Which of 24 latency buckets a model call's duration falls in, for its percentiles: bucket i
163
+ * holds calls up to 50·2^(i/2) ms (bucket 0 under 50 ms), and the last one everything slower.
164
+ */
165
+ declare function latencyBucket(ms: number): number;
166
+
167
+ export { ERROR_CODES, FALLBACK_HEADER, LATENCY_BUCKETS, PromptHelm, PromptHelmError, extractVariables, latencyBucket, render };
168
+ export type { GetOptions, PromptExperiment, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
package/dist/index.d.ts CHANGED
@@ -23,6 +23,18 @@ interface PromptPayload {
23
23
  version: number;
24
24
  content: string;
25
25
  variables: string[];
26
+ /** An A/B test running on the prompt: the live version above is its control. */
27
+ experiment?: PromptExperiment;
28
+ }
29
+ interface PromptExperiment {
30
+ id: string;
31
+ /** The variant's share of calls, in percent (1–99). */
32
+ weight: number;
33
+ variant: {
34
+ version: number;
35
+ content: string;
36
+ variables: string[];
37
+ };
26
38
  }
27
39
  /**
28
40
  * Sent with the next request for a slug after `get()` couldn't serve its live text: the error's
@@ -33,6 +45,35 @@ declare const FALLBACK_HEADER = "x-prompthelm-fallback";
33
45
  interface PromptListPayload {
34
46
  prompts: PromptPayload[];
35
47
  }
48
+ /**
49
+ * Body of `POST /v1/reports`: what an app counted under running A/B tests since its last report.
50
+ * Numbers only, never any text, variable value or unit.
51
+ */
52
+ interface ReportPayload {
53
+ experiments: {
54
+ id: string;
55
+ /** The variant's share when these calls were assigned. */
56
+ weight: number;
57
+ /** By version number. */
58
+ versions: Record<string, ReportCounts>;
59
+ }[];
60
+ }
61
+ interface ReportCounts {
62
+ served: number;
63
+ /** Picked for the variant, served the live version because a variable wasn't passed. */
64
+ skipped: number;
65
+ outcomes: number;
66
+ /** The outcomes' scores (true = 1, false = 0), summed, and their squares summed. */
67
+ scoreSum: number;
68
+ scoreSqSum: number;
69
+ /** Calls whose model usage was reported, their tokens and their durations in ms, summed. */
70
+ usage: number;
71
+ tokensIn: number;
72
+ tokensOut: number;
73
+ latencyMs: number;
74
+ /** Those calls per latency bucket (`latencyBucket`). */
75
+ latency: number[];
76
+ }
36
77
  interface PromptHelmOptions {
37
78
  /** Defaults to the PROMPTHELM_API_KEY environment variable. */
38
79
  apiKey?: string;
@@ -50,6 +91,14 @@ interface PromptHelmOptions {
50
91
  * prompt's page shows it. Defaults to true.
51
92
  */
52
93
  reportFallbacks?: boolean;
94
+ /**
95
+ * Whether to send PromptHelm, as totals only, what happened under an A/B test: how often each
96
+ * version was served, the outcomes and the model usage you report. Defaults to
97
+ * `reportFallbacks`.
98
+ */
99
+ reportResults?: boolean;
100
+ /** Called on every `get` and `resolve`, so your analytics or traces can carry the version. */
101
+ onResolve?: (event: ResolveEvent) => void;
53
102
  }
54
103
  interface GetOptions {
55
104
  vars?: Vars;
@@ -58,13 +107,62 @@ interface GetOptions {
58
107
  * version uses a variable this call doesn't pass. Without it those cases throw.
59
108
  */
60
109
  fallback?: string;
110
+ /**
111
+ * Who the call is for, such as a user or conversation id: during an A/B test the same unit
112
+ * always gets the same version. Never sent to PromptHelm.
113
+ */
114
+ unit?: string | number;
115
+ /** During an A/B test, serves this side, such as the variant to your own team. Never counted. */
116
+ force?: "control" | "variant";
117
+ }
118
+ interface ResolveEvent {
119
+ slug: string;
120
+ /** The version served, or null when the fallback was. */
121
+ version: number | null;
122
+ /** The A/B test running on the prompt, if any. */
123
+ experiment: string | null;
124
+ /** Whether the call got the test's variant. */
125
+ variant: boolean;
126
+ }
127
+ interface Resolved {
128
+ text: string;
129
+ /** The version served, or null when the fallback was. */
130
+ version: number | null;
131
+ /**
132
+ * During an A/B test, names the test and the version this call got, so its outcome can be
133
+ * reported later from any process: `ph.outcome(ticket, true)`. No user in it. Null otherwise.
134
+ */
135
+ ticket: string | null;
136
+ /** How the call went: true or false, or a score from 0 to 1. The first report counts. */
137
+ outcome(value: boolean | number): void;
138
+ /**
139
+ * The model call's tokens, from your provider's `usage` object, and how long it took: `ms`,
140
+ * else the time since `resolve` returned. The first report counts.
141
+ */
142
+ usage(usage: object | null | undefined, ms?: number): void;
61
143
  }
62
144
  declare class PromptHelm {
63
145
  #private;
64
146
  constructor(options?: PromptHelmOptions);
65
147
  /** The live version of a prompt with its variables filled in. */
66
148
  get(slug: string, options?: GetOptions): Promise<string>;
149
+ /** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
150
+ resolve(slug: string, options?: GetOptions): Promise<Resolved>;
151
+ /** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
152
+ outcome(ticket: string, value: boolean | number): void;
153
+ /**
154
+ * Sends what was counted for A/B tests now, instead of within a ttl: call it before a
155
+ * serverless function freezes. Never throws; what can't be sent is kept for the next try.
156
+ */
157
+ flush(): Promise<void>;
67
158
  }
68
159
 
69
- export { ERROR_CODES, FALLBACK_HEADER, PromptHelm, PromptHelmError, extractVariables, render };
70
- export type { GetOptions, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, Vars };
160
+ declare const LATENCY_BUCKETS = 24;
161
+ /**
162
+ * Which of 24 latency buckets a model call's duration falls in, for its percentiles: bucket i
163
+ * holds calls up to 50·2^(i/2) ms (bucket 0 under 50 ms), and the last one everything slower.
164
+ */
165
+ declare function latencyBucket(ms: number): number;
166
+
167
+ export { ERROR_CODES, FALLBACK_HEADER, LATENCY_BUCKETS, PromptHelm, PromptHelmError, extractVariables, latencyBucket, render };
168
+ export type { GetOptions, PromptExperiment, PromptHelmErrorCode, PromptHelmOptions, PromptListPayload, PromptPayload, ReportCounts, ReportPayload, ResolveEvent, Resolved, Vars };
package/dist/index.js CHANGED
@@ -18,6 +18,20 @@ var PromptHelmError = class extends Error {
18
18
  name = "PromptHelmError";
19
19
  };
20
20
 
21
+ // src/experiment.ts
22
+ function fnv1a32(text) {
23
+ let hash = 2166136261;
24
+ for (const byte of new TextEncoder().encode(text)) hash = Math.imul(hash ^ byte, 16777619);
25
+ return hash >>> 0;
26
+ }
27
+ function bucket(experimentId, unit) {
28
+ return unit === void 0 ? Math.floor(Math.random() * 100) : fnv1a32(`${experimentId}:${unit}`) % 100;
29
+ }
30
+ var LATENCY_BUCKETS = 24;
31
+ function latencyBucket(ms) {
32
+ return ms < 50 ? 0 : Math.min(LATENCY_BUCKETS - 1, 1 + Math.floor(2 * Math.log2(ms / 50)));
33
+ }
34
+
21
35
  // src/template.ts
22
36
  var VARIABLE = /\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
23
37
  function extractVariables(template) {
@@ -38,6 +52,27 @@ function render(template, vars = {}) {
38
52
  // src/client.ts
39
53
  var DEFAULT_BASE_URL = "https://prompthelm.app";
40
54
  var FALLBACK_HEADER = "x-prompthelm-fallback";
55
+ var FIELDS = [
56
+ "served",
57
+ "skipped",
58
+ "outcomes",
59
+ "scoreSum",
60
+ "scoreSqSum",
61
+ "usage",
62
+ "tokensIn",
63
+ "tokensOut",
64
+ "latencyMs"
65
+ ];
66
+ var SERVED = 0;
67
+ var SKIPPED = 1;
68
+ var OUTCOMES = 2;
69
+ var SCORE = 3;
70
+ var SCORE_SQ = 4;
71
+ var USAGE = 5;
72
+ var TOKENS_IN = 6;
73
+ var TOKENS_OUT = 7;
74
+ var LATENCY = 8;
75
+ var BUCKETS = FIELDS.length;
41
76
  var PromptHelm = class {
42
77
  #apiKey;
43
78
  #baseUrl;
@@ -45,12 +80,16 @@ var PromptHelm = class {
45
80
  #timeout;
46
81
  #fetch;
47
82
  #onError;
83
+ #onResolve;
48
84
  #cache = /* @__PURE__ */ new Map();
49
85
  #inflight = /* @__PURE__ */ new Map();
50
86
  #reportFallbacks;
87
+ #reportResults;
51
88
  // Per slug, why the live text last couldn't be served, until a request carries it. An object,
52
89
  // so a report made while that request is in flight isn't cleared with the one it carried.
53
90
  #unreported = /* @__PURE__ */ new Map();
91
+ #pending = /* @__PURE__ */ new Map();
92
+ #timer;
54
93
  constructor(options = {}) {
55
94
  const apiKey = options.apiKey ?? env("PROMPTHELM_API_KEY");
56
95
  if (!apiKey) throw new Error("PromptHelm: pass apiKey or set PROMPTHELM_API_KEY");
@@ -61,22 +100,159 @@ var PromptHelm = class {
61
100
  this.#timeout = options.timeout ?? 3e3;
62
101
  this.#fetch = options.fetch ?? ((input, init) => globalThis.fetch(input, init));
63
102
  this.#onError = options.onError ?? ((error, slug) => console.warn(`[PromptHelm] ${slug}: ${error.message}`));
103
+ this.#onResolve = options.onResolve;
64
104
  this.#reportFallbacks = options.reportFallbacks ?? true;
105
+ this.#reportResults = options.reportResults ?? this.#reportFallbacks;
65
106
  }
66
107
  /** The live version of a prompt with its variables filled in. */
67
108
  async get(slug, options = {}) {
109
+ return (await this.resolve(slug, options)).text;
110
+ }
111
+ /** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
112
+ async resolve(slug, options = {}) {
68
113
  const { vars, fallback } = options;
69
114
  try {
70
- const entry = await this.#entry(slug);
71
- return render(entry.content, vars);
115
+ return this.#pick(slug, await this.#entry(slug), options);
72
116
  } catch (error) {
73
117
  const failure = toPromptHelmError(error);
74
118
  this.#remember(slug, failure.code);
75
119
  if (fallback === void 0) throw failure;
76
120
  this.#onError(failure, slug);
77
- return render(fallback, vars);
121
+ return this.#served(slug, render(fallback, vars), null);
78
122
  }
79
123
  }
124
+ /** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
125
+ outcome(ticket, value) {
126
+ const [id = "", version, weight] = ticket.split(".");
127
+ const test = { id, weight: Number(weight) };
128
+ if (id && test.weight > 0) this.#outcome(test, Number(version), value);
129
+ }
130
+ /**
131
+ * Sends what was counted for A/B tests now, instead of within a ttl: call it before a
132
+ * serverless function freezes. Never throws; what can't be sent is kept for the next try.
133
+ */
134
+ async flush() {
135
+ clearTimeout(this.#timer);
136
+ this.#timer = void 0;
137
+ const sent = this.#pending;
138
+ if (sent.size === 0) return;
139
+ this.#pending = /* @__PURE__ */ new Map();
140
+ const body = {
141
+ experiments: [...sent.values()].map(({ id, weight, versions }) => ({
142
+ id,
143
+ weight,
144
+ versions: Object.fromEntries(
145
+ [...versions].map(([version, counts]) => [
146
+ version,
147
+ {
148
+ ...Object.fromEntries(FIELDS.map((field, i) => [field, counts[i]])),
149
+ latency: counts.slice(BUCKETS)
150
+ }
151
+ ])
152
+ )
153
+ }))
154
+ };
155
+ try {
156
+ const response = await this.#fetch(`${this.#baseUrl}/v1/reports`, {
157
+ method: "POST",
158
+ headers: { authorization: `Bearer ${this.#apiKey}`, "content-type": "application/json" },
159
+ body: JSON.stringify(body),
160
+ signal: AbortSignal.timeout(this.#timeout)
161
+ });
162
+ if (response.status < 500) return;
163
+ } catch {
164
+ }
165
+ for (const { id, weight, versions } of sent.values()) {
166
+ for (const [version, counts] of versions) {
167
+ const total = this.#counts({ id, weight }, version);
168
+ counts.forEach((n, i) => {
169
+ add(total, i, n);
170
+ });
171
+ }
172
+ }
173
+ }
174
+ #pick(slug, entry, options) {
175
+ const { vars, unit, force } = options;
176
+ const { experiment } = entry;
177
+ const test = experiment && !force ? { id: experiment.id, weight: experiment.weight } : void 0;
178
+ let skipped;
179
+ if (experiment && (force ? force === "variant" : bucket(experiment.id, unit) < experiment.weight)) {
180
+ const { version, content } = experiment.variant;
181
+ try {
182
+ return this.#served(slug, render(content, vars), version, experiment.id, test, true);
183
+ } catch (error) {
184
+ skipped = toPromptHelmError(error);
185
+ }
186
+ }
187
+ const text = render(entry.content, vars);
188
+ if (skipped && experiment) {
189
+ this.#onError(skipped, slug);
190
+ if (test) add(this.#counts(test, experiment.variant.version), SKIPPED, 1);
191
+ }
192
+ return this.#served(slug, text, entry.version, experiment?.id, test);
193
+ }
194
+ /** Counts the call, tells `onResolve`, and hands the caller its text and what to report. */
195
+ #served(slug, text, version, experiment, test, variant = false) {
196
+ if (test && version !== null) add(this.#counts(test, version), SERVED, 1);
197
+ try {
198
+ this.#onResolve?.({ slug, version, experiment: experiment ?? null, variant });
199
+ } catch {
200
+ }
201
+ const started = Date.now();
202
+ let reported = 0;
203
+ return {
204
+ text,
205
+ version,
206
+ ticket: test && version !== null ? `${test.id}.${version}.${test.weight}` : null,
207
+ outcome: (value) => {
208
+ if (!test || version === null || reported & 1) return;
209
+ reported |= 1;
210
+ this.#outcome(test, version, value);
211
+ },
212
+ usage: (usage, ms = Date.now() - started) => {
213
+ if (!test || version === null || reported & 2) return;
214
+ reported |= 2;
215
+ const counts = this.#counts(test, version);
216
+ const fields = usage ?? {};
217
+ add(counts, USAGE, 1);
218
+ add(counts, TOKENS_IN, tokens(fields, IN));
219
+ add(counts, TOKENS_OUT, tokens(fields, OUT));
220
+ add(counts, LATENCY, Math.max(0, ms));
221
+ add(counts, BUCKETS + latencyBucket(ms), 1);
222
+ }
223
+ };
224
+ }
225
+ #outcome(test, version, value) {
226
+ const score = value === true ? 1 : value === false ? 0 : value;
227
+ if (!Number.isInteger(version) || !(score >= 0 && score <= 1)) return;
228
+ const counts = this.#counts(test, version);
229
+ add(counts, OUTCOMES, 1);
230
+ add(counts, SCORE, score);
231
+ add(counts, SCORE_SQ, score * score);
232
+ }
233
+ /** The counts to add a call to, kept until the next report; none when reporting is off. */
234
+ #counts(test, version) {
235
+ if (!this.#reportResults) return;
236
+ const key = `${test.id}.${test.weight}`;
237
+ let pending = this.#pending.get(key);
238
+ if (!pending) {
239
+ pending = { ...test, versions: /* @__PURE__ */ new Map() };
240
+ this.#pending.set(key, pending);
241
+ }
242
+ let counts = pending.versions.get(version);
243
+ if (!counts) {
244
+ counts = new Array(BUCKETS + LATENCY_BUCKETS).fill(0);
245
+ pending.versions.set(version, counts);
246
+ }
247
+ this.#schedule();
248
+ return counts;
249
+ }
250
+ /** Reports within a ttl of the first count, without keeping the process alive for it. */
251
+ #schedule() {
252
+ if (this.#timer) return;
253
+ this.#timer = setTimeout(() => this.flush(), this.#ttl);
254
+ this.#timer.unref?.();
255
+ }
80
256
  #entry(slug) {
81
257
  const cached = this.#cache.get(slug);
82
258
  if (!cached) return this.#load(slug);
@@ -133,7 +309,9 @@ var PromptHelm = class {
133
309
  if (!response.ok) throw await errorFromResponse(response);
134
310
  const payload = await response.json();
135
311
  const entry = {
312
+ version: payload.version,
136
313
  content: payload.content,
314
+ experiment: payload.experiment,
137
315
  etag: response.headers.get("etag"),
138
316
  fetchedAt: Date.now()
139
317
  };
@@ -141,6 +319,18 @@ var PromptHelm = class {
141
319
  return entry;
142
320
  }
143
321
  };
322
+ var IN = ["input_tokens", "prompt_tokens", "inputTokens", "promptTokenCount"];
323
+ var OUT = ["output_tokens", "completion_tokens", "outputTokens", "candidatesTokenCount"];
324
+ function tokens(usage, keys) {
325
+ for (const key of keys) {
326
+ const value = usage[key];
327
+ if (typeof value === "number" && value >= 0 && Number.isFinite(value)) return value;
328
+ }
329
+ return 0;
330
+ }
331
+ function add(counts, index, amount) {
332
+ if (counts) counts[index] = (counts[index] ?? 0) + amount;
333
+ }
144
334
  async function errorFromResponse(response) {
145
335
  const body = await response.json().catch(() => null);
146
336
  const message = body?.error?.message ?? `HTTP ${response.status}`;
@@ -157,8 +347,10 @@ function toPromptHelmError(error) {
157
347
  export {
158
348
  ERROR_CODES,
159
349
  FALLBACK_HEADER,
350
+ LATENCY_BUCKETS,
160
351
  PromptHelm,
161
352
  PromptHelmError,
162
353
  extractVariables,
354
+ latencyBucket,
163
355
  render
164
356
  };
package/package.json CHANGED
@@ -1,8 +1,15 @@
1
1
  {
2
2
  "name": "@prompt-helm/sdk",
3
- "version": "1.2.0",
3
+ "version": "1.3.0",
4
4
  "description": "Fetch your PromptHelm prompts at runtime, so changing a prompt needs no redeploy.",
5
5
  "homepage": "https://prompthelm.app/quickstart",
6
+ "repository": {
7
+ "type": "git",
8
+ "url": "git+https://github.com/Runivox/prompthelm-node.git"
9
+ },
10
+ "bugs": {
11
+ "url": "https://github.com/Runivox/prompthelm-node/issues"
12
+ },
6
13
  "keywords": [
7
14
  "prompt",
8
15
  "prompt-management",
@@ -30,7 +37,7 @@
30
37
  }
31
38
  },
32
39
  "bin": {
33
- "prompthelm": "./dist/cli.js"
40
+ "prompthelm": "dist/cli.js"
34
41
  },
35
42
  "files": [
36
43
  "dist"
@@ -41,9 +48,13 @@
41
48
  "dev": "tsup --watch",
42
49
  "test": "vitest run",
43
50
  "typecheck": "tsc --noEmit",
51
+ "lint": "biome check",
52
+ "format": "biome format --write",
53
+ "smoke": "node scripts/smoke.mjs",
44
54
  "prepublishOnly": "npm run typecheck && npm test && npm run build"
45
55
  },
46
56
  "devDependencies": {
57
+ "@biomejs/biome": "2.4.2",
47
58
  "@types/node": "^22",
48
59
  "tsup": "^9.0.0",
49
60
  "typescript": "~6.0.3",