@prompt-helm/sdk 1.4.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -20,17 +20,229 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
20
20
  // src/index.ts
21
21
  var index_exports = {};
22
22
  __export(index_exports, {
23
+ API_HEADER: () => API_HEADER,
23
24
  ERROR_CODES: () => ERROR_CODES,
24
25
  FALLBACK_HEADER: () => FALLBACK_HEADER,
25
26
  LATENCY_BUCKETS: () => LATENCY_BUCKETS,
27
+ PROMPT_APIS: () => PROMPT_APIS,
26
28
  PromptHelm: () => PromptHelm,
27
29
  PromptHelmError: () => PromptHelmError,
30
+ clientOptions: () => clientOptions,
28
31
  extractVariables: () => extractVariables,
29
32
  latencyBucket: () => latencyBucket,
30
- render: () => render
33
+ render: () => render,
34
+ toAnthropic: () => toAnthropic,
35
+ toBedrockConverse: () => toBedrockConverse,
36
+ toChatCompletions: () => toChatCompletions,
37
+ toGemini: () => toGemini,
38
+ toOpenAIResponses: () => toOpenAIResponses
31
39
  });
32
40
  module.exports = __toCommonJS(index_exports);
33
41
 
42
+ // src/adapters.ts
43
+ var obj = (value) => value !== null && typeof value === "object" && !Array.isArray(value) ? value : {};
44
+ function requestOptions(c) {
45
+ const options = {};
46
+ if (c.timeoutMs !== void 0) options.timeout = c.timeoutMs;
47
+ if (c.maxRetries !== void 0) options.maxRetries = c.maxRetries;
48
+ return options;
49
+ }
50
+ function put(target, values) {
51
+ for (const [key, value] of Object.entries(values)) if (value !== void 0) target[key] = value;
52
+ return target;
53
+ }
54
+ function turns(yours, user, wrap) {
55
+ if (user === null) return yours;
56
+ return Array.isArray(yours) ? [...yours, wrap(user)] : [wrap(user)];
57
+ }
58
+ function toOpenAIResponses(r, base) {
59
+ const c = r.config;
60
+ const yours = base;
61
+ const params = put(
62
+ { ...yours, ...c.extra },
63
+ {
64
+ model: c.model,
65
+ instructions: r.system,
66
+ // Input as text stays text; a list of items is history the user message follows.
67
+ input: r.user !== null && !Array.isArray(yours.input) ? r.user : turns(yours.input, r.user, userMessage),
68
+ max_output_tokens: c.maxTokens,
69
+ temperature: c.temperature,
70
+ text: c.response && {
71
+ ...obj(yours.text),
72
+ format: {
73
+ type: "json_schema",
74
+ name: c.response.name,
75
+ schema: c.response.schema,
76
+ strict: true
77
+ }
78
+ }
79
+ }
80
+ );
81
+ return { params, options: requestOptions(c) };
82
+ }
83
+ function toChatCompletions(r, base) {
84
+ const c = r.config;
85
+ const yours = base;
86
+ const history = (Array.isArray(yours.messages) ? yours.messages : []).filter((m) => {
87
+ const role = obj(m).role;
88
+ return role !== "system" && role !== "developer";
89
+ });
90
+ const own = c.provider === "openai" || c.provider === "azure";
91
+ const params = put(
92
+ { ...yours, ...c.extra },
93
+ {
94
+ model: c.model,
95
+ messages: [
96
+ { role: "system", content: r.system },
97
+ ...turns(history, r.user, userMessage)
98
+ ],
99
+ [own ? "max_completion_tokens" : "max_tokens"]: c.maxTokens,
100
+ temperature: c.temperature,
101
+ stop: c.stop,
102
+ response_format: c.response && {
103
+ type: "json_schema",
104
+ json_schema: { name: c.response.name, schema: c.response.schema, strict: true }
105
+ }
106
+ }
107
+ );
108
+ return { params, options: requestOptions(c) };
109
+ }
110
+ function toAnthropic(r, base) {
111
+ const c = r.config;
112
+ const yours = base;
113
+ const params = put(
114
+ { ...yours, ...c.extra },
115
+ {
116
+ model: c.model,
117
+ system: r.system,
118
+ messages: turns(yours.messages, r.user, userMessage),
119
+ max_tokens: c.maxTokens,
120
+ temperature: c.temperature,
121
+ stop_sequences: c.stop,
122
+ output_config: c.response && {
123
+ ...obj(yours.output_config),
124
+ format: { type: "json_schema", schema: c.response.schema }
125
+ }
126
+ }
127
+ );
128
+ return { params, options: requestOptions(c) };
129
+ }
130
+ function toGemini(r, base) {
131
+ const c = r.config;
132
+ const yours = base;
133
+ const config = obj(yours.config);
134
+ const http = obj(config.httpOptions);
135
+ const params = put(
136
+ { ...yours },
137
+ {
138
+ model: c.model,
139
+ contents: turns(yours.contents, r.user, (text) => ({ role: "user", parts: [{ text }] })),
140
+ config: put(
141
+ { ...config },
142
+ {
143
+ systemInstruction: r.system,
144
+ maxOutputTokens: c.maxTokens,
145
+ temperature: c.temperature,
146
+ stopSequences: c.stop,
147
+ responseMimeType: c.response && "application/json",
148
+ responseJsonSchema: c.response?.schema,
149
+ httpOptions: c.timeoutMs !== void 0 || c.extra ? put(
150
+ { ...http },
151
+ {
152
+ timeout: c.timeoutMs,
153
+ extraBody: c.extra && { ...obj(http.extraBody), ...c.extra }
154
+ }
155
+ ) : void 0
156
+ }
157
+ )
158
+ }
159
+ );
160
+ return { params, options: {} };
161
+ }
162
+ function toBedrockConverse(r, base) {
163
+ const c = r.config;
164
+ const yours = base;
165
+ const inference = obj(yours.inferenceConfig);
166
+ const params = put(
167
+ { ...yours },
168
+ {
169
+ modelId: c.model,
170
+ system: [{ text: r.system }],
171
+ messages: turns(yours.messages, r.user, (text) => ({ role: "user", content: [{ text }] })),
172
+ inferenceConfig: c.maxTokens !== void 0 || c.temperature !== void 0 || c.stop ? put(
173
+ { ...inference },
174
+ { maxTokens: c.maxTokens, temperature: c.temperature, stopSequences: c.stop }
175
+ ) : void 0,
176
+ outputConfig: c.response && {
177
+ ...obj(yours.outputConfig),
178
+ textFormat: {
179
+ type: "json_schema",
180
+ structure: {
181
+ jsonSchema: { name: c.response.name, schema: JSON.stringify(c.response.schema) }
182
+ }
183
+ }
184
+ },
185
+ additionalModelRequestFields: c.extra && {
186
+ ...obj(yours.additionalModelRequestFields),
187
+ ...c.extra
188
+ }
189
+ }
190
+ );
191
+ return { params, options: {} };
192
+ }
193
+ function clientOptions(r, base) {
194
+ const c = r.config;
195
+ const to = c.connection ?? {};
196
+ const out = { ...base };
197
+ const trimmed = (url) => url.replace(/\/+$/, "");
198
+ switch (r.api ?? c.apis?.[0]) {
199
+ case "openai-responses":
200
+ case "chat-completions": {
201
+ const baseURL = to.baseUrl ?? (c.provider === "azure" && to.endpointUrl ? withPath(trimmed(to.endpointUrl), "/openai/v1/") : c.provider === "databricks" && to.workspaceUrl ? withPath(trimmed(to.workspaceUrl), "/serving-endpoints") : void 0);
202
+ return put(out, { baseURL, ...requestOptions(c) });
203
+ }
204
+ case "anthropic-messages": {
205
+ const baseURL = to.baseUrl ?? (c.provider === "azure" && to.endpointUrl ? withPath(trimmed(to.endpointUrl), "/anthropic/") : void 0);
206
+ const region = c.provider === "google-vertex" ? to.location : void 0;
207
+ return put(out, { baseURL, region, ...requestOptions(c) });
208
+ }
209
+ case "gemini": {
210
+ const http = obj(out.httpOptions);
211
+ const httpOptions = to.baseUrl !== void 0 || c.timeoutMs !== void 0 || c.maxRetries !== void 0 ? put(
212
+ { ...http },
213
+ {
214
+ baseUrl: to.baseUrl,
215
+ timeout: c.timeoutMs,
216
+ retryOptions: c.maxRetries !== void 0 ? { ...obj(http.retryOptions), attempts: c.maxRetries + 1 } : void 0
217
+ }
218
+ ) : void 0;
219
+ const vertex = c.provider === "google-vertex";
220
+ return put(out, {
221
+ vertexai: vertex || void 0,
222
+ location: vertex ? to.location : void 0,
223
+ httpOptions
224
+ });
225
+ }
226
+ case "bedrock-converse": {
227
+ const handler = obj(out.requestHandler);
228
+ return put(out, {
229
+ region: to.region,
230
+ endpoint: to.endpointUrl,
231
+ maxAttempts: c.maxRetries !== void 0 ? c.maxRetries + 1 : void 0,
232
+ requestHandler: c.timeoutMs !== void 0 ? { ...handler, requestTimeout: c.timeoutMs, throwOnRequestTimeout: true } : void 0
233
+ });
234
+ }
235
+ default:
236
+ return out;
237
+ }
238
+ }
239
+ function withPath(url, path) {
240
+ return /^https?:\/\/[^/]+\/./.test(url) ? url : `${url}${path}`;
241
+ }
242
+ function userMessage(text) {
243
+ return { role: "user", content: text };
244
+ }
245
+
34
246
  // src/errors.ts
35
247
  var ERROR_CODES = [
36
248
  "unauthorized",
@@ -38,7 +250,9 @@ var ERROR_CODES = [
38
250
  "missing_variables",
39
251
  "timeout",
40
252
  "network",
41
- "server"
253
+ "server",
254
+ // The version is for another API than the one the call asked for (`api`, SDK 1.6).
255
+ "api_mismatch"
42
256
  ];
43
257
  var PromptHelmError = class extends Error {
44
258
  constructor(code, message, status) {
@@ -84,6 +298,14 @@ function render(template, vars = {}) {
84
298
 
85
299
  // src/client.ts
86
300
  var DEFAULT_BASE_URL = "https://prompthelm.app";
301
+ var PROMPT_APIS = [
302
+ "openai-responses",
303
+ "chat-completions",
304
+ "anthropic-messages",
305
+ "gemini",
306
+ "bedrock-converse"
307
+ ];
308
+ var API_HEADER = "x-prompthelm-api";
87
309
  var FALLBACK_HEADER = "x-prompthelm-fallback";
88
310
  var FIELDS = [
89
311
  "served",
@@ -106,6 +328,13 @@ var TOKENS_IN = 6;
106
328
  var TOKENS_OUT = 7;
107
329
  var LATENCY = 8;
108
330
  var BUCKETS = FIELDS.length;
331
+ function fill(system, user, vars, withUser) {
332
+ const text = typeof user === "string" ? user : user?.content;
333
+ return {
334
+ system: render(system, vars),
335
+ user: withUser && text !== void 0 ? render(text, vars) : null
336
+ };
337
+ }
109
338
  var PromptHelm = class {
110
339
  #apiKey;
111
340
  #baseUrl;
@@ -118,6 +347,10 @@ var PromptHelm = class {
118
347
  #inflight = /* @__PURE__ */ new Map();
119
348
  #reportFallbacks;
120
349
  #reportResults;
350
+ #reportFailures;
351
+ // Per slug, the API a call last asked for, sent with its requests (`API_HEADER`).
352
+ #apis = /* @__PURE__ */ new Map();
353
+ #failures = /* @__PURE__ */ new Map();
121
354
  // Per slug, why the live text last couldn't be served, until a request carries it. An object,
122
355
  // so a report made while that request is in flight isn't cleared with the one it carried.
123
356
  #unreported = /* @__PURE__ */ new Map();
@@ -136,22 +369,37 @@ var PromptHelm = class {
136
369
  this.#onResolve = options.onResolve;
137
370
  this.#reportFallbacks = options.reportFallbacks ?? true;
138
371
  this.#reportResults = options.reportResults ?? this.#reportFallbacks;
372
+ this.#reportFailures = options.reportFailures ?? this.#reportFallbacks;
139
373
  }
140
- /** The live version of a prompt with its variables filled in. */
374
+ /**
375
+ * The live version's system message with its variables filled in. It never needs the user
376
+ * message's variables: an app that calls `get` sends its own.
377
+ */
141
378
  async get(slug, options = {}) {
142
- return (await this.resolve(slug, options)).text;
379
+ return (await this.#resolve(slug, options, false)).text;
143
380
  }
144
- /** Like `get`, and says which version was served, so you can tag your logs or traces with it. */
145
- async resolve(slug, options = {}) {
146
- const { vars, fallback } = options;
381
+ /**
382
+ * Like `get`, with the user message too, the model settings, and which version was served, so
383
+ * you can tag your logs or traces with it.
384
+ */
385
+ resolve(slug, options = {}) {
386
+ return this.#resolve(slug, options, true);
387
+ }
388
+ async #resolve(slug, options, withUser) {
389
+ const { vars, fallback, api } = options;
390
+ if (api) this.#apis.set(slug, api);
147
391
  try {
148
- return this.#pick(slug, await this.#entry(slug), options);
392
+ return { ...this.#pick(slug, await this.#entry(slug), options, withUser), api: api ?? null };
149
393
  } catch (error) {
150
394
  const failure = toPromptHelmError(error);
151
395
  this.#remember(slug, failure.code);
152
396
  if (fallback === void 0) throw failure;
153
397
  this.#onError(failure, slug);
154
- return this.#served(slug, render(fallback, vars), {}, null);
398
+ const own = typeof fallback === "string" ? { system: fallback } : fallback;
399
+ return {
400
+ ...this.#served(slug, fill(own.system, own.user, vars, withUser), {}, null),
401
+ api: api ?? null
402
+ };
155
403
  }
156
404
  }
157
405
  /** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
@@ -168,8 +416,10 @@ var PromptHelm = class {
168
416
  clearTimeout(this.#timer);
169
417
  this.#timer = void 0;
170
418
  const sent = this.#pending;
171
- if (sent.size === 0) return;
419
+ const failures = this.#failures;
420
+ if (sent.size === 0 && failures.size === 0) return;
172
421
  this.#pending = /* @__PURE__ */ new Map();
422
+ this.#failures = /* @__PURE__ */ new Map();
173
423
  const body = {
174
424
  experiments: [...sent.values()].map(({ id, weight, versions }) => ({
175
425
  id,
@@ -185,6 +435,7 @@ var PromptHelm = class {
185
435
  )
186
436
  }))
187
437
  };
438
+ if (failures.size > 0) body.failures = [...failures.values()];
188
439
  try {
189
440
  const response = await this.#fetch(`${this.#baseUrl}/v1/reports`, {
190
441
  method: "POST",
@@ -195,6 +446,11 @@ var PromptHelm = class {
195
446
  if (response.status < 500) return;
196
447
  } catch {
197
448
  }
449
+ for (const [key, failure] of failures) {
450
+ const now = this.#failures.get(key);
451
+ if (now) now.count += failure.count;
452
+ else this.#failures.set(key, failure);
453
+ }
198
454
  for (const { id, weight, versions } of sent.values()) {
199
455
  for (const [version, counts] of versions) {
200
456
  const total = this.#counts({ id, weight }, version);
@@ -204,17 +460,26 @@ var PromptHelm = class {
204
460
  }
205
461
  }
206
462
  }
207
- #pick(slug, entry, options) {
208
- const { vars, unit, force } = options;
463
+ #pick(slug, entry, options, withUser) {
464
+ const { vars, unit, force, api } = options;
209
465
  const { experiment } = entry;
466
+ const check = (version, config) => {
467
+ if (api && config.apis && !config.apis.includes(api)) {
468
+ throw new PromptHelmError(
469
+ "api_mismatch",
470
+ `v${version} is called with ${config.apis.join(" or ")}, not ${api}`
471
+ );
472
+ }
473
+ };
210
474
  const test = experiment && !force ? { id: experiment.id, weight: experiment.weight } : void 0;
211
475
  let skipped;
212
476
  if (experiment && (force ? force === "variant" : bucket(experiment.id, unit) < experiment.weight)) {
213
- const { version, content, config = {} } = experiment.variant;
477
+ const { version, content, user, config = {} } = experiment.variant;
214
478
  try {
479
+ check(version, config);
215
480
  return this.#served(
216
481
  slug,
217
- render(content, vars),
482
+ fill(content, user, vars, withUser),
218
483
  config,
219
484
  version,
220
485
  experiment.id,
@@ -225,15 +490,16 @@ var PromptHelm = class {
225
490
  skipped = toPromptHelmError(error);
226
491
  }
227
492
  }
228
- const text = render(entry.content, vars);
493
+ check(entry.version, entry.config);
494
+ const messages = fill(entry.content, entry.user, vars, withUser);
229
495
  if (skipped && experiment) {
230
496
  this.#onError(skipped, slug);
231
497
  if (test) add(this.#counts(test, experiment.variant.version), SKIPPED, 1);
232
498
  }
233
- return this.#served(slug, text, entry.config, entry.version, experiment?.id, test);
499
+ return this.#served(slug, messages, entry.config, entry.version, experiment?.id, test);
234
500
  }
235
- /** Counts the call, tells `onResolve`, and hands the caller its text, settings and what to report. */
236
- #served(slug, text, config, version, experiment, test, variant = false) {
501
+ /** Counts the call, tells `onResolve`, and hands the caller its messages, settings and what to report. */
502
+ #served(slug, { system, user }, config, version, experiment, test, variant = false) {
237
503
  if (test && version !== null) add(this.#counts(test, version), SERVED, 1);
238
504
  try {
239
505
  this.#onResolve?.({ slug, version, experiment: experiment ?? null, variant });
@@ -242,9 +508,11 @@ var PromptHelm = class {
242
508
  const started = Date.now();
243
509
  let reported = 0;
244
510
  return {
245
- text,
246
- // A copy: the cached one serves every later call.
247
- config: { ...config },
511
+ text: system,
512
+ system,
513
+ user,
514
+ // A copy, schema and all: the cached one serves every later call.
515
+ config: structuredClone(config),
248
516
  version,
249
517
  ticket: test && version !== null ? `${test.id}.${version}.${test.weight}` : null,
250
518
  outcome: (value) => {
@@ -252,6 +520,11 @@ var PromptHelm = class {
252
520
  reported |= 1;
253
521
  this.#outcome(test, version, value);
254
522
  },
523
+ failed: (error) => {
524
+ if (version === null || reported & 4) return;
525
+ reported |= 4;
526
+ this.#failed(slug, version, error);
527
+ },
255
528
  usage: (usage, ms = Date.now() - started) => {
256
529
  if (!test || version === null || reported & 2) return;
257
530
  reported |= 2;
@@ -273,6 +546,18 @@ var PromptHelm = class {
273
546
  add(counts, SCORE, score);
274
547
  add(counts, SCORE_SQ, score * score);
275
548
  }
549
+ /** Counts a refused model call by its HTTP status and codes; anything else is ignored. */
550
+ #failed(slug, version, error) {
551
+ if (!this.#reportFailures) return;
552
+ const failure = refusal(error);
553
+ if (!failure) return;
554
+ const { status, type, code, param } = failure;
555
+ const key = JSON.stringify([slug, version, status, type, code, param]);
556
+ const counted = this.#failures.get(key);
557
+ if (counted) counted.count += 1;
558
+ else this.#failures.set(key, { slug, version, ...failure, count: 1 });
559
+ this.#schedule();
560
+ }
276
561
  /** The counts to add a call to, kept until the next report; none when reporting is off. */
277
562
  #counts(test, version) {
278
563
  if (!this.#reportResults) return;
@@ -329,6 +614,8 @@ var PromptHelm = class {
329
614
  if (cached?.etag) headers["if-none-match"] = cached.etag;
330
615
  const report = this.#unreported.get(slug);
331
616
  if (report) headers[FALLBACK_HEADER] = report.code;
617
+ const api = this.#apis.get(slug);
618
+ if (api) headers[API_HEADER] = api;
332
619
  let response;
333
620
  try {
334
621
  response = await this.#fetch(`${this.#baseUrl}/v1/prompts/${encodeURIComponent(slug)}`, {
@@ -354,6 +641,7 @@ var PromptHelm = class {
354
641
  const entry = {
355
642
  version: payload.version,
356
643
  content: payload.content,
644
+ user: payload.user,
357
645
  config: payload.config ?? {},
358
646
  experiment: payload.experiment,
359
647
  etag: response.headers.get("etag"),
@@ -372,6 +660,27 @@ function tokens(usage, keys) {
372
660
  }
373
661
  return 0;
374
662
  }
663
+ var CODE = /^[A-Za-z0-9_.:[\]-]{1,64}$/;
664
+ var word = (value) => typeof value === "string" && CODE.test(value) ? value : void 0;
665
+ function refusal(error) {
666
+ if (error === null || typeof error !== "object") return null;
667
+ const e = error;
668
+ const meta = e.$metadata;
669
+ const response = e.response;
670
+ const status = [e.status, e.statusCode, meta?.httpStatusCode, response?.status].find(
671
+ (value) => Number.isInteger(value) && Number(value) >= 400 && Number(value) < 600
672
+ );
673
+ if (status === void 0) return null;
674
+ const nested = e.error !== null && typeof e.error === "object" ? e.error : {};
675
+ const failure = { status };
676
+ const type = word(e.type) ?? word(nested.type) ?? (meta ? word(e.name) : void 0);
677
+ const code = word(e.code) ?? word(e.errorCode) ?? word(nested.code);
678
+ const param = word(e.param) ?? word(nested.param);
679
+ if (type) failure.type = type;
680
+ if (code) failure.code = code;
681
+ if (param) failure.param = param;
682
+ return failure;
683
+ }
375
684
  function add(counts, index, amount) {
376
685
  if (counts) counts[index] = (counts[index] ?? 0) + amount;
377
686
  }
@@ -390,12 +699,20 @@ function toPromptHelmError(error) {
390
699
  }
391
700
  // Annotate the CommonJS export names for ESM import in node:
392
701
  0 && (module.exports = {
702
+ API_HEADER,
393
703
  ERROR_CODES,
394
704
  FALLBACK_HEADER,
395
705
  LATENCY_BUCKETS,
706
+ PROMPT_APIS,
396
707
  PromptHelm,
397
708
  PromptHelmError,
709
+ clientOptions,
398
710
  extractVariables,
399
711
  latencyBucket,
400
- render
712
+ render,
713
+ toAnthropic,
714
+ toBedrockConverse,
715
+ toChatCompletions,
716
+ toGemini,
717
+ toOpenAIResponses
401
718
  });