@prompt-helm/sdk 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -36
- package/dist/index.cjs +305 -8
- package/dist/index.d.cts +126 -6
- package/dist/index.d.ts +126 -6
- package/dist/index.js +296 -7
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -89,6 +89,7 @@ All optional when the environment variables are set.
|
|
|
89
89
|
| `onError` | `console.warn` | Called with the error and the slug when a fallback or a cached prompt was served instead of failing. |
|
|
90
90
|
| `reportFallbacks` | `true` | Tells PromptHelm, by error code only, when a fallback or a cached prompt was served, so the prompt's page shows which app fell back and why. |
|
|
91
91
|
| `reportResults` | `reportFallbacks` | During an [A/B test](#ab-tests), sends PromptHelm totals only: calls served per version, and the outcomes and usage you report. |
|
|
92
|
+
| `reportFailures` | `reportFallbacks` | Sends what [`failed`](#refused-calls) learns: the provider's HTTP status and error codes, by version, never a message. |
|
|
92
93
|
| `onResolve` | none | Called on every `get` and `resolve` with the version served, for your analytics or traces. |
|
|
93
94
|
| `fetch` | global `fetch` | Your own fetch, for tests or a proxy. |
|
|
94
95
|
|
|
@@ -104,6 +105,7 @@ Without a `fallback`, `get` throws them; with one, `onError` receives them.
|
|
|
104
105
|
| `missing_variables` | The live version uses a variable the call doesn't pass. |
|
|
105
106
|
| `timeout` | No response within the timeout. |
|
|
106
107
|
| `network` | PromptHelm couldn't be reached. |
|
|
108
|
+
| `api_mismatch` | The live version's model can't be called with the call's `api`. |
|
|
107
109
|
| `server` | Anything else, such as a 5xx response. |
|
|
108
110
|
|
|
109
111
|
After the first successful fetch, a failed background refresh keeps serving the cached prompt
|
|
@@ -140,57 +142,88 @@ Run it in CI before the build, so every deploy carries the text that was live th
|
|
|
140
142
|
|
|
141
143
|
## Model settings
|
|
142
144
|
|
|
143
|
-
A version can carry the model to call it with
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
145
|
+
A version can carry the model to call it with and how: its temperature and output limit, stop
|
|
146
|
+
sequences, an answer in JSON, the provider and how to reach it, a timeout, retries and parameters
|
|
147
|
+
of your own, all set next to the text in the dashboard. Moving to a cheaper model is then a publish
|
|
148
|
+
like any other: one click rolls it back, and an A/B test can tell whether it does as well.
|
|
149
|
+
|
|
150
|
+
Each API wants them in its own place, so the SDK has a request builder for each:
|
|
151
|
+
`toOpenAIResponses`, `toChatCompletions` (OpenAI's and every compatible host's), `toAnthropic`,
|
|
152
|
+
`toGemini` (`@google/genai`) and `toBedrockConverse`. Give it what `resolve` returned and your own
|
|
153
|
+
request; it returns that request with the version's messages and settings in, and the options for
|
|
154
|
+
`create()`:
|
|
147
155
|
|
|
148
156
|
```ts
|
|
157
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
158
|
+
import { PromptHelm, toAnthropic } from "@prompt-helm/sdk";
|
|
159
|
+
|
|
149
160
|
const r = await ph.resolve("support-reply", {
|
|
150
161
|
vars: { company: "Acme", customer_name: user.name },
|
|
151
162
|
fallback: SUPPORT_REPLY,
|
|
163
|
+
api: "anthropic-messages",
|
|
152
164
|
});
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
max_output_tokens: r.config.maxTokens,
|
|
158
|
-
input: r.text,
|
|
165
|
+
const { params, options } = toAnthropic(r, {
|
|
166
|
+
model: "claude-sonnet-5-5", // when the dashboard sets none, and for the fallback
|
|
167
|
+
max_tokens: 1024,
|
|
168
|
+
messages: history, // your conversation so far; the version's user message follows it
|
|
159
169
|
});
|
|
170
|
+
try {
|
|
171
|
+
const reply = await anthropic.messages.create(params, options);
|
|
172
|
+
} catch (error) {
|
|
173
|
+
r.failed(error);
|
|
174
|
+
throw error;
|
|
175
|
+
}
|
|
160
176
|
```
|
|
161
177
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
178
|
+
What the dashboard leaves unset stays yours; what it sets wins. The builders import no provider
|
|
179
|
+
package and call nothing. When the fallback is served, `config` is empty and your request goes out
|
|
180
|
+
as you wrote it, with the fallback's messages. `r.config` holds the same settings (`model`,
|
|
181
|
+
`temperature`, `maxTokens`, `stop`, `response`, `provider`, `apis`, `connection`, `timeoutMs`,
|
|
182
|
+
`maxRetries`, `extra`) if you'd rather map them yourself. PromptHelm doesn't check a model name
|
|
183
|
+
with your provider, and a name your provider doesn't know fails your calls: try a new model in an
|
|
184
|
+
A/B test first. SDK versions before 1.4 ignore the settings, and before 1.6 receive only the first
|
|
185
|
+
four.
|
|
167
186
|
|
|
168
|
-
|
|
187
|
+
`api` names the API your app calls the model with: `openai-responses`, `chat-completions`,
|
|
188
|
+
`anthropic-messages`, `gemini` or `bedrock-converse`. `config.apis` lists the ones the version's
|
|
189
|
+
model takes where it's called (OpenAI's models take Responses and Chat Completions); a version
|
|
190
|
+
whose model your API can't reach isn't served, and the call gets its fallback (`api_mismatch`).
|
|
191
|
+
PromptHelm learns which API each key's app uses, so the prompt's page warns before such a version
|
|
192
|
+
is published.
|
|
169
193
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
194
|
+
### Reaching the provider
|
|
195
|
+
|
|
196
|
+
`clientOptions(r, yours)` adds what the dashboard set for reaching the provider (an address, a
|
|
197
|
+
Region or location, a timeout and retries) to your client's options, in the names the official SDK
|
|
198
|
+
of the API you named in `resolve` uses. Keys stay in your code:
|
|
174
199
|
|
|
175
200
|
```ts
|
|
176
|
-
const {
|
|
177
|
-
const reply = await openai.responses.create({
|
|
178
|
-
model: r.config.model ?? "gpt-5",
|
|
179
|
-
instructions: r.system,
|
|
180
|
-
input: r.user ?? ticket.text,
|
|
181
|
-
...(response && {
|
|
182
|
-
text: {
|
|
183
|
-
format: { type: "json_schema", name: response.name, schema: response.schema, strict: true },
|
|
184
|
-
},
|
|
185
|
-
}),
|
|
186
|
-
});
|
|
201
|
+
const bedrock = new BedrockRuntimeClient(clientOptions(r, { credentials }));
|
|
187
202
|
```
|
|
188
203
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
204
|
+
Build a client again when `r.config.connection` changes; most apps keep one per connection. A
|
|
205
|
+
Microsoft Foundry resource's address becomes its `/openai/v1/` or `/anthropic/` URL and a
|
|
206
|
+
Databricks workspace its `/serving-endpoints`; for anything else, the dashboard's Base URL is used
|
|
207
|
+
as it is.
|
|
208
|
+
|
|
209
|
+
### Answers in JSON
|
|
210
|
+
|
|
211
|
+
A version can ask for an answer in JSON with the fields set in the dashboard. `config.response` is
|
|
212
|
+
then `{ type: "json", name, schema }`: a JSON Schema kept inside what OpenAI, Anthropic, Gemini and
|
|
213
|
+
Amazon Bedrock all accept (and, for GPT on Microsoft Foundry, what Azure accepts), so the same
|
|
214
|
+
schema works with each. The request builders put it where each API wants it: `text.format` on
|
|
215
|
+
Responses, `response_format` on Chat Completions, `output_config.format` on Anthropic,
|
|
216
|
+
`responseJsonSchema` on Gemini and `outputConfig.textFormat` on Bedrock, as text. PromptHelm never
|
|
217
|
+
sees the answer, so check it in your code. Without a response format the model answers in text;
|
|
218
|
+
SDK versions before 1.5 ignore it.
|
|
219
|
+
|
|
220
|
+
### Refused calls
|
|
221
|
+
|
|
222
|
+
`r.failed(error)` tells PromptHelm that your provider refused the call: the HTTP status, the
|
|
223
|
+
error's type and code, and the parameter it names (OpenAI's SDK gives one), never its message.
|
|
224
|
+
The prompt's page shows them per version, so a setting your provider rejects shows up while it's
|
|
225
|
+
still one publish to roll back. Anything without an HTTP status, such as a network error, is
|
|
226
|
+
ignored. `reportFailures: false` turns it off.
|
|
194
227
|
|
|
195
228
|
## A/B tests
|
|
196
229
|
|
package/dist/index.cjs
CHANGED
|
@@ -20,17 +20,229 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
20
20
|
// src/index.ts
|
|
21
21
|
var index_exports = {};
|
|
22
22
|
__export(index_exports, {
|
|
23
|
+
API_HEADER: () => API_HEADER,
|
|
23
24
|
ERROR_CODES: () => ERROR_CODES,
|
|
24
25
|
FALLBACK_HEADER: () => FALLBACK_HEADER,
|
|
25
26
|
LATENCY_BUCKETS: () => LATENCY_BUCKETS,
|
|
27
|
+
PROMPT_APIS: () => PROMPT_APIS,
|
|
26
28
|
PromptHelm: () => PromptHelm,
|
|
27
29
|
PromptHelmError: () => PromptHelmError,
|
|
30
|
+
clientOptions: () => clientOptions,
|
|
28
31
|
extractVariables: () => extractVariables,
|
|
29
32
|
latencyBucket: () => latencyBucket,
|
|
30
|
-
render: () => render
|
|
33
|
+
render: () => render,
|
|
34
|
+
toAnthropic: () => toAnthropic,
|
|
35
|
+
toBedrockConverse: () => toBedrockConverse,
|
|
36
|
+
toChatCompletions: () => toChatCompletions,
|
|
37
|
+
toGemini: () => toGemini,
|
|
38
|
+
toOpenAIResponses: () => toOpenAIResponses
|
|
31
39
|
});
|
|
32
40
|
module.exports = __toCommonJS(index_exports);
|
|
33
41
|
|
|
42
|
+
// src/adapters.ts
|
|
43
|
+
var obj = (value) => value !== null && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
44
|
+
function requestOptions(c) {
|
|
45
|
+
const options = {};
|
|
46
|
+
if (c.timeoutMs !== void 0) options.timeout = c.timeoutMs;
|
|
47
|
+
if (c.maxRetries !== void 0) options.maxRetries = c.maxRetries;
|
|
48
|
+
return options;
|
|
49
|
+
}
|
|
50
|
+
function put(target, values) {
|
|
51
|
+
for (const [key, value] of Object.entries(values)) if (value !== void 0) target[key] = value;
|
|
52
|
+
return target;
|
|
53
|
+
}
|
|
54
|
+
function turns(yours, user, wrap) {
|
|
55
|
+
if (user === null) return yours;
|
|
56
|
+
return Array.isArray(yours) ? [...yours, wrap(user)] : [wrap(user)];
|
|
57
|
+
}
|
|
58
|
+
function toOpenAIResponses(r, base) {
|
|
59
|
+
const c = r.config;
|
|
60
|
+
const yours = base;
|
|
61
|
+
const params = put(
|
|
62
|
+
{ ...yours, ...c.extra },
|
|
63
|
+
{
|
|
64
|
+
model: c.model,
|
|
65
|
+
instructions: r.system,
|
|
66
|
+
// Input as text stays text; a list of items is history the user message follows.
|
|
67
|
+
input: r.user !== null && !Array.isArray(yours.input) ? r.user : turns(yours.input, r.user, userMessage),
|
|
68
|
+
max_output_tokens: c.maxTokens,
|
|
69
|
+
temperature: c.temperature,
|
|
70
|
+
text: c.response && {
|
|
71
|
+
...obj(yours.text),
|
|
72
|
+
format: {
|
|
73
|
+
type: "json_schema",
|
|
74
|
+
name: c.response.name,
|
|
75
|
+
schema: c.response.schema,
|
|
76
|
+
strict: true
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
);
|
|
81
|
+
return { params, options: requestOptions(c) };
|
|
82
|
+
}
|
|
83
|
+
function toChatCompletions(r, base) {
|
|
84
|
+
const c = r.config;
|
|
85
|
+
const yours = base;
|
|
86
|
+
const history = (Array.isArray(yours.messages) ? yours.messages : []).filter((m) => {
|
|
87
|
+
const role = obj(m).role;
|
|
88
|
+
return role !== "system" && role !== "developer";
|
|
89
|
+
});
|
|
90
|
+
const own = c.provider === "openai" || c.provider === "azure";
|
|
91
|
+
const params = put(
|
|
92
|
+
{ ...yours, ...c.extra },
|
|
93
|
+
{
|
|
94
|
+
model: c.model,
|
|
95
|
+
messages: [
|
|
96
|
+
{ role: "system", content: r.system },
|
|
97
|
+
...turns(history, r.user, userMessage)
|
|
98
|
+
],
|
|
99
|
+
[own ? "max_completion_tokens" : "max_tokens"]: c.maxTokens,
|
|
100
|
+
temperature: c.temperature,
|
|
101
|
+
stop: c.stop,
|
|
102
|
+
response_format: c.response && {
|
|
103
|
+
type: "json_schema",
|
|
104
|
+
json_schema: { name: c.response.name, schema: c.response.schema, strict: true }
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
);
|
|
108
|
+
return { params, options: requestOptions(c) };
|
|
109
|
+
}
|
|
110
|
+
function toAnthropic(r, base) {
|
|
111
|
+
const c = r.config;
|
|
112
|
+
const yours = base;
|
|
113
|
+
const params = put(
|
|
114
|
+
{ ...yours, ...c.extra },
|
|
115
|
+
{
|
|
116
|
+
model: c.model,
|
|
117
|
+
system: r.system,
|
|
118
|
+
messages: turns(yours.messages, r.user, userMessage),
|
|
119
|
+
max_tokens: c.maxTokens,
|
|
120
|
+
temperature: c.temperature,
|
|
121
|
+
stop_sequences: c.stop,
|
|
122
|
+
output_config: c.response && {
|
|
123
|
+
...obj(yours.output_config),
|
|
124
|
+
format: { type: "json_schema", schema: c.response.schema }
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
);
|
|
128
|
+
return { params, options: requestOptions(c) };
|
|
129
|
+
}
|
|
130
|
+
function toGemini(r, base) {
|
|
131
|
+
const c = r.config;
|
|
132
|
+
const yours = base;
|
|
133
|
+
const config = obj(yours.config);
|
|
134
|
+
const http = obj(config.httpOptions);
|
|
135
|
+
const params = put(
|
|
136
|
+
{ ...yours },
|
|
137
|
+
{
|
|
138
|
+
model: c.model,
|
|
139
|
+
contents: turns(yours.contents, r.user, (text) => ({ role: "user", parts: [{ text }] })),
|
|
140
|
+
config: put(
|
|
141
|
+
{ ...config },
|
|
142
|
+
{
|
|
143
|
+
systemInstruction: r.system,
|
|
144
|
+
maxOutputTokens: c.maxTokens,
|
|
145
|
+
temperature: c.temperature,
|
|
146
|
+
stopSequences: c.stop,
|
|
147
|
+
responseMimeType: c.response && "application/json",
|
|
148
|
+
responseJsonSchema: c.response?.schema,
|
|
149
|
+
httpOptions: c.timeoutMs !== void 0 || c.extra ? put(
|
|
150
|
+
{ ...http },
|
|
151
|
+
{
|
|
152
|
+
timeout: c.timeoutMs,
|
|
153
|
+
extraBody: c.extra && { ...obj(http.extraBody), ...c.extra }
|
|
154
|
+
}
|
|
155
|
+
) : void 0
|
|
156
|
+
}
|
|
157
|
+
)
|
|
158
|
+
}
|
|
159
|
+
);
|
|
160
|
+
return { params, options: {} };
|
|
161
|
+
}
|
|
162
|
+
function toBedrockConverse(r, base) {
|
|
163
|
+
const c = r.config;
|
|
164
|
+
const yours = base;
|
|
165
|
+
const inference = obj(yours.inferenceConfig);
|
|
166
|
+
const params = put(
|
|
167
|
+
{ ...yours },
|
|
168
|
+
{
|
|
169
|
+
modelId: c.model,
|
|
170
|
+
system: [{ text: r.system }],
|
|
171
|
+
messages: turns(yours.messages, r.user, (text) => ({ role: "user", content: [{ text }] })),
|
|
172
|
+
inferenceConfig: c.maxTokens !== void 0 || c.temperature !== void 0 || c.stop ? put(
|
|
173
|
+
{ ...inference },
|
|
174
|
+
{ maxTokens: c.maxTokens, temperature: c.temperature, stopSequences: c.stop }
|
|
175
|
+
) : void 0,
|
|
176
|
+
outputConfig: c.response && {
|
|
177
|
+
...obj(yours.outputConfig),
|
|
178
|
+
textFormat: {
|
|
179
|
+
type: "json_schema",
|
|
180
|
+
structure: {
|
|
181
|
+
jsonSchema: { name: c.response.name, schema: JSON.stringify(c.response.schema) }
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
},
|
|
185
|
+
additionalModelRequestFields: c.extra && {
|
|
186
|
+
...obj(yours.additionalModelRequestFields),
|
|
187
|
+
...c.extra
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
);
|
|
191
|
+
return { params, options: {} };
|
|
192
|
+
}
|
|
193
|
+
function clientOptions(r, base) {
|
|
194
|
+
const c = r.config;
|
|
195
|
+
const to = c.connection ?? {};
|
|
196
|
+
const out = { ...base };
|
|
197
|
+
const trimmed = (url) => url.replace(/\/+$/, "");
|
|
198
|
+
switch (r.api ?? c.apis?.[0]) {
|
|
199
|
+
case "openai-responses":
|
|
200
|
+
case "chat-completions": {
|
|
201
|
+
const baseURL = to.baseUrl ?? (c.provider === "azure" && to.endpointUrl ? withPath(trimmed(to.endpointUrl), "/openai/v1/") : c.provider === "databricks" && to.workspaceUrl ? withPath(trimmed(to.workspaceUrl), "/serving-endpoints") : void 0);
|
|
202
|
+
return put(out, { baseURL, ...requestOptions(c) });
|
|
203
|
+
}
|
|
204
|
+
case "anthropic-messages": {
|
|
205
|
+
const baseURL = to.baseUrl ?? (c.provider === "azure" && to.endpointUrl ? withPath(trimmed(to.endpointUrl), "/anthropic/") : void 0);
|
|
206
|
+
const region = c.provider === "google-vertex" ? to.location : void 0;
|
|
207
|
+
return put(out, { baseURL, region, ...requestOptions(c) });
|
|
208
|
+
}
|
|
209
|
+
case "gemini": {
|
|
210
|
+
const http = obj(out.httpOptions);
|
|
211
|
+
const httpOptions = to.baseUrl !== void 0 || c.timeoutMs !== void 0 || c.maxRetries !== void 0 ? put(
|
|
212
|
+
{ ...http },
|
|
213
|
+
{
|
|
214
|
+
baseUrl: to.baseUrl,
|
|
215
|
+
timeout: c.timeoutMs,
|
|
216
|
+
retryOptions: c.maxRetries !== void 0 ? { ...obj(http.retryOptions), attempts: c.maxRetries + 1 } : void 0
|
|
217
|
+
}
|
|
218
|
+
) : void 0;
|
|
219
|
+
const vertex = c.provider === "google-vertex";
|
|
220
|
+
return put(out, {
|
|
221
|
+
vertexai: vertex || void 0,
|
|
222
|
+
location: vertex ? to.location : void 0,
|
|
223
|
+
httpOptions
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
case "bedrock-converse": {
|
|
227
|
+
const handler = obj(out.requestHandler);
|
|
228
|
+
return put(out, {
|
|
229
|
+
region: to.region,
|
|
230
|
+
endpoint: to.endpointUrl,
|
|
231
|
+
maxAttempts: c.maxRetries !== void 0 ? c.maxRetries + 1 : void 0,
|
|
232
|
+
requestHandler: c.timeoutMs !== void 0 ? { ...handler, requestTimeout: c.timeoutMs, throwOnRequestTimeout: true } : void 0
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
default:
|
|
236
|
+
return out;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
function withPath(url, path) {
|
|
240
|
+
return /^https?:\/\/[^/]+\/./.test(url) ? url : `${url}${path}`;
|
|
241
|
+
}
|
|
242
|
+
function userMessage(text) {
|
|
243
|
+
return { role: "user", content: text };
|
|
244
|
+
}
|
|
245
|
+
|
|
34
246
|
// src/errors.ts
|
|
35
247
|
var ERROR_CODES = [
|
|
36
248
|
"unauthorized",
|
|
@@ -38,7 +250,9 @@ var ERROR_CODES = [
|
|
|
38
250
|
"missing_variables",
|
|
39
251
|
"timeout",
|
|
40
252
|
"network",
|
|
41
|
-
"server"
|
|
253
|
+
"server",
|
|
254
|
+
// The version is for another API than the one the call asked for (`api`, SDK 1.6).
|
|
255
|
+
"api_mismatch"
|
|
42
256
|
];
|
|
43
257
|
var PromptHelmError = class extends Error {
|
|
44
258
|
constructor(code, message, status) {
|
|
@@ -84,6 +298,14 @@ function render(template, vars = {}) {
|
|
|
84
298
|
|
|
85
299
|
// src/client.ts
|
|
86
300
|
var DEFAULT_BASE_URL = "https://prompthelm.app";
|
|
301
|
+
var PROMPT_APIS = [
|
|
302
|
+
"openai-responses",
|
|
303
|
+
"chat-completions",
|
|
304
|
+
"anthropic-messages",
|
|
305
|
+
"gemini",
|
|
306
|
+
"bedrock-converse"
|
|
307
|
+
];
|
|
308
|
+
var API_HEADER = "x-prompthelm-api";
|
|
87
309
|
var FALLBACK_HEADER = "x-prompthelm-fallback";
|
|
88
310
|
var FIELDS = [
|
|
89
311
|
"served",
|
|
@@ -125,6 +347,10 @@ var PromptHelm = class {
|
|
|
125
347
|
#inflight = /* @__PURE__ */ new Map();
|
|
126
348
|
#reportFallbacks;
|
|
127
349
|
#reportResults;
|
|
350
|
+
#reportFailures;
|
|
351
|
+
// Per slug, the API a call last asked for, sent with its requests (`API_HEADER`).
|
|
352
|
+
#apis = /* @__PURE__ */ new Map();
|
|
353
|
+
#failures = /* @__PURE__ */ new Map();
|
|
128
354
|
// Per slug, why the live text last couldn't be served, until a request carries it. An object,
|
|
129
355
|
// so a report made while that request is in flight isn't cleared with the one it carried.
|
|
130
356
|
#unreported = /* @__PURE__ */ new Map();
|
|
@@ -143,6 +369,7 @@ var PromptHelm = class {
|
|
|
143
369
|
this.#onResolve = options.onResolve;
|
|
144
370
|
this.#reportFallbacks = options.reportFallbacks ?? true;
|
|
145
371
|
this.#reportResults = options.reportResults ?? this.#reportFallbacks;
|
|
372
|
+
this.#reportFailures = options.reportFailures ?? this.#reportFallbacks;
|
|
146
373
|
}
|
|
147
374
|
/**
|
|
148
375
|
* The live version's system message with its variables filled in. It never needs the user
|
|
@@ -159,16 +386,20 @@ var PromptHelm = class {
|
|
|
159
386
|
return this.#resolve(slug, options, true);
|
|
160
387
|
}
|
|
161
388
|
async #resolve(slug, options, withUser) {
|
|
162
|
-
const { vars, fallback } = options;
|
|
389
|
+
const { vars, fallback, api } = options;
|
|
390
|
+
if (api) this.#apis.set(slug, api);
|
|
163
391
|
try {
|
|
164
|
-
return this.#pick(slug, await this.#entry(slug), options, withUser);
|
|
392
|
+
return { ...this.#pick(slug, await this.#entry(slug), options, withUser), api: api ?? null };
|
|
165
393
|
} catch (error) {
|
|
166
394
|
const failure = toPromptHelmError(error);
|
|
167
395
|
this.#remember(slug, failure.code);
|
|
168
396
|
if (fallback === void 0) throw failure;
|
|
169
397
|
this.#onError(failure, slug);
|
|
170
398
|
const own = typeof fallback === "string" ? { system: fallback } : fallback;
|
|
171
|
-
return
|
|
399
|
+
return {
|
|
400
|
+
...this.#served(slug, fill(own.system, own.user, vars, withUser), {}, null),
|
|
401
|
+
api: api ?? null
|
|
402
|
+
};
|
|
172
403
|
}
|
|
173
404
|
}
|
|
174
405
|
/** Reports the outcome of a call made earlier, by its `ticket`. Ignores anything else. */
|
|
@@ -185,8 +416,10 @@ var PromptHelm = class {
|
|
|
185
416
|
clearTimeout(this.#timer);
|
|
186
417
|
this.#timer = void 0;
|
|
187
418
|
const sent = this.#pending;
|
|
188
|
-
|
|
419
|
+
const failures = this.#failures;
|
|
420
|
+
if (sent.size === 0 && failures.size === 0) return;
|
|
189
421
|
this.#pending = /* @__PURE__ */ new Map();
|
|
422
|
+
this.#failures = /* @__PURE__ */ new Map();
|
|
190
423
|
const body = {
|
|
191
424
|
experiments: [...sent.values()].map(({ id, weight, versions }) => ({
|
|
192
425
|
id,
|
|
@@ -202,6 +435,7 @@ var PromptHelm = class {
|
|
|
202
435
|
)
|
|
203
436
|
}))
|
|
204
437
|
};
|
|
438
|
+
if (failures.size > 0) body.failures = [...failures.values()];
|
|
205
439
|
try {
|
|
206
440
|
const response = await this.#fetch(`${this.#baseUrl}/v1/reports`, {
|
|
207
441
|
method: "POST",
|
|
@@ -212,6 +446,11 @@ var PromptHelm = class {
|
|
|
212
446
|
if (response.status < 500) return;
|
|
213
447
|
} catch {
|
|
214
448
|
}
|
|
449
|
+
for (const [key, failure] of failures) {
|
|
450
|
+
const now = this.#failures.get(key);
|
|
451
|
+
if (now) now.count += failure.count;
|
|
452
|
+
else this.#failures.set(key, failure);
|
|
453
|
+
}
|
|
215
454
|
for (const { id, weight, versions } of sent.values()) {
|
|
216
455
|
for (const [version, counts] of versions) {
|
|
217
456
|
const total = this.#counts({ id, weight }, version);
|
|
@@ -222,13 +461,22 @@ var PromptHelm = class {
|
|
|
222
461
|
}
|
|
223
462
|
}
|
|
224
463
|
#pick(slug, entry, options, withUser) {
|
|
225
|
-
const { vars, unit, force } = options;
|
|
464
|
+
const { vars, unit, force, api } = options;
|
|
226
465
|
const { experiment } = entry;
|
|
466
|
+
const check = (version, config) => {
|
|
467
|
+
if (api && config.apis && !config.apis.includes(api)) {
|
|
468
|
+
throw new PromptHelmError(
|
|
469
|
+
"api_mismatch",
|
|
470
|
+
`v${version} is called with ${config.apis.join(" or ")}, not ${api}`
|
|
471
|
+
);
|
|
472
|
+
}
|
|
473
|
+
};
|
|
227
474
|
const test = experiment && !force ? { id: experiment.id, weight: experiment.weight } : void 0;
|
|
228
475
|
let skipped;
|
|
229
476
|
if (experiment && (force ? force === "variant" : bucket(experiment.id, unit) < experiment.weight)) {
|
|
230
477
|
const { version, content, user, config = {} } = experiment.variant;
|
|
231
478
|
try {
|
|
479
|
+
check(version, config);
|
|
232
480
|
return this.#served(
|
|
233
481
|
slug,
|
|
234
482
|
fill(content, user, vars, withUser),
|
|
@@ -242,6 +490,7 @@ var PromptHelm = class {
|
|
|
242
490
|
skipped = toPromptHelmError(error);
|
|
243
491
|
}
|
|
244
492
|
}
|
|
493
|
+
check(entry.version, entry.config);
|
|
245
494
|
const messages = fill(entry.content, entry.user, vars, withUser);
|
|
246
495
|
if (skipped && experiment) {
|
|
247
496
|
this.#onError(skipped, slug);
|
|
@@ -271,6 +520,11 @@ var PromptHelm = class {
|
|
|
271
520
|
reported |= 1;
|
|
272
521
|
this.#outcome(test, version, value);
|
|
273
522
|
},
|
|
523
|
+
failed: (error) => {
|
|
524
|
+
if (version === null || reported & 4) return;
|
|
525
|
+
reported |= 4;
|
|
526
|
+
this.#failed(slug, version, error);
|
|
527
|
+
},
|
|
274
528
|
usage: (usage, ms = Date.now() - started) => {
|
|
275
529
|
if (!test || version === null || reported & 2) return;
|
|
276
530
|
reported |= 2;
|
|
@@ -292,6 +546,18 @@ var PromptHelm = class {
|
|
|
292
546
|
add(counts, SCORE, score);
|
|
293
547
|
add(counts, SCORE_SQ, score * score);
|
|
294
548
|
}
|
|
549
|
+
/** Counts a refused model call by its HTTP status and codes; anything else is ignored. */
|
|
550
|
+
#failed(slug, version, error) {
|
|
551
|
+
if (!this.#reportFailures) return;
|
|
552
|
+
const failure = refusal(error);
|
|
553
|
+
if (!failure) return;
|
|
554
|
+
const { status, type, code, param } = failure;
|
|
555
|
+
const key = JSON.stringify([slug, version, status, type, code, param]);
|
|
556
|
+
const counted = this.#failures.get(key);
|
|
557
|
+
if (counted) counted.count += 1;
|
|
558
|
+
else this.#failures.set(key, { slug, version, ...failure, count: 1 });
|
|
559
|
+
this.#schedule();
|
|
560
|
+
}
|
|
295
561
|
/** The counts to add a call to, kept until the next report; none when reporting is off. */
|
|
296
562
|
#counts(test, version) {
|
|
297
563
|
if (!this.#reportResults) return;
|
|
@@ -348,6 +614,8 @@ var PromptHelm = class {
|
|
|
348
614
|
if (cached?.etag) headers["if-none-match"] = cached.etag;
|
|
349
615
|
const report = this.#unreported.get(slug);
|
|
350
616
|
if (report) headers[FALLBACK_HEADER] = report.code;
|
|
617
|
+
const api = this.#apis.get(slug);
|
|
618
|
+
if (api) headers[API_HEADER] = api;
|
|
351
619
|
let response;
|
|
352
620
|
try {
|
|
353
621
|
response = await this.#fetch(`${this.#baseUrl}/v1/prompts/${encodeURIComponent(slug)}`, {
|
|
@@ -392,6 +660,27 @@ function tokens(usage, keys) {
|
|
|
392
660
|
}
|
|
393
661
|
return 0;
|
|
394
662
|
}
|
|
663
|
+
var CODE = /^[A-Za-z0-9_.:[\]-]{1,64}$/;
|
|
664
|
+
var word = (value) => typeof value === "string" && CODE.test(value) ? value : void 0;
|
|
665
|
+
function refusal(error) {
|
|
666
|
+
if (error === null || typeof error !== "object") return null;
|
|
667
|
+
const e = error;
|
|
668
|
+
const meta = e.$metadata;
|
|
669
|
+
const response = e.response;
|
|
670
|
+
const status = [e.status, e.statusCode, meta?.httpStatusCode, response?.status].find(
|
|
671
|
+
(value) => Number.isInteger(value) && Number(value) >= 400 && Number(value) < 600
|
|
672
|
+
);
|
|
673
|
+
if (status === void 0) return null;
|
|
674
|
+
const nested = e.error !== null && typeof e.error === "object" ? e.error : {};
|
|
675
|
+
const failure = { status };
|
|
676
|
+
const type = word(e.type) ?? word(nested.type) ?? (meta ? word(e.name) : void 0);
|
|
677
|
+
const code = word(e.code) ?? word(e.errorCode) ?? word(nested.code);
|
|
678
|
+
const param = word(e.param) ?? word(nested.param);
|
|
679
|
+
if (type) failure.type = type;
|
|
680
|
+
if (code) failure.code = code;
|
|
681
|
+
if (param) failure.param = param;
|
|
682
|
+
return failure;
|
|
683
|
+
}
|
|
395
684
|
function add(counts, index, amount) {
|
|
396
685
|
if (counts) counts[index] = (counts[index] ?? 0) + amount;
|
|
397
686
|
}
|
|
@@ -410,12 +699,20 @@ function toPromptHelmError(error) {
|
|
|
410
699
|
}
|
|
411
700
|
// Annotate the CommonJS export names for ESM import in node:
|
|
412
701
|
0 && (module.exports = {
|
|
702
|
+
API_HEADER,
|
|
413
703
|
ERROR_CODES,
|
|
414
704
|
FALLBACK_HEADER,
|
|
415
705
|
LATENCY_BUCKETS,
|
|
706
|
+
PROMPT_APIS,
|
|
416
707
|
PromptHelm,
|
|
417
708
|
PromptHelmError,
|
|
709
|
+
clientOptions,
|
|
418
710
|
extractVariables,
|
|
419
711
|
latencyBucket,
|
|
420
|
-
render
|
|
712
|
+
render,
|
|
713
|
+
toAnthropic,
|
|
714
|
+
toBedrockConverse,
|
|
715
|
+
toChatCompletions,
|
|
716
|
+
toGemini,
|
|
717
|
+
toOpenAIResponses
|
|
421
718
|
});
|