@spendgraph/prompt 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -238,3 +238,6 @@ Two subpaths hold the rest:
238
238
  Reach for `internals` only to build your own client; `PromptClient` already
239
239
  wires all of it.
240
240
 
241
+ ## License
242
+
243
+ MIT
@@ -1,23 +1,8 @@
1
1
  import { BudgetExceededError } from "./errors.js";
2
- /** Warned once per process, not per client — a library that nags gets muted. */
3
2
  let warned = false;
4
- /**
5
- * What this client has spent, and whether it may spend more.
6
- *
7
- * A `sample` loop over a dataset is the shape of an expensive accident, and the
8
- * guard is a counter and a comparison.
9
- *
10
- * Governs what this client initiates — `run`, `sample`, `runAll`. Not `report`,
11
- * which records money already spent elsewhere.
12
- */
13
3
  export class Budget {
14
4
  used = 0;
15
5
  limit;
16
- /**
17
- * `undefined` means no ceiling and warns once — an unbounded client in a
18
- * scheduled script is how a surprise bill happens. `null` means the same
19
- * thing deliberately, and says nothing.
20
- */
21
6
  constructor(maxCostMicros) {
22
7
  this.limit = maxCostMicros ?? Infinity;
23
8
  if (maxCostMicros === undefined && !warned) {
@@ -36,18 +21,11 @@ export class Budget {
36
21
  if (Number.isFinite(costMicros) && costMicros > 0)
37
22
  this.used += costMicros;
38
23
  }
39
- /**
40
- * Throws if the ceiling is already reached.
41
- *
42
- * Checked before a request rather than after: stopping once the bill has been
43
- * incurred is a report, not a limit.
44
- */
45
24
  assertAffordable() {
46
25
  if (this.used >= this.limit)
47
26
  throw new BudgetExceededError(this.used, this.limit);
48
27
  }
49
28
  }
50
- /** Test seam: lets the once-only warning be exercised more than once. */
51
29
  export function resetBudgetWarning() {
52
30
  warned = false;
53
31
  }
@@ -1,10 +1,3 @@
1
- /**
2
- * Thrown when a client has spent its ceiling.
3
- *
4
- * Deliberately not a `SpendgraphError`: nothing was refused by the server, so a
5
- * caller inspecting `status` or `retryable` would be told a story about HTTP
6
- * that never happened.
7
- */
8
1
  export class BudgetExceededError extends Error {
9
2
  spentMicros;
10
3
  limitMicros;
@@ -12,14 +12,6 @@ function warnOnEmptyTokens(entry) {
12
12
  "the provider response, so return inputTokens and outputTokens from your callback " +
13
13
  "or every cost on this prompt reads as zero.");
14
14
  }
15
- /**
16
- * Usage, not a rollout.
17
- *
18
- * A custom prompt has no row on the server and so no version to reference, and
19
- * `POST /api/v1/prompts/:id/rollouts` needs both. The tokens are still real, so
20
- * they go to ingest — the spend lands on the dashboard, attributed to the model
21
- * and tagged with the prompt's name; only the wording goes unversioned.
22
- */
23
15
  function usageRecorder(sdk, name, opts) {
24
16
  const onEmpty = opts.onEmptyTokens ?? warnOnEmptyTokens;
25
17
  return {
@@ -46,14 +38,6 @@ function usageRecorder(sdk, name, opts) {
46
38
  },
47
39
  };
48
40
  }
49
- /**
50
- * A prompt written in code rather than pulled from the server.
51
- *
52
- * The same `Prompt` `pullPrompt` returns — same `render`, same `serialize`,
53
- * same `trace` — so nothing downstream has to know which one it was handed.
54
- * Give it an `sdk` and it records what the call cost; leave it off and it
55
- * touches the network never.
56
- */
57
41
  export function buildCustomPrompt(spec, opts = {}) {
58
42
  const variables = spec.variables ?? {};
59
43
  const blocks = (spec.blocks ?? []).map((b) => ({
@@ -3,37 +3,16 @@ export class PullCache {
3
3
  ttlMs;
4
4
  maxSize;
5
5
  now;
6
- /** Keys with a refresh in flight, so N concurrent readers cause one fetch. */
7
6
  refreshing = new Set();
8
- /** When a key's refresh last failed, so a dead handle is not retried per call. */
9
7
  failures = new Map();
10
8
  retryAfterMs;
11
9
  onRefreshError;
12
10
  identify;
13
- /**
14
- * A monotonic counter and the tick at which each name was last invalidated.
15
- *
16
- * A fetch records the counter when it starts and, when it lands, asks whether
17
- * anything it turned out to be — its key, its id, its slug — has been
18
- * invalidated since. That is the whole protocol, and it holds for the cold
19
- * path, the background refresh, and a handle nobody has seen before, none of
20
- * which the previous per-key epoch could cover.
21
- */
22
11
  tick = 0;
23
12
  invalidatedAt = new Map();
24
- /**
25
- * The tick of the last `clear()`.
26
- *
27
- * `clear` cannot name a fetch it has never seen — a cold one is in neither
28
- * `store` nor `refreshing` — so it raises a floor instead: anything that
29
- * started before it is out of date whatever it turns out to be.
30
- */
31
13
  clearedAt = -1;
32
- /** Fetches outstanding, so the log above can be dropped when none remain. */
33
14
  outstanding = 0;
34
15
  constructor(opts = {}) {
35
- // A TTL of 0 disables caching without a separate flag; only undefined
36
- // falls back to the default.
37
16
  this.ttlMs = (opts.ttlSeconds ?? 300) * 1000;
38
17
  this.maxSize = Math.max(1, opts.maxSize ?? 100);
39
18
  this.now = opts.now ?? (() => Date.now());
@@ -45,7 +24,6 @@ export class PullCache {
45
24
  const entry = this.store.get(key);
46
25
  if (!entry)
47
26
  return undefined;
48
- // Map iterates in insertion order, so re-setting marks most-recently-used.
49
27
  this.store.delete(key);
50
28
  this.store.set(key, entry);
51
29
  const ageMs = this.now() - entry.at;
@@ -59,27 +37,13 @@ export class PullCache {
59
37
  if (oldest === undefined)
60
38
  break;
61
39
  this.store.delete(oldest);
62
- // The bookkeeping goes with the entry unless a fetch for it is still
63
- // running, otherwise these grow forever in a class whose contract is a
64
- // bounded LRU — one permanent entry per distinct key ever seen.
65
40
  if (!this.refreshing.has(oldest))
66
41
  this.failures.delete(oldest);
67
42
  }
68
43
  }
69
- /**
70
- * Runs `refresh` unless one is already in flight for this key.
71
- *
72
- * Without the guard, a burst of requests arriving just after the TTL lapses
73
- * each start their own refresh — the stampede the cache exists to prevent.
74
- * Failures are swallowed on purpose: a background refresh that throws must not
75
- * surface in a request that was already answered from cache.
76
- */
77
44
  refreshInBackground(key, refresh) {
78
45
  if (this.refreshing.has(key))
79
46
  return;
80
- // Backed off after a failure. A handle that stopped resolving — routine
81
- // since a rename re-mints the slug — would otherwise fire a doomed request
82
- // on every pull, forever. The entry stays; the stampede stops.
83
47
  const failedAt = this.failures.get(key);
84
48
  if (failedAt !== undefined && this.now() - failedAt < this.retryAfterMs)
85
49
  return;
@@ -89,19 +53,10 @@ export class PullCache {
89
53
  .then((value) => this.settle(key, value, startedAt))
90
54
  .catch((err) => {
91
55
  this.abandon(key);
92
- // Surfaced rather than swallowed. `report` has onReportError for exactly
93
- // this reason; without a counterpart here a service can serve a frozen
94
- // prompt indefinitely with nothing anywhere saying so.
95
- //
96
- // Wrapped, because nobody is holding this promise: `pullWithCache` has
97
- // already returned the stale value, so a callback that throws would be
98
- // an unhandled rejection — process exit in Node, isolate abort in a
99
- // Worker.
100
56
  try {
101
57
  this.onRefreshError?.(err, key);
102
58
  }
103
59
  catch {
104
- /* a reporter that cannot report is not worth an outage */
105
60
  }
106
61
  })
107
62
  .finally(() => {
@@ -109,45 +64,34 @@ export class PullCache {
109
64
  this.forgetLogIfIdle();
110
65
  });
111
66
  }
112
- /** Records a fetch starting, and the tick it must be judged against. */
113
67
  beginFetch() {
114
68
  this.outstanding++;
115
69
  return this.tick;
116
70
  }
117
- /** Stores a landed fetch unless what it fetched was invalidated meanwhile. */
118
71
  settle(key, value, startedAt) {
119
72
  this.outstanding--;
120
73
  const names = [key, ...(this.identify?.(value) ?? [])];
121
74
  const overtaken = startedAt < this.clearedAt ||
122
75
  names.some((n) => (this.invalidatedAt.get(n) ?? -1) > startedAt);
123
76
  if (!overtaken) {
124
- // Only once the value is kept: clearing first meant a fetch that lost the
125
- // race still wiped a live backoff, turning bounded retry into a retry on
126
- // every call.
127
77
  this.failures.delete(key);
128
78
  this.set(key, value);
129
79
  }
130
80
  this.forgetLogIfIdle();
131
81
  }
132
- /** Ends a fetch that threw, and backs the handle off. */
133
82
  abandon(key) {
134
83
  this.outstanding--;
135
- // Recorded on the cold path too — the one a fresh isolate takes — or a
136
- // permanently 404ing handle fires one upstream request per pull, forever.
137
84
  this.failures.set(key, this.now());
138
85
  this.boundFailures();
139
86
  this.forgetLogIfIdle();
140
87
  }
141
- /** Records that everything under this name is out of date. */
142
88
  invalidate(name) {
143
89
  this.invalidatedAt.set(name, ++this.tick);
144
90
  }
145
- /** The log only matters while a fetch could still be judged against it. */
146
91
  forgetLogIfIdle() {
147
92
  if (this.outstanding === 0 && this.refreshing.size === 0)
148
93
  this.invalidatedAt.clear();
149
94
  }
150
- /** Keeps `failures` inside the same bound as the entries themselves. */
151
95
  boundFailures() {
152
96
  while (this.failures.size > this.maxSize) {
153
97
  const oldest = this.failures.keys().next().value;
@@ -156,7 +100,6 @@ export class PullCache {
156
100
  this.failures.delete(oldest);
157
101
  }
158
102
  }
159
- /** Every key whose entry satisfies `match`. */
160
103
  keysWhere(match) {
161
104
  const out = [];
162
105
  for (const [key, entry] of this.store)
@@ -166,17 +109,11 @@ export class PullCache {
166
109
  }
167
110
  delete(key) {
168
111
  this.store.delete(key);
169
- // The backoff goes with it, or a handle that has demonstrably recovered
170
- // keeps skipping its refresh for the whole retry window.
171
112
  this.failures.delete(key);
172
113
  this.invalidate(key);
173
- // Nothing outstanding means nothing to judge against it — residue that
174
- // grew by one per prompt ever invalidated.
175
114
  this.forgetLogIfIdle();
176
115
  }
177
116
  clear() {
178
- // Every key the cache has an opinion about, not just stored ones: a cold
179
- // fetch is in flight under a key not in `store` yet.
180
117
  this.clearedAt = ++this.tick;
181
118
  this.store.clear();
182
119
  this.failures.clear();
@@ -185,13 +122,6 @@ export class PullCache {
185
122
  get size() {
186
123
  return this.store.size;
187
124
  }
188
- /**
189
- * How many keys the cache holds bookkeeping for, entries aside.
190
- *
191
- * Exposed for the test that this stays bounded: `epochs` and `failures` are
192
- * private and grow on paths `size` cannot see, so a test written against
193
- * `size` passes whether or not they leak — which is what the first one did.
194
- */
195
125
  get trackedKeys() {
196
126
  return new Set([...this.invalidatedAt.keys(), ...this.failures.keys()]).size;
197
127
  }
package/dist/client.js CHANGED
@@ -7,14 +7,7 @@ import { pullWithCache } from "./pull/index.js";
7
7
  import { newRolloutId } from "./run/id.js";
8
8
  import { runAcrossModels, runOnce, sampleRuns } from "./run/index.js";
9
9
  const DEFAULT_CONCURRENCY = 4;
10
- /**
11
- * Stored prompts, and the spend they account for.
12
- *
13
- * Everything that leaves the process goes through `@spendgraph/sdk`; this class
14
- * owns the caching, the budget and the rollout bookkeeping on top of it.
15
- */
16
10
  export class PromptClient {
17
- /** The SDK underneath. Reach for it for anything this class does not wrap. */
18
11
  sdk;
19
12
  cache;
20
13
  hasKey;
@@ -44,15 +37,6 @@ export class PromptClient {
44
37
  },
45
38
  };
46
39
  }
47
- /**
48
- * A prompt, from cache when one is warm.
49
- *
50
- * Serves a stale copy immediately and refreshes behind the caller, so an edit
51
- * takes effect within a TTL without any request ever paying for the fetch.
52
- *
53
- * Takes an id or a slug — the server resolves either. Cached under whatever
54
- * you passed, so pulling the same prompt both ways keeps two entries.
55
- */
56
40
  async pull(handle) {
57
41
  const payload = await pullWithCache(this.cache, handle, async () => {
58
42
  const res = await this.sdk.prompts.get(handle);
@@ -69,17 +53,9 @@ export class PromptClient {
69
53
  models: payload.models ?? [],
70
54
  }, "server", this.recorder());
71
55
  }
72
- /** A prompt written in code, wired to record its usage through this client. */
73
56
  build(spec, opts = {}) {
74
57
  return buildCustomPrompt(spec, { ...opts, sdk: this.sdk });
75
58
  }
76
- /**
77
- * Said once, because this package cannot count tokens it never saw.
78
- *
79
- * A callback that forgets to return them records a rollout reading zero in
80
- * and zero out, which prices at nothing and quietly drags every average that
81
- * includes it. Silence here is how that goes unnoticed for a month.
82
- */
83
59
  warnOnEmptyTokens(outcome) {
84
60
  if (this.warnedAboutTokens)
85
61
  return;
@@ -92,22 +68,12 @@ export class PromptClient {
92
68
  "the provider response, so return inputTokens and outputTokens from your callback " +
93
69
  "or every cost on this prompt reads as zero.");
94
70
  }
95
- /**
96
- * Runs the prompt server-side and records the rollout.
97
- *
98
- * The batch path. A production request should pull, render and report
99
- * instead — this adds a hop and makes spendgraph a dependency of the caller's
100
- * uptime, which is the right trade for evaluation and the wrong one for a
101
- * user waiting on a response.
102
- */
103
71
  async run(promptId, values = {}, opts = {}) {
104
72
  return runOnce(this.sdk.prompts, promptId, values, opts, this.budget);
105
73
  }
106
- /** k repetitions of one case. Always k results, failures included. */
107
74
  async sample(promptId, values = {}, opts = {}) {
108
75
  return sampleRuns(this.sdk.prompts, promptId, values, { concurrency: this.concurrency, ...opts }, this.budget);
109
76
  }
110
- /** One rollout per model — the playground's comparison, headless. */
111
77
  async runAll(promptId, values = {}, opts = {}) {
112
78
  const models = opts.models ?? (await this.pull(promptId)).models;
113
79
  if (models.length === 0) {
@@ -115,52 +81,26 @@ export class PromptClient {
115
81
  }
116
82
  return runAcrossModels(this.sdk.prompts, promptId, values, models, { concurrency: this.concurrency, ...opts }, this.budget);
117
83
  }
118
- /** The dataset a prompt is evaluated against, with its split counts. */
119
84
  async cases(promptId) {
120
85
  return this.sdk.prompts.cases(promptId);
121
86
  }
122
- /**
123
- * Replaces the whole dataset.
124
- *
125
- * A dataset is a set: "which cases am I evaluating against" has one answer at
126
- * a time, and a merge would leave no way to remove a case or to know from the
127
- * outside what the set currently is.
128
- */
129
87
  async setCases(promptId, cases) {
130
88
  return this.sdk.prompts.putCases(promptId, cases);
131
89
  }
132
- /** Micro-dollars this client has spent on runs it initiated. */
133
90
  spent() {
134
91
  return this.budget.spent();
135
92
  }
136
- /** What is left of the ceiling, or Infinity when none was set. */
137
93
  remaining() {
138
94
  return this.budget.remaining();
139
95
  }
140
- /** Every wording this prompt has had, newest first. */
141
96
  async versions(promptId, opts = {}) {
142
97
  return this.sdk.prompts.versions(promptId, opts);
143
98
  }
144
- /**
145
- * Serves this version from now on.
146
- *
147
- * Invalidates the cache: a promotion that left a warm entry in place would
148
- * keep sending the old wording for up to a TTL, which is the one moment a
149
- * caller is actively watching for the change.
150
- */
151
99
  async promote(promptId, versionId) {
152
100
  const res = await this.sdk.prompts.promote(promptId, versionId);
153
101
  this.invalidate(promptId);
154
102
  return res;
155
103
  }
156
- /**
157
- * Drops whatever is cached, so the next pull refetches.
158
- *
159
- * Both handles, not just the one passed. `pull` keys the cache on whatever
160
- * string it was given, so the same prompt can sit under its uuid and its
161
- * slug at once — and `promote` is only reachable with a uuid while the
162
- * dashboard teaches pulling by slug.
163
- */
164
104
  invalidate(promptId) {
165
105
  if (!promptId) {
166
106
  this.cache.clear();
@@ -172,25 +112,6 @@ export class PromptClient {
172
112
  this.cache.delete(promptId);
173
113
  this.cache.invalidate(promptId);
174
114
  }
175
- /**
176
- * Records a rollout the caller executed.
177
- *
178
- * Never throws and never rejects. This sits beside a user-facing request that
179
- * has already been answered; telemetry that can break the thing it measures
180
- * is worse than no telemetry.
181
- */
182
- /**
183
- * Records a rollout and returns what the server priced it at, in micro-USD,
184
- * or undefined where the model had no price to compute one from.
185
- *
186
- * The price is computed there, from the tokens and the project's pricing
187
- * table, so this is the only place it exists — and it is why `report` cannot
188
- * simply be reused: that one answers with the rollout id.
189
- *
190
- * `priced: false` is answered as undefined rather than as the stored zero.
191
- * The column is `notNull`, so an unpriced rollout is written as zero and the
192
- * flag beside it is the only thing that tells free apart from unknown.
193
- */
194
115
  async priced(promptId, input) {
195
116
  const rolloutId = input.rolloutId ?? newRolloutId();
196
117
  if (!this.hasKey)
@@ -219,12 +140,6 @@ export class PromptClient {
219
140
  return rolloutId;
220
141
  }
221
142
  }
222
- /**
223
- * One prompt from the server, without keeping a client around.
224
- *
225
- * Convenience over `new PromptClient(...).pull(...)`, and it builds a client
226
- * per call — a service pulling repeatedly wants the class, which caches.
227
- */
228
143
  export function pullPrompt(handle, opts) {
229
144
  return new PromptClient(opts).pull(handle);
230
145
  }
package/dist/internals.js CHANGED
@@ -1,10 +1,3 @@
1
- /**
2
- * The pieces `PromptClient` is built from.
3
- *
4
- * Not in the main entry on purpose: everything here is already wired for you,
5
- * and a caller reaching for it is building their own client rather than using
6
- * this one. Kept exported so that stays possible.
7
- */
8
1
  export { serializeFields, validateFields } from "@spendgraph/sdk";
9
2
  export { Budget } from "./budget/index.js";
10
3
  export { PullCache } from "./cache/index.js";
@@ -1,11 +1,5 @@
1
1
  import { mapLimit } from "../run/concurrency.js";
2
2
  const DEFAULT_CONCURRENCY = 4;
3
- /**
4
- * Attaches a model, so the prompt can run itself.
5
- *
6
- * Without this you hand the call to `prompt.call`; with it the call is already
7
- * known and `invoke` is the whole thing.
8
- */
9
3
  export function bind(prompt, model) {
10
4
  const once = (values, opts, streaming) => prompt.call(values, ({ messages, turn }) => {
11
5
  const modelOpts = { tools: turn, onText: opts.onText };
@@ -6,13 +6,6 @@ export const noRecorder = {
6
6
  enabled: false,
7
7
  write: () => { },
8
8
  };
9
- /**
10
- * What the rollout route accepts in one record.
11
- *
12
- * A conversation longer than this still runs — only its rollout is refused, and
13
- * that refusal reaches `onReportError` rather than the caller, so it is said
14
- * here instead while there is still somebody watching.
15
- */
16
9
  export const MAX_RECORDED_MESSAGES = 50;
17
10
  let warnedAboutLength = false;
18
11
  function warnIfUnrecordable(messages) {
@@ -29,13 +22,6 @@ function lastUserMessage(messages) {
29
22
  }
30
23
  return "";
31
24
  }
32
- /**
33
- * The `Prompt` both constructors return.
34
- *
35
- * `pullPrompt` and `buildCustomPrompt` differ only in where the wording came
36
- * from and what the recorder does with the result — everything a caller
37
- * touches is built here, once.
38
- */
39
25
  export function makePrompt(shape, source, recorder = noRecorder) {
40
26
  const serialize = (values = {}) => serializeFields(values, shape.fields);
41
27
  const format = (values = {}, opts = {}) => renderMessages(shape.blocks, shape.question, serialize(values), opts.history);
@@ -104,10 +90,6 @@ export function makePrompt(shape, source, recorder = noRecorder) {
104
90
  offeredTools: outcome.offeredTools ?? recorded?.offeredTools,
105
91
  steps: outcome.steps ?? recorded?.steps,
106
92
  });
107
- // Not awaited. The write stays fire-and-forget — a stage that stalled
108
- // on a report round trip would pay it fifteen times over a run — and
109
- // the promise is handed back so a caller can settle every price at the
110
- // end, when the total is what it needs.
111
93
  pricing = written instanceof Promise ? written : undefined;
112
94
  }
113
95
  return { ...outcome, rolloutId, ...(pricing ? { pricing } : {}) };
package/dist/pull/pull.js CHANGED
@@ -1,13 +1,3 @@
1
- /**
2
- * Fetch-by-id with stale-while-revalidate.
3
- *
4
- * fresh → return it, no request
5
- * stale → return it now, refresh behind the caller
6
- * miss → fetch, and only then answer
7
- *
8
- * The middle path is why a prompt edit takes effect within a TTL without any
9
- * request paying for the fetch. Generic because tools and graphs pull the same.
10
- */
11
1
  export async function pullWithCache(cache, key, fetchOne) {
12
2
  const hit = cache.get(key);
13
3
  if (hit && !hit.stale)
@@ -16,8 +6,6 @@ export async function pullWithCache(cache, key, fetchOne) {
16
6
  cache.refreshInBackground(key, fetchOne);
17
7
  return hit.value;
18
8
  }
19
- // Guarded like the background refresh: an invalidation can land mid-fetch,
20
- // and storing afterwards marks pre-invalidation state fresh for a whole TTL.
21
9
  const startedAt = cache.beginFetch();
22
10
  try {
23
11
  const value = await fetchOne();
@@ -1,15 +1,5 @@
1
1
  import { compileSystemPrompt } from "./system.js";
2
2
  import { applyVariables } from "./variables.js";
3
- /**
4
- * The messages a request will send.
5
- *
6
- * A prompt with no blocks sends one message, not two: an empty system message is
7
- * a different request from no system message at all.
8
- *
9
- * `history` is placed between the system turn and the question, oldest first,
10
- * and is passed through verbatim — it is a record of what was already said, not
11
- * a template, so `{placeholders}` inside it are left alone.
12
- */
13
3
  export function renderMessages(blocks, question, values, history = []) {
14
4
  const system = compileSystemPrompt(blocks, values);
15
5
  const user = applyVariables(question, values);
@@ -1,10 +1,4 @@
1
1
  import { applyVariables } from "./variables.js";
2
- /**
3
- * Blocks in order, each under its title.
4
- *
5
- * An empty body drops the block — a heading over nothing is noise. An untitled
6
- * block is emitted bare rather than under a blank `##`.
7
- */
8
2
  export function compileSystemPrompt(blocks, values) {
9
3
  const out = [];
10
4
  for (const b of blocks ?? []) {
@@ -1,16 +1,8 @@
1
- /**
2
- * Substitutes `{name}` placeholders, leaving unknown ones verbatim — emptying
3
- * one would send a prompt with a hole the caller cannot see.
4
- *
5
- * Mirrors `lib/prompt.ts` in the app. Duplicated because this package depends on
6
- * nothing; `rendered` on every rollout makes a divergence visible.
7
- */
8
1
  export function applyVariables(text, values) {
9
2
  if (!text || !values)
10
3
  return text;
11
4
  return text.replace(/\{(\w+)\}/g, (whole, name) => name in values ? values[name] : whole);
12
5
  }
13
- /** Placeholder names used anywhere in the prompt, in first-seen order. */
14
6
  export function findVariables(blocks, question) {
15
7
  const seen = [];
16
8
  const scan = (text) => {
@@ -19,7 +11,6 @@ export function findVariables(blocks, question) {
19
11
  seen.push(m[1]);
20
12
  }
21
13
  };
22
- // titles too — a heading can carry a placeholder as readily as a body
23
14
  for (const b of blocks ?? []) {
24
15
  scan(b.title ?? "");
25
16
  scan(b.body ?? "");
@@ -1,6 +1,5 @@
1
1
  import { mapLimit } from "./concurrency.js";
2
2
  import { runOnce } from "./once.js";
3
- /** One rollout per model — the playground's comparison, headless. */
4
3
  export async function runAcrossModels(prompts, promptId, values, models, opts = {}, budget) {
5
4
  return mapLimit(models, opts.concurrency ?? 4, (model) => runOnce(prompts, promptId, values, { ...opts, model, rolloutId: undefined }, budget));
6
5
  }
@@ -1,9 +1,3 @@
1
- /**
2
- * Runs `fn` over `items`, at most `limit` in flight.
3
- *
4
- * Bounded on purpose: a whole dataset at once becomes 429s you still wait out,
5
- * so the unbounded version finishes later and flakier.
6
- */
7
1
  export async function mapLimit(items, limit, fn) {
8
2
  const out = new Array(items.length);
9
3
  let next = 0;
package/dist/run/id.js CHANGED
@@ -1,4 +1,3 @@
1
- /** Generated per attempt so a retry of the same call lands on one row. */
2
1
  export function newRolloutId() {
3
2
  return `ro_${crypto.randomUUID().replace(/-/g, "")}`;
4
3
  }
package/dist/run/once.js CHANGED
@@ -15,7 +15,6 @@ export async function runOnce(prompts, promptId, values, opts = {}, budget) {
15
15
  generation: opts.generation,
16
16
  record: opts.record ?? true,
17
17
  });
18
- // A deduped reply costs nothing new — the attempt that wrote it already paid.
19
18
  if (!res.deduped)
20
19
  budget?.add(res.rollout.costMicros ?? 0);
21
20
  return res.rollout;
@@ -1,16 +1,6 @@
1
1
  import { BudgetExceededError } from "../budget/index.js";
2
2
  import { mapLimit } from "./concurrency.js";
3
3
  import { runOnce } from "./once.js";
4
- /**
5
- * k repetitions of one case, so `pass^k` is computable.
6
- *
7
- * Always k results. A failed seed is present with `status: "failed"` rather than
8
- * missing — three results for k = 5 would have the metric compute over three and
9
- * report better consistency than the run showed.
10
- *
11
- * A budget stop throws instead. Money running out says nothing about the prompt,
12
- * and recording those seeds as failures would put the wallet into pass^k.
13
- */
14
4
  export async function sampleRuns(prompts, promptId, values, opts = {}, budget) {
15
5
  const k = Math.max(1, opts.k ?? 5);
16
6
  const seeds = Array.from({ length: k }, (_, i) => i);
@@ -21,7 +11,6 @@ export async function sampleRuns(prompts, promptId, values, opts = {}, budget) {
21
11
  catch (err) {
22
12
  if (err instanceof BudgetExceededError)
23
13
  throw err;
24
- // Still an outcome for this seed; throwing discards the k-1 that landed.
25
14
  return {
26
15
  id: null,
27
16
  model: opts.model ?? "",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@spendgraph/prompt",
3
- "version": "0.2.0",
3
+ "version": "0.2.1",
4
4
  "description": "Pull a stored prompt or build one in code, render it, and record what it cost.",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -42,16 +42,16 @@
42
42
  "README.md"
43
43
  ],
44
44
  "scripts": {
45
- "build": "tsc -p tsconfig.json",
45
+ "build": "tsc -p tsconfig.json --emitDeclarationOnly && tsc -p tsconfig.json --declaration false --removeComments",
46
46
  "test": "npm run build && vitest run",
47
47
  "pretest": "npm run build --workspace @spendgraph/tools"
48
48
  },
49
49
  "dependencies": {
50
- "@spendgraph/sdk": "^0.2.0"
50
+ "@spendgraph/sdk": "^0.2.1"
51
51
  },
52
52
  "devDependencies": {
53
- "@spendgraph/llms": "^0.2.0",
54
- "@spendgraph/tools": "^0.2.0",
53
+ "@spendgraph/llms": "^0.2.1",
54
+ "@spendgraph/tools": "^0.2.1",
55
55
  "typescript": "^5"
56
56
  },
57
57
  "engines": {