@spendgraph/prompt 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/budget/budget.js +0 -22
- package/dist/budget/errors.js +0 -7
- package/dist/build/build.js +0 -16
- package/dist/cache/cache.js +0 -70
- package/dist/client.js +0 -85
- package/dist/internals.js +0 -7
- package/dist/prompt/bind.js +0 -6
- package/dist/prompt/factory.js +0 -18
- package/dist/pull/pull.js +0 -12
- package/dist/render/messages.js +0 -10
- package/dist/render/system.js +0 -6
- package/dist/render/variables.js +0 -9
- package/dist/run/across-models.js +0 -1
- package/dist/run/concurrency.js +0 -6
- package/dist/run/id.js +0 -1
- package/dist/run/once.js +0 -1
- package/dist/run/sample.js +0 -11
- package/package.json +5 -5
package/README.md
CHANGED
package/dist/budget/budget.js
CHANGED
|
@@ -1,23 +1,8 @@
|
|
|
1
1
|
import { BudgetExceededError } from "./errors.js";
|
|
2
|
-
/** Warned once per process, not per client — a library that nags gets muted. */
|
|
3
2
|
let warned = false;
|
|
4
|
-
/**
|
|
5
|
-
* What this client has spent, and whether it may spend more.
|
|
6
|
-
*
|
|
7
|
-
* A `sample` loop over a dataset is the shape of an expensive accident, and the
|
|
8
|
-
* guard is a counter and a comparison.
|
|
9
|
-
*
|
|
10
|
-
* Governs what this client initiates — `run`, `sample`, `runAll`. Not `report`,
|
|
11
|
-
* which records money already spent elsewhere.
|
|
12
|
-
*/
|
|
13
3
|
export class Budget {
|
|
14
4
|
used = 0;
|
|
15
5
|
limit;
|
|
16
|
-
/**
|
|
17
|
-
* `undefined` means no ceiling and warns once — an unbounded client in a
|
|
18
|
-
* scheduled script is how a surprise bill happens. `null` means the same
|
|
19
|
-
* thing deliberately, and says nothing.
|
|
20
|
-
*/
|
|
21
6
|
constructor(maxCostMicros) {
|
|
22
7
|
this.limit = maxCostMicros ?? Infinity;
|
|
23
8
|
if (maxCostMicros === undefined && !warned) {
|
|
@@ -36,18 +21,11 @@ export class Budget {
|
|
|
36
21
|
if (Number.isFinite(costMicros) && costMicros > 0)
|
|
37
22
|
this.used += costMicros;
|
|
38
23
|
}
|
|
39
|
-
/**
|
|
40
|
-
* Throws if the ceiling is already reached.
|
|
41
|
-
*
|
|
42
|
-
* Checked before a request rather than after: stopping once the bill has been
|
|
43
|
-
* incurred is a report, not a limit.
|
|
44
|
-
*/
|
|
45
24
|
assertAffordable() {
|
|
46
25
|
if (this.used >= this.limit)
|
|
47
26
|
throw new BudgetExceededError(this.used, this.limit);
|
|
48
27
|
}
|
|
49
28
|
}
|
|
50
|
-
/** Test seam: lets the once-only warning be exercised more than once. */
|
|
51
29
|
export function resetBudgetWarning() {
|
|
52
30
|
warned = false;
|
|
53
31
|
}
|
package/dist/budget/errors.js
CHANGED
|
@@ -1,10 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Thrown when a client has spent its ceiling.
|
|
3
|
-
*
|
|
4
|
-
* Deliberately not a `SpendgraphError`: nothing was refused by the server, so a
|
|
5
|
-
* caller inspecting `status` or `retryable` would be told a story about HTTP
|
|
6
|
-
* that never happened.
|
|
7
|
-
*/
|
|
8
1
|
export class BudgetExceededError extends Error {
|
|
9
2
|
spentMicros;
|
|
10
3
|
limitMicros;
|
package/dist/build/build.js
CHANGED
|
@@ -12,14 +12,6 @@ function warnOnEmptyTokens(entry) {
|
|
|
12
12
|
"the provider response, so return inputTokens and outputTokens from your callback " +
|
|
13
13
|
"or every cost on this prompt reads as zero.");
|
|
14
14
|
}
|
|
15
|
-
/**
|
|
16
|
-
* Usage, not a rollout.
|
|
17
|
-
*
|
|
18
|
-
* A custom prompt has no row on the server and so no version to reference, and
|
|
19
|
-
* `POST /api/v1/prompts/:id/rollouts` needs both. The tokens are still real, so
|
|
20
|
-
* they go to ingest — the spend lands on the dashboard, attributed to the model
|
|
21
|
-
* and tagged with the prompt's name; only the wording goes unversioned.
|
|
22
|
-
*/
|
|
23
15
|
function usageRecorder(sdk, name, opts) {
|
|
24
16
|
const onEmpty = opts.onEmptyTokens ?? warnOnEmptyTokens;
|
|
25
17
|
return {
|
|
@@ -46,14 +38,6 @@ function usageRecorder(sdk, name, opts) {
|
|
|
46
38
|
},
|
|
47
39
|
};
|
|
48
40
|
}
|
|
49
|
-
/**
|
|
50
|
-
* A prompt written in code rather than pulled from the server.
|
|
51
|
-
*
|
|
52
|
-
* The same `Prompt` `pullPrompt` returns — same `render`, same `serialize`,
|
|
53
|
-
* same `trace` — so nothing downstream has to know which one it was handed.
|
|
54
|
-
* Give it an `sdk` and it records what the call cost; leave it off and it
|
|
55
|
-
* touches the network never.
|
|
56
|
-
*/
|
|
57
41
|
export function buildCustomPrompt(spec, opts = {}) {
|
|
58
42
|
const variables = spec.variables ?? {};
|
|
59
43
|
const blocks = (spec.blocks ?? []).map((b) => ({
|
package/dist/cache/cache.js
CHANGED
|
@@ -3,37 +3,16 @@ export class PullCache {
|
|
|
3
3
|
ttlMs;
|
|
4
4
|
maxSize;
|
|
5
5
|
now;
|
|
6
|
-
/** Keys with a refresh in flight, so N concurrent readers cause one fetch. */
|
|
7
6
|
refreshing = new Set();
|
|
8
|
-
/** When a key's refresh last failed, so a dead handle is not retried per call. */
|
|
9
7
|
failures = new Map();
|
|
10
8
|
retryAfterMs;
|
|
11
9
|
onRefreshError;
|
|
12
10
|
identify;
|
|
13
|
-
/**
|
|
14
|
-
* A monotonic counter and the tick at which each name was last invalidated.
|
|
15
|
-
*
|
|
16
|
-
* A fetch records the counter when it starts and, when it lands, asks whether
|
|
17
|
-
* anything it turned out to be — its key, its id, its slug — has been
|
|
18
|
-
* invalidated since. That is the whole protocol, and it holds for the cold
|
|
19
|
-
* path, the background refresh, and a handle nobody has seen before, none of
|
|
20
|
-
* which the previous per-key epoch could cover.
|
|
21
|
-
*/
|
|
22
11
|
tick = 0;
|
|
23
12
|
invalidatedAt = new Map();
|
|
24
|
-
/**
|
|
25
|
-
* The tick of the last `clear()`.
|
|
26
|
-
*
|
|
27
|
-
* `clear` cannot name a fetch it has never seen — a cold one is in neither
|
|
28
|
-
* `store` nor `refreshing` — so it raises a floor instead: anything that
|
|
29
|
-
* started before it is out of date whatever it turns out to be.
|
|
30
|
-
*/
|
|
31
13
|
clearedAt = -1;
|
|
32
|
-
/** Fetches outstanding, so the log above can be dropped when none remain. */
|
|
33
14
|
outstanding = 0;
|
|
34
15
|
constructor(opts = {}) {
|
|
35
|
-
// A TTL of 0 disables caching without a separate flag; only undefined
|
|
36
|
-
// falls back to the default.
|
|
37
16
|
this.ttlMs = (opts.ttlSeconds ?? 300) * 1000;
|
|
38
17
|
this.maxSize = Math.max(1, opts.maxSize ?? 100);
|
|
39
18
|
this.now = opts.now ?? (() => Date.now());
|
|
@@ -45,7 +24,6 @@ export class PullCache {
|
|
|
45
24
|
const entry = this.store.get(key);
|
|
46
25
|
if (!entry)
|
|
47
26
|
return undefined;
|
|
48
|
-
// Map iterates in insertion order, so re-setting marks most-recently-used.
|
|
49
27
|
this.store.delete(key);
|
|
50
28
|
this.store.set(key, entry);
|
|
51
29
|
const ageMs = this.now() - entry.at;
|
|
@@ -59,27 +37,13 @@ export class PullCache {
|
|
|
59
37
|
if (oldest === undefined)
|
|
60
38
|
break;
|
|
61
39
|
this.store.delete(oldest);
|
|
62
|
-
// The bookkeeping goes with the entry unless a fetch for it is still
|
|
63
|
-
// running, otherwise these grow forever in a class whose contract is a
|
|
64
|
-
// bounded LRU — one permanent entry per distinct key ever seen.
|
|
65
40
|
if (!this.refreshing.has(oldest))
|
|
66
41
|
this.failures.delete(oldest);
|
|
67
42
|
}
|
|
68
43
|
}
|
|
69
|
-
/**
|
|
70
|
-
* Runs `refresh` unless one is already in flight for this key.
|
|
71
|
-
*
|
|
72
|
-
* Without the guard, a burst of requests arriving just after the TTL lapses
|
|
73
|
-
* each start their own refresh — the stampede the cache exists to prevent.
|
|
74
|
-
* Failures are swallowed on purpose: a background refresh that throws must not
|
|
75
|
-
* surface in a request that was already answered from cache.
|
|
76
|
-
*/
|
|
77
44
|
refreshInBackground(key, refresh) {
|
|
78
45
|
if (this.refreshing.has(key))
|
|
79
46
|
return;
|
|
80
|
-
// Backed off after a failure. A handle that stopped resolving — routine
|
|
81
|
-
// since a rename re-mints the slug — would otherwise fire a doomed request
|
|
82
|
-
// on every pull, forever. The entry stays; the stampede stops.
|
|
83
47
|
const failedAt = this.failures.get(key);
|
|
84
48
|
if (failedAt !== undefined && this.now() - failedAt < this.retryAfterMs)
|
|
85
49
|
return;
|
|
@@ -89,19 +53,10 @@ export class PullCache {
|
|
|
89
53
|
.then((value) => this.settle(key, value, startedAt))
|
|
90
54
|
.catch((err) => {
|
|
91
55
|
this.abandon(key);
|
|
92
|
-
// Surfaced rather than swallowed. `report` has onReportError for exactly
|
|
93
|
-
// this reason; without a counterpart here a service can serve a frozen
|
|
94
|
-
// prompt indefinitely with nothing anywhere saying so.
|
|
95
|
-
//
|
|
96
|
-
// Wrapped, because nobody is holding this promise: `pullWithCache` has
|
|
97
|
-
// already returned the stale value, so a callback that throws would be
|
|
98
|
-
// an unhandled rejection — process exit in Node, isolate abort in a
|
|
99
|
-
// Worker.
|
|
100
56
|
try {
|
|
101
57
|
this.onRefreshError?.(err, key);
|
|
102
58
|
}
|
|
103
59
|
catch {
|
|
104
|
-
/* a reporter that cannot report is not worth an outage */
|
|
105
60
|
}
|
|
106
61
|
})
|
|
107
62
|
.finally(() => {
|
|
@@ -109,45 +64,34 @@ export class PullCache {
|
|
|
109
64
|
this.forgetLogIfIdle();
|
|
110
65
|
});
|
|
111
66
|
}
|
|
112
|
-
/** Records a fetch starting, and the tick it must be judged against. */
|
|
113
67
|
beginFetch() {
|
|
114
68
|
this.outstanding++;
|
|
115
69
|
return this.tick;
|
|
116
70
|
}
|
|
117
|
-
/** Stores a landed fetch unless what it fetched was invalidated meanwhile. */
|
|
118
71
|
settle(key, value, startedAt) {
|
|
119
72
|
this.outstanding--;
|
|
120
73
|
const names = [key, ...(this.identify?.(value) ?? [])];
|
|
121
74
|
const overtaken = startedAt < this.clearedAt ||
|
|
122
75
|
names.some((n) => (this.invalidatedAt.get(n) ?? -1) > startedAt);
|
|
123
76
|
if (!overtaken) {
|
|
124
|
-
// Only once the value is kept: clearing first meant a fetch that lost the
|
|
125
|
-
// race still wiped a live backoff, turning bounded retry into a retry on
|
|
126
|
-
// every call.
|
|
127
77
|
this.failures.delete(key);
|
|
128
78
|
this.set(key, value);
|
|
129
79
|
}
|
|
130
80
|
this.forgetLogIfIdle();
|
|
131
81
|
}
|
|
132
|
-
/** Ends a fetch that threw, and backs the handle off. */
|
|
133
82
|
abandon(key) {
|
|
134
83
|
this.outstanding--;
|
|
135
|
-
// Recorded on the cold path too — the one a fresh isolate takes — or a
|
|
136
|
-
// permanently 404ing handle fires one upstream request per pull, forever.
|
|
137
84
|
this.failures.set(key, this.now());
|
|
138
85
|
this.boundFailures();
|
|
139
86
|
this.forgetLogIfIdle();
|
|
140
87
|
}
|
|
141
|
-
/** Records that everything under this name is out of date. */
|
|
142
88
|
invalidate(name) {
|
|
143
89
|
this.invalidatedAt.set(name, ++this.tick);
|
|
144
90
|
}
|
|
145
|
-
/** The log only matters while a fetch could still be judged against it. */
|
|
146
91
|
forgetLogIfIdle() {
|
|
147
92
|
if (this.outstanding === 0 && this.refreshing.size === 0)
|
|
148
93
|
this.invalidatedAt.clear();
|
|
149
94
|
}
|
|
150
|
-
/** Keeps `failures` inside the same bound as the entries themselves. */
|
|
151
95
|
boundFailures() {
|
|
152
96
|
while (this.failures.size > this.maxSize) {
|
|
153
97
|
const oldest = this.failures.keys().next().value;
|
|
@@ -156,7 +100,6 @@ export class PullCache {
|
|
|
156
100
|
this.failures.delete(oldest);
|
|
157
101
|
}
|
|
158
102
|
}
|
|
159
|
-
/** Every key whose entry satisfies `match`. */
|
|
160
103
|
keysWhere(match) {
|
|
161
104
|
const out = [];
|
|
162
105
|
for (const [key, entry] of this.store)
|
|
@@ -166,17 +109,11 @@ export class PullCache {
|
|
|
166
109
|
}
|
|
167
110
|
delete(key) {
|
|
168
111
|
this.store.delete(key);
|
|
169
|
-
// The backoff goes with it, or a handle that has demonstrably recovered
|
|
170
|
-
// keeps skipping its refresh for the whole retry window.
|
|
171
112
|
this.failures.delete(key);
|
|
172
113
|
this.invalidate(key);
|
|
173
|
-
// Nothing outstanding means nothing to judge against it — residue that
|
|
174
|
-
// grew by one per prompt ever invalidated.
|
|
175
114
|
this.forgetLogIfIdle();
|
|
176
115
|
}
|
|
177
116
|
clear() {
|
|
178
|
-
// Every key the cache has an opinion about, not just stored ones: a cold
|
|
179
|
-
// fetch is in flight under a key not in `store` yet.
|
|
180
117
|
this.clearedAt = ++this.tick;
|
|
181
118
|
this.store.clear();
|
|
182
119
|
this.failures.clear();
|
|
@@ -185,13 +122,6 @@ export class PullCache {
|
|
|
185
122
|
get size() {
|
|
186
123
|
return this.store.size;
|
|
187
124
|
}
|
|
188
|
-
/**
|
|
189
|
-
* How many keys the cache holds bookkeeping for, entries aside.
|
|
190
|
-
*
|
|
191
|
-
* Exposed for the test that this stays bounded: `epochs` and `failures` are
|
|
192
|
-
* private and grow on paths `size` cannot see, so a test written against
|
|
193
|
-
* `size` passes whether or not they leak — which is what the first one did.
|
|
194
|
-
*/
|
|
195
125
|
get trackedKeys() {
|
|
196
126
|
return new Set([...this.invalidatedAt.keys(), ...this.failures.keys()]).size;
|
|
197
127
|
}
|
package/dist/client.js
CHANGED
|
@@ -7,14 +7,7 @@ import { pullWithCache } from "./pull/index.js";
|
|
|
7
7
|
import { newRolloutId } from "./run/id.js";
|
|
8
8
|
import { runAcrossModels, runOnce, sampleRuns } from "./run/index.js";
|
|
9
9
|
const DEFAULT_CONCURRENCY = 4;
|
|
10
|
-
/**
|
|
11
|
-
* Stored prompts, and the spend they account for.
|
|
12
|
-
*
|
|
13
|
-
* Everything that leaves the process goes through `@spendgraph/sdk`; this class
|
|
14
|
-
* owns the caching, the budget and the rollout bookkeeping on top of it.
|
|
15
|
-
*/
|
|
16
10
|
export class PromptClient {
|
|
17
|
-
/** The SDK underneath. Reach for it for anything this class does not wrap. */
|
|
18
11
|
sdk;
|
|
19
12
|
cache;
|
|
20
13
|
hasKey;
|
|
@@ -44,15 +37,6 @@ export class PromptClient {
|
|
|
44
37
|
},
|
|
45
38
|
};
|
|
46
39
|
}
|
|
47
|
-
/**
|
|
48
|
-
* A prompt, from cache when one is warm.
|
|
49
|
-
*
|
|
50
|
-
* Serves a stale copy immediately and refreshes behind the caller, so an edit
|
|
51
|
-
* takes effect within a TTL without any request ever paying for the fetch.
|
|
52
|
-
*
|
|
53
|
-
* Takes an id or a slug — the server resolves either. Cached under whatever
|
|
54
|
-
* you passed, so pulling the same prompt both ways keeps two entries.
|
|
55
|
-
*/
|
|
56
40
|
async pull(handle) {
|
|
57
41
|
const payload = await pullWithCache(this.cache, handle, async () => {
|
|
58
42
|
const res = await this.sdk.prompts.get(handle);
|
|
@@ -69,17 +53,9 @@ export class PromptClient {
|
|
|
69
53
|
models: payload.models ?? [],
|
|
70
54
|
}, "server", this.recorder());
|
|
71
55
|
}
|
|
72
|
-
/** A prompt written in code, wired to record its usage through this client. */
|
|
73
56
|
build(spec, opts = {}) {
|
|
74
57
|
return buildCustomPrompt(spec, { ...opts, sdk: this.sdk });
|
|
75
58
|
}
|
|
76
|
-
/**
|
|
77
|
-
* Said once, because this package cannot count tokens it never saw.
|
|
78
|
-
*
|
|
79
|
-
* A callback that forgets to return them records a rollout reading zero in
|
|
80
|
-
* and zero out, which prices at nothing and quietly drags every average that
|
|
81
|
-
* includes it. Silence here is how that goes unnoticed for a month.
|
|
82
|
-
*/
|
|
83
59
|
warnOnEmptyTokens(outcome) {
|
|
84
60
|
if (this.warnedAboutTokens)
|
|
85
61
|
return;
|
|
@@ -92,22 +68,12 @@ export class PromptClient {
|
|
|
92
68
|
"the provider response, so return inputTokens and outputTokens from your callback " +
|
|
93
69
|
"or every cost on this prompt reads as zero.");
|
|
94
70
|
}
|
|
95
|
-
/**
|
|
96
|
-
* Runs the prompt server-side and records the rollout.
|
|
97
|
-
*
|
|
98
|
-
* The batch path. A production request should pull, render and report
|
|
99
|
-
* instead — this adds a hop and makes spendgraph a dependency of the caller's
|
|
100
|
-
* uptime, which is the right trade for evaluation and the wrong one for a
|
|
101
|
-
* user waiting on a response.
|
|
102
|
-
*/
|
|
103
71
|
async run(promptId, values = {}, opts = {}) {
|
|
104
72
|
return runOnce(this.sdk.prompts, promptId, values, opts, this.budget);
|
|
105
73
|
}
|
|
106
|
-
/** k repetitions of one case. Always k results, failures included. */
|
|
107
74
|
async sample(promptId, values = {}, opts = {}) {
|
|
108
75
|
return sampleRuns(this.sdk.prompts, promptId, values, { concurrency: this.concurrency, ...opts }, this.budget);
|
|
109
76
|
}
|
|
110
|
-
/** One rollout per model — the playground's comparison, headless. */
|
|
111
77
|
async runAll(promptId, values = {}, opts = {}) {
|
|
112
78
|
const models = opts.models ?? (await this.pull(promptId)).models;
|
|
113
79
|
if (models.length === 0) {
|
|
@@ -115,52 +81,26 @@ export class PromptClient {
|
|
|
115
81
|
}
|
|
116
82
|
return runAcrossModels(this.sdk.prompts, promptId, values, models, { concurrency: this.concurrency, ...opts }, this.budget);
|
|
117
83
|
}
|
|
118
|
-
/** The dataset a prompt is evaluated against, with its split counts. */
|
|
119
84
|
async cases(promptId) {
|
|
120
85
|
return this.sdk.prompts.cases(promptId);
|
|
121
86
|
}
|
|
122
|
-
/**
|
|
123
|
-
* Replaces the whole dataset.
|
|
124
|
-
*
|
|
125
|
-
* A dataset is a set: "which cases am I evaluating against" has one answer at
|
|
126
|
-
* a time, and a merge would leave no way to remove a case or to know from the
|
|
127
|
-
* outside what the set currently is.
|
|
128
|
-
*/
|
|
129
87
|
async setCases(promptId, cases) {
|
|
130
88
|
return this.sdk.prompts.putCases(promptId, cases);
|
|
131
89
|
}
|
|
132
|
-
/** Micro-dollars this client has spent on runs it initiated. */
|
|
133
90
|
spent() {
|
|
134
91
|
return this.budget.spent();
|
|
135
92
|
}
|
|
136
|
-
/** What is left of the ceiling, or Infinity when none was set. */
|
|
137
93
|
remaining() {
|
|
138
94
|
return this.budget.remaining();
|
|
139
95
|
}
|
|
140
|
-
/** Every wording this prompt has had, newest first. */
|
|
141
96
|
async versions(promptId, opts = {}) {
|
|
142
97
|
return this.sdk.prompts.versions(promptId, opts);
|
|
143
98
|
}
|
|
144
|
-
/**
|
|
145
|
-
* Serves this version from now on.
|
|
146
|
-
*
|
|
147
|
-
* Invalidates the cache: a promotion that left a warm entry in place would
|
|
148
|
-
* keep sending the old wording for up to a TTL, which is the one moment a
|
|
149
|
-
* caller is actively watching for the change.
|
|
150
|
-
*/
|
|
151
99
|
async promote(promptId, versionId) {
|
|
152
100
|
const res = await this.sdk.prompts.promote(promptId, versionId);
|
|
153
101
|
this.invalidate(promptId);
|
|
154
102
|
return res;
|
|
155
103
|
}
|
|
156
|
-
/**
|
|
157
|
-
* Drops whatever is cached, so the next pull refetches.
|
|
158
|
-
*
|
|
159
|
-
* Both handles, not just the one passed. `pull` keys the cache on whatever
|
|
160
|
-
* string it was given, so the same prompt can sit under its uuid and its
|
|
161
|
-
* slug at once — and `promote` is only reachable with a uuid while the
|
|
162
|
-
* dashboard teaches pulling by slug.
|
|
163
|
-
*/
|
|
164
104
|
invalidate(promptId) {
|
|
165
105
|
if (!promptId) {
|
|
166
106
|
this.cache.clear();
|
|
@@ -172,25 +112,6 @@ export class PromptClient {
|
|
|
172
112
|
this.cache.delete(promptId);
|
|
173
113
|
this.cache.invalidate(promptId);
|
|
174
114
|
}
|
|
175
|
-
/**
|
|
176
|
-
* Records a rollout the caller executed.
|
|
177
|
-
*
|
|
178
|
-
* Never throws and never rejects. This sits beside a user-facing request that
|
|
179
|
-
* has already been answered; telemetry that can break the thing it measures
|
|
180
|
-
* is worse than no telemetry.
|
|
181
|
-
*/
|
|
182
|
-
/**
|
|
183
|
-
* Records a rollout and returns what the server priced it at, in micro-USD,
|
|
184
|
-
* or undefined where the model had no price to compute one from.
|
|
185
|
-
*
|
|
186
|
-
* The price is computed there, from the tokens and the project's pricing
|
|
187
|
-
* table, so this is the only place it exists — and it is why `report` cannot
|
|
188
|
-
* simply be reused: that one answers with the rollout id.
|
|
189
|
-
*
|
|
190
|
-
* `priced: false` is answered as undefined rather than as the stored zero.
|
|
191
|
-
* The column is `notNull`, so an unpriced rollout is written as zero and the
|
|
192
|
-
* flag beside it is the only thing that tells free apart from unknown.
|
|
193
|
-
*/
|
|
194
115
|
async priced(promptId, input) {
|
|
195
116
|
const rolloutId = input.rolloutId ?? newRolloutId();
|
|
196
117
|
if (!this.hasKey)
|
|
@@ -219,12 +140,6 @@ export class PromptClient {
|
|
|
219
140
|
return rolloutId;
|
|
220
141
|
}
|
|
221
142
|
}
|
|
222
|
-
/**
|
|
223
|
-
* One prompt from the server, without keeping a client around.
|
|
224
|
-
*
|
|
225
|
-
* Convenience over `new PromptClient(...).pull(...)`, and it builds a client
|
|
226
|
-
* per call — a service pulling repeatedly wants the class, which caches.
|
|
227
|
-
*/
|
|
228
143
|
export function pullPrompt(handle, opts) {
|
|
229
144
|
return new PromptClient(opts).pull(handle);
|
|
230
145
|
}
|
package/dist/internals.js
CHANGED
|
@@ -1,10 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The pieces `PromptClient` is built from.
|
|
3
|
-
*
|
|
4
|
-
* Not in the main entry on purpose: everything here is already wired for you,
|
|
5
|
-
* and a caller reaching for it is building their own client rather than using
|
|
6
|
-
* this one. Kept exported so that stays possible.
|
|
7
|
-
*/
|
|
8
1
|
export { serializeFields, validateFields } from "@spendgraph/sdk";
|
|
9
2
|
export { Budget } from "./budget/index.js";
|
|
10
3
|
export { PullCache } from "./cache/index.js";
|
package/dist/prompt/bind.js
CHANGED
|
@@ -1,11 +1,5 @@
|
|
|
1
1
|
import { mapLimit } from "../run/concurrency.js";
|
|
2
2
|
const DEFAULT_CONCURRENCY = 4;
|
|
3
|
-
/**
|
|
4
|
-
* Attaches a model, so the prompt can run itself.
|
|
5
|
-
*
|
|
6
|
-
* Without this you hand the call to `prompt.call`; with it the call is already
|
|
7
|
-
* known and `invoke` is the whole thing.
|
|
8
|
-
*/
|
|
9
3
|
export function bind(prompt, model) {
|
|
10
4
|
const once = (values, opts, streaming) => prompt.call(values, ({ messages, turn }) => {
|
|
11
5
|
const modelOpts = { tools: turn, onText: opts.onText };
|
package/dist/prompt/factory.js
CHANGED
|
@@ -6,13 +6,6 @@ export const noRecorder = {
|
|
|
6
6
|
enabled: false,
|
|
7
7
|
write: () => { },
|
|
8
8
|
};
|
|
9
|
-
/**
|
|
10
|
-
* What the rollout route accepts in one record.
|
|
11
|
-
*
|
|
12
|
-
* A conversation longer than this still runs — only its rollout is refused, and
|
|
13
|
-
* that refusal reaches `onReportError` rather than the caller, so it is said
|
|
14
|
-
* here instead while there is still somebody watching.
|
|
15
|
-
*/
|
|
16
9
|
export const MAX_RECORDED_MESSAGES = 50;
|
|
17
10
|
let warnedAboutLength = false;
|
|
18
11
|
function warnIfUnrecordable(messages) {
|
|
@@ -29,13 +22,6 @@ function lastUserMessage(messages) {
|
|
|
29
22
|
}
|
|
30
23
|
return "";
|
|
31
24
|
}
|
|
32
|
-
/**
|
|
33
|
-
* The `Prompt` both constructors return.
|
|
34
|
-
*
|
|
35
|
-
* `pullPrompt` and `buildCustomPrompt` differ only in where the wording came
|
|
36
|
-
* from and what the recorder does with the result — everything a caller
|
|
37
|
-
* touches is built here, once.
|
|
38
|
-
*/
|
|
39
25
|
export function makePrompt(shape, source, recorder = noRecorder) {
|
|
40
26
|
const serialize = (values = {}) => serializeFields(values, shape.fields);
|
|
41
27
|
const format = (values = {}, opts = {}) => renderMessages(shape.blocks, shape.question, serialize(values), opts.history);
|
|
@@ -104,10 +90,6 @@ export function makePrompt(shape, source, recorder = noRecorder) {
|
|
|
104
90
|
offeredTools: outcome.offeredTools ?? recorded?.offeredTools,
|
|
105
91
|
steps: outcome.steps ?? recorded?.steps,
|
|
106
92
|
});
|
|
107
|
-
// Not awaited. The write stays fire-and-forget — a stage that stalled
|
|
108
|
-
// on a report round trip would pay it fifteen times over a run — and
|
|
109
|
-
// the promise is handed back so a caller can settle every price at the
|
|
110
|
-
// end, when the total is what it needs.
|
|
111
93
|
pricing = written instanceof Promise ? written : undefined;
|
|
112
94
|
}
|
|
113
95
|
return { ...outcome, rolloutId, ...(pricing ? { pricing } : {}) };
|
package/dist/pull/pull.js
CHANGED
|
@@ -1,13 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Fetch-by-id with stale-while-revalidate.
|
|
3
|
-
*
|
|
4
|
-
* fresh → return it, no request
|
|
5
|
-
* stale → return it now, refresh behind the caller
|
|
6
|
-
* miss → fetch, and only then answer
|
|
7
|
-
*
|
|
8
|
-
* The middle path is why a prompt edit takes effect within a TTL without any
|
|
9
|
-
* request paying for the fetch. Generic because tools and graphs pull the same.
|
|
10
|
-
*/
|
|
11
1
|
export async function pullWithCache(cache, key, fetchOne) {
|
|
12
2
|
const hit = cache.get(key);
|
|
13
3
|
if (hit && !hit.stale)
|
|
@@ -16,8 +6,6 @@ export async function pullWithCache(cache, key, fetchOne) {
|
|
|
16
6
|
cache.refreshInBackground(key, fetchOne);
|
|
17
7
|
return hit.value;
|
|
18
8
|
}
|
|
19
|
-
// Guarded like the background refresh: an invalidation can land mid-fetch,
|
|
20
|
-
// and storing afterwards marks pre-invalidation state fresh for a whole TTL.
|
|
21
9
|
const startedAt = cache.beginFetch();
|
|
22
10
|
try {
|
|
23
11
|
const value = await fetchOne();
|
package/dist/render/messages.js
CHANGED
|
@@ -1,15 +1,5 @@
|
|
|
1
1
|
import { compileSystemPrompt } from "./system.js";
|
|
2
2
|
import { applyVariables } from "./variables.js";
|
|
3
|
-
/**
|
|
4
|
-
* The messages a request will send.
|
|
5
|
-
*
|
|
6
|
-
* A prompt with no blocks sends one message, not two: an empty system message is
|
|
7
|
-
* a different request from no system message at all.
|
|
8
|
-
*
|
|
9
|
-
* `history` is placed between the system turn and the question, oldest first,
|
|
10
|
-
* and is passed through verbatim — it is a record of what was already said, not
|
|
11
|
-
* a template, so `{placeholders}` inside it are left alone.
|
|
12
|
-
*/
|
|
13
3
|
export function renderMessages(blocks, question, values, history = []) {
|
|
14
4
|
const system = compileSystemPrompt(blocks, values);
|
|
15
5
|
const user = applyVariables(question, values);
|
package/dist/render/system.js
CHANGED
|
@@ -1,10 +1,4 @@
|
|
|
1
1
|
import { applyVariables } from "./variables.js";
|
|
2
|
-
/**
|
|
3
|
-
* Blocks in order, each under its title.
|
|
4
|
-
*
|
|
5
|
-
* An empty body drops the block — a heading over nothing is noise. An untitled
|
|
6
|
-
* block is emitted bare rather than under a blank `##`.
|
|
7
|
-
*/
|
|
8
2
|
export function compileSystemPrompt(blocks, values) {
|
|
9
3
|
const out = [];
|
|
10
4
|
for (const b of blocks ?? []) {
|
package/dist/render/variables.js
CHANGED
|
@@ -1,16 +1,8 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Substitutes `{name}` placeholders, leaving unknown ones verbatim — emptying
|
|
3
|
-
* one would send a prompt with a hole the caller cannot see.
|
|
4
|
-
*
|
|
5
|
-
* Mirrors `lib/prompt.ts` in the app. Duplicated because this package depends on
|
|
6
|
-
* nothing; `rendered` on every rollout makes a divergence visible.
|
|
7
|
-
*/
|
|
8
1
|
export function applyVariables(text, values) {
|
|
9
2
|
if (!text || !values)
|
|
10
3
|
return text;
|
|
11
4
|
return text.replace(/\{(\w+)\}/g, (whole, name) => name in values ? values[name] : whole);
|
|
12
5
|
}
|
|
13
|
-
/** Placeholder names used anywhere in the prompt, in first-seen order. */
|
|
14
6
|
export function findVariables(blocks, question) {
|
|
15
7
|
const seen = [];
|
|
16
8
|
const scan = (text) => {
|
|
@@ -19,7 +11,6 @@ export function findVariables(blocks, question) {
|
|
|
19
11
|
seen.push(m[1]);
|
|
20
12
|
}
|
|
21
13
|
};
|
|
22
|
-
// titles too — a heading can carry a placeholder as readily as a body
|
|
23
14
|
for (const b of blocks ?? []) {
|
|
24
15
|
scan(b.title ?? "");
|
|
25
16
|
scan(b.body ?? "");
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { mapLimit } from "./concurrency.js";
|
|
2
2
|
import { runOnce } from "./once.js";
|
|
3
|
-
/** One rollout per model — the playground's comparison, headless. */
|
|
4
3
|
export async function runAcrossModels(prompts, promptId, values, models, opts = {}, budget) {
|
|
5
4
|
return mapLimit(models, opts.concurrency ?? 4, (model) => runOnce(prompts, promptId, values, { ...opts, model, rolloutId: undefined }, budget));
|
|
6
5
|
}
|
package/dist/run/concurrency.js
CHANGED
|
@@ -1,9 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Runs `fn` over `items`, at most `limit` in flight.
|
|
3
|
-
*
|
|
4
|
-
* Bounded on purpose: a whole dataset at once becomes 429s you still wait out,
|
|
5
|
-
* so the unbounded version finishes later and flakier.
|
|
6
|
-
*/
|
|
7
1
|
export async function mapLimit(items, limit, fn) {
|
|
8
2
|
const out = new Array(items.length);
|
|
9
3
|
let next = 0;
|
package/dist/run/id.js
CHANGED
package/dist/run/once.js
CHANGED
|
@@ -15,7 +15,6 @@ export async function runOnce(prompts, promptId, values, opts = {}, budget) {
|
|
|
15
15
|
generation: opts.generation,
|
|
16
16
|
record: opts.record ?? true,
|
|
17
17
|
});
|
|
18
|
-
// A deduped reply costs nothing new — the attempt that wrote it already paid.
|
|
19
18
|
if (!res.deduped)
|
|
20
19
|
budget?.add(res.rollout.costMicros ?? 0);
|
|
21
20
|
return res.rollout;
|
package/dist/run/sample.js
CHANGED
|
@@ -1,16 +1,6 @@
|
|
|
1
1
|
import { BudgetExceededError } from "../budget/index.js";
|
|
2
2
|
import { mapLimit } from "./concurrency.js";
|
|
3
3
|
import { runOnce } from "./once.js";
|
|
4
|
-
/**
|
|
5
|
-
* k repetitions of one case, so `pass^k` is computable.
|
|
6
|
-
*
|
|
7
|
-
* Always k results. A failed seed is present with `status: "failed"` rather than
|
|
8
|
-
* missing — three results for k = 5 would have the metric compute over three and
|
|
9
|
-
* report better consistency than the run showed.
|
|
10
|
-
*
|
|
11
|
-
* A budget stop throws instead. Money running out says nothing about the prompt,
|
|
12
|
-
* and recording those seeds as failures would put the wallet into pass^k.
|
|
13
|
-
*/
|
|
14
4
|
export async function sampleRuns(prompts, promptId, values, opts = {}, budget) {
|
|
15
5
|
const k = Math.max(1, opts.k ?? 5);
|
|
16
6
|
const seeds = Array.from({ length: k }, (_, i) => i);
|
|
@@ -21,7 +11,6 @@ export async function sampleRuns(prompts, promptId, values, opts = {}, budget) {
|
|
|
21
11
|
catch (err) {
|
|
22
12
|
if (err instanceof BudgetExceededError)
|
|
23
13
|
throw err;
|
|
24
|
-
// Still an outcome for this seed; throwing discards the k-1 that landed.
|
|
25
14
|
return {
|
|
26
15
|
id: null,
|
|
27
16
|
model: opts.model ?? "",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spendgraph/prompt",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Pull a stored prompt or build one in code, render it, and record what it cost.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -42,16 +42,16 @@
|
|
|
42
42
|
"README.md"
|
|
43
43
|
],
|
|
44
44
|
"scripts": {
|
|
45
|
-
"build": "tsc -p tsconfig.json",
|
|
45
|
+
"build": "tsc -p tsconfig.json --emitDeclarationOnly && tsc -p tsconfig.json --declaration false --removeComments",
|
|
46
46
|
"test": "npm run build && vitest run",
|
|
47
47
|
"pretest": "npm run build --workspace @spendgraph/tools"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@spendgraph/sdk": "^0.
|
|
50
|
+
"@spendgraph/sdk": "^0.3.0"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
|
-
"@spendgraph/llms": "^0.
|
|
54
|
-
"@spendgraph/tools": "^0.
|
|
53
|
+
"@spendgraph/llms": "^0.3.0",
|
|
54
|
+
"@spendgraph/tools": "^0.3.0",
|
|
55
55
|
"typescript": "^5"
|
|
56
56
|
},
|
|
57
57
|
"engines": {
|