@edgehero/pi-dispatch 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +44 -7
- package/README.md +14 -6
- package/deploy/com.pi-dispatch.worker.plist +1 -1
- package/deploy/docker-compose.yml +12 -0
- package/deploy/egress-proxy.conf +28 -3
- package/deploy/pi-dispatch-egress-proxy.container +8 -2
- package/deploy/worker-env-wrapper.cmd +1 -1
- package/deploy/worker-env-wrapper.sh +3 -3
- package/package.json +9 -2
- package/src/allocation.mjs +731 -0
- package/src/backends.mjs +243 -0
- package/src/budget.mjs +40 -4
- package/src/cli.mjs +222 -11
- package/src/config.mjs +126 -5
- package/src/daemon-facts.mjs +3 -0
- package/src/deployment-venue.mjs +1 -0
- package/src/doctor.mjs +2316 -183
- package/src/dollar-budget.mjs +373 -0
- package/src/dollar-fingerprint.mjs +83 -0
- package/src/egress-cli.mjs +316 -0
- package/src/egress-proxy-state.mjs +35 -5
- package/src/egress.mjs +16 -3
- package/src/env-allowlist.mjs +142 -18
- package/src/env-file.mjs +194 -25
- package/src/envelope.mjs +413 -0
- package/src/exit-code.mjs +22 -0
- package/src/fleet-lease.mjs +85 -25
- package/src/get-token.mjs +16 -5
- package/src/git-dirty.mjs +67 -0
- package/src/github-app-setup.mjs +6 -3
- package/src/github-host.mjs +5 -3
- package/src/host-pi.mjs +19 -3
- package/src/identity.mjs +2 -1
- package/src/image-preflight.mjs +98 -24
- package/src/image-ref.mjs +37 -0
- package/src/import-pi.mjs +4 -2
- package/src/index.mjs +407 -62
- package/src/init.mjs +18 -0
- package/src/job-id.mjs +26 -3
- package/src/live-probes.mjs +24 -9
- package/src/model-catalog.mjs +297 -0
- package/src/model-endpoints.mjs +649 -0
- package/src/model-ref.mjs +151 -0
- package/src/models-json.mjs +262 -0
- package/src/money.mjs +144 -0
- package/src/octokit-log.mjs +65 -0
- package/src/outbox-plan.mjs +218 -0
- package/src/outbox.mjs +29 -9
- package/src/output-cap.mjs +157 -0
- package/src/packages.mjs +2 -2
- package/src/pause-windows.mjs +81 -2
- package/src/pi-model-loader.mjs +77 -0
- package/src/podman-stack.mjs +16 -3
- package/src/portfolio-snapshot.mjs +304 -0
- package/src/prepare-local.mjs +247 -12
- package/src/prepare.mjs +35 -3
- package/src/pricing.mjs +9 -5
- package/src/priorities.mjs +569 -0
- package/src/processor.mjs +603 -173
- package/src/project-id.mjs +17 -0
- package/src/projects.mjs +238 -0
- package/src/provider-key.mjs +32 -7
- package/src/provider-steering.mjs +214 -59
- package/src/queue.mjs +111 -6
- package/src/reserved-env.mjs +30 -0
- package/src/run-container.mjs +59 -5
- package/src/run-history.mjs +379 -24
- package/src/run-mirror.mjs +30 -0
- package/src/runtime-settings.mjs +104 -9
- package/src/schedules.mjs +33 -1
- package/src/scoped-limits.mjs +447 -27
- package/src/secrets.mjs +2 -1
- package/src/service.mjs +15 -4
- package/src/session-store.mjs +131 -6
- package/src/start.mjs +528 -40
- package/src/subscriptions.mjs +7 -3
- package/src/triggers-file.mjs +65 -4
- package/src/triggers.mjs +140 -9
- package/src/up.mjs +308 -34
- package/src/valkey-endpoint.mjs +3 -2
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dollar windows (issue #501, parts 3 and 4; DES-DOLLAR-RESERVE-AND-SETTLE, the worker half).
|
|
3
|
+
*
|
|
4
|
+
* A deployment may cap what its jobs spend per UTC day, Monday week and month, in dollars. Before a container
|
|
5
|
+
* starts, the worker RESERVES the job's per-job cost cap against every active window (`reserveDollars`); after
|
|
6
|
+
* the run it SETTLES that reservation to what the job really cost (`settleDollars`), or keeps it whole when the
|
|
7
|
+
* cost is not fully known. The runner enforces the per-job cap before every provider call, so the reservation is
|
|
8
|
+
* a true bound on what the run can spend, and the worker needs no prices.
|
|
9
|
+
*
|
|
10
|
+
* Every amount is an INTEGER number of micro-dollars (money.mjs). Valkey's INCRBY adds integers exactly; nothing
|
|
11
|
+
* here stores, adds or compares a float. The one float this module ever reads is the runner's metered cost, and
|
|
12
|
+
* `meteredMicros` turns it into an integer once, rounding up.
|
|
13
|
+
*
|
|
14
|
+
* Keys, all under `budget:usd`, built by budget.mjs's own key functions so a dollar window and a job-count window
|
|
15
|
+
* share their UTC boundaries (the day, the week from Monday, the month) and their TTLs:
|
|
16
|
+
* - `budget:usd:YYYY-MM-DD`, `budget:usd:w:<Monday>`, `budget:usd:m:YYYY-MM`: the deployment's windows;
|
|
17
|
+
* - `budget:usd:s:<hash16>` (a repo or folder, and a project: the hash of its row scope `project:<id>`) and
|
|
18
|
+
* `budget:usd:mdl:<hash16>` (a model): the scoped, project and per-model windows of `scoped-limits.json` version 2,
|
|
19
|
+
* which reach `reserveDollars` as further ledgers with their own `keyPrefix` (scoped-limits.mjs builds them).
|
|
20
|
+
* `budget:usd:p:` was reserved for project windows and is WITHDRAWN (issue #499 part B): a project row's keys come
|
|
21
|
+
* from its row scope like every other row's, one key rule and no second keyspace.
|
|
22
|
+
* None of those sub-namespaces can collide with a day key, whose first segment after the prefix is a 4-digit year.
|
|
23
|
+
*
|
|
24
|
+
* Unlike the job-count ledger, a REFUSED dollar reservation is given back at once (`budget.mjs` keeps a refused
|
|
25
|
+
* slot on purpose): one lost slot out of 25 is tolerable, while five refused $2 reservations would empty a $10
|
|
26
|
+
* window with nothing run.
|
|
27
|
+
*
|
|
28
|
+
* `redis` is any ioredis-compatible client, injected so the logic is testable without a server.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import { DAY_TTL_SECONDS, MONTH_TTL_SECONDS, WEEK_TTL_SECONDS, dayKey, monthKey, weekKey } from "./budget.mjs";
|
|
32
|
+
import { MICROS_PER_USD, optionalUsdMicros } from "./money.mjs";
|
|
33
|
+
|
|
34
|
+
/** The deployment's dollar ledger prefix. */
|
|
35
|
+
export const DOLLAR_KEY_PREFIX = "budget:usd";
|
|
36
|
+
/** The refusal reason a full dollar window gives, in the record, the log and the result. */
|
|
37
|
+
export const DOLLAR_CAP_REASON = "dollar-cap";
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The three `basis` values a settled record carries, plus `refunded` and `unreserved` (INT-RUN-HISTORY-FILE-CONTRACT):
|
|
41
|
+
* - `metered`: the window was charged the runner's metered cost, which was complete;
|
|
42
|
+
* - `floor`: the cost was not fully known, so the window keeps the whole reservation;
|
|
43
|
+
* - `refunded`: no container ran (never started, refused by configuration, or the reservation itself refused),
|
|
44
|
+
* so the reservation was given back in full;
|
|
45
|
+
* - `unreserved`: the job could not spend, so nothing was reserved (a zero-rated local model, issue #503 part 7).
|
|
46
|
+
*/
|
|
47
|
+
export const DOLLAR_BASIS = Object.freeze(["metered", "floor", "refunded", "unreserved"]);
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* The deployment's dollar window caps in micro-dollars, from the effective settings, or `null` when no window is
|
|
51
|
+
* set. `null` is the switch that keeps a deployment with no dollar setting byte-identical: the processor reserves
|
|
52
|
+
* and settles nothing, and no `budget:usd:*` key is ever written. The values were validated where they were read
|
|
53
|
+
* (config.mjs at boot, `validateOverlay` per job), so a throw here is a defect, not a state.
|
|
54
|
+
*/
|
|
55
|
+
export function dollarWindowCaps(settings) {
|
|
56
|
+
const day = optionalUsdMicros(settings?.dailyCostUsd, "dailyCostUsd");
|
|
57
|
+
const week = optionalUsdMicros(settings?.weeklyCostUsd, "weeklyCostUsd");
|
|
58
|
+
const month = optionalUsdMicros(settings?.monthlyCostUsd, "monthlyCostUsd");
|
|
59
|
+
if (day === null && week === null && month === null) return null;
|
|
60
|
+
return { day, week, month };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* The ledgers for one job, in reservation order: the deployment's (when it has any window), then its repo or folder
|
|
65
|
+
* row's (`dollarCapsFor`), then its project row's (`projectDollarCapsFor`, issue #499 part B), then `_other`'s under an
|
|
66
|
+
* envelope (issue #504 part B: a job in no envelope project), then each model row it reserves in (`modelDollarRows`).
|
|
67
|
+
* One `reserveDollars` call over all of them, so a refusal in any window gives back every key, the deployment's included.
|
|
68
|
+
*/
|
|
69
|
+
export function dollarLedgers(caps, { scope = null, project = null, other = null, models = [] } = {}) {
|
|
70
|
+
const out = caps ? [{ keyPrefix: DOLLAR_KEY_PREFIX, caps }] : [];
|
|
71
|
+
if (scope) out.push({ keyPrefix: scope.keyPrefix, caps: scope.caps });
|
|
72
|
+
if (project) out.push({ keyPrefix: project.keyPrefix, caps: project.caps });
|
|
73
|
+
if (other) out.push({ keyPrefix: other.keyPrefix, caps: other.caps });
|
|
74
|
+
for (const m of models ?? []) out.push({ keyPrefix: m.keyPrefix, caps: m.caps });
|
|
75
|
+
return out;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Every ACTIVE window of every ledger, in ledger order and day, week, month within each. A null cap is no window. */
|
|
79
|
+
function activeWindows(ledgers, now) {
|
|
80
|
+
const out = [];
|
|
81
|
+
for (const ledger of ledgers ?? []) {
|
|
82
|
+
const { keyPrefix, caps } = ledger;
|
|
83
|
+
for (const [name, key, ttl] of [
|
|
84
|
+
["day", dayKey(now, keyPrefix), DAY_TTL_SECONDS],
|
|
85
|
+
["week", weekKey(now, keyPrefix), WEEK_TTL_SECONDS],
|
|
86
|
+
["month", monthKey(now, keyPrefix), MONTH_TTL_SECONDS],
|
|
87
|
+
]) {
|
|
88
|
+
const cap = caps?.[name];
|
|
89
|
+
if (cap !== null && cap !== undefined) out.push({ ledger: keyPrefix, window: name, key, cap, ttl });
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return out;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function assertMicros(value, what, { positive }) {
|
|
96
|
+
if (!Number.isSafeInteger(value) || value < (positive ? 1 : 0)) throw new TypeError(`${what} must be a ${positive ? "positive" : "non-negative"} safe integer of micro-dollars`);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* DECRBY every key already added to, BEST EFFORT PER KEY: each key is tried whatever happened to the one before it,
|
|
101
|
+
* and a key whose DECRBY fails is logged (`dollar_giveback_error`, the error's code and the key, which is a date and
|
|
102
|
+
* a fixed prefix, never a value) and left holding the amount. Never throws. Returns how many keys it could not give
|
|
103
|
+
* back. One fault stranding every later key would leave a refused job's whole cap in windows it never ran in.
|
|
104
|
+
*/
|
|
105
|
+
async function giveBack(redis, touched, amountMicros, log) {
|
|
106
|
+
let failed = 0;
|
|
107
|
+
for (const w of touched) {
|
|
108
|
+
try {
|
|
109
|
+
await redis.decrby(w.key, amountMicros);
|
|
110
|
+
} catch (error) {
|
|
111
|
+
failed++;
|
|
112
|
+
log("dollar_giveback_error", { code: typeof error?.code === "string" ? error.code : "error", key: w.key });
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return failed;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Reserve `amountMicros` in every active window of every ledger, or refuse.
|
|
120
|
+
*
|
|
121
|
+
* Per window: `INCRBY key amount`, the TTL set when the result equals the amount (the first write; budget.mjs's
|
|
122
|
+
* set-once idiom, so a busy window cannot push its own expiry forward), then the new total against the cap. A
|
|
123
|
+
* total ABOVE the cap refuses (a total equal to it fits: five $2 jobs fill a $10 window exactly). INCRBY is atomic,
|
|
124
|
+
* so concurrent reservations need no lock: each sees a total that includes every reservation before it.
|
|
125
|
+
*
|
|
126
|
+
* On refusal it tries to give back EVERY key it touched, the refusing one included, one key at a time and best
|
|
127
|
+
* effort (`giveBack`), and returns `{ allowed: false, reason: "dollar-cap", ledger, window, reservedMicros, capMicros }`,
|
|
128
|
+
* where `reservedMicros` is the total that went over, plus `stranded`, the number of keys whose give-back failed. On success it returns `{ allowed: true, hold }`: `hold` is
|
|
129
|
+
* `{ amountMicros, keys: [key] }`, the exact keys it added to, which is all `settleDollars` and `releaseDollars`
|
|
130
|
+
* ever touch. A settlement therefore lands on the reservation's own day, week and month, whenever it runs.
|
|
131
|
+
*
|
|
132
|
+
* An amount of 0 reserves NOTHING and touches no key: a job under a per-job cap of 0 cannot make a priced call, so
|
|
133
|
+
* there is nothing to hold (`{ allowed: true, hold: { amountMicros: 0, keys: [] } }`). It is answered here, not only
|
|
134
|
+
* by the caller, so a caller that forgets the case gets an empty hold rather than a TypeError, which the processor
|
|
135
|
+
* would retry as never-started for ever.
|
|
136
|
+
*
|
|
137
|
+
* A Valkey fault part-way gives back what it had added (best effort, per key) and rethrows, so a fault strands
|
|
138
|
+
* nothing it can avoid stranding. `ledgers` with no active window returns an empty hold.
|
|
139
|
+
*/
|
|
140
|
+
export async function reserveDollars(redis, { ledgers, amountMicros, now = new Date(), log = () => {} }) {
|
|
141
|
+
assertMicros(amountMicros, "amountMicros", { positive: false });
|
|
142
|
+
if (amountMicros === 0) return { allowed: true, hold: { amountMicros: 0, keys: [] } };
|
|
143
|
+
const touched = [];
|
|
144
|
+
let refusal = null;
|
|
145
|
+
try {
|
|
146
|
+
for (const w of activeWindows(ledgers, now)) {
|
|
147
|
+
const total = Number(await redis.incrby(w.key, amountMicros));
|
|
148
|
+
touched.push(w);
|
|
149
|
+
if (total === amountMicros) await redis.expire(w.key, w.ttl);
|
|
150
|
+
if (total > w.cap) {
|
|
151
|
+
refusal = { allowed: false, reason: DOLLAR_CAP_REASON, ledger: w.ledger, window: w.window, reservedMicros: total, capMicros: w.cap };
|
|
152
|
+
break;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
} catch (error) {
|
|
156
|
+
await giveBack(redis, touched, amountMicros, log);
|
|
157
|
+
throw error;
|
|
158
|
+
}
|
|
159
|
+
if (refusal) {
|
|
160
|
+
// `stranded`: keys whose give-back failed and still hold the amount (0 normally), so the record can say so.
|
|
161
|
+
const stranded = await giveBack(redis, touched, amountMicros, log);
|
|
162
|
+
return { ...refusal, stranded };
|
|
163
|
+
}
|
|
164
|
+
return { allowed: true, hold: { amountMicros, keys: touched.map((w) => w.key), ledgers: holdLedgers(touched) } };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* The hold's keys grouped by ledger, `[{ keyPrefix, keys }]` in ledger order (issues #501 part 5, #502 part 6), so a
|
|
169
|
+
* caller can settle each ledger to its own amount: the deployment and scope windows to the job's cost, a model window
|
|
170
|
+
* to that model's. Grouped by the ledger each window came from, never by matching key text: the deployment's prefix
|
|
171
|
+
* `budget:usd` is a prefix of every other dollar key.
|
|
172
|
+
*/
|
|
173
|
+
function holdLedgers(touched) {
|
|
174
|
+
const out = [];
|
|
175
|
+
for (const w of touched) {
|
|
176
|
+
const last = out[out.length - 1];
|
|
177
|
+
if (last && last.keyPrefix === w.ledger) last.keys.push(w.key);
|
|
178
|
+
else out.push({ keyPrefix: w.ledger, keys: [w.key] });
|
|
179
|
+
}
|
|
180
|
+
return out;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* The part of `hold` that belongs to the ledgers `keep(keyPrefix)` selects, as a hold of its own
|
|
185
|
+
* (`{ amountMicros, keys, ledgers }`) for `settleDollars`. A hold with no `ledgers` (built by hand, never by
|
|
186
|
+
* `reserveDollars`) has no parts: every selection is empty, so a settlement through it adjusts nothing and its keys
|
|
187
|
+
* keep the whole reservation, the money-safe side.
|
|
188
|
+
*/
|
|
189
|
+
export function holdPart(hold, keep) {
|
|
190
|
+
const ledgers = (Array.isArray(hold?.ledgers) ? hold.ledgers : []).filter((l) => keep(l.keyPrefix));
|
|
191
|
+
return { amountMicros: hold?.amountMicros, keys: ledgers.flatMap((l) => l.keys), ledgers };
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* One key's settlement, ATOMIC in Valkey (a Lua script runs whole): a key that no longer EXISTS is skipped, never
|
|
196
|
+
* recreated, and a result below 0 is clamped back to 0 (an INCRBY of its own negative, which keeps the key's TTL).
|
|
197
|
+
* Returns `[status, total]`: 0 adjusted, 1 missing, 2 clamped.
|
|
198
|
+
*
|
|
199
|
+
* Why not a bare INCRBY: a key that expired or was evicted between the reserve and the settle (a clock jump, a hand
|
|
200
|
+
* edit, `maxmemory` eviction) would be recreated by a negative INCRBY as a NEGATIVE counter with no TTL, or with a fresh
|
|
201
|
+
* one, and every later job in that window would then fit under a cap it should not. Missing means the window's history
|
|
202
|
+
* is gone; there is nothing true to subtract from.
|
|
203
|
+
*/
|
|
204
|
+
export const SETTLE_SCRIPT = `if redis.call('EXISTS', KEYS[1]) == 0 then return {1, 0} end
|
|
205
|
+
local total = redis.call('INCRBY', KEYS[1], ARGV[1])
|
|
206
|
+
if total < 0 then redis.call('INCRBY', KEYS[1], -total) return {2, 0} end
|
|
207
|
+
return {0, total}`;
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Apply one atomic `INCRBY(settled - reserved)` to each key of `hold` (`SETTLE_SCRIPT`: a missing key is skipped, a
|
|
211
|
+
* negative result clamped at 0, each logged as `dollar_settle_key_missing` / `dollar_settle_clamped` with the key), and
|
|
212
|
+
* NEVER throw. Returns `{ applied, of }`: how many of the hold's keys the step completed on. A Valkey fault logs
|
|
213
|
+
* `dollar_settle_error` (the error's code and the counts, never a key's value) and stops there: the keys not yet
|
|
214
|
+
* adjusted keep the whole reservation, which errs toward overcounting, the money-safe side.
|
|
215
|
+
*
|
|
216
|
+
* Exactly the hold's keys, never keys rebuilt from the clock: a job reserved at 23:59:59 UTC and settled after
|
|
217
|
+
* midnight settles into the day it reserved. A settled amount ABOVE the reservation is charged in full (the delta
|
|
218
|
+
* is positive).
|
|
219
|
+
*
|
|
220
|
+
* `settledMicros` must be a non-negative safe integer; anything else (a float, NaN, a negative) is refused here and
|
|
221
|
+
* leaves the reservation standing, logged as `dollar_settle_error` with `code: "not-micros"`.
|
|
222
|
+
*/
|
|
223
|
+
export async function settleDollars(redis, hold, settledMicros, { log = () => {}, event = "dollar_settle_error" } = {}) {
|
|
224
|
+
const keys = hold?.keys ?? [];
|
|
225
|
+
if (!Number.isSafeInteger(settledMicros) || settledMicros < 0 || !Number.isSafeInteger(hold?.amountMicros)) {
|
|
226
|
+
log(event, { code: "not-micros", applied: 0, of: keys.length });
|
|
227
|
+
return { applied: 0, of: keys.length };
|
|
228
|
+
}
|
|
229
|
+
const delta = settledMicros - hold.amountMicros;
|
|
230
|
+
if (delta === 0) return { applied: keys.length, of: keys.length };
|
|
231
|
+
let applied = 0;
|
|
232
|
+
try {
|
|
233
|
+
for (const key of keys) {
|
|
234
|
+
const [status] = await redis.eval(SETTLE_SCRIPT, 1, key, String(delta));
|
|
235
|
+
applied++;
|
|
236
|
+
if (Number(status) === 1) log("dollar_settle_key_missing", { key });
|
|
237
|
+
else if (Number(status) === 2) log("dollar_settle_clamped", { key });
|
|
238
|
+
}
|
|
239
|
+
} catch (error) {
|
|
240
|
+
log(event, { code: typeof error?.code === "string" ? error.code : "error", applied, of: keys.length });
|
|
241
|
+
}
|
|
242
|
+
return { applied, of: keys.length };
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Give the whole reservation back: a settlement at 0. For a job whose container never ran (never started, or
|
|
247
|
+
* refused by configuration before it could). Never throws; a fault logs `dollar_release_error`.
|
|
248
|
+
*/
|
|
249
|
+
export function releaseDollars(redis, hold, { log = () => {} } = {}) {
|
|
250
|
+
return settleDollars(redis, hold, 0, { log, event: "dollar_release_error" });
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* The runner's metered cost (a float of US dollars off the exit line) as integer micro-dollars, rounded UP:
|
|
255
|
+
* `Math.ceil(cost * 1e6)`. Rounding up is the money-safe direction, and the rule DES-DOLLAR-RESERVE-AND-SETTLE
|
|
256
|
+
* names: the float product can sit a hair above the exact decimal (0.1 + 0.2 is 0.30000000000000004, which is
|
|
257
|
+
* 300001 micro-dollars), so a cost may read up to one micro-dollar high, never low. `null` for anything that is
|
|
258
|
+
* not a finite, non-negative number whose micro-dollars are a safe integer: the caller settles that at the floor.
|
|
259
|
+
*/
|
|
260
|
+
export function meteredMicros(costUsd) {
|
|
261
|
+
if (typeof costUsd !== "number" || !Number.isFinite(costUsd) || costUsd < 0) return null;
|
|
262
|
+
const micros = Math.ceil(costUsd * MICROS_PER_USD);
|
|
263
|
+
return Number.isSafeInteger(micros) ? micros : null;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* The exit-line counters that each say "some of this job's cost is not in the metered number" (issue #501 and
|
|
268
|
+
* PR #534's review). Each must be PRESENT and 0 for a metered settlement: an ABSENT counter reads as non-zero,
|
|
269
|
+
* because a runner that did not write it did not measure it, and an absent number read as 0 would settle a partial
|
|
270
|
+
* count as a cheap job.
|
|
271
|
+
*
|
|
272
|
+
* `unmeteredChildren` (issue #500 part F) counts the job's pi child processes whose spend the runner could not count
|
|
273
|
+
* (`DES-USAGE-METER-VIA-API-PROVIDER-REGISTRY`). The same absent-means-floor rule holds for it, so a worker from part F
|
|
274
|
+
* running an image from before issue #500 part E settles every capped job at the floor: such a runner never writes
|
|
275
|
+
* the key. That is the `costUnjudged` precedent, an overcharge until the image is rebuilt, never an undercharge.
|
|
276
|
+
*
|
|
277
|
+
* `costUnreported` (issue #571) counts the calls on a priced model whose answer carried broken usage (pi fills a
|
|
278
|
+
* missing usage block with zeros, so the metered cost holds about $0 for them); the runner's meter writes it on every
|
|
279
|
+
* line. Absent means floor here too, so a worker from issue #571 running an older image settles every capped job at
|
|
280
|
+
* the floor: upgrade the image before the worker.
|
|
281
|
+
*/
|
|
282
|
+
export const FLOOR_COUNTERS = Object.freeze(["unresolved", "unpriced", "boundExceeded", "longContext", "costUnjudged", "costUnanswered", "costUnreported", "unmeteredChildren"]);
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* How a job's reservation settles, from what the container reported. PURE. Returns `{ settledMicros, basis }`.
|
|
286
|
+
*
|
|
287
|
+
* FIRST, whether the exit line may be believed at all (PR #542's review, round 3). The job's own tools can write to
|
|
288
|
+
* the runner's stdout (a child reaches `/proc/<ppid>/fd/1`), so a forged `{"event":"exit",...}` with a $0 cost is a
|
|
289
|
+
* line like any other in the tail. `trusted` is the processor's verdict that the exit line is the runner's own: the
|
|
290
|
+
* worker did NOT stop the container (no timeout, cancel, shutdown or detach: a stopped runner writes no exit line, so
|
|
291
|
+
* the last one in the tail can only be a forgery or a stale one), AND the line's own `code` equals the container's
|
|
292
|
+
* real exit code. Not trusted is the floor, whatever the line says, the zero-call rule included.
|
|
293
|
+
*
|
|
294
|
+
* - `metered`, `settledMicros = meteredMicros(tokens.cost)`, only when the line is trusted and ALL of these hold: `tokens` is an object with
|
|
295
|
+
* `metered: true`; `costCapMicros` is present and NOT above the reservation (a runner that ran under a wider cap
|
|
296
|
+
* than was reserved was not bounded by the reservation); every `FLOOR_COUNTERS` member is present and 0; the cost
|
|
297
|
+
* converts; and either the per-model `usage` ledger is not null, or the run made NO provider call (`calls: 0` and
|
|
298
|
+
* a cost of 0: the runner omits the ledger when it observed no call, and a call the cost guard refused is never
|
|
299
|
+
* sent, so `costRefused` may be above 0). A metered cost above the reservation is charged in full;
|
|
300
|
+
* - otherwise `floor`: AT LEAST the reservation, and never less than a metered cost the runner did report.
|
|
301
|
+
* `settledMicros = max(reservedMicros, meteredMicros(tokens.cost))` when that cost is a valid number, else the
|
|
302
|
+
* reservation. The floor means "the cost is not fully known", and a known part of it above the reservation is
|
|
303
|
+
* still known: discarding it would undercharge the window.
|
|
304
|
+
* Floor cases: no exit line (`tokens: null`, a job killed before it wrote one), the fallback meter, a missing or
|
|
305
|
+
* non-zero counter, a cap wider than the reservation, or calls with no ledger.
|
|
306
|
+
*/
|
|
307
|
+
export function dollarSettlement({ tokens, usage, reservedMicros, trusted = false }) {
|
|
308
|
+
assertMicros(reservedMicros, "reservedMicros", { positive: false });
|
|
309
|
+
const reported = tokens !== null && typeof tokens === "object" ? meteredMicros(tokens.cost) : null;
|
|
310
|
+
const floor = { settledMicros: reported === null ? reservedMicros : Math.max(reservedMicros, reported), basis: "floor" };
|
|
311
|
+
if (trusted !== true) return floor;
|
|
312
|
+
if (tokens === null || typeof tokens !== "object" || tokens.metered !== true) return floor;
|
|
313
|
+
if (typeof tokens.costCapMicros !== "number" || tokens.costCapMicros > reservedMicros) return floor;
|
|
314
|
+
for (const key of FLOOR_COUNTERS) if (tokens[key] !== 0) return floor;
|
|
315
|
+
if (reported === null) return floor;
|
|
316
|
+
if (usage === null || usage === undefined) {
|
|
317
|
+
// No ledger: complete only for a run that made no provider call at all.
|
|
318
|
+
return tokens.calls === 0 && reported === 0 ? { settledMicros: 0, basis: "metered" } : floor;
|
|
319
|
+
}
|
|
320
|
+
return { settledMicros: reported, basis: "metered" };
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* How ONE model window settles (issue #502 part 6). PURE. Returns `{ settledMicros, basis }`. A model window settles
|
|
325
|
+
* from that model's own row in the exit line's `usage.models`, never from the job's total: a job that spent $1.80 on
|
|
326
|
+
* one model and $0.20 on another charges each model window its own share.
|
|
327
|
+
*
|
|
328
|
+
* `basis` is the job's deployment basis (`dollarSettlement`), which already folds in the trusted exit line, every
|
|
329
|
+
* floor counter and "calls with no ledger". The model window is at the FLOOR when ANY of these holds:
|
|
330
|
+
* - the deployment basis is not `metered`;
|
|
331
|
+
* - the exit line is not trusted (checked here too, so the rule does not rest on the caller's order);
|
|
332
|
+
* - `usage` is null while the run made calls (`tokens.calls` not 0, an absent count included): the per-model
|
|
333
|
+
* split is unknown;
|
|
334
|
+
* - `usage.truncated` is not PRESENT and 0: rows past the ledger's limit were folded together, so a model's own
|
|
335
|
+
* row may be missing part of its spend. Absent is not 0, the `FLOOR_COUNTERS` rule;
|
|
336
|
+
* - an `other/other` row has a cost above 0: some spend is attributed to no model.
|
|
337
|
+
* At the floor it settles at least the reservation and never less than the model row's own reported cost. Metered,
|
|
338
|
+
* it settles `ceil(row.cost x 1e6)`, 0 when the model has no row (it made no call), and an overshoot is charged in
|
|
339
|
+
* full. `ref` is the lowercased `provider/model`; the ledger's ids are lowercased by `parseExitUsage`.
|
|
340
|
+
*/
|
|
341
|
+
export function modelDollarSettlement({ ref, basis, tokens, usage, reservedMicros, trusted = false }) {
|
|
342
|
+
assertMicros(reservedMicros, "reservedMicros", { positive: false });
|
|
343
|
+
const rows = usage !== null && typeof usage === "object" && Array.isArray(usage.models) ? usage.models : [];
|
|
344
|
+
// EVERY row whose lowercased ref is this model's, summed (PR #549's review): the parser lowercases ids, so two rows
|
|
345
|
+
// that differ only in case are one model, and charging the first alone would undercount it. Each row's cost is
|
|
346
|
+
// turned into integer micro-dollars on its own, so the sum is integers; one unconvertible row makes it unknown.
|
|
347
|
+
let reported = 0;
|
|
348
|
+
for (const r of rows) {
|
|
349
|
+
if (typeof r?.provider !== "string" || typeof r?.model !== "string" || `${r.provider}/${r.model}`.toLowerCase() !== ref) continue;
|
|
350
|
+
const micros = meteredMicros(r.cost);
|
|
351
|
+
reported = reported === null || micros === null ? null : reported + micros;
|
|
352
|
+
}
|
|
353
|
+
const floor = { settledMicros: reported === null ? reservedMicros : Math.max(reservedMicros, reported), basis: "floor" };
|
|
354
|
+
if (basis !== "metered" || trusted !== true) return floor;
|
|
355
|
+
if (usage === null || usage === undefined) return tokens?.calls === 0 ? { settledMicros: 0, basis: "metered" } : floor;
|
|
356
|
+
if (typeof usage !== "object" || usage.truncated !== 0) return floor;
|
|
357
|
+
if (rows.some((r) => r?.provider === "other" && r?.model === "other" && !(r.cost === 0))) return floor;
|
|
358
|
+
if (reported === null) return floor;
|
|
359
|
+
return { settledMicros: reported, basis: "metered" };
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/** The `modelBasis` tokens a record may carry (INT-RUN-HISTORY-FILE-CONTRACT), or null when no model window applied. */
|
|
363
|
+
export const MODEL_BASIS = Object.freeze(["metered", "floor", "refunded"]);
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* The run record's `dollars` object (INT-RUN-HISTORY-FILE-CONTRACT): an explicit literal of integers and fixed
|
|
367
|
+
* tokens. `modelBasis` is how the job's MODEL windows settled (issue #502 part 6): `metered` when every one settled
|
|
368
|
+
* to its own usage row, `floor` when any kept at least its reservation, `refunded` when they were given back, and
|
|
369
|
+
* `null` when the job held no model window.
|
|
370
|
+
*/
|
|
371
|
+
export function dollarsRecord({ reservedMicros, settledMicros, basis, modelBasis = null }) {
|
|
372
|
+
return { reservedMicros, settledMicros, basis, modelBasis: MODEL_BASIS.includes(modelBasis) ? modelBasis : null };
|
|
373
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The fingerprint of a host's dollar caps (issue #501, part 6), published on its host registry row as `fpUsd`.
|
|
3
|
+
*
|
|
4
|
+
* WHY THE FLEET NEEDS IT. The dollar counters are shared keys (`budget:usd:*`), as the job-count windows are, but
|
|
5
|
+
* each host reads its CAPS from its own env, its own settings overlay and its own scoped-limits file. Two hosts
|
|
6
|
+
* on one Valkey can therefore judge one counter against two different caps, and the host with the larger cap
|
|
7
|
+
* admits a job the other would refuse. Nothing on the paid path can see that, so `doctor` compares the hosts'
|
|
8
|
+
* fingerprints and warns when they differ, the way it treats the image digest and the timezone.
|
|
9
|
+
*
|
|
10
|
+
* WHAT IT COVERS, which is exactly what decides a dollar admission on a host:
|
|
11
|
+
* - the four dollar settings (`maxCostUsd` and the three windows): the overlay over env, as a job resolves them.
|
|
12
|
+
* When the overlay is invalid, or the merged values break the dollar rule, it hashes the env values; such a host
|
|
13
|
+
* refuses every job (`settings-overlay-invalid`), and env is what it will run under once the file is fixed or
|
|
14
|
+
* removed (`start.mjs`, the fallback its slot count and its scoped-limits warning take);
|
|
15
|
+
* - every scoped-limits row that carries a dollar window: the COUNTER it reserves in and its three caps;
|
|
16
|
+
* - the model rows a job with no list of its own reserves in (PR #551's review): `PI_ALLOWED_MODELS` decides
|
|
17
|
+
* them (`modelDollarRows`), so two hosts with equal caps and different env lists admit differently. Their
|
|
18
|
+
* counter prefixes, sorted. With no env list that is every model row; listed as the counters themselves, so
|
|
19
|
+
* two hosts that reserve in the same counters agree whatever their lists say about models with no row.
|
|
20
|
+
*
|
|
21
|
+
* NUMBERS AND HASHES ONLY, the registry's content rule (`INT-HOST-REGISTRY-CONTRACT`). An amount is hashed as its integer
|
|
22
|
+
* micro-dollars and never appears in clear. A row is named by its dollar key prefix (`budget:usd:s:<hash16>` or `budget:usd:mdl:<hash16>`,
|
|
23
|
+
* `dollarKeyPrefixFor`), never by its scope string: a folder scope is a host path and a repo scope a repository
|
|
24
|
+
* name, and neither may reach a Valkey value even inside a digest's input. The prefix is also the right identity
|
|
25
|
+
* for the comparison, because it is the counter two hosts share: two rows that spell one scope differently but
|
|
26
|
+
* reserve in one counter are one row here.
|
|
27
|
+
*
|
|
28
|
+
* Pure apart from the hash, and it never throws: an amount that does not parse (a value the worker refuses at
|
|
29
|
+
* boot or per job) hashes as the word `invalid`, so a host still publishes a fingerprint and a peer still sees
|
|
30
|
+
* that it differs.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
import { fingerprint } from "./fingerprint.mjs";
|
|
34
|
+
import { DOLLAR_SETTING_KEYS, optionalUsdMicros } from "./money.mjs";
|
|
35
|
+
import { USD_LIMIT_FIELDS, dollarKeyPrefixFor, modelDollarRows } from "./scoped-limits.mjs";
|
|
36
|
+
|
|
37
|
+
/** An amount as integer micro-dollars, `null` when unset, or `"invalid"`: never the value as written. */
|
|
38
|
+
function amount(value, key) {
|
|
39
|
+
try {
|
|
40
|
+
return optionalUsdMicros(value, key);
|
|
41
|
+
} catch {
|
|
42
|
+
return "invalid";
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* What the fingerprint hashes, exported so a test can read it: `{ settings, rows, untargeted }`, where `settings` maps
|
|
48
|
+
* each dollar key to micro-dollars or null, `rows` lists every dollar-carrying row as `{ counter, dayUsd, weekUsd,
|
|
49
|
+
* monthUsd }`, sorted by counter, and `untargeted` is the sorted counter prefixes of the model rows a job with no list
|
|
50
|
+
* of its own reserves in under `envList` (the parsed `PI_ALLOWED_MODELS`, or null). Row order in the file is not a
|
|
51
|
+
* cap, so two hosts listing the same rows in a different order agree.
|
|
52
|
+
*/
|
|
53
|
+
export function usdFingerprintInput(settings, limits, envList = null) {
|
|
54
|
+
const caps = {};
|
|
55
|
+
for (const key of DOLLAR_SETTING_KEYS) caps[key] = amount(settings?.[key], key);
|
|
56
|
+
const rows = [];
|
|
57
|
+
for (const row of Array.isArray(limits) ? limits : []) {
|
|
58
|
+
if (!USD_LIMIT_FIELDS.some((field) => row?.[field] !== null && row?.[field] !== undefined)) continue;
|
|
59
|
+
const windows = {};
|
|
60
|
+
for (const field of USD_LIMIT_FIELDS) windows[field] = amount(row[field], field);
|
|
61
|
+
rows.push({ counter: dollarKeyPrefixFor(row), ...windows });
|
|
62
|
+
}
|
|
63
|
+
rows.sort((a, b) => (a.counter < b.counter ? -1 : a.counter > b.counter ? 1 : 0));
|
|
64
|
+
let untargeted;
|
|
65
|
+
try {
|
|
66
|
+
untargeted = modelDollarRows(Array.isArray(limits) ? limits : [], Array.isArray(envList) ? envList : null).map((row) => row.keyPrefix).sort();
|
|
67
|
+
} catch {
|
|
68
|
+
untargeted = "invalid"; // a row whose amount does not parse: the rows above already say so
|
|
69
|
+
}
|
|
70
|
+
return { settings: caps, rows, untargeted };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** The 16-hex fingerprint `fpUsd` (`fingerprint.mjs`) of a host's dollar settings, scoped-limits rows and env list. */
|
|
74
|
+
export function usdFingerprint(settings, limits, envList = null) {
|
|
75
|
+
return fingerprint(usdFingerprintInput(settings, limits, envList));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* The fingerprint of a host with no dollar setting and no dollar row. A peer that publishes no `fpUsd` (it
|
|
80
|
+
* predates this field) is worth a warning only when dollar caps are in use somewhere on the fleet, and this is
|
|
81
|
+
* how a reader tells.
|
|
82
|
+
*/
|
|
83
|
+
export const EMPTY_USD_FINGERPRINT = usdFingerprint({}, []);
|