omp-conductor 0.3.25 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +733 -30
- package/package.json +1 -1
- package/src/board.ts +24 -5
- package/src/briefs/orchestrator.md +106 -20
- package/src/briefs/policy.md +48 -27
- package/src/briefs/worker.md +82 -31
- package/src/cli.ts +155 -2
- package/src/config.ts +669 -16
- package/src/confinement.ts +506 -25
- package/src/credentials.ts +2029 -0
- package/src/daemon.ts +1000 -32
- package/src/diff-flags.ts +696 -0
- package/src/escalate.ts +136 -9
- package/src/fleet.ts +71 -3
- package/src/omp.ts +471 -15
- package/src/orchestrator-tick.ts +42 -19
- package/src/orchestrator.ts +32 -5
- package/src/plugin.ts +267 -15
- package/src/release-policy.ts +191 -29
- package/src/reports.ts +440 -0
- package/src/session-host.ts +307 -0
- package/src/setup-host.ts +19 -1
- package/src/setup.ts +207 -18
- package/src/store.ts +506 -3
- package/src/tracker/github.ts +45 -0
- package/src/types.ts +847 -10
- package/src/usage.ts +726 -0
- package/src/verbs/actions.ts +142 -0
- package/src/verbs/client.ts +207 -0
- package/src/verbs/ledger.ts +77 -0
- package/src/verbs/protocol.ts +465 -0
- package/src/verbs/server.ts +1098 -0
- package/src/verbs/socket.ts +446 -0
- package/src/worker.ts +36 -7
- package/src/worktree.ts +202 -109
- package/systemd/omp-conductor.service.example +96 -8
package/src/usage.ts
ADDED
|
@@ -0,0 +1,726 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The plan-allowance guard: a second economic control beside `dailySpendUsd`.
|
|
3
|
+
*
|
|
4
|
+
* `dailySpendUsd` meters money, which is the only thing an API-billed account
|
|
5
|
+
* can run out of. A fixed-price subscription cannot be expressed that way at
|
|
6
|
+
* all — the fleet this package was written for burns a *weekly token
|
|
7
|
+
* allowance* whose marginal dollar cost is zero and whose exhaustion stops
|
|
8
|
+
* every session on the host. Pricing that allowance to fit the dollar meter
|
|
9
|
+
* would mean inventing a number, and a ceiling built on an invented number is
|
|
10
|
+
* worse than no ceiling (#110).
|
|
11
|
+
*
|
|
12
|
+
* So the guard reads the provider's own figures — `omp usage --json`, which is
|
|
13
|
+
* the structured form of the harness `/usage` view — and normalises them into
|
|
14
|
+
* windows the admission gate compares. Three properties of the real payload
|
|
15
|
+
* shape every decision in this file, and each of them is a way an obvious
|
|
16
|
+
* implementation gets it wrong:
|
|
17
|
+
*
|
|
18
|
+
* - **`limits` is a list, not a scalar.** Anthropic alone reports
|
|
19
|
+
* `anthropic:5h`, `anthropic:7d` and the tier-scoped `anthropic:7d:fable` in
|
|
20
|
+
* one reply, and other providers add their own. A gate that read "the first
|
|
21
|
+
* limit" would bind a config that meant the weekly window to whichever entry
|
|
22
|
+
* the provider happened to serialise first, and would rebind itself silently
|
|
23
|
+
* when that order changed. The cap therefore names its window.
|
|
24
|
+
* - **`unit` is not comparable across providers.** Anthropic reports
|
|
25
|
+
* `unit: "percent"` with `used: 5`; `xai-oauth` reports `unit: "unknown"`
|
|
26
|
+
* with `used: 4709` against a limit of 20000. A threshold compared against
|
|
27
|
+
* `used` reads the second account as 4709% spent. Nothing here ever compares
|
|
28
|
+
* a raw count: the only number the gate looks at is `usedFraction`.
|
|
29
|
+
* - **A source can legitimately report nothing.** A provider with no readable
|
|
30
|
+
* allowance simply contributes no limits. That is the honest origin of
|
|
31
|
+
* "unmetered/unavailable", and it is why no code path here manufactures a
|
|
32
|
+
* zero to fill the gap: a fabricated `0% used` renders as a guard that is
|
|
33
|
+
* working when it is in fact blind.
|
|
34
|
+
*
|
|
35
|
+
* Availability policy — which way each failure falls — is decided in
|
|
36
|
+
* {@link planUsageStatus} and documented beside it.
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
import type { PlanUsageCap } from "./types.ts";
|
|
40
|
+
|
|
41
|
+
/** Bounded so a wedged provider call cannot hold a dispatch tick open. A warm
|
|
42
|
+
* `omp usage --json` answers in well under a second; this leaves room for a
|
|
43
|
+
* cold fetch across several provider endpoints without becoming a stall. */
|
|
44
|
+
export const USAGE_PROBE_TIMEOUT_MS = 15_000;
|
|
45
|
+
|
|
46
|
+
/** How long one reading serves every caller. One admission pass asks the gate
|
|
47
|
+
* once, but `status` and `board` re-render far faster than any allowance
|
|
48
|
+
* moves, so the provider is asked at most once a minute per process. */
|
|
49
|
+
export const USAGE_TTL_MS = 60_000;
|
|
50
|
+
|
|
51
|
+
/** A failed read is retried sooner than a good one is refreshed. The guard is
|
|
52
|
+
* open while the source is unreadable (see below), so sitting on a stale
|
|
53
|
+
* failure keeps the fleet unguarded for longer than the failure lasted. */
|
|
54
|
+
export const USAGE_ERROR_TTL_MS = 15_000;
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* How long an unreadable source is tolerated before the guard stops claiming.
|
|
58
|
+
*
|
|
59
|
+
* This is the whole of the "which way does it fail" question. A short outage —
|
|
60
|
+
* a token refresh, a provider 502, `omp` briefly absent mid-upgrade — must not
|
|
61
|
+
* stall a fleet: the other ceilings (spend, turns, wall clock, concurrency)
|
|
62
|
+
* are all still enforced, so admitting through a blip costs at most one extra
|
|
63
|
+
* worker. A source that has been unreadable for half an hour is not an outage,
|
|
64
|
+
* it is a broken meter, and running a plan-capped fleet on a broken meter is
|
|
65
|
+
* exactly how the allowance gets spent to zero with nobody watching.
|
|
66
|
+
*/
|
|
67
|
+
export const USAGE_BLIND_GRACE_MS = 30 * 60_000;
|
|
68
|
+
|
|
69
|
+
/** Past this age a rendered reading says how old it is. Anything an operator
|
|
70
|
+
* reads as current has to actually be current — a stale figure presented as
|
|
71
|
+
* live is the dishonesty this guard was specified to avoid. */
|
|
72
|
+
export const USAGE_STALE_MS = 10 * 60_000;
|
|
73
|
+
|
|
74
|
+
/** Bound on how many allowance ids an error message lists back at an operator. */
|
|
75
|
+
const NAMED_WINDOW_LIMIT = 12;
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* One allowance window, normalised away from any single provider's shape.
|
|
79
|
+
*
|
|
80
|
+
* `id` is the primary key and `windowId` deliberately is not: `anthropic:7d`
|
|
81
|
+
* and `anthropic:7d:fable` share the window key `7d` and are different
|
|
82
|
+
* allowances with different numbers.
|
|
83
|
+
*/
|
|
84
|
+
export interface UsageWindow {
|
|
85
|
+
/** Fully-qualified allowance id, unique within a reading (`anthropic:7d`). */
|
|
86
|
+
id: string;
|
|
87
|
+
provider: string;
|
|
88
|
+
/** Provider-scoped window key (`7d`, `1mo`). Not unique — see above. */
|
|
89
|
+
windowId?: string;
|
|
90
|
+
/** The provider's own name for the window, for display only. */
|
|
91
|
+
label?: string;
|
|
92
|
+
used: number;
|
|
93
|
+
limit?: number;
|
|
94
|
+
remaining?: number;
|
|
95
|
+
/**
|
|
96
|
+
* Consumed share of the allowance. The only figure the gate compares, and
|
|
97
|
+
* absent when the provider gave nothing a fraction can be derived from —
|
|
98
|
+
* which is a state the guard reports rather than papers over.
|
|
99
|
+
*/
|
|
100
|
+
usedFraction?: number;
|
|
101
|
+
remainingFraction?: number;
|
|
102
|
+
/** The provider's word for the counts (`percent`, `unknown`). Display only:
|
|
103
|
+
* treating it as a scale is the bug this field exists to warn about. */
|
|
104
|
+
unit: string;
|
|
105
|
+
/** Derived from the provider's reset instant and window length when both are
|
|
106
|
+
* reported; the payload carries no explicit start. */
|
|
107
|
+
windowStart?: number;
|
|
108
|
+
/** The provider's own reset instant. Never a locally assumed calendar week:
|
|
109
|
+
* a weekly plan resets on the account's clock, not on Monday. */
|
|
110
|
+
resetsAt?: number;
|
|
111
|
+
/** Per-limit health the provider attached. Carried and displayed, never
|
|
112
|
+
* turned into arithmetic — the vocabulary is the provider's, not ours. */
|
|
113
|
+
status?: string;
|
|
114
|
+
/** When the provider fetched this figure (its `fetchedAt`, not our clock). */
|
|
115
|
+
observedAt: number;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* One reading of the whole source.
|
|
120
|
+
*
|
|
121
|
+
* `ok` carries at least one window by construction: a payload that parsed but
|
|
122
|
+
* yielded no allowances is `unavailable`, because "nothing is metered" and
|
|
123
|
+
* "the meter is fine and reads zero" are different facts and only one of them
|
|
124
|
+
* may be displayed as a percentage.
|
|
125
|
+
*/
|
|
126
|
+
export type UsageReading =
|
|
127
|
+
| { kind: "ok"; observedAt: number; windows: UsageWindow[] }
|
|
128
|
+
| {
|
|
129
|
+
kind: "unavailable";
|
|
130
|
+
observedAt: number;
|
|
131
|
+
/** Operator-facing cause, already bounded for a status line. */
|
|
132
|
+
reason: string;
|
|
133
|
+
/**
|
|
134
|
+
* Start of the current unbroken run of unreadable results. A single
|
|
135
|
+
* failure and a fortnight of failures are the same object otherwise, and
|
|
136
|
+
* the grace period in {@link USAGE_BLIND_GRACE_MS} needs to tell them
|
|
137
|
+
* apart.
|
|
138
|
+
*/
|
|
139
|
+
since: number;
|
|
140
|
+
};
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* A source of readings. The one contract that matters: `read` never throws. An
|
|
144
|
+
* unreadable provider is a *reading* the caller applies policy to, not an
|
|
145
|
+
* exception that aborts a dispatch tick.
|
|
146
|
+
*/
|
|
147
|
+
export interface UsageProvider {
|
|
148
|
+
read(now: number): Promise<UsageReading>;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** A provider with a cache in front of it. */
|
|
152
|
+
export interface UsageSource {
|
|
153
|
+
read(now?: number): Promise<UsageReading>;
|
|
154
|
+
/** Drop the cached reading so the next `read` asks the provider. The
|
|
155
|
+
* operator-facing half of the same idea is `omp usage invalidate`. */
|
|
156
|
+
invalidate(): void;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// ------------------------------------------------------------------ normalising
|
|
160
|
+
|
|
161
|
+
function asObject(v: unknown): Record<string, unknown> | undefined {
|
|
162
|
+
return typeof v === "object" && v !== null && !Array.isArray(v)
|
|
163
|
+
? (v as Record<string, unknown>)
|
|
164
|
+
: undefined;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function asNumber(v: unknown): number | undefined {
|
|
168
|
+
return typeof v === "number" && Number.isFinite(v) ? v : undefined;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function asString(v: unknown): string | undefined {
|
|
172
|
+
return typeof v === "string" && v.length > 0 ? v : undefined;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* The single place a threshold-comparable number is produced.
|
|
177
|
+
*
|
|
178
|
+
* Preference order is deliberate. `usedFraction` is what the provider itself
|
|
179
|
+
* computed. `remainingFraction` is the same statement inverted. `used/limit`
|
|
180
|
+
* is our own arithmetic and only valid when the limit is a positive count — a
|
|
181
|
+
* limit of `0` divides to `Infinity`/`NaN`, which would render as a plausible
|
|
182
|
+
* percentage and hold a fleet on nonsense.
|
|
183
|
+
*/
|
|
184
|
+
function usedFractionOf(amount: Record<string, unknown>): number | undefined {
|
|
185
|
+
const direct = asNumber(amount["usedFraction"]);
|
|
186
|
+
if (direct !== undefined) return direct;
|
|
187
|
+
const remaining = asNumber(amount["remainingFraction"]);
|
|
188
|
+
if (remaining !== undefined) return 1 - remaining;
|
|
189
|
+
const used = asNumber(amount["used"]);
|
|
190
|
+
const limit = asNumber(amount["limit"]);
|
|
191
|
+
if (used !== undefined && limit !== undefined && limit > 0) return used / limit;
|
|
192
|
+
return undefined;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* `omp usage --json` → normalised windows.
|
|
197
|
+
*
|
|
198
|
+
* Pure and exported so the traps in the real payload can be pinned without a
|
|
199
|
+
* provider on the machine running the tests. Unrecognisable input is never
|
|
200
|
+
* partially believed: it becomes `unavailable` with a reason an operator can
|
|
201
|
+
* act on.
|
|
202
|
+
*/
|
|
203
|
+
export function normalizeUsagePayload(payload: unknown, observedAt: number): UsageReading {
|
|
204
|
+
const root = asObject(payload);
|
|
205
|
+
const reports = root?.["reports"];
|
|
206
|
+
if (root === undefined || !Array.isArray(reports)) {
|
|
207
|
+
return {
|
|
208
|
+
kind: "unavailable",
|
|
209
|
+
observedAt,
|
|
210
|
+
reason: "usage payload has no reports array",
|
|
211
|
+
since: observedAt,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const windows: UsageWindow[] = [];
|
|
216
|
+
reports.forEach((entry: unknown, reportIndex) => {
|
|
217
|
+
const report = asObject(entry);
|
|
218
|
+
if (report === undefined) return;
|
|
219
|
+
const provider = asString(report["provider"]) ?? "unknown";
|
|
220
|
+
const fetchedAt = asNumber(report["fetchedAt"]) ?? observedAt;
|
|
221
|
+
const limits = report["limits"];
|
|
222
|
+
// A provider that contributes no limits is not an error. It is the
|
|
223
|
+
// ordinary shape of "this account exposes no readable allowance", and it
|
|
224
|
+
// must reach the caller as absence rather than as a zero.
|
|
225
|
+
if (!Array.isArray(limits)) return;
|
|
226
|
+
|
|
227
|
+
limits.forEach((limitEntry: unknown, limitIndex) => {
|
|
228
|
+
const limit = asObject(limitEntry);
|
|
229
|
+
const amount = asObject(limit?.["amount"]);
|
|
230
|
+
if (limit === undefined || amount === undefined) return;
|
|
231
|
+
const scope = asObject(limit["scope"]);
|
|
232
|
+
const window = asObject(limit["window"]);
|
|
233
|
+
const windowId = asString(scope?.["windowId"]) ?? asString(window?.["id"]);
|
|
234
|
+
const resetsAt = asNumber(window?.["resetsAt"]);
|
|
235
|
+
const durationMs = asNumber(window?.["durationMs"]);
|
|
236
|
+
const normalized: UsageWindow = {
|
|
237
|
+
// Synthesised only as a last resort, and deterministically: a window
|
|
238
|
+
// nothing can name is a window a config can never select.
|
|
239
|
+
id: asString(limit["id"]) ?? `${provider}:${windowId ?? String(reportIndex)}.${String(limitIndex)}`,
|
|
240
|
+
provider,
|
|
241
|
+
unit: asString(amount["unit"]) ?? "unknown",
|
|
242
|
+
used: asNumber(amount["used"]) ?? 0,
|
|
243
|
+
observedAt: fetchedAt,
|
|
244
|
+
};
|
|
245
|
+
if (windowId !== undefined) normalized.windowId = windowId;
|
|
246
|
+
const label = asString(limit["label"]) ?? asString(window?.["label"]);
|
|
247
|
+
if (label !== undefined) normalized.label = label;
|
|
248
|
+
const limitAmount = asNumber(amount["limit"]);
|
|
249
|
+
if (limitAmount !== undefined) normalized.limit = limitAmount;
|
|
250
|
+
const remaining = asNumber(amount["remaining"]);
|
|
251
|
+
if (remaining !== undefined) normalized.remaining = remaining;
|
|
252
|
+
const usedFraction = usedFractionOf(amount);
|
|
253
|
+
if (usedFraction !== undefined) normalized.usedFraction = usedFraction;
|
|
254
|
+
const remainingFraction = asNumber(amount["remainingFraction"]);
|
|
255
|
+
if (remainingFraction !== undefined) normalized.remainingFraction = remainingFraction;
|
|
256
|
+
if (resetsAt !== undefined) normalized.resetsAt = resetsAt;
|
|
257
|
+
if (resetsAt !== undefined && durationMs !== undefined) {
|
|
258
|
+
normalized.windowStart = resetsAt - durationMs;
|
|
259
|
+
}
|
|
260
|
+
const status = asString(limit["status"]);
|
|
261
|
+
if (status !== undefined) normalized.status = status;
|
|
262
|
+
windows.push(normalized);
|
|
263
|
+
});
|
|
264
|
+
});
|
|
265
|
+
|
|
266
|
+
if (windows.length === 0) {
|
|
267
|
+
return {
|
|
268
|
+
kind: "unavailable",
|
|
269
|
+
observedAt,
|
|
270
|
+
reason: "no provider reported a readable allowance",
|
|
271
|
+
since: observedAt,
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
return { kind: "ok", observedAt, windows };
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// -------------------------------------------------------------------- adapters
|
|
278
|
+
|
|
279
|
+
export type UsageCommandResult =
|
|
280
|
+
| { kind: "completed"; exitCode: number; stdout: string; stderr: string }
|
|
281
|
+
| { kind: "timeout" }
|
|
282
|
+
| { kind: "unavailable"; reason: string };
|
|
283
|
+
|
|
284
|
+
async function runOmpUsage(timeoutMs: number): Promise<UsageCommandResult> {
|
|
285
|
+
try {
|
|
286
|
+
const proc = Bun.spawn(["omp", "usage", "--json"], {
|
|
287
|
+
// Closed stdin: a command that never reads it sees EOF instead of an
|
|
288
|
+
// open pipe nobody ends.
|
|
289
|
+
stdin: "ignore",
|
|
290
|
+
stdout: "pipe",
|
|
291
|
+
stderr: "pipe",
|
|
292
|
+
});
|
|
293
|
+
let timedOut = false;
|
|
294
|
+
const timer = setTimeout(() => {
|
|
295
|
+
timedOut = true;
|
|
296
|
+
proc.kill("SIGKILL");
|
|
297
|
+
}, timeoutMs);
|
|
298
|
+
try {
|
|
299
|
+
const [exitCode, stdout, stderr] = await Promise.all([
|
|
300
|
+
proc.exited,
|
|
301
|
+
new Response(proc.stdout).text(),
|
|
302
|
+
new Response(proc.stderr).text(),
|
|
303
|
+
]);
|
|
304
|
+
if (timedOut) return { kind: "timeout" };
|
|
305
|
+
return { kind: "completed", exitCode, stdout, stderr };
|
|
306
|
+
} finally {
|
|
307
|
+
clearTimeout(timer);
|
|
308
|
+
}
|
|
309
|
+
} catch (err) {
|
|
310
|
+
// `omp` missing from PATH lands here, and it is a perfectly ordinary state
|
|
311
|
+
// for a conductor CLI invoked from a bare shell.
|
|
312
|
+
return { kind: "unavailable", reason: err instanceof Error ? err.message : String(err) };
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/** Bounded so a provider that dumps a stack trace cannot own the status line. */
|
|
317
|
+
function firstLine(text: string, max = 120): string {
|
|
318
|
+
const line = text.replace(/\s+/g, " ").trim();
|
|
319
|
+
return line.length > max ? `${line.slice(0, max - 1)}…` : line;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* The first adapter: `omp usage --json`.
|
|
324
|
+
*
|
|
325
|
+
* The command is injectable because every interesting case — a non-zero exit,
|
|
326
|
+
* a truncated stdout, a provider that reports three windows for one account —
|
|
327
|
+
* has to be pinned without the machine running the tests owning a
|
|
328
|
+
* subscription.
|
|
329
|
+
*/
|
|
330
|
+
export function ompUsageProvider(
|
|
331
|
+
run: (timeoutMs: number) => Promise<UsageCommandResult> = runOmpUsage,
|
|
332
|
+
timeoutMs = USAGE_PROBE_TIMEOUT_MS,
|
|
333
|
+
): UsageProvider {
|
|
334
|
+
return {
|
|
335
|
+
async read(now: number): Promise<UsageReading> {
|
|
336
|
+
const result = await run(timeoutMs);
|
|
337
|
+
if (result.kind === "timeout") {
|
|
338
|
+
return {
|
|
339
|
+
kind: "unavailable",
|
|
340
|
+
observedAt: now,
|
|
341
|
+
reason: `omp usage --json did not answer within ${String(Math.round(timeoutMs / 1000))}s`,
|
|
342
|
+
since: now,
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
if (result.kind === "unavailable") {
|
|
346
|
+
return {
|
|
347
|
+
kind: "unavailable",
|
|
348
|
+
observedAt: now,
|
|
349
|
+
reason: `omp usage --json could not run: ${firstLine(result.reason)}`,
|
|
350
|
+
since: now,
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
if (result.exitCode !== 0) {
|
|
354
|
+
const detail = firstLine(result.stderr);
|
|
355
|
+
return {
|
|
356
|
+
kind: "unavailable",
|
|
357
|
+
observedAt: now,
|
|
358
|
+
reason: `omp usage --json exited ${String(result.exitCode)}${detail === "" ? "" : `: ${detail}`}`,
|
|
359
|
+
since: now,
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
let payload: unknown;
|
|
363
|
+
try {
|
|
364
|
+
payload = JSON.parse(result.stdout);
|
|
365
|
+
} catch (err) {
|
|
366
|
+
return {
|
|
367
|
+
kind: "unavailable",
|
|
368
|
+
observedAt: now,
|
|
369
|
+
reason: `omp usage --json returned unparseable output: ${firstLine(err instanceof Error ? err.message : String(err))}`,
|
|
370
|
+
since: now,
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
return normalizeUsagePayload(payload, now);
|
|
374
|
+
},
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
// ----------------------------------------------------------------------- cache
|
|
379
|
+
|
|
380
|
+
/**
|
|
381
|
+
* TTL cache with explicit invalidation, plus one rule the TTL alone cannot
|
|
382
|
+
* express: a reading whose window has already reset is stale regardless of its
|
|
383
|
+
* age. That is what makes "admission resumes once the allowance rolls over"
|
|
384
|
+
* true at the rollover rather than up to a TTL after it.
|
|
385
|
+
*
|
|
386
|
+
* Concurrent callers share one in-flight request. An admission pass and a
|
|
387
|
+
* board repaint landing in the same second must cost one provider call, not
|
|
388
|
+
* two — "do not make every candidate trigger a provider request" is an
|
|
389
|
+
* acceptance criterion of #110, not an optimisation.
|
|
390
|
+
*/
|
|
391
|
+
export function cacheUsage(
|
|
392
|
+
provider: UsageProvider,
|
|
393
|
+
opts: { ttlMs?: number; errorTtlMs?: number } = {},
|
|
394
|
+
): UsageSource {
|
|
395
|
+
const ttlMs = opts.ttlMs ?? USAGE_TTL_MS;
|
|
396
|
+
const errorTtlMs = opts.errorTtlMs ?? USAGE_ERROR_TTL_MS;
|
|
397
|
+
let cached: UsageReading | undefined;
|
|
398
|
+
let inFlight: Promise<UsageReading> | undefined;
|
|
399
|
+
let unreadableSince: number | undefined;
|
|
400
|
+
|
|
401
|
+
const usable = (reading: UsageReading, now: number): boolean => {
|
|
402
|
+
const ttl = reading.kind === "ok" ? ttlMs : errorTtlMs;
|
|
403
|
+
if (now - reading.observedAt >= ttl) return false;
|
|
404
|
+
if (reading.kind === "ok") {
|
|
405
|
+
return !reading.windows.some((w) => w.resetsAt !== undefined && w.resetsAt <= now);
|
|
406
|
+
}
|
|
407
|
+
return true;
|
|
408
|
+
};
|
|
409
|
+
|
|
410
|
+
return {
|
|
411
|
+
async read(now = Date.now()): Promise<UsageReading> {
|
|
412
|
+
if (cached !== undefined && usable(cached, now)) return cached;
|
|
413
|
+
inFlight ??= provider
|
|
414
|
+
.read(now)
|
|
415
|
+
.catch(
|
|
416
|
+
(err: unknown): UsageReading => ({
|
|
417
|
+
kind: "unavailable",
|
|
418
|
+
observedAt: now,
|
|
419
|
+
// The provider contract says it never throws. If one ever does,
|
|
420
|
+
// the tick still has to survive it as data.
|
|
421
|
+
reason: `usage provider threw: ${firstLine(err instanceof Error ? err.message : String(err))}`,
|
|
422
|
+
since: now,
|
|
423
|
+
}),
|
|
424
|
+
)
|
|
425
|
+
.then((reading) => {
|
|
426
|
+
if (reading.kind === "unavailable") {
|
|
427
|
+
unreadableSince ??= reading.since;
|
|
428
|
+
cached = { ...reading, since: unreadableSince };
|
|
429
|
+
} else {
|
|
430
|
+
unreadableSince = undefined;
|
|
431
|
+
cached = reading;
|
|
432
|
+
}
|
|
433
|
+
return cached;
|
|
434
|
+
})
|
|
435
|
+
.finally(() => {
|
|
436
|
+
inFlight = undefined;
|
|
437
|
+
});
|
|
438
|
+
return inFlight;
|
|
439
|
+
},
|
|
440
|
+
invalidate(): void {
|
|
441
|
+
cached = undefined;
|
|
442
|
+
// The unreadable streak deliberately survives. An operator asking for a
|
|
443
|
+
// re-read has not repaired anything, and forgetting how long the meter
|
|
444
|
+
// has been broken is how a permanently blind guard stays open forever.
|
|
445
|
+
},
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
let shared: UsageSource | undefined;
|
|
450
|
+
|
|
451
|
+
/** One cache per process. `status`, `board` and the dispatch tick all want the
|
|
452
|
+
* same reading, and three caches would mean three provider calls. */
|
|
453
|
+
export function sharedUsageSource(): UsageSource {
|
|
454
|
+
shared ??= cacheUsage(ompUsageProvider());
|
|
455
|
+
return shared;
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
// -------------------------------------------------------------------- decision
|
|
459
|
+
|
|
460
|
+
/**
|
|
461
|
+
* Every distinguishable answer the guard can give. They exist separately
|
|
462
|
+
* because each one sends an operator somewhere different: `at-cap` is normal
|
|
463
|
+
* flow control, `window-missing` is a config edit, `blind` is a broken host.
|
|
464
|
+
* Collapsing them into a boolean is how "the fleet is not claiming" becomes
|
|
465
|
+
* unexplainable.
|
|
466
|
+
*/
|
|
467
|
+
export type PlanUsageState =
|
|
468
|
+
| "unmetered"
|
|
469
|
+
| "ok"
|
|
470
|
+
| "at-cap"
|
|
471
|
+
| "window-missing"
|
|
472
|
+
| "window-ambiguous"
|
|
473
|
+
| "window-uncomparable"
|
|
474
|
+
| "unreadable"
|
|
475
|
+
| "blind";
|
|
476
|
+
|
|
477
|
+
export interface PlanUsageStatus {
|
|
478
|
+
state: PlanUsageState;
|
|
479
|
+
/**
|
|
480
|
+
* Whether new claims are held. Stated once, here, so no renderer and no gate
|
|
481
|
+
* re-derives the availability policy and disagrees with the other.
|
|
482
|
+
*/
|
|
483
|
+
blocking: boolean;
|
|
484
|
+
cap?: PlanUsageCap;
|
|
485
|
+
window?: UsageWindow;
|
|
486
|
+
/** Present only when a real figure was read. Never a filled-in zero. */
|
|
487
|
+
usedFraction?: number;
|
|
488
|
+
resetsAt?: number;
|
|
489
|
+
/** When the underlying reading was taken. */
|
|
490
|
+
observedAt?: number;
|
|
491
|
+
/** The operator-facing sentence `status` and `board` render verbatim. */
|
|
492
|
+
detail: string;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/** Where a configured window id landed in a reading. */
|
|
496
|
+
export type WindowSelection =
|
|
497
|
+
| { kind: "found"; window: UsageWindow }
|
|
498
|
+
| { kind: "missing"; available: string[] }
|
|
499
|
+
| { kind: "ambiguous"; candidates: string[] };
|
|
500
|
+
|
|
501
|
+
/**
|
|
502
|
+
* Resolve a configured window id against a reading.
|
|
503
|
+
*
|
|
504
|
+
* Exact `id` first, because that is the only unique key the payload has. A
|
|
505
|
+
* bare window key (`7d`) is accepted as a convenience, but *only* when it
|
|
506
|
+
* resolves to exactly one allowance: on an Anthropic account `7d` matches both
|
|
507
|
+
* `anthropic:7d` and the tier-scoped `anthropic:7d:fable`, and picking either
|
|
508
|
+
* is the silent binding this guard was specified to refuse.
|
|
509
|
+
*/
|
|
510
|
+
export function selectUsageWindow(
|
|
511
|
+
windows: readonly UsageWindow[],
|
|
512
|
+
windowId: string,
|
|
513
|
+
): WindowSelection {
|
|
514
|
+
const exact = windows.filter((w) => w.id === windowId);
|
|
515
|
+
if (exact.length === 1) return { kind: "found", window: exact[0]! };
|
|
516
|
+
if (exact.length > 1) return { kind: "ambiguous", candidates: exact.map((w) => w.id) };
|
|
517
|
+
|
|
518
|
+
const byWindow = windows.filter((w) => w.windowId === windowId);
|
|
519
|
+
if (byWindow.length === 1) return { kind: "found", window: byWindow[0]! };
|
|
520
|
+
if (byWindow.length > 1) return { kind: "ambiguous", candidates: byWindow.map((w) => w.id) };
|
|
521
|
+
|
|
522
|
+
return { kind: "missing", available: windows.slice(0, NAMED_WINDOW_LIMIT).map((w) => w.id) };
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
function percent(fraction: number): string {
|
|
526
|
+
const whole = Math.round(fraction * 100);
|
|
527
|
+
// A live reading of 0.4% must not render as `0%`: that is character-for-
|
|
528
|
+
// character the fabricated zero this guard refuses to display (#110).
|
|
529
|
+
if (whole === 0 && fraction > 0) return "<1%";
|
|
530
|
+
return `${String(whole)}%`;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function duration(ms: number): string {
|
|
534
|
+
const minutes = Math.max(0, Math.floor(ms / 60_000));
|
|
535
|
+
if (minutes < 1) return "<1m";
|
|
536
|
+
if (minutes < 60) return `${String(minutes)}m`;
|
|
537
|
+
const hours = Math.floor(minutes / 60);
|
|
538
|
+
if (hours < 24) return `${String(hours)}h ${String(minutes % 60)}m`;
|
|
539
|
+
return `${String(Math.floor(hours / 24))}d ${String(hours % 24)}h`;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
function resetsText(resetsAt: number | undefined, now: number): string {
|
|
543
|
+
if (resetsAt === undefined) return "no reset time reported";
|
|
544
|
+
return resetsAt <= now ? "resetting now" : `resets in ${duration(resetsAt - now)}`;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/** Appended only when the figure is old enough that calling it current would
|
|
548
|
+
* be a lie. Silence below the threshold keeps the common line short. */
|
|
549
|
+
function ageSuffix(observedAt: number, now: number): string {
|
|
550
|
+
const age = now - observedAt;
|
|
551
|
+
return age >= USAGE_STALE_MS ? ` · stale, read ${duration(age)} ago` : "";
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
function unmeteredStatus(): PlanUsageStatus {
|
|
555
|
+
return {
|
|
556
|
+
state: "unmetered",
|
|
557
|
+
blocking: false,
|
|
558
|
+
detail: "unmetered — no plan allowance cap configured",
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* The whole availability policy, in one pure function.
|
|
564
|
+
*
|
|
565
|
+
* | state | holds new claims | why |
|
|
566
|
+
* | --- | --- | --- |
|
|
567
|
+
* | `unmetered` | no | the operator did not ask for this guard |
|
|
568
|
+
* | `ok` | no | below the threshold |
|
|
569
|
+
* | `at-cap` | **yes** | the guard doing its job |
|
|
570
|
+
* | `window-missing` | **yes** | fail closed on config, like the rest of this package |
|
|
571
|
+
* | `window-ambiguous` | **yes** | as above; never guess which allowance was meant |
|
|
572
|
+
* | `window-uncomparable` | **yes** | the window exists and cannot be compared |
|
|
573
|
+
* | `unreadable` | no | a transient read error must not stall a fleet |
|
|
574
|
+
* | `blind` | **yes** | unreadable for longer than {@link USAGE_BLIND_GRACE_MS} |
|
|
575
|
+
*
|
|
576
|
+
* The split that matters is between the last two rows and the config rows. A
|
|
577
|
+
* *read error* is transient by nature, so it is tolerated for a bounded time
|
|
578
|
+
* and then stops being tolerated. A *successful read that does not contain the
|
|
579
|
+
* configured window* is not a read error: the source answered, and it says the
|
|
580
|
+
* operator named something that is not there. That is a config fault, and this
|
|
581
|
+
* package fails closed on config faults everywhere else (see
|
|
582
|
+
* `normalizeAuthority` in `config.ts`). It recovers by itself the moment a
|
|
583
|
+
* reading contains the window again, so one bad tick costs one tick.
|
|
584
|
+
*/
|
|
585
|
+
export function planUsageStatus(
|
|
586
|
+
cap: PlanUsageCap | null,
|
|
587
|
+
reading: UsageReading,
|
|
588
|
+
now: number,
|
|
589
|
+
graceMs = USAGE_BLIND_GRACE_MS,
|
|
590
|
+
): PlanUsageStatus {
|
|
591
|
+
if (cap === null) return unmeteredStatus();
|
|
592
|
+
|
|
593
|
+
if (reading.kind === "unavailable") {
|
|
594
|
+
const blindFor = now - reading.since;
|
|
595
|
+
if (blindFor >= graceMs) {
|
|
596
|
+
return {
|
|
597
|
+
state: "blind",
|
|
598
|
+
blocking: true,
|
|
599
|
+
cap,
|
|
600
|
+
observedAt: reading.observedAt,
|
|
601
|
+
detail: `unavailable for ${duration(blindFor)} — ${reading.reason}; holding new claims`,
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
return {
|
|
605
|
+
state: "unreadable",
|
|
606
|
+
blocking: false,
|
|
607
|
+
cap,
|
|
608
|
+
observedAt: reading.observedAt,
|
|
609
|
+
detail: `unavailable — ${reading.reason}; retrying, guard open for up to ${duration(graceMs - blindFor)}`,
|
|
610
|
+
};
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
const selection = selectUsageWindow(reading.windows, cap.windowId);
|
|
614
|
+
if (selection.kind === "missing") {
|
|
615
|
+
return {
|
|
616
|
+
state: "window-missing",
|
|
617
|
+
blocking: true,
|
|
618
|
+
cap,
|
|
619
|
+
observedAt: reading.observedAt,
|
|
620
|
+
detail:
|
|
621
|
+
`window "${cap.windowId}" is not in this reading — holding new claims. ` +
|
|
622
|
+
`Reported: ${selection.available.join(", ")}`,
|
|
623
|
+
};
|
|
624
|
+
}
|
|
625
|
+
if (selection.kind === "ambiguous") {
|
|
626
|
+
return {
|
|
627
|
+
state: "window-ambiguous",
|
|
628
|
+
blocking: true,
|
|
629
|
+
cap,
|
|
630
|
+
observedAt: reading.observedAt,
|
|
631
|
+
detail:
|
|
632
|
+
`window "${cap.windowId}" matches ${selection.candidates.join(", ")} — ` +
|
|
633
|
+
"holding new claims until the cap names exactly one",
|
|
634
|
+
};
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
const window = selection.window;
|
|
638
|
+
const threshold = percent(cap.maxUsedFraction);
|
|
639
|
+
if (window.usedFraction === undefined) {
|
|
640
|
+
return {
|
|
641
|
+
state: "window-uncomparable",
|
|
642
|
+
blocking: true,
|
|
643
|
+
cap,
|
|
644
|
+
window,
|
|
645
|
+
observedAt: window.observedAt,
|
|
646
|
+
detail:
|
|
647
|
+
`window "${window.id}" reports no comparable fraction ` +
|
|
648
|
+
`(unit ${window.unit}, used ${String(window.used)}, limit ${window.limit === undefined ? "unreported" : String(window.limit)}) — ` +
|
|
649
|
+
"holding new claims",
|
|
650
|
+
};
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
const used = percent(window.usedFraction);
|
|
654
|
+
const statusNote = window.status === undefined || window.status === "ok" ? "" : ` · ${window.status}`;
|
|
655
|
+
const base: PlanUsageStatus = {
|
|
656
|
+
state: window.usedFraction >= cap.maxUsedFraction ? "at-cap" : "ok",
|
|
657
|
+
blocking: window.usedFraction >= cap.maxUsedFraction,
|
|
658
|
+
cap,
|
|
659
|
+
window,
|
|
660
|
+
usedFraction: window.usedFraction,
|
|
661
|
+
observedAt: window.observedAt,
|
|
662
|
+
detail: "",
|
|
663
|
+
};
|
|
664
|
+
if (window.resetsAt !== undefined) base.resetsAt = window.resetsAt;
|
|
665
|
+
base.detail =
|
|
666
|
+
base.state === "at-cap"
|
|
667
|
+
? `${used} / ${threshold} of ${window.id} used — holding new claims · ${resetsText(window.resetsAt, now)}${statusNote}${ageSuffix(window.observedAt, now)}`
|
|
668
|
+
: `${used} / ${threshold} of ${window.id} used · ${resetsText(window.resetsAt, now)}${statusNote}${ageSuffix(window.observedAt, now)}`;
|
|
669
|
+
return base;
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
/**
|
|
673
|
+
* Read and decide in one call. An unmetered project never spawns the provider:
|
|
674
|
+
* a fleet that does not use this guard must not pay a subprocess for it every
|
|
675
|
+
* time somebody runs `status`.
|
|
676
|
+
*/
|
|
677
|
+
export async function readPlanUsage(
|
|
678
|
+
cap: PlanUsageCap | null,
|
|
679
|
+
source: UsageSource,
|
|
680
|
+
now = Date.now(),
|
|
681
|
+
): Promise<PlanUsageStatus> {
|
|
682
|
+
if (cap === null) return unmeteredStatus();
|
|
683
|
+
return planUsageStatus(cap, await source.read(now), now);
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
/**
|
|
687
|
+
* The compact form for the board's admission line, where the whole fleet has
|
|
688
|
+
* to fit on one row. Derived from the same status object as the long form so
|
|
689
|
+
* the two cannot disagree about whether the guard is active.
|
|
690
|
+
*/
|
|
691
|
+
export function planUsageBadge(status: PlanUsageStatus | undefined): string {
|
|
692
|
+
if (status === undefined) return "plan not read";
|
|
693
|
+
let core: string;
|
|
694
|
+
switch (status.state) {
|
|
695
|
+
case "unmetered":
|
|
696
|
+
core = "plan unmetered";
|
|
697
|
+
break;
|
|
698
|
+
case "ok":
|
|
699
|
+
case "at-cap":
|
|
700
|
+
// No `?? 0` fallback anywhere on this path. Both fields are always set
|
|
701
|
+
// in these two states, and a default would put the one number this
|
|
702
|
+
// guard must never invent — a zero — on an operator's screen (#110).
|
|
703
|
+
core =
|
|
704
|
+
status.usedFraction === undefined || status.cap === undefined
|
|
705
|
+
? "plan read incomplete"
|
|
706
|
+
: `plan ${percent(status.usedFraction)}/${percent(status.cap.maxUsedFraction)}`;
|
|
707
|
+
break;
|
|
708
|
+
case "unreadable":
|
|
709
|
+
case "blind":
|
|
710
|
+
core = "plan unavailable";
|
|
711
|
+
break;
|
|
712
|
+
default:
|
|
713
|
+
// Every remaining state is "the configured window did not resolve", and
|
|
714
|
+
// naming it is the point: an operator who sees `plan unmetered` when
|
|
715
|
+
// they configured a cap will not go looking for a typo.
|
|
716
|
+
core = `plan window "${status.cap?.windowId ?? "?"}" ${status.state.replace("window-", "")}`;
|
|
717
|
+
break;
|
|
718
|
+
}
|
|
719
|
+
return status.blocking ? `${core} · HELD` : core;
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
/** The long form for `status`. Absent means nobody read the source on this
|
|
723
|
+
* path, which is its own honest answer and not "0% used". */
|
|
724
|
+
export function planUsageLine(status: PlanUsageStatus | undefined): string {
|
|
725
|
+
return status?.detail ?? "not read";
|
|
726
|
+
}
|