@opsee/cli 0.11.12 → 0.11.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +143 -9
- package/package.json +2 -2
- package/src/args.ts +167 -15
- package/src/cli.ts +62 -6
- package/src/commands/account-usage.ts +106 -0
- package/src/commands/account.ts +134 -1
- package/src/commands/claude-launcher.ts +183 -0
- package/src/commands/foreman-debug.ts +11 -4
- package/src/commands/foreman-up.ts +184 -4
- package/src/commands/foreman.ts +133 -11
- package/src/commands/init.ts +4 -8
- package/src/foreman/account.ts +106 -13
- package/src/foreman/claude-worker-adapter.ts +100 -1
- package/src/foreman/config-skeleton.ts +167 -0
- package/src/foreman/core/run.ts +157 -7
- package/src/foreman/core/scheduler.ts +139 -0
- package/src/foreman/core/short-turn.ts +80 -0
- package/src/foreman/core/start-discovery.ts +158 -0
- package/src/foreman/core/text.ts +29 -0
- package/src/foreman/core/triage.ts +10 -47
- package/src/foreman/core/verifier.ts +51 -4
- package/src/foreman/credential-store.ts +247 -0
- package/src/foreman/run-recipe.ts +37 -8
- package/src/foreman/usage-activity.ts +102 -0
- package/src/foreman/usage-format.ts +107 -0
- package/src/foreman/usage-poller.ts +489 -0
- package/src/foreman/usage-store.ts +213 -0
- package/src/foreman/usage.ts +257 -0
- package/src/foreman/vendor.ts +23 -0
- package/src/foreman/worker-adapter.ts +12 -0
- package/src/init/prompt.ts +17 -18
- package/src/init/run-recipe-config.ts +11 -12
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { mergeObservation, type AccountUsage, type ActivityTurn, type UsageWindow } from "./usage.js";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Where per-Account usage is kept (the multi-account headroom design, §2).
|
|
8
|
+
*
|
|
9
|
+
* Deliberately **not** a field on `Account`. The accounts file is a registry: written rarely, by
|
|
10
|
+
* humans, through `FileAccountStore.save`, whose own comment says what it does not buy — the write
|
|
11
|
+
* is atomic but the read-modify-write is not, so "the later save wins whole". Usage is telemetry,
|
|
12
|
+
* written constantly by a poller and by every Run that sees a `rate_limit_event`, and putting it
|
|
13
|
+
* there would lose an operator's `account set --cap` to a poll.
|
|
14
|
+
*
|
|
15
|
+
* There is no `save(all)` on this interface on purpose: a writer states what it observed about one
|
|
16
|
+
* Account and the store folds it in (`mergeObservation`), so no writer can write a snapshot of the
|
|
17
|
+
* whole world and flatten another's Account.
|
|
18
|
+
*/
|
|
19
|
+
export interface UsageStore {
|
|
20
|
+
/** Every Account the store knows about. Unreadable records are skipped, never thrown. */
|
|
21
|
+
load(): AccountUsage[];
|
|
22
|
+
loadOne(account: string): AccountUsage | undefined;
|
|
23
|
+
/** Folds one observation in and returns the merged record. `activity` carries the turns this
|
|
24
|
+
* Foreman ran
|
|
25
|
+
* (usage-activity.ts); a write may carry windows, turns, or both. */
|
|
26
|
+
observe(account: string, windows: Record<string, UsageWindow>, now: number, activity?: { turns: ActivityTurn[] }): AccountUsage;
|
|
27
|
+
/** Notes that a poll failed, touching no window (a failing poll means the Foreman knows less, not
|
|
28
|
+
* that the Account is spent). `backoffUntil` is set when the endpoint rate-limited us; nothing
|
|
29
|
+
* asks again before it. */
|
|
30
|
+
noteFailure(account: string, at: Date, reason: string, backoffUntil?: Date): AccountUsage;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** The store's own directory, overridable for tests and for a daemon run out of tree. Sibling of
|
|
34
|
+
* the accounts file and of the credentials file the mcp auth module keeps, never inside either. */
|
|
35
|
+
export function usageDirPath(): string {
|
|
36
|
+
return process.env.OPSEE_FOREMAN_USAGE_DIR || join(homedir(), ".opsee", "foreman-usage");
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
interface UsageFile {
|
|
40
|
+
version: 1;
|
|
41
|
+
usage: AccountUsage;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* The Account names this store will touch: exactly `NAME_PATTERN` from account.ts, re-stated as a
|
|
46
|
+
* file-name guard rather than imported, because what matters here is not "is this a valid Account"
|
|
47
|
+
* but "is this safe to join onto a path". A registered Account always passes; a name from a
|
|
48
|
+
* hand-edited file or a future caller that does not is refused rather than escaping the directory.
|
|
49
|
+
*/
|
|
50
|
+
const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
51
|
+
|
|
52
|
+
function safeName(account: string): string {
|
|
53
|
+
if (!SAFE_NAME.test(account) || account.includes("..")) {
|
|
54
|
+
throw new Error(`Account name "${account}" cannot be used as a usage file name`);
|
|
55
|
+
}
|
|
56
|
+
return account;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export class MemoryUsageStore implements UsageStore {
|
|
60
|
+
private records = new Map<string, AccountUsage>();
|
|
61
|
+
|
|
62
|
+
load(): AccountUsage[] {
|
|
63
|
+
return [...this.records.values()].map(clone);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
loadOne(account: string): AccountUsage | undefined {
|
|
67
|
+
const record = this.records.get(account);
|
|
68
|
+
return record ? clone(record) : undefined;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
observe(account: string, windows: Record<string, UsageWindow>, now: number, activity?: { turns: ActivityTurn[] }): AccountUsage {
|
|
72
|
+
const merged = mergeObservation(this.records.get(account) ?? empty(account), windows, now, activity);
|
|
73
|
+
this.records.set(account, merged);
|
|
74
|
+
return clone(merged);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
noteFailure(account: string, at: Date, reason: string, backoffUntil?: Date): AccountUsage {
|
|
78
|
+
const record = {
|
|
79
|
+
...(this.records.get(account) ?? empty(account)),
|
|
80
|
+
pollFailedAt: at.toISOString(),
|
|
81
|
+
pollFailure: reason,
|
|
82
|
+
...(backoffUntil ? { pollBackoffUntil: backoffUntil.toISOString() } : {}),
|
|
83
|
+
};
|
|
84
|
+
this.records.set(account, record);
|
|
85
|
+
return clone(record);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* One JSON file per Account under one directory, written through a temp file that is renamed over
|
|
91
|
+
* the target — the same idiom, envelope and modes as `FileAccountStore`.
|
|
92
|
+
*
|
|
93
|
+
* One file per Account rather than one file for all of them because this store has two writers: the
|
|
94
|
+
* poller in the daemon, and whichever Run saw an in-band observation. Separate files mean they can
|
|
95
|
+
* never lose each other's Accounts; `mergeObservation` is what settles the case where they write
|
|
96
|
+
* the same Account's window.
|
|
97
|
+
*
|
|
98
|
+
* A record it cannot read is no record. Usage is telemetry, and telemetry must not be able to stop
|
|
99
|
+
* a Run: the Foreman's fallback is the reactive behaviour it had before any of this existed.
|
|
100
|
+
*/
|
|
101
|
+
export class FileUsageStore implements UsageStore {
|
|
102
|
+
constructor(private readonly dir: string) {}
|
|
103
|
+
|
|
104
|
+
load(): AccountUsage[] {
|
|
105
|
+
if (!existsSync(this.dir)) return [];
|
|
106
|
+
let entries: string[];
|
|
107
|
+
try {
|
|
108
|
+
entries = readdirSync(this.dir);
|
|
109
|
+
} catch {
|
|
110
|
+
return [];
|
|
111
|
+
}
|
|
112
|
+
const records: AccountUsage[] = [];
|
|
113
|
+
for (const entry of entries) {
|
|
114
|
+
if (!entry.endsWith(".json")) continue;
|
|
115
|
+
const record = this.read(join(this.dir, entry));
|
|
116
|
+
if (record) records.push(record);
|
|
117
|
+
}
|
|
118
|
+
return records;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
loadOne(account: string): AccountUsage | undefined {
|
|
122
|
+
return this.read(this.pathFor(account));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
observe(account: string, windows: Record<string, UsageWindow>, now: number, activity?: { turns: ActivityTurn[] }): AccountUsage {
|
|
126
|
+
// Read-modify-write against this Account's own file. Two writers on one Account can still
|
|
127
|
+
// interleave here; the merge rule is what makes the outcome of that deterministic rather than
|
|
128
|
+
// whichever process happened to finish last, and a lost in-band observation costs one tick.
|
|
129
|
+
const merged = mergeObservation(this.loadOne(account) ?? empty(account), windows, now, activity);
|
|
130
|
+
this.write(account, merged);
|
|
131
|
+
return merged;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
noteFailure(account: string, at: Date, reason: string, backoffUntil?: Date): AccountUsage {
|
|
135
|
+
const record = {
|
|
136
|
+
...(this.loadOne(account) ?? empty(account)),
|
|
137
|
+
pollFailedAt: at.toISOString(),
|
|
138
|
+
pollFailure: reason,
|
|
139
|
+
...(backoffUntil ? { pollBackoffUntil: backoffUntil.toISOString() } : {}),
|
|
140
|
+
};
|
|
141
|
+
this.write(account, record);
|
|
142
|
+
return record;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
private pathFor(account: string): string {
|
|
146
|
+
return join(this.dir, `${safeName(account)}.json`);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
private read(path: string): AccountUsage | undefined {
|
|
150
|
+
let text: string;
|
|
151
|
+
try {
|
|
152
|
+
// A directory named `*.json`, a dangling symlink, a file another user owns: all of them are
|
|
153
|
+
// "no usage", the same as a file whose JSON does not parse.
|
|
154
|
+
if (!statSync(path).isFile()) return undefined;
|
|
155
|
+
text = readFileSync(path, "utf-8");
|
|
156
|
+
} catch {
|
|
157
|
+
return undefined;
|
|
158
|
+
}
|
|
159
|
+
let parsed: unknown;
|
|
160
|
+
try {
|
|
161
|
+
parsed = JSON.parse(text);
|
|
162
|
+
} catch {
|
|
163
|
+
return undefined;
|
|
164
|
+
}
|
|
165
|
+
const file = parsed as Partial<UsageFile> | null;
|
|
166
|
+
if (!file || file.version !== 1) return undefined;
|
|
167
|
+
const usage = file.usage as Partial<AccountUsage> | undefined;
|
|
168
|
+
if (!usage || typeof usage.account !== "string" || typeof usage.windows !== "object" || usage.windows === null) {
|
|
169
|
+
return undefined;
|
|
170
|
+
}
|
|
171
|
+
return usage as AccountUsage;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
private write(account: string, usage: AccountUsage): void {
|
|
175
|
+
const path = this.pathFor(account);
|
|
176
|
+
mkdirSync(this.dir, { recursive: true, mode: 0o700 });
|
|
177
|
+
const file: UsageFile = { version: 1, usage };
|
|
178
|
+
const temp = `${path}.${process.pid}.tmp`;
|
|
179
|
+
try {
|
|
180
|
+
writeFileSync(temp, JSON.stringify(file, null, 2) + "\n", { mode: 0o600, encoding: "utf-8" });
|
|
181
|
+
renameSync(temp, path);
|
|
182
|
+
} catch (error) {
|
|
183
|
+
try {
|
|
184
|
+
rmSync(temp, { force: true });
|
|
185
|
+
} catch {
|
|
186
|
+
/* the write's error is the one worth reporting */
|
|
187
|
+
}
|
|
188
|
+
throw error;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function empty(account: string): AccountUsage {
|
|
194
|
+
return { account, windows: {} };
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function clone(usage: AccountUsage): AccountUsage {
|
|
198
|
+
return { ...usage, windows: { ...usage.windows }, ...(usage.activity ? { activity: { turns: [...usage.activity.turns] } } : {}) };
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Everything the store knows, keyed by Account name.
|
|
203
|
+
*
|
|
204
|
+
* Every reader wants this shape and none wants the list: the Run ranks Lanes by it, `account usage`
|
|
205
|
+
* and `account list` print by it, and the launcher chooses by it. All three had built the same Map
|
|
206
|
+
* by hand. An absent store is an empty Map rather than an error, which is what keeps usage an
|
|
207
|
+
* optimisation and never a dependency.
|
|
208
|
+
*/
|
|
209
|
+
export function usageByAccount(store: UsageStore | undefined): Map<string, AccountUsage> {
|
|
210
|
+
const known = new Map<string, AccountUsage>();
|
|
211
|
+
for (const record of store?.load() ?? []) known.set(record.account, record);
|
|
212
|
+
return known;
|
|
213
|
+
}
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import { mergeActivity } from "./usage-activity.js";
|
|
2
|
+
/**
|
|
3
|
+
* What the Foreman knows about how much of an Account's quota is gone (the multi-account headroom
|
|
4
|
+
* design, docs/superpowers/specs/2026-09-12-multi-account-headroom-design.md).
|
|
5
|
+
*
|
|
6
|
+
* Pure: no file system, no clock of its own, no vendor. Everything here is a function of a stored
|
|
7
|
+
* record and a `now`, so the ranking and holding decisions built on it (core/scheduler.ts,
|
|
8
|
+
* core/run.ts) are testable without a network or a credential.
|
|
9
|
+
*
|
|
10
|
+
* Nothing in this module reads a credential. The one module that does is usage-poller.ts, and it
|
|
11
|
+
* hands its result here as plain numbers (ADR-0013 and its amendment).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/** Where an observation came from, and — via `SOURCE_RANK` — how much it is trusted. */
|
|
15
|
+
export type UsageSourceKind = "polled" | "in_band";
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* How far an observation outranks another of the same window.
|
|
20
|
+
*
|
|
21
|
+
* `in_band` is highest because it is not an observation at all but a report: the vendor said it to a
|
|
22
|
+
* Worker that was running at that moment. `polled` is a measurement of an Account nobody is using.
|
|
23
|
+
*
|
|
24
|
+
* There is deliberately no third, estimated source. The design sketched one, from the Foreman's own
|
|
25
|
+
* turn counts, and usage-activity.ts says why it is not built: turns and dollars cannot be turned
|
|
26
|
+
* into a share of a window without the plan's limit, which nothing here knows.
|
|
27
|
+
*/
|
|
28
|
+
const SOURCE_RANK: Record<UsageSourceKind, number> = { in_band: 2, polled: 1 };
|
|
29
|
+
|
|
30
|
+
/** One rate-limit window as the vendor names it. */
|
|
31
|
+
export interface UsageWindow {
|
|
32
|
+
/** How much of the window is spent, 0–100. A vendor number above 100 is kept as given and
|
|
33
|
+
* clamped where it is used (`headroom`), so the stored record stays what was actually reported. */
|
|
34
|
+
usedPct: number;
|
|
35
|
+
/** When this window resets, ISO 8601. A window whose reset has passed is not a window. */
|
|
36
|
+
resetsAt: string;
|
|
37
|
+
/** When the observation was made, ISO 8601, so a reader can say "6m ago". */
|
|
38
|
+
observedAt: string;
|
|
39
|
+
source: UsageSourceKind;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** One turn the Foreman ran on an Account (usage-activity.ts). */
|
|
43
|
+
export interface ActivityTurn {
|
|
44
|
+
at: string;
|
|
45
|
+
/** What the vendor said the turn cost, when it said. A turn with no cost is still a turn. */
|
|
46
|
+
costUsd?: number;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Everything known about one Account's quota. Keyed by name, one file each (usage-store.ts). */
|
|
50
|
+
export interface AccountUsage {
|
|
51
|
+
account: string;
|
|
52
|
+
/**
|
|
53
|
+
* Keyed by the vendor's own `rateLimitType` — "five_hour", "seven_day", a per-model weekly — so a
|
|
54
|
+
* window the vendor invents needs no change here. Claude Code emits that string on
|
|
55
|
+
* `rate_limit_event`, and the poller reports whatever the endpoint names.
|
|
56
|
+
*/
|
|
57
|
+
windows: Record<string, UsageWindow>;
|
|
58
|
+
/** The last poll that failed, and its words. Carried so `account usage` can show why a number is
|
|
59
|
+
* stale, and deliberately never a reason to hold an Account back: a failing poll means the
|
|
60
|
+
* Foreman knows less, not that the Account is spent. */
|
|
61
|
+
pollFailedAt?: string;
|
|
62
|
+
pollFailure?: string;
|
|
63
|
+
/** Set when the usage endpoint rate-limited us: nothing asks again before this moment, not the
|
|
64
|
+
* poller and not a human running `account usage --refresh` (usage-poller.ts). */
|
|
65
|
+
pollBackoffUntil?: string;
|
|
66
|
+
/** What this Foreman has itself spent on the Account (usage-activity.ts). Advisory: it is never a
|
|
67
|
+
* window, never a percentage, and nothing ranks or holds an Account by it. */
|
|
68
|
+
activity?: { turns: ActivityTurn[] };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* How much of an Account is left to spend, or that nothing is known.
|
|
73
|
+
*
|
|
74
|
+
* `unknown` is not zero. An API-key Account reports no windows at all and is not rate limited by
|
|
75
|
+
* anything the Foreman can see; a subscription Account nobody has run yet is in the same position.
|
|
76
|
+
* Both must stay dispatchable — they rank below a measured Account (core/scheduler.ts) and are
|
|
77
|
+
* never held back.
|
|
78
|
+
*/
|
|
79
|
+
export type Headroom =
|
|
80
|
+
| { kind: "known"; remainingPct: number; tightest: string; resetsAt: string; observedAt: string }
|
|
81
|
+
| { kind: "unknown" };
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The furthest ahead a window's reset is believed.
|
|
85
|
+
*
|
|
86
|
+
* The longest window any vendor here reports is seven days, so anything past eight is not a window
|
|
87
|
+
* — it is a corrupt record, a hand-edited file, or a unit mistaken for another. Trusting one would
|
|
88
|
+
* hold an Account back until that date: `thresholdHold` has no natural ceiling of its own, where
|
|
89
|
+
* the vendor's own pauses are capped at `MAX_PAUSE_MS` (account.ts) for exactly this reason.
|
|
90
|
+
*/
|
|
91
|
+
export const MAX_WINDOW_MS = 8 * 24 * 60 * 60_000;
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* True while this window still describes something: it holds real numbers, and its reset is ahead
|
|
95
|
+
* of `now` but not absurdly so.
|
|
96
|
+
*
|
|
97
|
+
* Everything that is not believable reads as expired, which loses one observation rather than
|
|
98
|
+
* acting on it. A `usedPct` that is not a number, an unparseable `resetsAt`, a reset years out — a
|
|
99
|
+
* hand-edited file, a truncated write, a future vendor field — all of them mean the Foreman knows
|
|
100
|
+
* less about this Account, never that the Account should stop being used.
|
|
101
|
+
*/
|
|
102
|
+
function isLive(window: UsageWindow, now: number): boolean {
|
|
103
|
+
if (typeof window?.usedPct !== "number" || !Number.isFinite(window.usedPct)) return false;
|
|
104
|
+
const resetsAt = Date.parse(window?.resetsAt);
|
|
105
|
+
return Number.isFinite(resetsAt) && resetsAt > now && resetsAt - now <= MAX_WINDOW_MS;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The Account's headroom: what is left of its **tightest** live window.
|
|
110
|
+
*
|
|
111
|
+
* The minimum, not an average, because a limit is a limit — an Account at 20% of its five-hour and
|
|
112
|
+
* 98% of its seven-day has 2% to spend, and dispatching it as though it had 41% would walk it into
|
|
113
|
+
* the weekly wall. Windows that have already reset are ignored rather than trusted.
|
|
114
|
+
*/
|
|
115
|
+
export function headroom(usage: AccountUsage | undefined, now: number): Headroom {
|
|
116
|
+
if (!usage) return { kind: "unknown" };
|
|
117
|
+
let tightest: { name: string; window: UsageWindow } | undefined;
|
|
118
|
+
for (const [name, window] of Object.entries(usage.windows)) {
|
|
119
|
+
if (!isLive(window, now)) continue;
|
|
120
|
+
if (!tightest || window.usedPct > tightest.window.usedPct) tightest = { name, window };
|
|
121
|
+
}
|
|
122
|
+
if (!tightest) return { kind: "unknown" };
|
|
123
|
+
return {
|
|
124
|
+
kind: "known",
|
|
125
|
+
// Clamped both ways: a vendor percentage past 100 would otherwise become negative headroom,
|
|
126
|
+
// which sorts below an exhausted Account and reads as nonsense in `account usage`.
|
|
127
|
+
remainingPct: Math.max(0, Math.min(100, 100 - tightest.window.usedPct)),
|
|
128
|
+
tightest: tightest.name,
|
|
129
|
+
resetsAt: tightest.window.resetsAt,
|
|
130
|
+
observedAt: tightest.window.observedAt,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Folds one observation into a stored record, window by window, and returns a new record.
|
|
136
|
+
*
|
|
137
|
+
* This is where "in-band always wins" lives, once, rather than at each call site. Two processes
|
|
138
|
+
* write an Account's usage — the poller in the daemon and whichever Run saw a `rate_limit_event` —
|
|
139
|
+
* and per-Account files (usage-store.ts) keep them from clobbering each other's Accounts, but not
|
|
140
|
+
* each other's windows. This rule is what makes that collision deterministic instead of
|
|
141
|
+
* last-write-wins:
|
|
142
|
+
*
|
|
143
|
+
* - a window the record does not have is taken;
|
|
144
|
+
* - a window whose stored reset has passed is replaced by anything, since the stored observation
|
|
145
|
+
* describes a window that no longer exists and the incoming one describes the new one;
|
|
146
|
+
* - otherwise the incoming window must come from a source that ranks at least as high, and within
|
|
147
|
+
* one source it must be newer.
|
|
148
|
+
*/
|
|
149
|
+
export function mergeObservation(
|
|
150
|
+
stored: AccountUsage,
|
|
151
|
+
observed: Record<string, UsageWindow>,
|
|
152
|
+
now: number,
|
|
153
|
+
activity?: { turns: ActivityTurn[] },
|
|
154
|
+
): AccountUsage {
|
|
155
|
+
const windows: Record<string, UsageWindow> = { ...stored.windows };
|
|
156
|
+
for (const [name, incoming] of Object.entries(observed)) {
|
|
157
|
+
const current = windows[name];
|
|
158
|
+
if (current === undefined || !isLive(current, now) || wins(incoming, current)) {
|
|
159
|
+
windows[name] = incoming;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
// An observation arriving at all is the answer the failed poll was after, so the failure stops
|
|
163
|
+
// being worth reporting; `account usage` shows a live number instead of a stale one and a reason.
|
|
164
|
+
// A write that carries only activity turns is not an observation, so it leaves the failure standing.
|
|
165
|
+
//
|
|
166
|
+
// `pollBackoffUntil` goes with them. It used to ride through in the rest, so an Account that was
|
|
167
|
+
// rate limited once kept the window even after a poll succeeded — and with an unbounded
|
|
168
|
+
// `Retry-After` that was a state nothing could leave (`MAX_BACKOFF_MS`, usage-poller.ts).
|
|
169
|
+
const carriesWindows = Object.keys(observed).length > 0;
|
|
170
|
+
const { pollFailedAt, pollFailure, pollBackoffUntil, ...rest } = stored;
|
|
171
|
+
const base = carriesWindows ? rest : stored;
|
|
172
|
+
if (!activity) return { ...base, windows };
|
|
173
|
+
// Both sides' turns, so a Run recording one and the poller writing a window at the same moment
|
|
174
|
+
// cannot lose each other's work (usage-activity.ts `mergeActivity`).
|
|
175
|
+
return { ...base, windows, activity: { turns: mergeActivity(stored.activity?.turns, activity.turns, now) } };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Whether `incoming` replaces `current` for the same live window: a higher-ranked source, or the
|
|
179
|
+
* same source with something newer to say. */
|
|
180
|
+
function wins(incoming: UsageWindow, current: UsageWindow): boolean {
|
|
181
|
+
const byRank = SOURCE_RANK[incoming.source] - SOURCE_RANK[current.source];
|
|
182
|
+
if (byRank !== 0) return byRank > 0;
|
|
183
|
+
return Date.parse(incoming.observedAt) >= Date.parse(current.observedAt);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** A hold the Foreman has decided on: which window breached, by how much, and when it resets. */
|
|
187
|
+
export interface ThresholdHold {
|
|
188
|
+
window: string;
|
|
189
|
+
usedPct: number;
|
|
190
|
+
until: Date;
|
|
191
|
+
/** When the breaching observation was made. Carried here so a caller that reports the hold does
|
|
192
|
+
* not have to reach back into the record's windows to find out how old the number is. */
|
|
193
|
+
observedAt: string;
|
|
194
|
+
/** The breach in the words a human reads on `account list`: "93% of seven_day". */
|
|
195
|
+
reason: string;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Whether this Account should be held back before the vendor refuses it, and until when.
|
|
200
|
+
*
|
|
201
|
+
* **Derived, never stored.** A hold is a function of what is known about usage and of the policy in
|
|
202
|
+
* force at the moment it is asked, so raising `--threshold` lifts one with nothing to clear, a
|
|
203
|
+
* window resetting lifts one by itself, and there is no way to leave a stale hold on an Account
|
|
204
|
+
* that is fine. That is the whole reason this is not written into the Paused state beside a vendor
|
|
205
|
+
* refusal, which *is* stored because it is a fact about something that already happened.
|
|
206
|
+
*
|
|
207
|
+
* The tightest breaching window wins, and the hold lasts until **that** window's reset — not the
|
|
208
|
+
* soonest reset of any window, which would put the Account back in rotation still spent.
|
|
209
|
+
*
|
|
210
|
+
* Nothing known is never a hold (`headroom` answers `unknown`, an API-key Account reports nothing),
|
|
211
|
+
* and neither is a failed poll: knowing less about an Account is not evidence against it.
|
|
212
|
+
*/
|
|
213
|
+
export function thresholdHold(usage: AccountUsage | undefined, thresholdPct: number, now: number): ThresholdHold | undefined {
|
|
214
|
+
if (!usage) return undefined;
|
|
215
|
+
let worst: { name: string; window: UsageWindow } | undefined;
|
|
216
|
+
for (const [name, window] of Object.entries(usage.windows)) {
|
|
217
|
+
if (!isLive(window, now)) continue;
|
|
218
|
+
if (window.usedPct < thresholdPct) continue;
|
|
219
|
+
if (!worst || window.usedPct > worst.window.usedPct) worst = { name, window };
|
|
220
|
+
}
|
|
221
|
+
if (!worst) return undefined;
|
|
222
|
+
const usedPct = worst.window.usedPct;
|
|
223
|
+
return { window: worst.name, usedPct, until: new Date(worst.window.resetsAt), observedAt: worst.window.observedAt, reason: `${usedPct}% of ${worst.name}` };
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* How far off a moment is, for a reader: "now", "in 14m", "in 2h14m", "in 5d".
|
|
228
|
+
*
|
|
229
|
+
* Beside `describeAge` and for the same reason: a reset printed as an ISO timestamp makes a reader
|
|
230
|
+
* do arithmetic against a clock in another timezone to answer "can I use this Account yet".
|
|
231
|
+
*/
|
|
232
|
+
export function describeUntil(at: string, now: number): string {
|
|
233
|
+
const ms = Date.parse(at) - now;
|
|
234
|
+
if (!Number.isFinite(ms)) return "at an unknown time";
|
|
235
|
+
if (ms <= 0) return "now";
|
|
236
|
+
const minutes = Math.floor(ms / 60_000);
|
|
237
|
+
if (minutes < 60) return `in ${Math.max(1, minutes)}m`;
|
|
238
|
+
const hours = Math.floor(minutes / 60);
|
|
239
|
+
if (hours < 48) return `in ${hours}h${minutes % 60 === 0 ? "" : `${minutes % 60}m`}`;
|
|
240
|
+
return `in ${Math.floor(hours / 24)}d`;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** How long ago an observation was made, for a reader: "just now", "6m ago", "3h ago". Rounded
|
|
244
|
+
* down, so a number is never reported as fresher than it is. */
|
|
245
|
+
export function describeAge(observedAt: string, now: number): string {
|
|
246
|
+
const ms = now - Date.parse(observedAt);
|
|
247
|
+
// A timestamp that is not a date is not a fresh one: reporting it as "just now" would print an
|
|
248
|
+
// unknown freshness as the freshest there is, which is the one reading that misleads. A stamp in
|
|
249
|
+
// the *future* is the same mistake wearing a sign: `ms < 60_000` is true of every negative
|
|
250
|
+
// number, so a skewed clock read as the freshest possible number.
|
|
251
|
+
if (!Number.isFinite(ms) || ms < 0) return "at an unknown time";
|
|
252
|
+
if (ms < 60_000) return "just now";
|
|
253
|
+
const minutes = Math.floor(ms / 60_000);
|
|
254
|
+
if (minutes < 60) return `${minutes}m ago`;
|
|
255
|
+
const hours = Math.floor(minutes / 60);
|
|
256
|
+
return hours < 48 ? `${hours}h ago` : `${Math.floor(hours / 24)}d ago`;
|
|
257
|
+
}
|
package/src/foreman/vendor.ts
CHANGED
|
@@ -21,3 +21,26 @@ export const VENDOR_CONFIG_DIR_ENV: Readonly<Record<Vendor, string>> = {
|
|
|
21
21
|
export function isVendor(value: string): value is Vendor {
|
|
22
22
|
return (VENDORS as readonly string[]).includes(value);
|
|
23
23
|
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The binary each vendor's CLI is invoked as. The Worker Adapters already know theirs; this is for
|
|
27
|
+
* the one place outside them that has to launch a vendor — `account add --login`, which runs the
|
|
28
|
+
* vendor's own interactive login in a fresh config directory.
|
|
29
|
+
*/
|
|
30
|
+
export const VENDOR_BINARY: Readonly<Record<Vendor, string>> = {
|
|
31
|
+
claude: "claude",
|
|
32
|
+
codex: "codex",
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* The file a vendor keeps its credential in, **where it keeps one in a file at all**.
|
|
37
|
+
*
|
|
38
|
+
* Not a universal truth, and assuming it was is what shipped a broken `account add --login`: on
|
|
39
|
+
* macOS, Claude Code writes no such file and keeps the credential in a Keychain item it derives
|
|
40
|
+
* from the config directory instead. `credential-store.ts` decides which store applies, and is the
|
|
41
|
+
* only module that should use this.
|
|
42
|
+
*/
|
|
43
|
+
export const VENDOR_CREDENTIAL_FILE: Readonly<Record<Vendor, string>> = {
|
|
44
|
+
claude: ".credentials.json",
|
|
45
|
+
codex: "auth.json",
|
|
46
|
+
};
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import type { Account } from "./account.js";
|
|
12
12
|
import type { CompletionReport } from "./completion-report.js";
|
|
13
|
+
import type { UsageWindow } from "./usage.js";
|
|
13
14
|
|
|
14
15
|
/** One unattended turn of a Worker. The Foreman pins the working directory to the Workspace and
|
|
15
16
|
* says which Account the Worker runs under; the adapter turns that into a vendor launch. */
|
|
@@ -105,6 +106,17 @@ export type AdapterEvent =
|
|
|
105
106
|
/** The vendor reported a rate limit. `resetAt` is ISO 8601 when the vendor said when it lifts;
|
|
106
107
|
* the scheduler marks the Account Paused until then (or a default when unknown). */
|
|
107
108
|
| { type: "rate_limited"; resetAt?: string; message: string }
|
|
109
|
+
/**
|
|
110
|
+
* What the vendor said about how much of this Account's quota is gone (the multi-account headroom
|
|
111
|
+
* design, §4). Advisory and non-terminal: it says nothing about whether the turn succeeded, and a
|
|
112
|
+
* turn may report it several times or never.
|
|
113
|
+
*
|
|
114
|
+
* Emitted from whatever the vendor volunteers in band — Claude Code puts a full set of windows
|
|
115
|
+
* with their utilizations on every `rate_limit_event`, including the 90% warning that precedes
|
|
116
|
+
* any refusal. It is the highest-trust usage source there is (`usage.ts` `SOURCE_RANK`): the
|
|
117
|
+
* vendor said it, about this identity, at that moment.
|
|
118
|
+
*/
|
|
119
|
+
| { type: "usage"; windows: Record<string, UsageWindow> }
|
|
108
120
|
/** No output for `stallTimeoutMs`. Always followed by `failed` with reason `stalled`. */
|
|
109
121
|
| { type: "stalled"; silentMs: number }
|
|
110
122
|
/** Terminal: the Worker ended with a valid final message under its contract. `report` is the
|
package/src/init/prompt.ts
CHANGED
|
@@ -3,26 +3,25 @@ import type { ProjectChoice } from "./project.js";
|
|
|
3
3
|
import type { RunRecipe } from "../foreman/run-recipe.js";
|
|
4
4
|
import type { RecipeDefaults } from "./run-recipe-config.js";
|
|
5
5
|
|
|
6
|
-
/**
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
6
|
+
/**
|
|
7
|
+
* Asks for the one Run Recipe value nothing else can infer: the typecheck Gate.
|
|
8
|
+
*
|
|
9
|
+
* It no longer asks how to start the app. That question has no answer at setup time in a
|
|
10
|
+
* repository with more than one program -- "the app" means whichever one a Task's Verification
|
|
11
|
+
* happens to exercise, which is not known until there is a Task. The analyzer's dev command is
|
|
12
|
+
* still written when it exists, and the Verifier works the rest out when a browser round is
|
|
13
|
+
* actually wanted.
|
|
14
|
+
*
|
|
15
|
+
* Without a terminal the inferred values stand as they are.
|
|
16
|
+
*/
|
|
17
|
+
export async function askRecipe(defaults: RecipeDefaults): Promise<RunRecipe> {
|
|
18
|
+
if (!process.stdin.isTTY) return { ...defaults };
|
|
13
19
|
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
14
|
-
const ask = async (label: string, fallback: string | undefined): Promise<string> => {
|
|
15
|
-
const answer = (await rl.question(`${label}${fallback ? ` [${fallback}]` : ""}: `)).trim();
|
|
16
|
-
return answer || fallback || "";
|
|
17
|
-
};
|
|
18
20
|
try {
|
|
19
|
-
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const portEnv = await ask("Port variable", defaults.portEnv);
|
|
24
|
-
const typecheck = await ask("Typecheck command (blank for none)", defaults.gates.typecheck);
|
|
25
|
-
return { start, readinessUrl, portEnv, gates: { ...defaults.gates, typecheck: typecheck || undefined } };
|
|
21
|
+
const fallback = defaults.gates.typecheck;
|
|
22
|
+
const answer = (await rl.question(`Typecheck command for the Foreman's Gates (blank for none)${fallback ? ` [${fallback}]` : ""}: `)).trim();
|
|
23
|
+
const typecheck = answer || fallback;
|
|
24
|
+
return { ...defaults, gates: { ...defaults.gates, typecheck: typecheck || undefined } };
|
|
26
25
|
} finally {
|
|
27
26
|
rl.close();
|
|
28
27
|
}
|
|
@@ -33,18 +33,16 @@ export function commandsOf(files: OpseeConfigFiles): Record<string, string> {
|
|
|
33
33
|
* Gate has no analyzer equivalent and is only ever a flag or a prompt answer. */
|
|
34
34
|
export function inferRecipe(files: OpseeConfigFiles, flags: RecipeFlags): RecipeDefaults {
|
|
35
35
|
const commands = commandsOf(files);
|
|
36
|
+
const start = flags.start ?? commands.dev ?? commands.start;
|
|
36
37
|
return {
|
|
37
|
-
|
|
38
|
-
|
|
38
|
+
// The pair travels together or not at all: a readiness URL with nothing to start is a line
|
|
39
|
+
// that answers a question nobody asked, and `recipeFromConfig` rejects the other half-pair.
|
|
40
|
+
...(start === undefined ? {} : { start, readinessUrl: flags.readinessUrl ?? DEFAULT_READINESS_URL }),
|
|
39
41
|
portEnv: flags.portEnv ?? DEFAULT_PORT_ENV,
|
|
40
42
|
gates: { test: commands.test, lint: commands.lint, typecheck: flags.typecheck },
|
|
41
43
|
};
|
|
42
44
|
}
|
|
43
45
|
|
|
44
|
-
/** Why `init` skipped the recipe when nothing said how to start the app. */
|
|
45
|
-
export const NO_START_COMMAND =
|
|
46
|
-
"no start command: pass --start <cmd>, answer the prompt on a terminal, or set commands.dev in .opsee/config";
|
|
47
|
-
|
|
48
46
|
/** The recipe a config file already carries, so a block present in one of the two files is copied
|
|
49
47
|
* into the other rather than asked for again; null when neither has a block that can serve as a
|
|
50
48
|
* recipe. The YAML is the analyzer's primary file and wins when both have one. */
|
|
@@ -68,12 +66,13 @@ function gateEntries(gates: RunRecipeGates): Array<[string, string]> {
|
|
|
68
66
|
}
|
|
69
67
|
|
|
70
68
|
function yamlRecipeBlock(recipe: RunRecipe, indent: string): string {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
69
|
+
// start and readiness_url only when something already knew them: the analyzer's dev command, or
|
|
70
|
+
// an --start flag. A block with neither is a complete recipe for a repository whose Verdict comes
|
|
71
|
+
// from its Gates.
|
|
72
|
+
const lines = [`${RECIPE_KEY}:`];
|
|
73
|
+
if (recipe.start) lines.push(`${indent}start: ${yamlScalar(recipe.start)}`);
|
|
74
|
+
if (recipe.readinessUrl) lines.push(`${indent}readiness_url: ${yamlScalar(recipe.readinessUrl)}`);
|
|
75
|
+
lines.push(`${indent}port_env: ${yamlScalar(recipe.portEnv)}`);
|
|
77
76
|
const gates = gateEntries(recipe.gates);
|
|
78
77
|
if (gates.length > 0) {
|
|
79
78
|
lines.push(`${indent}gates:`);
|