@indigoai-us/hq-cli 5.345.127 → 5.345.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/assets/scaffold/core/scripts/handoff-knowledge-commit.sh +123 -0
- package/assets/scaffold/core/scripts/hq-detach.sh +169 -0
- package/dist/command-catalog.generated.d.ts +40 -2
- package/dist/command-catalog.generated.js +48 -2
- package/dist/commands/auth.js +19 -2
- package/dist/commands/core-bucket-c-handoff-knowledge-fast.d.ts +2 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-fast.js +10 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-native.d.ts +46 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-native.js +284 -0
- package/dist/commands/core-bucket-c-hq-detach-fast.d.ts +2 -0
- package/dist/commands/core-bucket-c-hq-detach-fast.js +10 -0
- package/dist/commands/core-bucket-c-hq-detach-native.d.ts +43 -0
- package/dist/commands/core-bucket-c-hq-detach-native.js +241 -0
- package/dist/commands/core-bucket-c-work-mesh-offer-native.js +8 -2
- package/dist/commands/core.js +12 -0
- package/dist/commands/daemon.js +2 -0
- package/dist/commands/lanes-accounts.js +3 -1
- package/dist/commands/lanes-shared.d.ts +7 -0
- package/dist/commands/lanes.js +139 -17
- package/dist/commands/mesh.js +32 -19
- package/dist/commands/skill.js +9 -4
- package/dist/fast-scaffold.js +28 -0
- package/dist/lib/bot/inbox-poll-backoff-flag.d.ts +3 -5
- package/dist/lib/bot/inbox-poll-backoff-flag.js +20 -12
- package/dist/lib/bot/inbox-wake.d.ts +18 -0
- package/dist/lib/bot/inbox-wake.js +125 -0
- package/dist/lib/bot/run.d.ts +8 -0
- package/dist/lib/bot/run.js +51 -7
- package/dist/lib/bot/runtime/messages-stream.js +31 -3
- package/dist/lib/capacity/lane-login.js +1 -0
- package/dist/lib/daemon/catalog.d.ts +4 -0
- package/dist/lib/daemon/catalog.js +6 -3
- package/dist/lib/daemon/flag-monitor.d.ts +4 -1
- package/dist/lib/daemon/flag-monitor.js +12 -6
- package/dist/lib/daemon/flag.d.ts +2 -0
- package/dist/lib/daemon/flag.js +11 -5
- package/dist/lib/daemon/hq-child-bootstrap.mjs +47 -7
- package/dist/lib/daemon/kill-switch-flags.d.ts +5 -0
- package/dist/lib/daemon/kill-switch-flags.js +5 -0
- package/dist/lib/daemon/lanes-reconcile-loader.d.ts +7 -0
- package/dist/lib/daemon/lanes-reconcile-loader.js +24 -0
- package/dist/lib/daemon/lanes-reconcile-watcher.d.ts +17 -0
- package/dist/lib/daemon/lanes-reconcile-watcher.js +80 -0
- package/dist/lib/daemon/remote-control.d.ts +8 -5
- package/dist/lib/daemon/remote-control.js +8 -7
- package/dist/lib/daemon/run.d.ts +4 -1
- package/dist/lib/daemon/run.js +133 -20
- package/dist/lib/doctor/json-output.d.ts +15 -3
- package/dist/lib/doctor/json-output.js +13 -1
- package/dist/lib/lanes/accounts.js +2 -2
- package/dist/lib/lanes/daemon-reconcile.d.ts +3 -0
- package/dist/lib/lanes/daemon-reconcile.js +8 -0
- package/dist/lib/lanes/host-capacity.d.ts +4 -0
- package/dist/lib/lanes/host-capacity.js +10 -1
- package/dist/lib/lanes/index.d.ts +2 -2
- package/dist/lib/lanes/index.js +1 -1
- package/dist/lib/lanes/loop.d.ts +8 -0
- package/dist/lib/lanes/loop.js +35 -0
- package/dist/lib/lanes/quota-notice.d.ts +11 -11
- package/dist/lib/lanes/quota-notice.js +20 -29
- package/dist/lib/lanes/quota.d.ts +69 -50
- package/dist/lib/lanes/quota.js +146 -127
- package/dist/lib/lanes/reconcile-result.d.ts +10 -0
- package/dist/lib/lanes/reconcile-result.js +11 -0
- package/dist/lib/lanes/reconcile.d.ts +14 -0
- package/dist/lib/lanes/reconcile.js +77 -23
- package/dist/lib/lanes/watcher.d.ts +8 -0
- package/dist/lib/lanes/watcher.js +62 -20
- package/dist/lib/mesh/live/daemon/work-subscribe.d.ts +1 -1
- package/dist/lib/mesh/live/daemon/work-subscribe.js +8 -1
- package/dist/lib/mesh/live/paths.d.ts +2 -0
- package/dist/lib/mesh/live/paths.js +2 -0
- package/dist/lib/mesh/story-status.d.ts +2 -2
- package/dist/lib/mesh/story-status.js +2 -1
- package/dist/utils/resident-package-preload.js +18 -4
- package/package.json +1 -1
package/dist/lib/lanes/quota.js
CHANGED
|
@@ -17,55 +17,81 @@
|
|
|
17
17
|
* per (account, model), plus an account-wide reset when the text is a
|
|
18
18
|
* usage limit rather than a switch-model refusal.
|
|
19
19
|
*
|
|
20
|
-
* A
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* "You've hit your weekly limit · resets Oct 16 at 1am (America/Denver)"
|
|
26
|
-
* while the owner's /usage showed the week at 43%.)
|
|
20
|
+
* A recorded 429 never holds the account (owner decision 2026-10-14, after
|
|
21
|
+
* the 2026-10-13 incident where one 429 on account a blocked every Claude
|
|
22
|
+
* lane until Friday). The reset times and provider text stay here as
|
|
23
|
+
* information for `hq lanes capacity`, `hq lanes accounts list` and the
|
|
24
|
+
* senior notice; only a needs-credits marker skips an account, for that model.
|
|
27
25
|
*/
|
|
28
26
|
import * as fs from "node:fs";
|
|
29
27
|
import * as path from "node:path";
|
|
30
28
|
import { withLaneRecordFileLock } from "./store.js";
|
|
31
29
|
export const QUOTA_STATE_REL = path.join("personal", "lanes", "quota.json");
|
|
32
|
-
/**
|
|
33
|
-
* Claude quota holds are off. Removed by owner directive on 2026-10-14: a
|
|
34
|
-
* false weekly-limit 429 held every Claude lane for days while the account
|
|
35
|
-
* still had capacity. A Claude 429 still fails that lane over to another
|
|
36
|
-
* account; it never blocks the account for later lanes, and stored Claude
|
|
37
|
-
* holds are ignored on read. `HQ_LANES_CLAUDE_QUOTA_HOLDS=1` turns the old
|
|
38
|
-
* behavior back on; the test suite sets it to keep the hold machinery covered.
|
|
39
|
-
*/
|
|
40
|
-
export const CLAUDE_QUOTA_HOLDS_ENV = "HQ_LANES_CLAUDE_QUOTA_HOLDS";
|
|
41
|
-
export function providerHasQuotaHolds(provider) {
|
|
42
|
-
if (provider !== "claude")
|
|
43
|
-
return true;
|
|
44
|
-
return process.env[CLAUDE_QUOTA_HOLDS_ENV] === "1";
|
|
45
|
-
}
|
|
46
30
|
export const QUOTA_SCHEMA_VERSION = "hq-lanes-quota.v1";
|
|
47
|
-
/** A provider reset further away than this needs a second 429 before it holds. */
|
|
48
|
-
export const QUOTA_CONFIRM_HORIZON_MS = 24 * 60 * 60 * 1000;
|
|
49
31
|
/**
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
32
|
+
* Decide the hold a 429 records (incident 2026-10-13: one 429 on the wrong
|
|
33
|
+
* subscription held the only Claude account for three days).
|
|
34
|
+
*
|
|
35
|
+
* - A per-model credits refusal is untouched.
|
|
36
|
+
* - A 429 with no reset gets the provider's configured default hold
|
|
37
|
+
* (`quota_default_hold_ms`, fifteen minutes unless set), marked `derived`.
|
|
38
|
+
* - A 429 with a provider reset is capped to the same default hold on its
|
|
39
|
+
* own. The provider reset is kept on the signal as `provider_reset_at`;
|
|
40
|
+
* recordAccountQuota extends the hold to it only when a fresh probe hits a
|
|
41
|
+
* repeat 429 before that reset. A reset nearer than the cap is used as is
|
|
42
|
+
* and marked `parsed`.
|
|
56
43
|
*/
|
|
57
44
|
export function applyDefaultQuotaHold(signal, defaultHoldMs, now) {
|
|
58
45
|
if (signal.reason !== "account_quota_exhausted" || signal.needs_credits)
|
|
59
46
|
return signal;
|
|
60
|
-
|
|
61
|
-
|
|
47
|
+
const observed_at = now.toISOString();
|
|
48
|
+
const shortHoldMs = now.getTime() + defaultHoldMs;
|
|
49
|
+
const providerResetMs = signal.quota_reset_at ? Date.parse(signal.quota_reset_at) : Number.NaN;
|
|
50
|
+
if (Number.isFinite(providerResetMs)) {
|
|
51
|
+
const providerReset = new Date(providerResetMs).toISOString();
|
|
52
|
+
if (providerResetMs <= shortHoldMs) {
|
|
53
|
+
return { ...signal, quota_reset_at: providerReset, provider_reset_at: providerReset, observed_at, reset_source: "parsed" };
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
...signal,
|
|
57
|
+
quota_reset_at: new Date(shortHoldMs).toISOString(),
|
|
58
|
+
provider_reset_at: providerReset,
|
|
59
|
+
observed_at,
|
|
60
|
+
account_wide: true,
|
|
61
|
+
reset_source: "derived",
|
|
62
|
+
};
|
|
63
|
+
}
|
|
62
64
|
return {
|
|
63
65
|
...signal,
|
|
64
|
-
quota_reset_at: new Date(
|
|
66
|
+
quota_reset_at: new Date(shortHoldMs).toISOString(),
|
|
67
|
+
observed_at,
|
|
65
68
|
account_wide: true,
|
|
66
69
|
reset_source: "derived",
|
|
67
70
|
};
|
|
68
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* True when the signal is a fresh 429 on an account whose previous 429 named
|
|
74
|
+
* a provider reset that has not arrived yet: the short first-strike hold
|
|
75
|
+
* expired (or was cleared and re-recorded), a new probe ran, and the
|
|
76
|
+
* provider refused again. The provider's reset is then believed.
|
|
77
|
+
*/
|
|
78
|
+
export function isRepeatQuotaHit(current, signal, now) {
|
|
79
|
+
if (!current?.provider_reset_at || !current.last_quota_hit_at)
|
|
80
|
+
return false;
|
|
81
|
+
const priorResetMs = Date.parse(current.provider_reset_at);
|
|
82
|
+
const priorHitMs = Date.parse(current.last_quota_hit_at);
|
|
83
|
+
if (!Number.isFinite(priorResetMs) || !Number.isFinite(priorHitMs))
|
|
84
|
+
return false;
|
|
85
|
+
const observedMs = signal.observed_at ? Date.parse(signal.observed_at) : now.getTime();
|
|
86
|
+
if (!Number.isFinite(observedMs) || observedMs <= priorHitMs)
|
|
87
|
+
return false;
|
|
88
|
+
// A second lane that was already running when the first hold was written
|
|
89
|
+
// is the same strike, not a fresh probe: the hold must have lapsed first.
|
|
90
|
+
const priorHoldMs = current.all_models_quota_reset_at ? Date.parse(current.all_models_quota_reset_at) : Number.NaN;
|
|
91
|
+
if (Number.isFinite(priorHoldMs) && observedMs < priorHoldMs)
|
|
92
|
+
return false;
|
|
93
|
+
return observedMs < priorResetMs;
|
|
94
|
+
}
|
|
69
95
|
const MONTHS = {
|
|
70
96
|
jan: 0,
|
|
71
97
|
feb: 1,
|
|
@@ -80,6 +106,34 @@ const MONTHS = {
|
|
|
80
106
|
nov: 10,
|
|
81
107
|
dec: 11,
|
|
82
108
|
};
|
|
109
|
+
const zoneFormatters = new Map();
|
|
110
|
+
function zoneFormatter(zone) {
|
|
111
|
+
const cached = zoneFormatters.get(zone);
|
|
112
|
+
if (cached !== undefined)
|
|
113
|
+
return cached;
|
|
114
|
+
let formatter;
|
|
115
|
+
try {
|
|
116
|
+
formatter = new Intl.DateTimeFormat("en-US", {
|
|
117
|
+
timeZone: zone,
|
|
118
|
+
hourCycle: "h23",
|
|
119
|
+
year: "numeric",
|
|
120
|
+
month: "numeric",
|
|
121
|
+
day: "numeric",
|
|
122
|
+
hour: "numeric",
|
|
123
|
+
minute: "numeric",
|
|
124
|
+
second: "numeric",
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
formatter = null;
|
|
129
|
+
}
|
|
130
|
+
zoneFormatters.set(zone, formatter);
|
|
131
|
+
return formatter;
|
|
132
|
+
}
|
|
133
|
+
/** "UTC" or an IANA zone the runtime knows, e.g. America/Denver. */
|
|
134
|
+
export function isKnownTimeZone(zone) {
|
|
135
|
+
return zoneFormatter(zone) !== null;
|
|
136
|
+
}
|
|
83
137
|
function emptyState() {
|
|
84
138
|
return { schema_version: QUOTA_SCHEMA_VERSION, providers: {} };
|
|
85
139
|
}
|
|
@@ -113,8 +167,6 @@ export function loadQuotaState(hqRoot) {
|
|
|
113
167
|
const providers = {};
|
|
114
168
|
const rawProviders = isPlainObject(parsed.providers) ? parsed.providers : {};
|
|
115
169
|
for (const provider of ["claude", "codex", "grok"]) {
|
|
116
|
-
if (!providerHasQuotaHolds(provider))
|
|
117
|
-
continue;
|
|
118
170
|
const rawAccounts = rawProviders[provider];
|
|
119
171
|
if (!isPlainObject(rawAccounts))
|
|
120
172
|
continue;
|
|
@@ -157,8 +209,11 @@ export function loadQuotaState(hqRoot) {
|
|
|
157
209
|
account.reset_source = "unconfirmed";
|
|
158
210
|
}
|
|
159
211
|
}
|
|
160
|
-
if (typeof raw.
|
|
161
|
-
account.
|
|
212
|
+
if (typeof raw.provider_reset_at === "string" && raw.provider_reset_at.length > 0) {
|
|
213
|
+
account.provider_reset_at = raw.provider_reset_at;
|
|
214
|
+
}
|
|
215
|
+
if (typeof raw.last_quota_hit_at === "string" && raw.last_quota_hit_at.length > 0) {
|
|
216
|
+
account.last_quota_hit_at = raw.last_quota_hit_at;
|
|
162
217
|
}
|
|
163
218
|
if (typeof raw.provider_text === "string" && raw.provider_text.length > 0) {
|
|
164
219
|
account.provider_text = raw.provider_text;
|
|
@@ -176,18 +231,6 @@ function writeQuotaState(hqRoot, state) {
|
|
|
176
231
|
fs.writeFileSync(tmp, `${JSON.stringify(state, null, 2)}\n`, "utf8");
|
|
177
232
|
fs.renameSync(tmp, filePath);
|
|
178
233
|
}
|
|
179
|
-
/** True when `zone` is a timezone name the runtime's Intl data knows. */
|
|
180
|
-
function isKnownTimeZone(zone) {
|
|
181
|
-
try {
|
|
182
|
-
new Intl.DateTimeFormat("en-US", { timeZone: zone });
|
|
183
|
-
return true;
|
|
184
|
-
}
|
|
185
|
-
catch (err) {
|
|
186
|
-
if (err instanceof RangeError)
|
|
187
|
-
return false;
|
|
188
|
-
throw err;
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
234
|
/** Offset of `zone` from UTC at instant `ms`, in milliseconds. */
|
|
192
235
|
function zoneOffsetMs(zone, ms) {
|
|
193
236
|
const parts = new Intl.DateTimeFormat("en-US", {
|
|
@@ -377,6 +420,11 @@ export function normalizeStructuredReset(value) {
|
|
|
377
420
|
return null;
|
|
378
421
|
return new Date(ms).toISOString();
|
|
379
422
|
}
|
|
423
|
+
/**
|
|
424
|
+
* Read only `api_error_status`, `is_error`, `result`, and a structured
|
|
425
|
+
* reset off the last `type: "result"` object. Nothing else from the payload
|
|
426
|
+
* is returned, and this function does not log.
|
|
427
|
+
*/
|
|
380
428
|
export function quotaFieldsFromOutput(text) {
|
|
381
429
|
const lines = text.split("\n");
|
|
382
430
|
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
@@ -462,48 +510,25 @@ export function readQuotaSignal(runDir, model, now, workerOutOffset = 0) {
|
|
|
462
510
|
const observedAt = Number.isFinite(outputAt.getTime()) ? outputAt : now;
|
|
463
511
|
return quotaSignalFromFields(fields, model, observedAt);
|
|
464
512
|
}
|
|
465
|
-
|
|
466
|
-
* Hold a provider reset more than a day away to a short hold until a second
|
|
467
|
-
* 429 confirms it. The second 429 counts only when its lane ran after the
|
|
468
|
-
* short hold expired, so lanes that failed together on one 429 burst do not
|
|
469
|
-
* confirm each other.
|
|
470
|
-
*/
|
|
471
|
-
function gateFarReset(current, signal, options) {
|
|
472
|
-
if (!signal.account_wide ||
|
|
473
|
-
!signal.quota_reset_at ||
|
|
474
|
-
signal.needs_credits ||
|
|
475
|
-
signal.reset_source === "derived") {
|
|
476
|
-
return { signal };
|
|
477
|
-
}
|
|
478
|
-
const resetMs = Date.parse(signal.quota_reset_at);
|
|
479
|
-
const nowMs = options.now.getTime();
|
|
480
|
-
if (!Number.isFinite(resetMs) || resetMs - nowMs <= QUOTA_CONFIRM_HORIZON_MS)
|
|
481
|
-
return { signal };
|
|
482
|
-
const pending = current.unconfirmed_reset_at;
|
|
483
|
-
const shortHoldActive = resetStillAhead(current.all_models_quota_reset_at, options.now);
|
|
484
|
-
if (pending !== undefined && !shortHoldActive) {
|
|
485
|
-
return { signal: { ...signal, reset_source: "parsed" } };
|
|
486
|
-
}
|
|
487
|
-
return {
|
|
488
|
-
signal: {
|
|
489
|
-
...signal,
|
|
490
|
-
quota_reset_at: new Date(nowMs + options.unconfirmedHoldMs).toISOString(),
|
|
491
|
-
reset_source: "unconfirmed",
|
|
492
|
-
},
|
|
493
|
-
unconfirmed_reset_at: pending !== undefined && !shouldReplaceReset(pending, signal.quota_reset_at)
|
|
494
|
-
? pending
|
|
495
|
-
: signal.quota_reset_at,
|
|
496
|
-
};
|
|
497
|
-
}
|
|
498
|
-
export function recordAccountQuota(hqRoot, provider, accountId, incoming, options) {
|
|
499
|
-
if (!providerHasQuotaHolds(provider))
|
|
500
|
-
return loadQuotaState(hqRoot);
|
|
513
|
+
export function recordAccountQuota(hqRoot, provider, accountId, incoming, now = new Date()) {
|
|
501
514
|
return withLaneRecordFileLock(quotaStatePath(hqRoot), () => {
|
|
502
515
|
const state = loadQuotaState(hqRoot);
|
|
503
516
|
const accounts = state.providers[provider] ?? {};
|
|
504
517
|
const current = accounts[accountId] ?? { models: {} };
|
|
505
|
-
|
|
506
|
-
|
|
518
|
+
// A repeat 429 before the provider's reset promotes the capped hold to
|
|
519
|
+
// the provider's own reset (see applyDefaultQuotaHold).
|
|
520
|
+
let signal = incoming;
|
|
521
|
+
if (incoming.reason === "account_quota_exhausted" &&
|
|
522
|
+
!incoming.needs_credits &&
|
|
523
|
+
isRepeatQuotaHit(current, incoming, now)) {
|
|
524
|
+
const candidates = [incoming.provider_reset_at, current.provider_reset_at, incoming.quota_reset_at]
|
|
525
|
+
.map((value) => (value ? Date.parse(value) : Number.NaN))
|
|
526
|
+
.filter((ms) => Number.isFinite(ms));
|
|
527
|
+
if (candidates.length > 0) {
|
|
528
|
+
const promoted = new Date(Math.max(...candidates)).toISOString();
|
|
529
|
+
signal = { ...incoming, quota_reset_at: promoted, provider_reset_at: promoted, account_wide: true, reset_source: "parsed" };
|
|
530
|
+
}
|
|
531
|
+
}
|
|
507
532
|
const models = { ...current.models };
|
|
508
533
|
if (signal.model) {
|
|
509
534
|
const prior = models[signal.model] ?? {};
|
|
@@ -525,19 +550,18 @@ export function recordAccountQuota(hqRoot, provider, accountId, incoming, option
|
|
|
525
550
|
if (shouldReplaceReset(current.all_models_quota_reset_at, signal.quota_reset_at)) {
|
|
526
551
|
account.all_models_quota_reset_at = signal.quota_reset_at;
|
|
527
552
|
account.reset_source = signal.reset_source ?? "parsed";
|
|
528
|
-
if (signal.provider_text)
|
|
529
|
-
account.provider_text = signal.provider_text;
|
|
530
|
-
if (gated.unconfirmed_reset_at) {
|
|
531
|
-
account.unconfirmed_reset_at = gated.unconfirmed_reset_at;
|
|
532
|
-
}
|
|
533
|
-
else {
|
|
534
|
-
delete account.unconfirmed_reset_at;
|
|
535
|
-
}
|
|
536
553
|
}
|
|
537
554
|
else if (current.all_models_quota_reset_at) {
|
|
538
555
|
account.all_models_quota_reset_at = current.all_models_quota_reset_at;
|
|
539
556
|
account.reset_source = current.reset_source ?? "parsed";
|
|
540
557
|
}
|
|
558
|
+
if (signal.provider_text)
|
|
559
|
+
account.provider_text = signal.provider_text;
|
|
560
|
+
if (signal.provider_reset_at)
|
|
561
|
+
account.provider_reset_at = signal.provider_reset_at;
|
|
562
|
+
else
|
|
563
|
+
delete account.provider_reset_at;
|
|
564
|
+
account.last_quota_hit_at = signal.observed_at ?? now.toISOString();
|
|
541
565
|
}
|
|
542
566
|
accounts[accountId] = account;
|
|
543
567
|
state.providers[provider] = accounts;
|
|
@@ -560,16 +584,13 @@ export function clearAccountQuota(hqRoot, provider, accountId, options = {}) {
|
|
|
560
584
|
const shouldClear = (reset) => reset !== undefined &&
|
|
561
585
|
(!options.futureOnly || resetStillAhead(reset, now));
|
|
562
586
|
let changed = false;
|
|
563
|
-
// A successful turn or an operator clear disproves a pending far reset.
|
|
564
|
-
if (account.unconfirmed_reset_at !== undefined) {
|
|
565
|
-
delete account.unconfirmed_reset_at;
|
|
566
|
-
changed = true;
|
|
567
|
-
}
|
|
568
587
|
const accountWideReset = account.all_models_quota_reset_at;
|
|
569
588
|
if (shouldClear(accountWideReset)) {
|
|
570
589
|
cleared.all_models_quota_reset_at = accountWideReset;
|
|
571
590
|
delete account.all_models_quota_reset_at;
|
|
572
591
|
delete account.reset_source;
|
|
592
|
+
delete account.provider_reset_at;
|
|
593
|
+
delete account.last_quota_hit_at;
|
|
573
594
|
delete account.provider_text;
|
|
574
595
|
changed = true;
|
|
575
596
|
}
|
|
@@ -644,30 +665,20 @@ function resetStillAhead(iso, now) {
|
|
|
644
665
|
return Number.isFinite(ts) && ts > now.getTime();
|
|
645
666
|
}
|
|
646
667
|
/**
|
|
647
|
-
* Skip an account only for
|
|
648
|
-
*
|
|
649
|
-
*
|
|
668
|
+
* Skip an account only for a model it cannot run at all: a needs-credits
|
|
669
|
+
* marker blocks that model, with no expiry.
|
|
670
|
+
*
|
|
671
|
+
* A recorded 429 never blocks (owner decision, incident 2026-10-13). The
|
|
672
|
+
* lane that hit it ends resumable and its senior decides; other lanes keep
|
|
673
|
+
* using the account. The reset times stay in quota.json as information
|
|
674
|
+
* (`accountQuotaView`, `quotaHitInfo`).
|
|
650
675
|
*/
|
|
651
|
-
export function quotaBlockFor(state, provider, accountId, model,
|
|
652
|
-
if (!providerHasQuotaHolds(provider))
|
|
653
|
-
return { blocked: false };
|
|
676
|
+
export function quotaBlockFor(state, provider, accountId, model, _now) {
|
|
654
677
|
const account = state.providers[provider]?.[accountId];
|
|
655
|
-
if (!account)
|
|
656
|
-
return { blocked: false };
|
|
657
|
-
if (resetStillAhead(account.all_models_quota_reset_at, now)) {
|
|
658
|
-
return {
|
|
659
|
-
blocked: true,
|
|
660
|
-
reason: "account_quota_exhausted",
|
|
661
|
-
quota_reset_at: account.all_models_quota_reset_at,
|
|
662
|
-
model,
|
|
663
|
-
};
|
|
664
|
-
}
|
|
665
|
-
if (!model)
|
|
678
|
+
if (!account || !model)
|
|
666
679
|
return { blocked: false };
|
|
667
680
|
const entry = account.models[model];
|
|
668
|
-
if (
|
|
669
|
-
return { blocked: false };
|
|
670
|
-
if (entry.needs_credits) {
|
|
681
|
+
if (entry?.needs_credits) {
|
|
671
682
|
return {
|
|
672
683
|
blocked: true,
|
|
673
684
|
reason: "account_model_unavailable",
|
|
@@ -675,16 +686,21 @@ export function quotaBlockFor(state, provider, accountId, model, now) {
|
|
|
675
686
|
model,
|
|
676
687
|
};
|
|
677
688
|
}
|
|
678
|
-
if (resetStillAhead(entry.quota_reset_at, now)) {
|
|
679
|
-
return {
|
|
680
|
-
blocked: true,
|
|
681
|
-
reason: "account_quota_exhausted",
|
|
682
|
-
quota_reset_at: entry.quota_reset_at,
|
|
683
|
-
model,
|
|
684
|
-
};
|
|
685
|
-
}
|
|
686
689
|
return { blocked: false };
|
|
687
690
|
}
|
|
691
|
+
export function quotaHitInfo(state, provider, accountId, now) {
|
|
692
|
+
const account = state.providers[provider]?.[accountId];
|
|
693
|
+
if (!account || (!account.last_quota_hit_at && !account.provider_reset_at && !account.all_models_quota_reset_at)) {
|
|
694
|
+
return undefined;
|
|
695
|
+
}
|
|
696
|
+
const providerReset = account.provider_reset_at ??
|
|
697
|
+
(account.reset_source === "parsed" ? account.all_models_quota_reset_at : undefined);
|
|
698
|
+
return {
|
|
699
|
+
...(account.last_quota_hit_at ? { last_quota_hit_at: account.last_quota_hit_at } : {}),
|
|
700
|
+
...(providerReset ? { provider_reset_at: providerReset } : {}),
|
|
701
|
+
provider_reset_pending: resetStillAhead(providerReset, now),
|
|
702
|
+
};
|
|
703
|
+
}
|
|
688
704
|
export function accountQuotaView(state, provider, accountId) {
|
|
689
705
|
const account = state.providers[provider]?.[accountId];
|
|
690
706
|
if (!account)
|
|
@@ -703,6 +719,9 @@ export function accountQuotaView(state, provider, accountId) {
|
|
|
703
719
|
});
|
|
704
720
|
return {
|
|
705
721
|
quota_reset_at: account.all_models_quota_reset_at ?? null,
|
|
722
|
+
...(account.reset_source ? { quota_reset_source: account.reset_source } : {}),
|
|
723
|
+
...(account.provider_reset_at ? { provider_reset_at: account.provider_reset_at } : {}),
|
|
724
|
+
...(account.last_quota_hit_at ? { last_quota_hit_at: account.last_quota_hit_at } : {}),
|
|
706
725
|
model_quota,
|
|
707
726
|
};
|
|
708
727
|
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface ReconcilePassResult {
|
|
2
|
+
ok: boolean;
|
|
3
|
+
errors?: readonly {
|
|
4
|
+
code?: string;
|
|
5
|
+
message: string;
|
|
6
|
+
}[];
|
|
7
|
+
}
|
|
8
|
+
/** Preserve CLI exit semantics while letting the daemon report failures to its watcher. */
|
|
9
|
+
export declare function finishReconcilePass(summary: ReconcilePassResult, mode: "cli" | "daemon", exit?: (code: number) => void): void;
|
|
10
|
+
//# sourceMappingURL=reconcile-result.d.ts.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Preserve CLI exit semantics while letting the daemon report failures to its watcher. */
|
|
2
|
+
export function finishReconcilePass(summary, mode, exit = (code) => process.exit(code)) {
|
|
3
|
+
if (summary.ok)
|
|
4
|
+
return;
|
|
5
|
+
if (mode === "daemon") {
|
|
6
|
+
const detail = summary.errors?.map((error) => `${error.code ?? "unknown"}: ${error.message}`).join("; ") ?? "unknown reconcile error";
|
|
7
|
+
throw new Error(`In-process lanes reconcile failed: ${detail}`);
|
|
8
|
+
}
|
|
9
|
+
exit(1);
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=reconcile-result.js.map
|
|
@@ -178,4 +178,18 @@ export interface AnsweredLaneWakeResult {
|
|
|
178
178
|
}
|
|
179
179
|
/** Run the reconcile answer-resume path for one lane before questions answer returns. */
|
|
180
180
|
export declare function wakeAnsweredLaneNow(hqRoot: string, laneId: string, deps: ReconcileDeps): Promise<AnsweredLaneWakeResult>;
|
|
181
|
+
export interface AdmitQueuedNowResult {
|
|
182
|
+
admitted: ReconcileChange[];
|
|
183
|
+
errors: ReconcileSummary["errors"];
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Run queue admission in-process, without the rest of a reconcile pass.
|
|
187
|
+
*
|
|
188
|
+
* Commands that change capacity (create, stop, cancel, clear-quota, a lane
|
|
189
|
+
* exit) call this so queued lanes start without `hq daemon`. It takes the
|
|
190
|
+
* reconcile lock and every provider admission lock, so it is safe to run
|
|
191
|
+
* while a daemon is also reconciling: whichever runs second sees the lanes
|
|
192
|
+
* the first one already admitted.
|
|
193
|
+
*/
|
|
194
|
+
export declare function admitQueuedNow(hqRoot: string, deps: ReconcileDeps): Promise<AdmitQueuedNowResult>;
|
|
181
195
|
//# sourceMappingURL=reconcile.d.ts.map
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
import * as fs from "node:fs";
|
|
9
9
|
import * as path from "node:path";
|
|
10
10
|
import { parseUserYaml } from "../../utils/user-yaml-error.js";
|
|
11
|
-
import { compareLaneQueueOrder, compareLaneQueueOrderWithLeads, freeClaudeAccountSlots, leadLaneIds, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
|
|
11
|
+
import { compareLaneQueueOrder, compareLaneQueueOrderWithLeads, freeClaudeAccountSlots, leadLaneIds, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneOccupiesRunningSlot, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
|
|
12
12
|
import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
|
|
13
13
|
import { effectiveAccountHome } from "./account-home.js";
|
|
14
14
|
import { enforceRegistryApprovalGate } from "./approval-gate.js";
|
|
@@ -34,7 +34,7 @@ import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LE
|
|
|
34
34
|
import { isFailedQuotaKeepAliveLane, providerSupportsSessionResume } from "./delivery.js";
|
|
35
35
|
import { LANE_PROVIDERS, isIsoDateTime, isTerminalLaneState } from "./types.js";
|
|
36
36
|
import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, loadQuotaState, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
37
|
-
import {
|
|
37
|
+
import { deliverQuotaNotice, quotaLaneNotice } from "./quota-notice.js";
|
|
38
38
|
import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
39
39
|
import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
40
40
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
@@ -1724,32 +1724,16 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1724
1724
|
? classifyLaneExit({ envelope: null, hasPartialOutput: false, invalidEnvelope })
|
|
1725
1725
|
: classification;
|
|
1726
1726
|
let questionRefusalCode = null;
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
: null;
|
|
1727
|
+
// A usage limit fails only this lane; the senior decides on resume.
|
|
1728
|
+
const accountRetryReason = !quota && providerRejection && classification.reason === "account_not_logged_in"
|
|
1729
|
+
? "account_unavailable"
|
|
1730
|
+
: null;
|
|
1732
1731
|
if (!deps.dryRun &&
|
|
1733
1732
|
quota &&
|
|
1734
1733
|
classified.account &&
|
|
1735
1734
|
(classification.reason === "account_quota_exhausted" ||
|
|
1736
1735
|
classification.reason === "account_model_unavailable")) {
|
|
1737
|
-
|
|
1738
|
-
now,
|
|
1739
|
-
unconfirmedHoldMs: providerQuotaDefaultHoldMs(loadLanesConfig(hqRoot).providers[classified.provider]),
|
|
1740
|
-
});
|
|
1741
|
-
if (classification.reason === "account_quota_exhausted") {
|
|
1742
|
-
const notice = quotaHoldNotice(quotaState, classified.provider, classified.account, lane.lane_id);
|
|
1743
|
-
if (notice) {
|
|
1744
|
-
try {
|
|
1745
|
-
deliverQuotaHoldNotice(hqRoot, lane, notice, now.toISOString());
|
|
1746
|
-
}
|
|
1747
|
-
catch (err) {
|
|
1748
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
1749
|
-
process.stderr.write(`hq lanes reconcile: quota hold notice failed for ${lane.lane_id}: ${message}\n`);
|
|
1750
|
-
}
|
|
1751
|
-
}
|
|
1752
|
-
}
|
|
1736
|
+
recordAccountQuota(hqRoot, classified.provider, classified.account, quota, now);
|
|
1753
1737
|
}
|
|
1754
1738
|
if (!deps.dryRun &&
|
|
1755
1739
|
accountRetryReason &&
|
|
@@ -1886,6 +1870,22 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1886
1870
|
result.record.model) {
|
|
1887
1871
|
clearAccountQuotaAfterSuccessfulTurnBestEffort(hqRoot, result.record.provider, result.record.account, result.record.model, new Date(nowIso));
|
|
1888
1872
|
}
|
|
1873
|
+
if (!deps.dryRun &&
|
|
1874
|
+
result.changed &&
|
|
1875
|
+
quota &&
|
|
1876
|
+
(result.record.terminal_reason === "account_quota_exhausted" ||
|
|
1877
|
+
result.record.terminal_reason === "account_model_unavailable")) {
|
|
1878
|
+
// The dead worker's exit is classified here, not in its watcher, so
|
|
1879
|
+
// the senior notice goes out here, once, after the terminal write and
|
|
1880
|
+
// keyed to the new version. No account hold is set.
|
|
1881
|
+
try {
|
|
1882
|
+
deliverQuotaNotice(hqRoot, result.record, quotaLaneNotice(result.record, quota), nowIso);
|
|
1883
|
+
}
|
|
1884
|
+
catch (err) {
|
|
1885
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1886
|
+
process.stderr.write(`hq lanes reconcile: quota notice failed for ${lane.lane_id}: ${message}\n`);
|
|
1887
|
+
}
|
|
1888
|
+
}
|
|
1889
1889
|
if (result.changed) {
|
|
1890
1890
|
summary.dead_host.push({
|
|
1891
1891
|
id: lane.lane_id,
|
|
@@ -3875,4 +3875,58 @@ export async function wakeAnsweredLaneNow(hqRoot, laneId, deps) {
|
|
|
3875
3875
|
error: failure?.why ?? `Lane ${laneId} did not leave awaiting_input after its questions were answered`,
|
|
3876
3876
|
};
|
|
3877
3877
|
}
|
|
3878
|
+
/**
|
|
3879
|
+
* Run queue admission in-process, without the rest of a reconcile pass.
|
|
3880
|
+
*
|
|
3881
|
+
* Commands that change capacity (create, stop, cancel, clear-quota, a lane
|
|
3882
|
+
* exit) call this so queued lanes start without `hq daemon`. It takes the
|
|
3883
|
+
* reconcile lock and every provider admission lock, so it is safe to run
|
|
3884
|
+
* while a daemon is also reconciling: whichever runs second sees the lanes
|
|
3885
|
+
* the first one already admitted.
|
|
3886
|
+
*/
|
|
3887
|
+
export async function admitQueuedNow(hqRoot, deps) {
|
|
3888
|
+
const summary = emptySummary(deps.dryRun);
|
|
3889
|
+
const now = deps.now();
|
|
3890
|
+
// A queued lane is worth an admission attempt only while its provider has
|
|
3891
|
+
// a free running slot here. Account and profile caps are left to
|
|
3892
|
+
// admitQueued, which can move a lane to another free account.
|
|
3893
|
+
const hasAdmittableQueued = (lanes) => {
|
|
3894
|
+
const queued = lanes.filter((lane) => lane.state === "queued" && laneRunsHere(lane, deps.host));
|
|
3895
|
+
if (queued.length === 0)
|
|
3896
|
+
return false;
|
|
3897
|
+
const config = loadLanesConfig(hqRoot);
|
|
3898
|
+
const nowMs = now.getTime();
|
|
3899
|
+
return queued.some((lane) => {
|
|
3900
|
+
const providerRunning = lanes.filter((other) => other.lane_id !== lane.lane_id && other.provider === lane.provider &&
|
|
3901
|
+
laneOccupiesRunningSlot(other, nowMs) && laneRunsHere(other, deps.host)).length;
|
|
3902
|
+
return providerRunning < config.providers[lane.provider].max_running;
|
|
3903
|
+
});
|
|
3904
|
+
};
|
|
3905
|
+
// Skip the lock and the CPU sample when nothing can start here.
|
|
3906
|
+
if (!hasAdmittableQueued(loadLaneRecords(hqRoot, emptySummary(deps.dryRun)))) {
|
|
3907
|
+
return { admitted: [], errors: [] };
|
|
3908
|
+
}
|
|
3909
|
+
const throttleDeps = deps.throttleDeps ?? defaultThrottleDeps(deps.env ?? process.env);
|
|
3910
|
+
await withReconcileLock(hqRoot, async () => {
|
|
3911
|
+
const previousState = readThrottleState(throttleStatePath(hqRoot), throttleDeps);
|
|
3912
|
+
const previousSnapshot = previousState === "missing" || previousState === "unreadable"
|
|
3913
|
+
? undefined
|
|
3914
|
+
: previousState.cpu_snapshot;
|
|
3915
|
+
const cpuObservation = deps.cpuBusyFraction !== undefined
|
|
3916
|
+
? undefined
|
|
3917
|
+
: await sampleCpuBusyObservation(throttleDeps, previousSnapshot);
|
|
3918
|
+
const admissionDeps = {
|
|
3919
|
+
...deps,
|
|
3920
|
+
...(cpuObservation ? { cpuBusyFraction: cpuObservation.busy, cpuObservation } : {}),
|
|
3921
|
+
skipAdmissionWithoutCpuSample: false,
|
|
3922
|
+
};
|
|
3923
|
+
await withAllLaneAdmissionLocks(hqRoot, async () => {
|
|
3924
|
+
const lanes = loadLaneRecords(hqRoot, summary);
|
|
3925
|
+
if (!hasAdmittableQueued(lanes))
|
|
3926
|
+
return;
|
|
3927
|
+
await admitQueued(hqRoot, lanes, summary, admissionDeps, now);
|
|
3928
|
+
});
|
|
3929
|
+
});
|
|
3930
|
+
return { admitted: summary.admitted, errors: summary.errors };
|
|
3931
|
+
}
|
|
3878
3932
|
//# sourceMappingURL=reconcile.js.map
|
|
@@ -211,6 +211,14 @@ export interface WatcherLoopDeps {
|
|
|
211
211
|
retrySpawnDeps?: Omit<SpawnLaneDeps, "admissionHost">;
|
|
212
212
|
/** Active-company fallback when an unscoped lane asks from its envelope. */
|
|
213
213
|
resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
|
|
214
|
+
/**
|
|
215
|
+
* Admit queued lanes after this lane's exit frees its slot. Default: the
|
|
216
|
+
* in-process admitter (`admitQueuedNow`), so the queue drains without a
|
|
217
|
+
* daemon. Returns the admitted lane ids.
|
|
218
|
+
*/
|
|
219
|
+
admitQueued?: (hqRoot: string) => Promise<string[]>;
|
|
220
|
+
/** Test seam: deliver the one-time usage-limit notice to the lane's senior. */
|
|
221
|
+
deliverQuotaNotice?: (lane: LaneRecord, text: string) => Promise<void>;
|
|
214
222
|
/** Resume an atomically queued inbox after this watcher wins terminal CAS. */
|
|
215
223
|
recoverQueuedMessages?: (lane: LaneRecord) => Promise<void>;
|
|
216
224
|
/** Deliver a lane-senior escalation through the normal lane-message router. */
|