@indigoai-us/hq-cli 5.345.127 → 5.345.128
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/assets/scaffold/core/scripts/handoff-knowledge-commit.sh +123 -0
- package/assets/scaffold/core/scripts/hq-detach.sh +169 -0
- package/dist/command-catalog.generated.d.ts +40 -2
- package/dist/command-catalog.generated.js +48 -2
- package/dist/commands/auth.js +19 -2
- package/dist/commands/core-bucket-c-handoff-knowledge-fast.d.ts +2 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-fast.js +10 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-native.d.ts +46 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-native.js +284 -0
- package/dist/commands/core-bucket-c-hq-detach-fast.d.ts +2 -0
- package/dist/commands/core-bucket-c-hq-detach-fast.js +10 -0
- package/dist/commands/core-bucket-c-hq-detach-native.d.ts +43 -0
- package/dist/commands/core-bucket-c-hq-detach-native.js +241 -0
- package/dist/commands/core.js +12 -0
- package/dist/commands/lanes-accounts.js +3 -1
- package/dist/commands/lanes-shared.d.ts +7 -0
- package/dist/commands/lanes.js +113 -3
- package/dist/commands/skill.js +9 -4
- package/dist/fast-scaffold.js +28 -0
- package/dist/lib/capacity/lane-login.js +1 -0
- package/dist/lib/lanes/accounts.js +2 -2
- package/dist/lib/lanes/host-capacity.d.ts +4 -0
- package/dist/lib/lanes/host-capacity.js +10 -1
- package/dist/lib/lanes/index.d.ts +2 -2
- package/dist/lib/lanes/index.js +1 -1
- package/dist/lib/lanes/loop.d.ts +8 -0
- package/dist/lib/lanes/loop.js +35 -0
- package/dist/lib/lanes/quota-notice.d.ts +11 -11
- package/dist/lib/lanes/quota-notice.js +20 -29
- package/dist/lib/lanes/quota.d.ts +69 -50
- package/dist/lib/lanes/quota.js +146 -127
- package/dist/lib/lanes/reconcile.d.ts +14 -0
- package/dist/lib/lanes/reconcile.js +77 -23
- package/dist/lib/lanes/watcher.d.ts +8 -0
- package/dist/lib/lanes/watcher.js +62 -20
- package/dist/utils/resident-package-preload.js +12 -4
- package/package.json +1 -1
|
@@ -17,26 +17,14 @@
|
|
|
17
17
|
* per (account, model), plus an account-wide reset when the text is a
|
|
18
18
|
* usage limit rather than a switch-model refusal.
|
|
19
19
|
*
|
|
20
|
-
* A
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* "You've hit your weekly limit · resets Oct 16 at 1am (America/Denver)"
|
|
26
|
-
* while the owner's /usage showed the week at 43%.)
|
|
20
|
+
* A recorded 429 never holds the account (owner decision 2026-10-14, after
|
|
21
|
+
* the 2026-10-13 incident where one 429 on account a blocked every Claude
|
|
22
|
+
* lane until Friday). The reset times and provider text stay here as
|
|
23
|
+
* information for `hq lanes capacity`, `hq lanes accounts list` and the
|
|
24
|
+
* senior notice; only a needs-credits marker skips an account, for that model.
|
|
27
25
|
*/
|
|
28
26
|
import type { LaneProvider } from "./types.js";
|
|
29
27
|
export declare const QUOTA_STATE_REL: string;
|
|
30
|
-
/**
|
|
31
|
-
* Claude quota holds are off. Removed by owner directive on 2026-10-14: a
|
|
32
|
-
* false weekly-limit 429 held every Claude lane for days while the account
|
|
33
|
-
* still had capacity. A Claude 429 still fails that lane over to another
|
|
34
|
-
* account; it never blocks the account for later lanes, and stored Claude
|
|
35
|
-
* holds are ignored on read. `HQ_LANES_CLAUDE_QUOTA_HOLDS=1` turns the old
|
|
36
|
-
* behavior back on; the test suite sets it to keep the hold machinery covered.
|
|
37
|
-
*/
|
|
38
|
-
export declare const CLAUDE_QUOTA_HOLDS_ENV = "HQ_LANES_CLAUDE_QUOTA_HOLDS";
|
|
39
|
-
export declare function providerHasQuotaHolds(provider: LaneProvider): boolean;
|
|
40
28
|
export declare const QUOTA_SCHEMA_VERSION = "hq-lanes-quota.v1";
|
|
41
29
|
export interface ModelQuota {
|
|
42
30
|
quota_reset_at?: string;
|
|
@@ -46,26 +34,27 @@ export interface ModelQuota {
|
|
|
46
34
|
}
|
|
47
35
|
/**
|
|
48
36
|
* `parsed`: the provider's own reset. `derived`: the default hold for a 429
|
|
49
|
-
* with no parseable reset. `unconfirmed
|
|
50
|
-
* provider reset more than a day away that a second 429 has not confirmed.
|
|
37
|
+
* with no parseable reset. `unconfirmed` is read from older quota files only.
|
|
51
38
|
*/
|
|
52
39
|
export type QuotaResetSource = "parsed" | "derived" | "unconfirmed";
|
|
53
|
-
/** A provider reset further away than this needs a second 429 before it holds. */
|
|
54
|
-
export declare const QUOTA_CONFIRM_HORIZON_MS: number;
|
|
55
40
|
export interface AccountQuota {
|
|
56
41
|
/** Set when the 429 is an account-wide usage limit with a reset time. */
|
|
57
42
|
all_models_quota_reset_at?: string;
|
|
58
43
|
/**
|
|
59
|
-
* Whether `all_models_quota_reset_at`
|
|
60
|
-
*
|
|
44
|
+
* Whether `all_models_quota_reset_at` is the provider's own reset
|
|
45
|
+
* (`parsed`) or a shorter hold HQ chose (`derived`): the configured
|
|
46
|
+
* default hold for a 429 with no reset, or the capped first-strike hold
|
|
47
|
+
* when the provider's reset is further away than that.
|
|
61
48
|
*/
|
|
62
49
|
reset_source?: QuotaResetSource;
|
|
63
50
|
/**
|
|
64
|
-
* The provider's
|
|
65
|
-
*
|
|
51
|
+
* The provider's own reset time from the most recent 429, kept even when
|
|
52
|
+
* the recorded reset is shorter. Information only; nothing gates on it.
|
|
66
53
|
*/
|
|
67
|
-
|
|
68
|
-
/**
|
|
54
|
+
provider_reset_at?: string;
|
|
55
|
+
/** When the most recent account-wide 429 was observed. */
|
|
56
|
+
last_quota_hit_at?: string;
|
|
57
|
+
/** Raw provider text from the most recent account-wide 429. */
|
|
69
58
|
provider_text?: string;
|
|
70
59
|
models: Record<string, ModelQuota>;
|
|
71
60
|
}
|
|
@@ -83,18 +72,36 @@ export interface QuotaSignal {
|
|
|
83
72
|
account_wide?: boolean;
|
|
84
73
|
/** Set by applyDefaultQuotaHold: whether the reset is the provider's or the default hold. */
|
|
85
74
|
reset_source?: QuotaResetSource;
|
|
86
|
-
/** The provider's
|
|
75
|
+
/** The provider's own reset, parsed or structured, before any cap. */
|
|
76
|
+
provider_reset_at?: string;
|
|
77
|
+
/** When the 429 was observed. */
|
|
78
|
+
observed_at?: string;
|
|
79
|
+
/** The provider's raw result text, kept so the senior notice can quote it. */
|
|
87
80
|
provider_text?: string;
|
|
88
81
|
}
|
|
89
82
|
/**
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
83
|
+
* Decide the hold a 429 records (incident 2026-10-13: one 429 on the wrong
|
|
84
|
+
* subscription held the only Claude account for three days).
|
|
85
|
+
*
|
|
86
|
+
* - A per-model credits refusal is untouched.
|
|
87
|
+
* - A 429 with no reset gets the provider's configured default hold
|
|
88
|
+
* (`quota_default_hold_ms`, fifteen minutes unless set), marked `derived`.
|
|
89
|
+
* - A 429 with a provider reset is capped to the same default hold on its
|
|
90
|
+
* own. The provider reset is kept on the signal as `provider_reset_at`;
|
|
91
|
+
* recordAccountQuota extends the hold to it only when a fresh probe hits a
|
|
92
|
+
* repeat 429 before that reset. A reset nearer than the cap is used as is
|
|
93
|
+
* and marked `parsed`.
|
|
96
94
|
*/
|
|
97
95
|
export declare function applyDefaultQuotaHold(signal: QuotaSignal, defaultHoldMs: number, now: Date): QuotaSignal;
|
|
96
|
+
/**
|
|
97
|
+
* True when the signal is a fresh 429 on an account whose previous 429 named
|
|
98
|
+
* a provider reset that has not arrived yet: the short first-strike hold
|
|
99
|
+
* expired (or was cleared and re-recorded), a new probe ran, and the
|
|
100
|
+
* provider refused again. The provider's reset is then believed.
|
|
101
|
+
*/
|
|
102
|
+
export declare function isRepeatQuotaHit(current: AccountQuota | undefined, signal: QuotaSignal, now: Date): boolean;
|
|
103
|
+
/** "UTC" or an IANA zone the runtime knows, e.g. America/Denver. */
|
|
104
|
+
export declare function isKnownTimeZone(zone: string): boolean;
|
|
98
105
|
export declare function quotaStatePath(hqRoot: string): string;
|
|
99
106
|
export declare function loadQuotaState(hqRoot: string): QuotaState;
|
|
100
107
|
/**
|
|
@@ -112,11 +119,6 @@ export declare function loadQuotaState(hqRoot: string): QuotaState;
|
|
|
112
119
|
* past. An explicit provider year is kept as stated.
|
|
113
120
|
*/
|
|
114
121
|
export declare function parseProviderResetTime(text: string, now: Date): string | null;
|
|
115
|
-
/**
|
|
116
|
-
* Read only `api_error_status`, `is_error`, and `result` off the last
|
|
117
|
-
* `type: "result"` object. Nothing else from the payload is returned, and
|
|
118
|
-
* this function does not log.
|
|
119
|
-
*/
|
|
120
122
|
export interface QuotaResultFields {
|
|
121
123
|
api_error_status: unknown;
|
|
122
124
|
is_error: unknown;
|
|
@@ -129,16 +131,15 @@ export interface QuotaResultFields {
|
|
|
129
131
|
}
|
|
130
132
|
/** Epoch seconds, epoch milliseconds, or an ISO string, to ISO; null otherwise. */
|
|
131
133
|
export declare function normalizeStructuredReset(value: unknown): string | null;
|
|
134
|
+
/**
|
|
135
|
+
* Read only `api_error_status`, `is_error`, `result`, and a structured
|
|
136
|
+
* reset off the last `type: "result"` object. Nothing else from the payload
|
|
137
|
+
* is returned, and this function does not log.
|
|
138
|
+
*/
|
|
132
139
|
export declare function quotaFieldsFromOutput(text: string): QuotaResultFields | null;
|
|
133
140
|
export declare function quotaSignalFromFields(fields: QuotaResultFields, model: string | undefined, now: Date): QuotaSignal | null;
|
|
134
141
|
export declare function readQuotaSignal(runDir: string, model: string | undefined, now: Date, workerOutOffset?: number): QuotaSignal | null;
|
|
135
|
-
export
|
|
136
|
-
/** When the 429 was observed. Enables the far-reset confirmation gate. */
|
|
137
|
-
now: Date;
|
|
138
|
-
/** Hold applied while a far provider reset is unconfirmed (the default hold). */
|
|
139
|
-
unconfirmedHoldMs: number;
|
|
140
|
-
}
|
|
141
|
-
export declare function recordAccountQuota(hqRoot: string, provider: LaneProvider, accountId: string, incoming: QuotaSignal, options?: RecordAccountQuotaOptions): QuotaState;
|
|
142
|
+
export declare function recordAccountQuota(hqRoot: string, provider: LaneProvider, accountId: string, incoming: QuotaSignal, now?: Date): QuotaState;
|
|
142
143
|
export interface ClearedModelQuotaReset {
|
|
143
144
|
model: string;
|
|
144
145
|
quota_reset_at: string;
|
|
@@ -166,13 +167,31 @@ export interface QuotaBlock {
|
|
|
166
167
|
model?: string;
|
|
167
168
|
}
|
|
168
169
|
/**
|
|
169
|
-
* Skip an account only for
|
|
170
|
-
*
|
|
171
|
-
*
|
|
170
|
+
* Skip an account only for a model it cannot run at all: a needs-credits
|
|
171
|
+
* marker blocks that model, with no expiry.
|
|
172
|
+
*
|
|
173
|
+
* A recorded 429 never blocks (owner decision, incident 2026-10-13). The
|
|
174
|
+
* lane that hit it ends resumable and its senior decides; other lanes keep
|
|
175
|
+
* using the account. The reset times stay in quota.json as information
|
|
176
|
+
* (`accountQuotaView`, `quotaHitInfo`).
|
|
172
177
|
*/
|
|
173
|
-
export declare function quotaBlockFor(state: QuotaState, provider: LaneProvider, accountId: string, model: string | undefined,
|
|
178
|
+
export declare function quotaBlockFor(state: QuotaState, provider: LaneProvider, accountId: string, model: string | undefined, _now: Date): QuotaBlock;
|
|
179
|
+
/** The last recorded 429 on an account, for display only. */
|
|
180
|
+
export interface QuotaHitInfo {
|
|
181
|
+
last_quota_hit_at?: string;
|
|
182
|
+
/** The provider's own reset time from the last 429, when it named one. */
|
|
183
|
+
provider_reset_at?: string;
|
|
184
|
+
/** True while the provider's reset is still ahead. Informational; nothing gates on it. */
|
|
185
|
+
provider_reset_pending: boolean;
|
|
186
|
+
}
|
|
187
|
+
export declare function quotaHitInfo(state: QuotaState, provider: LaneProvider, accountId: string, now: Date): QuotaHitInfo | undefined;
|
|
174
188
|
export interface AccountQuotaView {
|
|
175
189
|
quota_reset_at: string | null;
|
|
190
|
+
/** "parsed" when the hold is the provider's reset, "derived" when HQ shortened it. */
|
|
191
|
+
quota_reset_source?: QuotaResetSource;
|
|
192
|
+
/** The provider's own reset from the last 429, when the hold was capped below it. */
|
|
193
|
+
provider_reset_at?: string;
|
|
194
|
+
last_quota_hit_at?: string;
|
|
176
195
|
model_quota: Array<{
|
|
177
196
|
model: string;
|
|
178
197
|
quota_reset_at?: string;
|
package/dist/lib/lanes/quota.js
CHANGED
|
@@ -17,55 +17,81 @@
|
|
|
17
17
|
* per (account, model), plus an account-wide reset when the text is a
|
|
18
18
|
* usage limit rather than a switch-model refusal.
|
|
19
19
|
*
|
|
20
|
-
* A
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* "You've hit your weekly limit · resets Oct 16 at 1am (America/Denver)"
|
|
26
|
-
* while the owner's /usage showed the week at 43%.)
|
|
20
|
+
* A recorded 429 never holds the account (owner decision 2026-10-14, after
|
|
21
|
+
* the 2026-10-13 incident where one 429 on account a blocked every Claude
|
|
22
|
+
* lane until Friday). The reset times and provider text stay here as
|
|
23
|
+
* information for `hq lanes capacity`, `hq lanes accounts list` and the
|
|
24
|
+
* senior notice; only a needs-credits marker skips an account, for that model.
|
|
27
25
|
*/
|
|
28
26
|
import * as fs from "node:fs";
|
|
29
27
|
import * as path from "node:path";
|
|
30
28
|
import { withLaneRecordFileLock } from "./store.js";
|
|
31
29
|
export const QUOTA_STATE_REL = path.join("personal", "lanes", "quota.json");
|
|
32
|
-
/**
|
|
33
|
-
* Claude quota holds are off. Removed by owner directive on 2026-10-14: a
|
|
34
|
-
* false weekly-limit 429 held every Claude lane for days while the account
|
|
35
|
-
* still had capacity. A Claude 429 still fails that lane over to another
|
|
36
|
-
* account; it never blocks the account for later lanes, and stored Claude
|
|
37
|
-
* holds are ignored on read. `HQ_LANES_CLAUDE_QUOTA_HOLDS=1` turns the old
|
|
38
|
-
* behavior back on; the test suite sets it to keep the hold machinery covered.
|
|
39
|
-
*/
|
|
40
|
-
export const CLAUDE_QUOTA_HOLDS_ENV = "HQ_LANES_CLAUDE_QUOTA_HOLDS";
|
|
41
|
-
export function providerHasQuotaHolds(provider) {
|
|
42
|
-
if (provider !== "claude")
|
|
43
|
-
return true;
|
|
44
|
-
return process.env[CLAUDE_QUOTA_HOLDS_ENV] === "1";
|
|
45
|
-
}
|
|
46
30
|
export const QUOTA_SCHEMA_VERSION = "hq-lanes-quota.v1";
|
|
47
|
-
/** A provider reset further away than this needs a second 429 before it holds. */
|
|
48
|
-
export const QUOTA_CONFIRM_HORIZON_MS = 24 * 60 * 60 * 1000;
|
|
49
31
|
/**
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
32
|
+
* Decide the hold a 429 records (incident 2026-10-13: one 429 on the wrong
|
|
33
|
+
* subscription held the only Claude account for three days).
|
|
34
|
+
*
|
|
35
|
+
* - A per-model credits refusal is untouched.
|
|
36
|
+
* - A 429 with no reset gets the provider's configured default hold
|
|
37
|
+
* (`quota_default_hold_ms`, fifteen minutes unless set), marked `derived`.
|
|
38
|
+
* - A 429 with a provider reset is capped to the same default hold on its
|
|
39
|
+
* own. The provider reset is kept on the signal as `provider_reset_at`;
|
|
40
|
+
* recordAccountQuota extends the hold to it only when a fresh probe hits a
|
|
41
|
+
* repeat 429 before that reset. A reset nearer than the cap is used as is
|
|
42
|
+
* and marked `parsed`.
|
|
56
43
|
*/
|
|
57
44
|
export function applyDefaultQuotaHold(signal, defaultHoldMs, now) {
|
|
58
45
|
if (signal.reason !== "account_quota_exhausted" || signal.needs_credits)
|
|
59
46
|
return signal;
|
|
60
|
-
|
|
61
|
-
|
|
47
|
+
const observed_at = now.toISOString();
|
|
48
|
+
const shortHoldMs = now.getTime() + defaultHoldMs;
|
|
49
|
+
const providerResetMs = signal.quota_reset_at ? Date.parse(signal.quota_reset_at) : Number.NaN;
|
|
50
|
+
if (Number.isFinite(providerResetMs)) {
|
|
51
|
+
const providerReset = new Date(providerResetMs).toISOString();
|
|
52
|
+
if (providerResetMs <= shortHoldMs) {
|
|
53
|
+
return { ...signal, quota_reset_at: providerReset, provider_reset_at: providerReset, observed_at, reset_source: "parsed" };
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
...signal,
|
|
57
|
+
quota_reset_at: new Date(shortHoldMs).toISOString(),
|
|
58
|
+
provider_reset_at: providerReset,
|
|
59
|
+
observed_at,
|
|
60
|
+
account_wide: true,
|
|
61
|
+
reset_source: "derived",
|
|
62
|
+
};
|
|
63
|
+
}
|
|
62
64
|
return {
|
|
63
65
|
...signal,
|
|
64
|
-
quota_reset_at: new Date(
|
|
66
|
+
quota_reset_at: new Date(shortHoldMs).toISOString(),
|
|
67
|
+
observed_at,
|
|
65
68
|
account_wide: true,
|
|
66
69
|
reset_source: "derived",
|
|
67
70
|
};
|
|
68
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* True when the signal is a fresh 429 on an account whose previous 429 named
|
|
74
|
+
* a provider reset that has not arrived yet: the short first-strike hold
|
|
75
|
+
* expired (or was cleared and re-recorded), a new probe ran, and the
|
|
76
|
+
* provider refused again. The provider's reset is then believed.
|
|
77
|
+
*/
|
|
78
|
+
export function isRepeatQuotaHit(current, signal, now) {
|
|
79
|
+
if (!current?.provider_reset_at || !current.last_quota_hit_at)
|
|
80
|
+
return false;
|
|
81
|
+
const priorResetMs = Date.parse(current.provider_reset_at);
|
|
82
|
+
const priorHitMs = Date.parse(current.last_quota_hit_at);
|
|
83
|
+
if (!Number.isFinite(priorResetMs) || !Number.isFinite(priorHitMs))
|
|
84
|
+
return false;
|
|
85
|
+
const observedMs = signal.observed_at ? Date.parse(signal.observed_at) : now.getTime();
|
|
86
|
+
if (!Number.isFinite(observedMs) || observedMs <= priorHitMs)
|
|
87
|
+
return false;
|
|
88
|
+
// A second lane that was already running when the first hold was written
|
|
89
|
+
// is the same strike, not a fresh probe: the hold must have lapsed first.
|
|
90
|
+
const priorHoldMs = current.all_models_quota_reset_at ? Date.parse(current.all_models_quota_reset_at) : Number.NaN;
|
|
91
|
+
if (Number.isFinite(priorHoldMs) && observedMs < priorHoldMs)
|
|
92
|
+
return false;
|
|
93
|
+
return observedMs < priorResetMs;
|
|
94
|
+
}
|
|
69
95
|
const MONTHS = {
|
|
70
96
|
jan: 0,
|
|
71
97
|
feb: 1,
|
|
@@ -80,6 +106,34 @@ const MONTHS = {
|
|
|
80
106
|
nov: 10,
|
|
81
107
|
dec: 11,
|
|
82
108
|
};
|
|
109
|
+
const zoneFormatters = new Map();
|
|
110
|
+
function zoneFormatter(zone) {
|
|
111
|
+
const cached = zoneFormatters.get(zone);
|
|
112
|
+
if (cached !== undefined)
|
|
113
|
+
return cached;
|
|
114
|
+
let formatter;
|
|
115
|
+
try {
|
|
116
|
+
formatter = new Intl.DateTimeFormat("en-US", {
|
|
117
|
+
timeZone: zone,
|
|
118
|
+
hourCycle: "h23",
|
|
119
|
+
year: "numeric",
|
|
120
|
+
month: "numeric",
|
|
121
|
+
day: "numeric",
|
|
122
|
+
hour: "numeric",
|
|
123
|
+
minute: "numeric",
|
|
124
|
+
second: "numeric",
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
formatter = null;
|
|
129
|
+
}
|
|
130
|
+
zoneFormatters.set(zone, formatter);
|
|
131
|
+
return formatter;
|
|
132
|
+
}
|
|
133
|
+
/** "UTC" or an IANA zone the runtime knows, e.g. America/Denver. */
|
|
134
|
+
export function isKnownTimeZone(zone) {
|
|
135
|
+
return zoneFormatter(zone) !== null;
|
|
136
|
+
}
|
|
83
137
|
function emptyState() {
|
|
84
138
|
return { schema_version: QUOTA_SCHEMA_VERSION, providers: {} };
|
|
85
139
|
}
|
|
@@ -113,8 +167,6 @@ export function loadQuotaState(hqRoot) {
|
|
|
113
167
|
const providers = {};
|
|
114
168
|
const rawProviders = isPlainObject(parsed.providers) ? parsed.providers : {};
|
|
115
169
|
for (const provider of ["claude", "codex", "grok"]) {
|
|
116
|
-
if (!providerHasQuotaHolds(provider))
|
|
117
|
-
continue;
|
|
118
170
|
const rawAccounts = rawProviders[provider];
|
|
119
171
|
if (!isPlainObject(rawAccounts))
|
|
120
172
|
continue;
|
|
@@ -157,8 +209,11 @@ export function loadQuotaState(hqRoot) {
|
|
|
157
209
|
account.reset_source = "unconfirmed";
|
|
158
210
|
}
|
|
159
211
|
}
|
|
160
|
-
if (typeof raw.
|
|
161
|
-
account.
|
|
212
|
+
if (typeof raw.provider_reset_at === "string" && raw.provider_reset_at.length > 0) {
|
|
213
|
+
account.provider_reset_at = raw.provider_reset_at;
|
|
214
|
+
}
|
|
215
|
+
if (typeof raw.last_quota_hit_at === "string" && raw.last_quota_hit_at.length > 0) {
|
|
216
|
+
account.last_quota_hit_at = raw.last_quota_hit_at;
|
|
162
217
|
}
|
|
163
218
|
if (typeof raw.provider_text === "string" && raw.provider_text.length > 0) {
|
|
164
219
|
account.provider_text = raw.provider_text;
|
|
@@ -176,18 +231,6 @@ function writeQuotaState(hqRoot, state) {
|
|
|
176
231
|
fs.writeFileSync(tmp, `${JSON.stringify(state, null, 2)}\n`, "utf8");
|
|
177
232
|
fs.renameSync(tmp, filePath);
|
|
178
233
|
}
|
|
179
|
-
/** True when `zone` is a timezone name the runtime's Intl data knows. */
|
|
180
|
-
function isKnownTimeZone(zone) {
|
|
181
|
-
try {
|
|
182
|
-
new Intl.DateTimeFormat("en-US", { timeZone: zone });
|
|
183
|
-
return true;
|
|
184
|
-
}
|
|
185
|
-
catch (err) {
|
|
186
|
-
if (err instanceof RangeError)
|
|
187
|
-
return false;
|
|
188
|
-
throw err;
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
234
|
/** Offset of `zone` from UTC at instant `ms`, in milliseconds. */
|
|
192
235
|
function zoneOffsetMs(zone, ms) {
|
|
193
236
|
const parts = new Intl.DateTimeFormat("en-US", {
|
|
@@ -377,6 +420,11 @@ export function normalizeStructuredReset(value) {
|
|
|
377
420
|
return null;
|
|
378
421
|
return new Date(ms).toISOString();
|
|
379
422
|
}
|
|
423
|
+
/**
|
|
424
|
+
* Read only `api_error_status`, `is_error`, `result`, and a structured
|
|
425
|
+
* reset off the last `type: "result"` object. Nothing else from the payload
|
|
426
|
+
* is returned, and this function does not log.
|
|
427
|
+
*/
|
|
380
428
|
export function quotaFieldsFromOutput(text) {
|
|
381
429
|
const lines = text.split("\n");
|
|
382
430
|
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
@@ -462,48 +510,25 @@ export function readQuotaSignal(runDir, model, now, workerOutOffset = 0) {
|
|
|
462
510
|
const observedAt = Number.isFinite(outputAt.getTime()) ? outputAt : now;
|
|
463
511
|
return quotaSignalFromFields(fields, model, observedAt);
|
|
464
512
|
}
|
|
465
|
-
|
|
466
|
-
* Hold a provider reset more than a day away to a short hold until a second
|
|
467
|
-
* 429 confirms it. The second 429 counts only when its lane ran after the
|
|
468
|
-
* short hold expired, so lanes that failed together on one 429 burst do not
|
|
469
|
-
* confirm each other.
|
|
470
|
-
*/
|
|
471
|
-
function gateFarReset(current, signal, options) {
|
|
472
|
-
if (!signal.account_wide ||
|
|
473
|
-
!signal.quota_reset_at ||
|
|
474
|
-
signal.needs_credits ||
|
|
475
|
-
signal.reset_source === "derived") {
|
|
476
|
-
return { signal };
|
|
477
|
-
}
|
|
478
|
-
const resetMs = Date.parse(signal.quota_reset_at);
|
|
479
|
-
const nowMs = options.now.getTime();
|
|
480
|
-
if (!Number.isFinite(resetMs) || resetMs - nowMs <= QUOTA_CONFIRM_HORIZON_MS)
|
|
481
|
-
return { signal };
|
|
482
|
-
const pending = current.unconfirmed_reset_at;
|
|
483
|
-
const shortHoldActive = resetStillAhead(current.all_models_quota_reset_at, options.now);
|
|
484
|
-
if (pending !== undefined && !shortHoldActive) {
|
|
485
|
-
return { signal: { ...signal, reset_source: "parsed" } };
|
|
486
|
-
}
|
|
487
|
-
return {
|
|
488
|
-
signal: {
|
|
489
|
-
...signal,
|
|
490
|
-
quota_reset_at: new Date(nowMs + options.unconfirmedHoldMs).toISOString(),
|
|
491
|
-
reset_source: "unconfirmed",
|
|
492
|
-
},
|
|
493
|
-
unconfirmed_reset_at: pending !== undefined && !shouldReplaceReset(pending, signal.quota_reset_at)
|
|
494
|
-
? pending
|
|
495
|
-
: signal.quota_reset_at,
|
|
496
|
-
};
|
|
497
|
-
}
|
|
498
|
-
export function recordAccountQuota(hqRoot, provider, accountId, incoming, options) {
|
|
499
|
-
if (!providerHasQuotaHolds(provider))
|
|
500
|
-
return loadQuotaState(hqRoot);
|
|
513
|
+
export function recordAccountQuota(hqRoot, provider, accountId, incoming, now = new Date()) {
|
|
501
514
|
return withLaneRecordFileLock(quotaStatePath(hqRoot), () => {
|
|
502
515
|
const state = loadQuotaState(hqRoot);
|
|
503
516
|
const accounts = state.providers[provider] ?? {};
|
|
504
517
|
const current = accounts[accountId] ?? { models: {} };
|
|
505
|
-
|
|
506
|
-
|
|
518
|
+
// A repeat 429 before the provider's reset promotes the capped hold to
|
|
519
|
+
// the provider's own reset (see applyDefaultQuotaHold).
|
|
520
|
+
let signal = incoming;
|
|
521
|
+
if (incoming.reason === "account_quota_exhausted" &&
|
|
522
|
+
!incoming.needs_credits &&
|
|
523
|
+
isRepeatQuotaHit(current, incoming, now)) {
|
|
524
|
+
const candidates = [incoming.provider_reset_at, current.provider_reset_at, incoming.quota_reset_at]
|
|
525
|
+
.map((value) => (value ? Date.parse(value) : Number.NaN))
|
|
526
|
+
.filter((ms) => Number.isFinite(ms));
|
|
527
|
+
if (candidates.length > 0) {
|
|
528
|
+
const promoted = new Date(Math.max(...candidates)).toISOString();
|
|
529
|
+
signal = { ...incoming, quota_reset_at: promoted, provider_reset_at: promoted, account_wide: true, reset_source: "parsed" };
|
|
530
|
+
}
|
|
531
|
+
}
|
|
507
532
|
const models = { ...current.models };
|
|
508
533
|
if (signal.model) {
|
|
509
534
|
const prior = models[signal.model] ?? {};
|
|
@@ -525,19 +550,18 @@ export function recordAccountQuota(hqRoot, provider, accountId, incoming, option
|
|
|
525
550
|
if (shouldReplaceReset(current.all_models_quota_reset_at, signal.quota_reset_at)) {
|
|
526
551
|
account.all_models_quota_reset_at = signal.quota_reset_at;
|
|
527
552
|
account.reset_source = signal.reset_source ?? "parsed";
|
|
528
|
-
if (signal.provider_text)
|
|
529
|
-
account.provider_text = signal.provider_text;
|
|
530
|
-
if (gated.unconfirmed_reset_at) {
|
|
531
|
-
account.unconfirmed_reset_at = gated.unconfirmed_reset_at;
|
|
532
|
-
}
|
|
533
|
-
else {
|
|
534
|
-
delete account.unconfirmed_reset_at;
|
|
535
|
-
}
|
|
536
553
|
}
|
|
537
554
|
else if (current.all_models_quota_reset_at) {
|
|
538
555
|
account.all_models_quota_reset_at = current.all_models_quota_reset_at;
|
|
539
556
|
account.reset_source = current.reset_source ?? "parsed";
|
|
540
557
|
}
|
|
558
|
+
if (signal.provider_text)
|
|
559
|
+
account.provider_text = signal.provider_text;
|
|
560
|
+
if (signal.provider_reset_at)
|
|
561
|
+
account.provider_reset_at = signal.provider_reset_at;
|
|
562
|
+
else
|
|
563
|
+
delete account.provider_reset_at;
|
|
564
|
+
account.last_quota_hit_at = signal.observed_at ?? now.toISOString();
|
|
541
565
|
}
|
|
542
566
|
accounts[accountId] = account;
|
|
543
567
|
state.providers[provider] = accounts;
|
|
@@ -560,16 +584,13 @@ export function clearAccountQuota(hqRoot, provider, accountId, options = {}) {
|
|
|
560
584
|
const shouldClear = (reset) => reset !== undefined &&
|
|
561
585
|
(!options.futureOnly || resetStillAhead(reset, now));
|
|
562
586
|
let changed = false;
|
|
563
|
-
// A successful turn or an operator clear disproves a pending far reset.
|
|
564
|
-
if (account.unconfirmed_reset_at !== undefined) {
|
|
565
|
-
delete account.unconfirmed_reset_at;
|
|
566
|
-
changed = true;
|
|
567
|
-
}
|
|
568
587
|
const accountWideReset = account.all_models_quota_reset_at;
|
|
569
588
|
if (shouldClear(accountWideReset)) {
|
|
570
589
|
cleared.all_models_quota_reset_at = accountWideReset;
|
|
571
590
|
delete account.all_models_quota_reset_at;
|
|
572
591
|
delete account.reset_source;
|
|
592
|
+
delete account.provider_reset_at;
|
|
593
|
+
delete account.last_quota_hit_at;
|
|
573
594
|
delete account.provider_text;
|
|
574
595
|
changed = true;
|
|
575
596
|
}
|
|
@@ -644,30 +665,20 @@ function resetStillAhead(iso, now) {
|
|
|
644
665
|
return Number.isFinite(ts) && ts > now.getTime();
|
|
645
666
|
}
|
|
646
667
|
/**
|
|
647
|
-
* Skip an account only for
|
|
648
|
-
*
|
|
649
|
-
*
|
|
668
|
+
* Skip an account only for a model it cannot run at all: a needs-credits
|
|
669
|
+
* marker blocks that model, with no expiry.
|
|
670
|
+
*
|
|
671
|
+
* A recorded 429 never blocks (owner decision, incident 2026-10-13). The
|
|
672
|
+
* lane that hit it ends resumable and its senior decides; other lanes keep
|
|
673
|
+
* using the account. The reset times stay in quota.json as information
|
|
674
|
+
* (`accountQuotaView`, `quotaHitInfo`).
|
|
650
675
|
*/
|
|
651
|
-
export function quotaBlockFor(state, provider, accountId, model,
|
|
652
|
-
if (!providerHasQuotaHolds(provider))
|
|
653
|
-
return { blocked: false };
|
|
676
|
+
export function quotaBlockFor(state, provider, accountId, model, _now) {
|
|
654
677
|
const account = state.providers[provider]?.[accountId];
|
|
655
|
-
if (!account)
|
|
656
|
-
return { blocked: false };
|
|
657
|
-
if (resetStillAhead(account.all_models_quota_reset_at, now)) {
|
|
658
|
-
return {
|
|
659
|
-
blocked: true,
|
|
660
|
-
reason: "account_quota_exhausted",
|
|
661
|
-
quota_reset_at: account.all_models_quota_reset_at,
|
|
662
|
-
model,
|
|
663
|
-
};
|
|
664
|
-
}
|
|
665
|
-
if (!model)
|
|
678
|
+
if (!account || !model)
|
|
666
679
|
return { blocked: false };
|
|
667
680
|
const entry = account.models[model];
|
|
668
|
-
if (
|
|
669
|
-
return { blocked: false };
|
|
670
|
-
if (entry.needs_credits) {
|
|
681
|
+
if (entry?.needs_credits) {
|
|
671
682
|
return {
|
|
672
683
|
blocked: true,
|
|
673
684
|
reason: "account_model_unavailable",
|
|
@@ -675,16 +686,21 @@ export function quotaBlockFor(state, provider, accountId, model, now) {
|
|
|
675
686
|
model,
|
|
676
687
|
};
|
|
677
688
|
}
|
|
678
|
-
if (resetStillAhead(entry.quota_reset_at, now)) {
|
|
679
|
-
return {
|
|
680
|
-
blocked: true,
|
|
681
|
-
reason: "account_quota_exhausted",
|
|
682
|
-
quota_reset_at: entry.quota_reset_at,
|
|
683
|
-
model,
|
|
684
|
-
};
|
|
685
|
-
}
|
|
686
689
|
return { blocked: false };
|
|
687
690
|
}
|
|
691
|
+
export function quotaHitInfo(state, provider, accountId, now) {
|
|
692
|
+
const account = state.providers[provider]?.[accountId];
|
|
693
|
+
if (!account || (!account.last_quota_hit_at && !account.provider_reset_at && !account.all_models_quota_reset_at)) {
|
|
694
|
+
return undefined;
|
|
695
|
+
}
|
|
696
|
+
const providerReset = account.provider_reset_at ??
|
|
697
|
+
(account.reset_source === "parsed" ? account.all_models_quota_reset_at : undefined);
|
|
698
|
+
return {
|
|
699
|
+
...(account.last_quota_hit_at ? { last_quota_hit_at: account.last_quota_hit_at } : {}),
|
|
700
|
+
...(providerReset ? { provider_reset_at: providerReset } : {}),
|
|
701
|
+
provider_reset_pending: resetStillAhead(providerReset, now),
|
|
702
|
+
};
|
|
703
|
+
}
|
|
688
704
|
export function accountQuotaView(state, provider, accountId) {
|
|
689
705
|
const account = state.providers[provider]?.[accountId];
|
|
690
706
|
if (!account)
|
|
@@ -703,6 +719,9 @@ export function accountQuotaView(state, provider, accountId) {
|
|
|
703
719
|
});
|
|
704
720
|
return {
|
|
705
721
|
quota_reset_at: account.all_models_quota_reset_at ?? null,
|
|
722
|
+
...(account.reset_source ? { quota_reset_source: account.reset_source } : {}),
|
|
723
|
+
...(account.provider_reset_at ? { provider_reset_at: account.provider_reset_at } : {}),
|
|
724
|
+
...(account.last_quota_hit_at ? { last_quota_hit_at: account.last_quota_hit_at } : {}),
|
|
706
725
|
model_quota,
|
|
707
726
|
};
|
|
708
727
|
}
|
|
@@ -178,4 +178,18 @@ export interface AnsweredLaneWakeResult {
|
|
|
178
178
|
}
|
|
179
179
|
/** Run the reconcile answer-resume path for one lane before questions answer returns. */
|
|
180
180
|
export declare function wakeAnsweredLaneNow(hqRoot: string, laneId: string, deps: ReconcileDeps): Promise<AnsweredLaneWakeResult>;
|
|
181
|
+
export interface AdmitQueuedNowResult {
|
|
182
|
+
admitted: ReconcileChange[];
|
|
183
|
+
errors: ReconcileSummary["errors"];
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Run queue admission in-process, without the rest of a reconcile pass.
|
|
187
|
+
*
|
|
188
|
+
* Commands that change capacity (create, stop, cancel, clear-quota, a lane
|
|
189
|
+
* exit) call this so queued lanes start without `hq daemon`. It takes the
|
|
190
|
+
* reconcile lock and every provider admission lock, so it is safe to run
|
|
191
|
+
* while a daemon is also reconciling: whichever runs second sees the lanes
|
|
192
|
+
* the first one already admitted.
|
|
193
|
+
*/
|
|
194
|
+
export declare function admitQueuedNow(hqRoot: string, deps: ReconcileDeps): Promise<AdmitQueuedNowResult>;
|
|
181
195
|
//# sourceMappingURL=reconcile.d.ts.map
|