@indigoai-us/hq-cli 5.345.126 → 5.345.128
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/assets/scaffold/core/scripts/handoff-knowledge-commit.sh +123 -0
- package/assets/scaffold/core/scripts/hq-detach.sh +169 -0
- package/dist/command-catalog.generated.d.ts +40 -2
- package/dist/command-catalog.generated.js +48 -2
- package/dist/commands/auth.js +19 -2
- package/dist/commands/core-bucket-c-handoff-knowledge-fast.d.ts +2 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-fast.js +10 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-native.d.ts +46 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-native.js +284 -0
- package/dist/commands/core-bucket-c-hq-detach-fast.d.ts +2 -0
- package/dist/commands/core-bucket-c-hq-detach-fast.js +10 -0
- package/dist/commands/core-bucket-c-hq-detach-native.d.ts +43 -0
- package/dist/commands/core-bucket-c-hq-detach-native.js +241 -0
- package/dist/commands/core.js +12 -0
- package/dist/commands/lanes-accounts.js +3 -1
- package/dist/commands/lanes-shared.d.ts +7 -0
- package/dist/commands/lanes.js +113 -3
- package/dist/commands/skill.js +9 -4
- package/dist/fast-scaffold.js +28 -0
- package/dist/lib/capacity/lane-login.js +1 -0
- package/dist/lib/lanes/accounts.js +2 -2
- package/dist/lib/lanes/host-capacity.d.ts +4 -0
- package/dist/lib/lanes/host-capacity.js +10 -1
- package/dist/lib/lanes/index.d.ts +2 -2
- package/dist/lib/lanes/index.js +1 -1
- package/dist/lib/lanes/loop.d.ts +8 -0
- package/dist/lib/lanes/loop.js +35 -0
- package/dist/lib/lanes/quota-notice.d.ts +11 -11
- package/dist/lib/lanes/quota-notice.js +20 -29
- package/dist/lib/lanes/quota.d.ts +69 -40
- package/dist/lib/lanes/quota.js +146 -107
- package/dist/lib/lanes/reconcile.d.ts +14 -0
- package/dist/lib/lanes/reconcile.js +77 -23
- package/dist/lib/lanes/remote-control-lane.js +10 -3
- package/dist/lib/lanes/watcher.d.ts +8 -0
- package/dist/lib/lanes/watcher.js +62 -20
- package/dist/utils/resident-package-preload.js +12 -4
- package/package.json +1 -1
|
@@ -17,13 +17,11 @@
|
|
|
17
17
|
* per (account, model), plus an account-wide reset when the text is a
|
|
18
18
|
* usage limit rather than a switch-model refusal.
|
|
19
19
|
*
|
|
20
|
-
* A
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* "You've hit your weekly limit · resets Oct 16 at 1am (America/Denver)"
|
|
26
|
-
* while the owner's /usage showed the week at 43%.)
|
|
20
|
+
* A recorded 429 never holds the account (owner decision 2026-10-14, after
|
|
21
|
+
* the 2026-10-13 incident where one 429 on account a blocked every Claude
|
|
22
|
+
* lane until Friday). The reset times and provider text stay here as
|
|
23
|
+
* information for `hq lanes capacity`, `hq lanes accounts list` and the
|
|
24
|
+
* senior notice; only a needs-credits marker skips an account, for that model.
|
|
27
25
|
*/
|
|
28
26
|
import type { LaneProvider } from "./types.js";
|
|
29
27
|
export declare const QUOTA_STATE_REL: string;
|
|
@@ -36,26 +34,27 @@ export interface ModelQuota {
|
|
|
36
34
|
}
|
|
37
35
|
/**
|
|
38
36
|
* `parsed`: the provider's own reset. `derived`: the default hold for a 429
|
|
39
|
-
* with no parseable reset. `unconfirmed
|
|
40
|
-
* provider reset more than a day away that a second 429 has not confirmed.
|
|
37
|
+
* with no parseable reset. `unconfirmed` is read from older quota files only.
|
|
41
38
|
*/
|
|
42
39
|
export type QuotaResetSource = "parsed" | "derived" | "unconfirmed";
|
|
43
|
-
/** A provider reset further away than this needs a second 429 before it holds. */
|
|
44
|
-
export declare const QUOTA_CONFIRM_HORIZON_MS: number;
|
|
45
40
|
export interface AccountQuota {
|
|
46
41
|
/** Set when the 429 is an account-wide usage limit with a reset time. */
|
|
47
42
|
all_models_quota_reset_at?: string;
|
|
48
43
|
/**
|
|
49
|
-
* Whether `all_models_quota_reset_at`
|
|
50
|
-
*
|
|
44
|
+
* Whether `all_models_quota_reset_at` is the provider's own reset
|
|
45
|
+
* (`parsed`) or a shorter hold HQ chose (`derived`): the configured
|
|
46
|
+
* default hold for a 429 with no reset, or the capped first-strike hold
|
|
47
|
+
* when the provider's reset is further away than that.
|
|
51
48
|
*/
|
|
52
49
|
reset_source?: QuotaResetSource;
|
|
53
50
|
/**
|
|
54
|
-
* The provider's
|
|
55
|
-
*
|
|
51
|
+
* The provider's own reset time from the most recent 429, kept even when
|
|
52
|
+
* the recorded reset is shorter. Information only; nothing gates on it.
|
|
56
53
|
*/
|
|
57
|
-
|
|
58
|
-
/**
|
|
54
|
+
provider_reset_at?: string;
|
|
55
|
+
/** When the most recent account-wide 429 was observed. */
|
|
56
|
+
last_quota_hit_at?: string;
|
|
57
|
+
/** Raw provider text from the most recent account-wide 429. */
|
|
59
58
|
provider_text?: string;
|
|
60
59
|
models: Record<string, ModelQuota>;
|
|
61
60
|
}
|
|
@@ -73,18 +72,36 @@ export interface QuotaSignal {
|
|
|
73
72
|
account_wide?: boolean;
|
|
74
73
|
/** Set by applyDefaultQuotaHold: whether the reset is the provider's or the default hold. */
|
|
75
74
|
reset_source?: QuotaResetSource;
|
|
76
|
-
/** The provider's
|
|
75
|
+
/** The provider's own reset, parsed or structured, before any cap. */
|
|
76
|
+
provider_reset_at?: string;
|
|
77
|
+
/** When the 429 was observed. */
|
|
78
|
+
observed_at?: string;
|
|
79
|
+
/** The provider's raw result text, kept so the senior notice can quote it. */
|
|
77
80
|
provider_text?: string;
|
|
78
81
|
}
|
|
79
82
|
/**
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
83
|
+
* Decide the hold a 429 records (incident 2026-10-13: one 429 on the wrong
|
|
84
|
+
* subscription held the only Claude account for three days).
|
|
85
|
+
*
|
|
86
|
+
* - A per-model credits refusal is untouched.
|
|
87
|
+
* - A 429 with no reset gets the provider's configured default hold
|
|
88
|
+
* (`quota_default_hold_ms`, fifteen minutes unless set), marked `derived`.
|
|
89
|
+
* - A 429 with a provider reset is capped to the same default hold on its
|
|
90
|
+
* own. The provider reset is kept on the signal as `provider_reset_at`;
|
|
91
|
+
* recordAccountQuota extends the hold to it only when a fresh probe hits a
|
|
92
|
+
* repeat 429 before that reset. A reset nearer than the cap is used as is
|
|
93
|
+
* and marked `parsed`.
|
|
86
94
|
*/
|
|
87
95
|
export declare function applyDefaultQuotaHold(signal: QuotaSignal, defaultHoldMs: number, now: Date): QuotaSignal;
|
|
96
|
+
/**
|
|
97
|
+
* True when the signal is a fresh 429 on an account whose previous 429 named
|
|
98
|
+
* a provider reset that has not arrived yet: the short first-strike hold
|
|
99
|
+
* expired (or was cleared and re-recorded), a new probe ran, and the
|
|
100
|
+
* provider refused again. The provider's reset is then believed.
|
|
101
|
+
*/
|
|
102
|
+
export declare function isRepeatQuotaHit(current: AccountQuota | undefined, signal: QuotaSignal, now: Date): boolean;
|
|
103
|
+
/** "UTC" or an IANA zone the runtime knows, e.g. America/Denver. */
|
|
104
|
+
export declare function isKnownTimeZone(zone: string): boolean;
|
|
88
105
|
export declare function quotaStatePath(hqRoot: string): string;
|
|
89
106
|
export declare function loadQuotaState(hqRoot: string): QuotaState;
|
|
90
107
|
/**
|
|
@@ -102,11 +119,6 @@ export declare function loadQuotaState(hqRoot: string): QuotaState;
|
|
|
102
119
|
* past. An explicit provider year is kept as stated.
|
|
103
120
|
*/
|
|
104
121
|
export declare function parseProviderResetTime(text: string, now: Date): string | null;
|
|
105
|
-
/**
|
|
106
|
-
* Read only `api_error_status`, `is_error`, and `result` off the last
|
|
107
|
-
* `type: "result"` object. Nothing else from the payload is returned, and
|
|
108
|
-
* this function does not log.
|
|
109
|
-
*/
|
|
110
122
|
export interface QuotaResultFields {
|
|
111
123
|
api_error_status: unknown;
|
|
112
124
|
is_error: unknown;
|
|
@@ -119,16 +131,15 @@ export interface QuotaResultFields {
|
|
|
119
131
|
}
|
|
120
132
|
/** Epoch seconds, epoch milliseconds, or an ISO string, to ISO; null otherwise. */
|
|
121
133
|
export declare function normalizeStructuredReset(value: unknown): string | null;
|
|
134
|
+
/**
|
|
135
|
+
* Read only `api_error_status`, `is_error`, `result`, and a structured
|
|
136
|
+
* reset off the last `type: "result"` object. Nothing else from the payload
|
|
137
|
+
* is returned, and this function does not log.
|
|
138
|
+
*/
|
|
122
139
|
export declare function quotaFieldsFromOutput(text: string): QuotaResultFields | null;
|
|
123
140
|
export declare function quotaSignalFromFields(fields: QuotaResultFields, model: string | undefined, now: Date): QuotaSignal | null;
|
|
124
141
|
export declare function readQuotaSignal(runDir: string, model: string | undefined, now: Date, workerOutOffset?: number): QuotaSignal | null;
|
|
125
|
-
export
|
|
126
|
-
/** When the 429 was observed. Enables the far-reset confirmation gate. */
|
|
127
|
-
now: Date;
|
|
128
|
-
/** Hold applied while a far provider reset is unconfirmed (the default hold). */
|
|
129
|
-
unconfirmedHoldMs: number;
|
|
130
|
-
}
|
|
131
|
-
export declare function recordAccountQuota(hqRoot: string, provider: LaneProvider, accountId: string, incoming: QuotaSignal, options?: RecordAccountQuotaOptions): QuotaState;
|
|
142
|
+
export declare function recordAccountQuota(hqRoot: string, provider: LaneProvider, accountId: string, incoming: QuotaSignal, now?: Date): QuotaState;
|
|
132
143
|
export interface ClearedModelQuotaReset {
|
|
133
144
|
model: string;
|
|
134
145
|
quota_reset_at: string;
|
|
@@ -156,13 +167,31 @@ export interface QuotaBlock {
|
|
|
156
167
|
model?: string;
|
|
157
168
|
}
|
|
158
169
|
/**
|
|
159
|
-
* Skip an account only for
|
|
160
|
-
*
|
|
161
|
-
*
|
|
170
|
+
* Skip an account only for a model it cannot run at all: a needs-credits
|
|
171
|
+
* marker blocks that model, with no expiry.
|
|
172
|
+
*
|
|
173
|
+
* A recorded 429 never blocks (owner decision, incident 2026-10-13). The
|
|
174
|
+
* lane that hit it ends resumable and its senior decides; other lanes keep
|
|
175
|
+
* using the account. The reset times stay in quota.json as information
|
|
176
|
+
* (`accountQuotaView`, `quotaHitInfo`).
|
|
162
177
|
*/
|
|
163
|
-
export declare function quotaBlockFor(state: QuotaState, provider: LaneProvider, accountId: string, model: string | undefined,
|
|
178
|
+
export declare function quotaBlockFor(state: QuotaState, provider: LaneProvider, accountId: string, model: string | undefined, _now: Date): QuotaBlock;
|
|
179
|
+
/** The last recorded 429 on an account, for display only. */
|
|
180
|
+
export interface QuotaHitInfo {
|
|
181
|
+
last_quota_hit_at?: string;
|
|
182
|
+
/** The provider's own reset time from the last 429, when it named one. */
|
|
183
|
+
provider_reset_at?: string;
|
|
184
|
+
/** True while the provider's reset is still ahead. Informational; nothing gates on it. */
|
|
185
|
+
provider_reset_pending: boolean;
|
|
186
|
+
}
|
|
187
|
+
export declare function quotaHitInfo(state: QuotaState, provider: LaneProvider, accountId: string, now: Date): QuotaHitInfo | undefined;
|
|
164
188
|
export interface AccountQuotaView {
|
|
165
189
|
quota_reset_at: string | null;
|
|
190
|
+
/** "parsed" when the hold is the provider's reset, "derived" when HQ shortened it. */
|
|
191
|
+
quota_reset_source?: QuotaResetSource;
|
|
192
|
+
/** The provider's own reset from the last 429, when the hold was capped below it. */
|
|
193
|
+
provider_reset_at?: string;
|
|
194
|
+
last_quota_hit_at?: string;
|
|
166
195
|
model_quota: Array<{
|
|
167
196
|
model: string;
|
|
168
197
|
quota_reset_at?: string;
|
package/dist/lib/lanes/quota.js
CHANGED
|
@@ -17,41 +17,81 @@
|
|
|
17
17
|
* per (account, model), plus an account-wide reset when the text is a
|
|
18
18
|
* usage limit rather than a switch-model refusal.
|
|
19
19
|
*
|
|
20
|
-
* A
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
* "You've hit your weekly limit · resets Oct 16 at 1am (America/Denver)"
|
|
26
|
-
* while the owner's /usage showed the week at 43%.)
|
|
20
|
+
* A recorded 429 never holds the account (owner decision 2026-10-14, after
|
|
21
|
+
* the 2026-10-13 incident where one 429 on account a blocked every Claude
|
|
22
|
+
* lane until Friday). The reset times and provider text stay here as
|
|
23
|
+
* information for `hq lanes capacity`, `hq lanes accounts list` and the
|
|
24
|
+
* senior notice; only a needs-credits marker skips an account, for that model.
|
|
27
25
|
*/
|
|
28
26
|
import * as fs from "node:fs";
|
|
29
27
|
import * as path from "node:path";
|
|
30
28
|
import { withLaneRecordFileLock } from "./store.js";
|
|
31
29
|
export const QUOTA_STATE_REL = path.join("personal", "lanes", "quota.json");
|
|
32
30
|
export const QUOTA_SCHEMA_VERSION = "hq-lanes-quota.v1";
|
|
33
|
-
/** A provider reset further away than this needs a second 429 before it holds. */
|
|
34
|
-
export const QUOTA_CONFIRM_HORIZON_MS = 24 * 60 * 60 * 1000;
|
|
35
31
|
/**
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
32
|
+
* Decide the hold a 429 records (incident 2026-10-13: one 429 on the wrong
|
|
33
|
+
* subscription held the only Claude account for three days).
|
|
34
|
+
*
|
|
35
|
+
* - A per-model credits refusal is untouched.
|
|
36
|
+
* - A 429 with no reset gets the provider's configured default hold
|
|
37
|
+
* (`quota_default_hold_ms`, fifteen minutes unless set), marked `derived`.
|
|
38
|
+
* - A 429 with a provider reset is capped to the same default hold on its
|
|
39
|
+
* own. The provider reset is kept on the signal as `provider_reset_at`;
|
|
40
|
+
* recordAccountQuota extends the hold to it only when a fresh probe hits a
|
|
41
|
+
* repeat 429 before that reset. A reset nearer than the cap is used as is
|
|
42
|
+
* and marked `parsed`.
|
|
42
43
|
*/
|
|
43
44
|
export function applyDefaultQuotaHold(signal, defaultHoldMs, now) {
|
|
44
45
|
if (signal.reason !== "account_quota_exhausted" || signal.needs_credits)
|
|
45
46
|
return signal;
|
|
46
|
-
|
|
47
|
-
|
|
47
|
+
const observed_at = now.toISOString();
|
|
48
|
+
const shortHoldMs = now.getTime() + defaultHoldMs;
|
|
49
|
+
const providerResetMs = signal.quota_reset_at ? Date.parse(signal.quota_reset_at) : Number.NaN;
|
|
50
|
+
if (Number.isFinite(providerResetMs)) {
|
|
51
|
+
const providerReset = new Date(providerResetMs).toISOString();
|
|
52
|
+
if (providerResetMs <= shortHoldMs) {
|
|
53
|
+
return { ...signal, quota_reset_at: providerReset, provider_reset_at: providerReset, observed_at, reset_source: "parsed" };
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
...signal,
|
|
57
|
+
quota_reset_at: new Date(shortHoldMs).toISOString(),
|
|
58
|
+
provider_reset_at: providerReset,
|
|
59
|
+
observed_at,
|
|
60
|
+
account_wide: true,
|
|
61
|
+
reset_source: "derived",
|
|
62
|
+
};
|
|
63
|
+
}
|
|
48
64
|
return {
|
|
49
65
|
...signal,
|
|
50
|
-
quota_reset_at: new Date(
|
|
66
|
+
quota_reset_at: new Date(shortHoldMs).toISOString(),
|
|
67
|
+
observed_at,
|
|
51
68
|
account_wide: true,
|
|
52
69
|
reset_source: "derived",
|
|
53
70
|
};
|
|
54
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* True when the signal is a fresh 429 on an account whose previous 429 named
|
|
74
|
+
* a provider reset that has not arrived yet: the short first-strike hold
|
|
75
|
+
* expired (or was cleared and re-recorded), a new probe ran, and the
|
|
76
|
+
* provider refused again. The provider's reset is then believed.
|
|
77
|
+
*/
|
|
78
|
+
export function isRepeatQuotaHit(current, signal, now) {
|
|
79
|
+
if (!current?.provider_reset_at || !current.last_quota_hit_at)
|
|
80
|
+
return false;
|
|
81
|
+
const priorResetMs = Date.parse(current.provider_reset_at);
|
|
82
|
+
const priorHitMs = Date.parse(current.last_quota_hit_at);
|
|
83
|
+
if (!Number.isFinite(priorResetMs) || !Number.isFinite(priorHitMs))
|
|
84
|
+
return false;
|
|
85
|
+
const observedMs = signal.observed_at ? Date.parse(signal.observed_at) : now.getTime();
|
|
86
|
+
if (!Number.isFinite(observedMs) || observedMs <= priorHitMs)
|
|
87
|
+
return false;
|
|
88
|
+
// A second lane that was already running when the first hold was written
|
|
89
|
+
// is the same strike, not a fresh probe: the hold must have lapsed first.
|
|
90
|
+
const priorHoldMs = current.all_models_quota_reset_at ? Date.parse(current.all_models_quota_reset_at) : Number.NaN;
|
|
91
|
+
if (Number.isFinite(priorHoldMs) && observedMs < priorHoldMs)
|
|
92
|
+
return false;
|
|
93
|
+
return observedMs < priorResetMs;
|
|
94
|
+
}
|
|
55
95
|
const MONTHS = {
|
|
56
96
|
jan: 0,
|
|
57
97
|
feb: 1,
|
|
@@ -66,6 +106,34 @@ const MONTHS = {
|
|
|
66
106
|
nov: 10,
|
|
67
107
|
dec: 11,
|
|
68
108
|
};
|
|
109
|
+
const zoneFormatters = new Map();
|
|
110
|
+
function zoneFormatter(zone) {
|
|
111
|
+
const cached = zoneFormatters.get(zone);
|
|
112
|
+
if (cached !== undefined)
|
|
113
|
+
return cached;
|
|
114
|
+
let formatter;
|
|
115
|
+
try {
|
|
116
|
+
formatter = new Intl.DateTimeFormat("en-US", {
|
|
117
|
+
timeZone: zone,
|
|
118
|
+
hourCycle: "h23",
|
|
119
|
+
year: "numeric",
|
|
120
|
+
month: "numeric",
|
|
121
|
+
day: "numeric",
|
|
122
|
+
hour: "numeric",
|
|
123
|
+
minute: "numeric",
|
|
124
|
+
second: "numeric",
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
formatter = null;
|
|
129
|
+
}
|
|
130
|
+
zoneFormatters.set(zone, formatter);
|
|
131
|
+
return formatter;
|
|
132
|
+
}
|
|
133
|
+
/** "UTC" or an IANA zone the runtime knows, e.g. America/Denver. */
|
|
134
|
+
export function isKnownTimeZone(zone) {
|
|
135
|
+
return zoneFormatter(zone) !== null;
|
|
136
|
+
}
|
|
69
137
|
function emptyState() {
|
|
70
138
|
return { schema_version: QUOTA_SCHEMA_VERSION, providers: {} };
|
|
71
139
|
}
|
|
@@ -141,8 +209,11 @@ export function loadQuotaState(hqRoot) {
|
|
|
141
209
|
account.reset_source = "unconfirmed";
|
|
142
210
|
}
|
|
143
211
|
}
|
|
144
|
-
if (typeof raw.
|
|
145
|
-
account.
|
|
212
|
+
if (typeof raw.provider_reset_at === "string" && raw.provider_reset_at.length > 0) {
|
|
213
|
+
account.provider_reset_at = raw.provider_reset_at;
|
|
214
|
+
}
|
|
215
|
+
if (typeof raw.last_quota_hit_at === "string" && raw.last_quota_hit_at.length > 0) {
|
|
216
|
+
account.last_quota_hit_at = raw.last_quota_hit_at;
|
|
146
217
|
}
|
|
147
218
|
if (typeof raw.provider_text === "string" && raw.provider_text.length > 0) {
|
|
148
219
|
account.provider_text = raw.provider_text;
|
|
@@ -160,18 +231,6 @@ function writeQuotaState(hqRoot, state) {
|
|
|
160
231
|
fs.writeFileSync(tmp, `${JSON.stringify(state, null, 2)}\n`, "utf8");
|
|
161
232
|
fs.renameSync(tmp, filePath);
|
|
162
233
|
}
|
|
163
|
-
/** True when `zone` is a timezone name the runtime's Intl data knows. */
|
|
164
|
-
function isKnownTimeZone(zone) {
|
|
165
|
-
try {
|
|
166
|
-
new Intl.DateTimeFormat("en-US", { timeZone: zone });
|
|
167
|
-
return true;
|
|
168
|
-
}
|
|
169
|
-
catch (err) {
|
|
170
|
-
if (err instanceof RangeError)
|
|
171
|
-
return false;
|
|
172
|
-
throw err;
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
234
|
/** Offset of `zone` from UTC at instant `ms`, in milliseconds. */
|
|
176
235
|
function zoneOffsetMs(zone, ms) {
|
|
177
236
|
const parts = new Intl.DateTimeFormat("en-US", {
|
|
@@ -361,6 +420,11 @@ export function normalizeStructuredReset(value) {
|
|
|
361
420
|
return null;
|
|
362
421
|
return new Date(ms).toISOString();
|
|
363
422
|
}
|
|
423
|
+
/**
|
|
424
|
+
* Read only `api_error_status`, `is_error`, `result`, and a structured
|
|
425
|
+
* reset off the last `type: "result"` object. Nothing else from the payload
|
|
426
|
+
* is returned, and this function does not log.
|
|
427
|
+
*/
|
|
364
428
|
export function quotaFieldsFromOutput(text) {
|
|
365
429
|
const lines = text.split("\n");
|
|
366
430
|
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
@@ -446,46 +510,25 @@ export function readQuotaSignal(runDir, model, now, workerOutOffset = 0) {
|
|
|
446
510
|
const observedAt = Number.isFinite(outputAt.getTime()) ? outputAt : now;
|
|
447
511
|
return quotaSignalFromFields(fields, model, observedAt);
|
|
448
512
|
}
|
|
449
|
-
|
|
450
|
-
* Hold a provider reset more than a day away to a short hold until a second
|
|
451
|
-
* 429 confirms it. The second 429 counts only when its lane ran after the
|
|
452
|
-
* short hold expired, so lanes that failed together on one 429 burst do not
|
|
453
|
-
* confirm each other.
|
|
454
|
-
*/
|
|
455
|
-
function gateFarReset(current, signal, options) {
|
|
456
|
-
if (!signal.account_wide ||
|
|
457
|
-
!signal.quota_reset_at ||
|
|
458
|
-
signal.needs_credits ||
|
|
459
|
-
signal.reset_source === "derived") {
|
|
460
|
-
return { signal };
|
|
461
|
-
}
|
|
462
|
-
const resetMs = Date.parse(signal.quota_reset_at);
|
|
463
|
-
const nowMs = options.now.getTime();
|
|
464
|
-
if (!Number.isFinite(resetMs) || resetMs - nowMs <= QUOTA_CONFIRM_HORIZON_MS)
|
|
465
|
-
return { signal };
|
|
466
|
-
const pending = current.unconfirmed_reset_at;
|
|
467
|
-
const shortHoldActive = resetStillAhead(current.all_models_quota_reset_at, options.now);
|
|
468
|
-
if (pending !== undefined && !shortHoldActive) {
|
|
469
|
-
return { signal: { ...signal, reset_source: "parsed" } };
|
|
470
|
-
}
|
|
471
|
-
return {
|
|
472
|
-
signal: {
|
|
473
|
-
...signal,
|
|
474
|
-
quota_reset_at: new Date(nowMs + options.unconfirmedHoldMs).toISOString(),
|
|
475
|
-
reset_source: "unconfirmed",
|
|
476
|
-
},
|
|
477
|
-
unconfirmed_reset_at: pending !== undefined && !shouldReplaceReset(pending, signal.quota_reset_at)
|
|
478
|
-
? pending
|
|
479
|
-
: signal.quota_reset_at,
|
|
480
|
-
};
|
|
481
|
-
}
|
|
482
|
-
export function recordAccountQuota(hqRoot, provider, accountId, incoming, options) {
|
|
513
|
+
export function recordAccountQuota(hqRoot, provider, accountId, incoming, now = new Date()) {
|
|
483
514
|
return withLaneRecordFileLock(quotaStatePath(hqRoot), () => {
|
|
484
515
|
const state = loadQuotaState(hqRoot);
|
|
485
516
|
const accounts = state.providers[provider] ?? {};
|
|
486
517
|
const current = accounts[accountId] ?? { models: {} };
|
|
487
|
-
|
|
488
|
-
|
|
518
|
+
// A repeat 429 before the provider's reset promotes the capped hold to
|
|
519
|
+
// the provider's own reset (see applyDefaultQuotaHold).
|
|
520
|
+
let signal = incoming;
|
|
521
|
+
if (incoming.reason === "account_quota_exhausted" &&
|
|
522
|
+
!incoming.needs_credits &&
|
|
523
|
+
isRepeatQuotaHit(current, incoming, now)) {
|
|
524
|
+
const candidates = [incoming.provider_reset_at, current.provider_reset_at, incoming.quota_reset_at]
|
|
525
|
+
.map((value) => (value ? Date.parse(value) : Number.NaN))
|
|
526
|
+
.filter((ms) => Number.isFinite(ms));
|
|
527
|
+
if (candidates.length > 0) {
|
|
528
|
+
const promoted = new Date(Math.max(...candidates)).toISOString();
|
|
529
|
+
signal = { ...incoming, quota_reset_at: promoted, provider_reset_at: promoted, account_wide: true, reset_source: "parsed" };
|
|
530
|
+
}
|
|
531
|
+
}
|
|
489
532
|
const models = { ...current.models };
|
|
490
533
|
if (signal.model) {
|
|
491
534
|
const prior = models[signal.model] ?? {};
|
|
@@ -507,19 +550,18 @@ export function recordAccountQuota(hqRoot, provider, accountId, incoming, option
|
|
|
507
550
|
if (shouldReplaceReset(current.all_models_quota_reset_at, signal.quota_reset_at)) {
|
|
508
551
|
account.all_models_quota_reset_at = signal.quota_reset_at;
|
|
509
552
|
account.reset_source = signal.reset_source ?? "parsed";
|
|
510
|
-
if (signal.provider_text)
|
|
511
|
-
account.provider_text = signal.provider_text;
|
|
512
|
-
if (gated.unconfirmed_reset_at) {
|
|
513
|
-
account.unconfirmed_reset_at = gated.unconfirmed_reset_at;
|
|
514
|
-
}
|
|
515
|
-
else {
|
|
516
|
-
delete account.unconfirmed_reset_at;
|
|
517
|
-
}
|
|
518
553
|
}
|
|
519
554
|
else if (current.all_models_quota_reset_at) {
|
|
520
555
|
account.all_models_quota_reset_at = current.all_models_quota_reset_at;
|
|
521
556
|
account.reset_source = current.reset_source ?? "parsed";
|
|
522
557
|
}
|
|
558
|
+
if (signal.provider_text)
|
|
559
|
+
account.provider_text = signal.provider_text;
|
|
560
|
+
if (signal.provider_reset_at)
|
|
561
|
+
account.provider_reset_at = signal.provider_reset_at;
|
|
562
|
+
else
|
|
563
|
+
delete account.provider_reset_at;
|
|
564
|
+
account.last_quota_hit_at = signal.observed_at ?? now.toISOString();
|
|
523
565
|
}
|
|
524
566
|
accounts[accountId] = account;
|
|
525
567
|
state.providers[provider] = accounts;
|
|
@@ -542,16 +584,13 @@ export function clearAccountQuota(hqRoot, provider, accountId, options = {}) {
|
|
|
542
584
|
const shouldClear = (reset) => reset !== undefined &&
|
|
543
585
|
(!options.futureOnly || resetStillAhead(reset, now));
|
|
544
586
|
let changed = false;
|
|
545
|
-
// A successful turn or an operator clear disproves a pending far reset.
|
|
546
|
-
if (account.unconfirmed_reset_at !== undefined) {
|
|
547
|
-
delete account.unconfirmed_reset_at;
|
|
548
|
-
changed = true;
|
|
549
|
-
}
|
|
550
587
|
const accountWideReset = account.all_models_quota_reset_at;
|
|
551
588
|
if (shouldClear(accountWideReset)) {
|
|
552
589
|
cleared.all_models_quota_reset_at = accountWideReset;
|
|
553
590
|
delete account.all_models_quota_reset_at;
|
|
554
591
|
delete account.reset_source;
|
|
592
|
+
delete account.provider_reset_at;
|
|
593
|
+
delete account.last_quota_hit_at;
|
|
555
594
|
delete account.provider_text;
|
|
556
595
|
changed = true;
|
|
557
596
|
}
|
|
@@ -626,28 +665,20 @@ function resetStillAhead(iso, now) {
|
|
|
626
665
|
return Number.isFinite(ts) && ts > now.getTime();
|
|
627
666
|
}
|
|
628
667
|
/**
|
|
629
|
-
* Skip an account only for
|
|
630
|
-
*
|
|
631
|
-
*
|
|
668
|
+
* Skip an account only for a model it cannot run at all: a needs-credits
|
|
669
|
+
* marker blocks that model, with no expiry.
|
|
670
|
+
*
|
|
671
|
+
* A recorded 429 never blocks (owner decision, incident 2026-10-13). The
|
|
672
|
+
* lane that hit it ends resumable and its senior decides; other lanes keep
|
|
673
|
+
* using the account. The reset times stay in quota.json as information
|
|
674
|
+
* (`accountQuotaView`, `quotaHitInfo`).
|
|
632
675
|
*/
|
|
633
|
-
export function quotaBlockFor(state, provider, accountId, model,
|
|
676
|
+
export function quotaBlockFor(state, provider, accountId, model, _now) {
|
|
634
677
|
const account = state.providers[provider]?.[accountId];
|
|
635
|
-
if (!account)
|
|
636
|
-
return { blocked: false };
|
|
637
|
-
if (resetStillAhead(account.all_models_quota_reset_at, now)) {
|
|
638
|
-
return {
|
|
639
|
-
blocked: true,
|
|
640
|
-
reason: "account_quota_exhausted",
|
|
641
|
-
quota_reset_at: account.all_models_quota_reset_at,
|
|
642
|
-
model,
|
|
643
|
-
};
|
|
644
|
-
}
|
|
645
|
-
if (!model)
|
|
678
|
+
if (!account || !model)
|
|
646
679
|
return { blocked: false };
|
|
647
680
|
const entry = account.models[model];
|
|
648
|
-
if (
|
|
649
|
-
return { blocked: false };
|
|
650
|
-
if (entry.needs_credits) {
|
|
681
|
+
if (entry?.needs_credits) {
|
|
651
682
|
return {
|
|
652
683
|
blocked: true,
|
|
653
684
|
reason: "account_model_unavailable",
|
|
@@ -655,16 +686,21 @@ export function quotaBlockFor(state, provider, accountId, model, now) {
|
|
|
655
686
|
model,
|
|
656
687
|
};
|
|
657
688
|
}
|
|
658
|
-
if (resetStillAhead(entry.quota_reset_at, now)) {
|
|
659
|
-
return {
|
|
660
|
-
blocked: true,
|
|
661
|
-
reason: "account_quota_exhausted",
|
|
662
|
-
quota_reset_at: entry.quota_reset_at,
|
|
663
|
-
model,
|
|
664
|
-
};
|
|
665
|
-
}
|
|
666
689
|
return { blocked: false };
|
|
667
690
|
}
|
|
691
|
+
export function quotaHitInfo(state, provider, accountId, now) {
|
|
692
|
+
const account = state.providers[provider]?.[accountId];
|
|
693
|
+
if (!account || (!account.last_quota_hit_at && !account.provider_reset_at && !account.all_models_quota_reset_at)) {
|
|
694
|
+
return undefined;
|
|
695
|
+
}
|
|
696
|
+
const providerReset = account.provider_reset_at ??
|
|
697
|
+
(account.reset_source === "parsed" ? account.all_models_quota_reset_at : undefined);
|
|
698
|
+
return {
|
|
699
|
+
...(account.last_quota_hit_at ? { last_quota_hit_at: account.last_quota_hit_at } : {}),
|
|
700
|
+
...(providerReset ? { provider_reset_at: providerReset } : {}),
|
|
701
|
+
provider_reset_pending: resetStillAhead(providerReset, now),
|
|
702
|
+
};
|
|
703
|
+
}
|
|
668
704
|
export function accountQuotaView(state, provider, accountId) {
|
|
669
705
|
const account = state.providers[provider]?.[accountId];
|
|
670
706
|
if (!account)
|
|
@@ -683,6 +719,9 @@ export function accountQuotaView(state, provider, accountId) {
|
|
|
683
719
|
});
|
|
684
720
|
return {
|
|
685
721
|
quota_reset_at: account.all_models_quota_reset_at ?? null,
|
|
722
|
+
...(account.reset_source ? { quota_reset_source: account.reset_source } : {}),
|
|
723
|
+
...(account.provider_reset_at ? { provider_reset_at: account.provider_reset_at } : {}),
|
|
724
|
+
...(account.last_quota_hit_at ? { last_quota_hit_at: account.last_quota_hit_at } : {}),
|
|
686
725
|
model_quota,
|
|
687
726
|
};
|
|
688
727
|
}
|
|
@@ -178,4 +178,18 @@ export interface AnsweredLaneWakeResult {
|
|
|
178
178
|
}
|
|
179
179
|
/** Run the reconcile answer-resume path for one lane before questions answer returns. */
|
|
180
180
|
export declare function wakeAnsweredLaneNow(hqRoot: string, laneId: string, deps: ReconcileDeps): Promise<AnsweredLaneWakeResult>;
|
|
181
|
+
export interface AdmitQueuedNowResult {
|
|
182
|
+
admitted: ReconcileChange[];
|
|
183
|
+
errors: ReconcileSummary["errors"];
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Run queue admission in-process, without the rest of a reconcile pass.
|
|
187
|
+
*
|
|
188
|
+
* Commands that change capacity (create, stop, cancel, clear-quota, a lane
|
|
189
|
+
* exit) call this so queued lanes start without `hq daemon`. It takes the
|
|
190
|
+
* reconcile lock and every provider admission lock, so it is safe to run
|
|
191
|
+
* while a daemon is also reconciling: whichever runs second sees the lanes
|
|
192
|
+
* the first one already admitted.
|
|
193
|
+
*/
|
|
194
|
+
export declare function admitQueuedNow(hqRoot: string, deps: ReconcileDeps): Promise<AdmitQueuedNowResult>;
|
|
181
195
|
//# sourceMappingURL=reconcile.d.ts.map
|