@indigoai-us/hq-cli 5.345.123 → 5.345.125
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/assets/scaffold/core/scripts/work-mesh-project-registration-offer.sh +77 -0
- package/assets/scaffold/core/scripts/workflow-gate.sh +247 -0
- package/dist/command-catalog.generated.d.ts +239 -0
- package/dist/command-catalog.generated.js +291 -0
- package/dist/commands/core-bucket-c-work-mesh-offer-native.d.ts +7 -0
- package/dist/commands/core-bucket-c-work-mesh-offer-native.js +147 -0
- package/dist/commands/core-bucket-c-workflow-gate-fast.d.ts +3 -0
- package/dist/commands/core-bucket-c-workflow-gate-fast.js +17 -0
- package/dist/commands/core-bucket-c-workflow-gate-native.d.ts +7 -0
- package/dist/commands/core-bucket-c-workflow-gate-native.js +250 -0
- package/dist/commands/core.js +12 -0
- package/dist/commands/lanes-accounts.js +22 -3
- package/dist/commands/lanes.js +25 -32
- package/dist/commands/mesh.js +457 -16
- package/dist/fast-scaffold.js +26 -0
- package/dist/lib/lanes/accounts.d.ts +48 -2
- package/dist/lib/lanes/accounts.js +160 -9
- package/dist/lib/lanes/index.d.ts +2 -2
- package/dist/lib/lanes/index.js +1 -1
- package/dist/lib/lanes/message-replay.js +6 -1
- package/dist/lib/lanes/provider-rejection.d.ts +2 -6
- package/dist/lib/lanes/provider-rejection.js +19 -0
- package/dist/lib/lanes/quota-notice.d.ts +24 -0
- package/dist/lib/lanes/quota-notice.js +61 -0
- package/dist/lib/lanes/quota.d.ts +55 -15
- package/dist/lib/lanes/quota.js +231 -49
- package/dist/lib/lanes/reconcile.js +17 -1
- package/dist/lib/lanes/session-relink.d.ts +2 -0
- package/dist/lib/lanes/session-relink.js +98 -4
- package/dist/lib/lanes/spawn.d.ts +1 -1
- package/dist/lib/lanes/spawn.js +58 -12
- package/dist/lib/lanes/types.d.ts +5 -0
- package/dist/lib/lanes/types.js +1 -1
- package/dist/lib/lanes/watcher.js +13 -1
- package/dist/lib/mesh/api.d.ts +2 -0
- package/dist/lib/mesh/api.js +3 -0
- package/dist/lib/mesh/client.d.ts +6 -1
- package/dist/lib/mesh/client.js +18 -5
- package/dist/lib/mesh/team-coordination.d.ts +106 -0
- package/dist/lib/mesh/team-coordination.js +297 -0
- package/dist/lib/work-context/auto-bind.d.ts +75 -0
- package/dist/lib/work-context/auto-bind.js +245 -0
- package/package.json +1 -1
|
@@ -16,6 +16,14 @@
|
|
|
16
16
|
* claude-fable-5-1 and still runs claude-opus-5). The record is therefore
|
|
17
17
|
* per (account, model), plus an account-wide reset when the text is a
|
|
18
18
|
* usage limit rather than a switch-model refusal.
|
|
19
|
+
*
|
|
20
|
+
* A single 429 is not trusted with a multi-day hold. When the provider names
|
|
21
|
+
* a reset more than a day away, the first 429 holds the account only for the
|
|
22
|
+
* short default hold and keeps the named reset as `unconfirmed_reset_at`. A
|
|
23
|
+
* second 429 from a lane that ran after that short hold expired confirms the
|
|
24
|
+
* reset; a successful turn drops it. (Measured 2026-10-13: account a returned
|
|
25
|
+
* "You've hit your weekly limit · resets Oct 16 at 1am (America/Denver)"
|
|
26
|
+
* while the owner's /usage showed the week at 43%.)
|
|
19
27
|
*/
|
|
20
28
|
import type { LaneProvider } from "./types.js";
|
|
21
29
|
export declare const QUOTA_STATE_REL: string;
|
|
@@ -26,7 +34,14 @@ export interface ModelQuota {
|
|
|
26
34
|
/** Operator-facing marker, e.g. "needs credits for claude-fable-5-1". */
|
|
27
35
|
marker?: string;
|
|
28
36
|
}
|
|
29
|
-
|
|
37
|
+
/**
|
|
38
|
+
* `parsed`: the provider's own reset. `derived`: the default hold for a 429
|
|
39
|
+
* with no parseable reset. `unconfirmed`: a short hold standing in for a
|
|
40
|
+
* provider reset more than a day away that a second 429 has not confirmed.
|
|
41
|
+
*/
|
|
42
|
+
export type QuotaResetSource = "parsed" | "derived" | "unconfirmed";
|
|
43
|
+
/** A provider reset further away than this needs a second 429 before it holds. */
|
|
44
|
+
export declare const QUOTA_CONFIRM_HORIZON_MS: number;
|
|
30
45
|
export interface AccountQuota {
|
|
31
46
|
/** Set when the 429 is an account-wide usage limit with a reset time. */
|
|
32
47
|
all_models_quota_reset_at?: string;
|
|
@@ -35,6 +50,13 @@ export interface AccountQuota {
|
|
|
35
50
|
* the configured default hold applied to a 429 with no parseable reset.
|
|
36
51
|
*/
|
|
37
52
|
reset_source?: QuotaResetSource;
|
|
53
|
+
/**
|
|
54
|
+
* The provider's named reset, held back until a second 429 on a fresh
|
|
55
|
+
* probe confirms it. Set only while `reset_source` is `unconfirmed`.
|
|
56
|
+
*/
|
|
57
|
+
unconfirmed_reset_at?: string;
|
|
58
|
+
/** Raw provider text from the 429 that set the current hold. */
|
|
59
|
+
provider_text?: string;
|
|
38
60
|
models: Record<string, ModelQuota>;
|
|
39
61
|
}
|
|
40
62
|
export interface QuotaState {
|
|
@@ -51,6 +73,8 @@ export interface QuotaSignal {
|
|
|
51
73
|
account_wide?: boolean;
|
|
52
74
|
/** Set by applyDefaultQuotaHold: whether the reset is the provider's or the default hold. */
|
|
53
75
|
reset_source?: QuotaResetSource;
|
|
76
|
+
/** The provider's raw result text, kept so the hold can be shown to the senior. */
|
|
77
|
+
provider_text?: string;
|
|
54
78
|
}
|
|
55
79
|
/**
|
|
56
80
|
* A 429 whose message names no reset time and no credits wording would
|
|
@@ -64,12 +88,18 @@ export declare function applyDefaultQuotaHold(signal: QuotaSignal, defaultHoldMs
|
|
|
64
88
|
export declare function quotaStatePath(hqRoot: string): string;
|
|
65
89
|
export declare function loadQuotaState(hqRoot: string): QuotaState;
|
|
66
90
|
/**
|
|
67
|
-
* Parse provider reset times
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
91
|
+
* Parse provider reset times into an ISO timestamp. Handled forms:
|
|
92
|
+
*
|
|
93
|
+
* - "resets 2am (UTC)" and "resets 1am (America/Denver)": the next such time.
|
|
94
|
+
* - "resets Sep 24, 2am (UTC)", "resets Oct 16 at 1am (America/Denver)",
|
|
95
|
+
* "resets Sep 24, 2026 2am (UTC)": a dated reset.
|
|
96
|
+
* - Codex's "try again at Sep 14th, 2026 9:24 PM", shown in the provider
|
|
97
|
+
* CLI's local timezone unless it names `(UTC)`.
|
|
98
|
+
*
|
|
99
|
+
* The zone is `UTC` or an IANA name; an unknown zone name fails the parse. A
|
|
100
|
+
* dated provider reset with no zone is read as UTC. The year is the zone's
|
|
101
|
+
* current year for an unyear-tagged reset, rolled forward when it is already
|
|
102
|
+
* past. An explicit provider year is kept as stated.
|
|
73
103
|
*/
|
|
74
104
|
export declare function parseProviderResetTime(text: string, now: Date): string | null;
|
|
75
105
|
/**
|
|
@@ -77,18 +107,28 @@ export declare function parseProviderResetTime(text: string, now: Date): string
|
|
|
77
107
|
* `type: "result"` object. Nothing else from the payload is returned, and
|
|
78
108
|
* this function does not log.
|
|
79
109
|
*/
|
|
80
|
-
export
|
|
110
|
+
export interface QuotaResultFields {
|
|
81
111
|
api_error_status: unknown;
|
|
82
112
|
is_error: unknown;
|
|
83
113
|
result: unknown;
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
}
|
|
114
|
+
/**
|
|
115
|
+
* The provider's structured reset (`resetsAt`, epoch seconds or ISO) when
|
|
116
|
+
* the result carries one; it beats the wording in `result`.
|
|
117
|
+
*/
|
|
118
|
+
resets_at?: unknown;
|
|
119
|
+
}
|
|
120
|
+
/** Epoch seconds, epoch milliseconds, or an ISO string, to ISO; null otherwise. */
|
|
121
|
+
export declare function normalizeStructuredReset(value: unknown): string | null;
|
|
122
|
+
export declare function quotaFieldsFromOutput(text: string): QuotaResultFields | null;
|
|
123
|
+
export declare function quotaSignalFromFields(fields: QuotaResultFields, model: string | undefined, now: Date): QuotaSignal | null;
|
|
90
124
|
export declare function readQuotaSignal(runDir: string, model: string | undefined, now: Date, workerOutOffset?: number): QuotaSignal | null;
|
|
91
|
-
export
|
|
125
|
+
export interface RecordAccountQuotaOptions {
|
|
126
|
+
/** When the 429 was observed. Enables the far-reset confirmation gate. */
|
|
127
|
+
now: Date;
|
|
128
|
+
/** Hold applied while a far provider reset is unconfirmed (the default hold). */
|
|
129
|
+
unconfirmedHoldMs: number;
|
|
130
|
+
}
|
|
131
|
+
export declare function recordAccountQuota(hqRoot: string, provider: LaneProvider, accountId: string, incoming: QuotaSignal, options?: RecordAccountQuotaOptions): QuotaState;
|
|
92
132
|
export interface ClearedModelQuotaReset {
|
|
93
133
|
model: string;
|
|
94
134
|
quota_reset_at: string;
|
package/dist/lib/lanes/quota.js
CHANGED
|
@@ -16,12 +16,22 @@
|
|
|
16
16
|
* claude-fable-5-1 and still runs claude-opus-5). The record is therefore
|
|
17
17
|
* per (account, model), plus an account-wide reset when the text is a
|
|
18
18
|
* usage limit rather than a switch-model refusal.
|
|
19
|
+
*
|
|
20
|
+
* A single 429 is not trusted with a multi-day hold. When the provider names
|
|
21
|
+
* a reset more than a day away, the first 429 holds the account only for the
|
|
22
|
+
* short default hold and keeps the named reset as `unconfirmed_reset_at`. A
|
|
23
|
+
* second 429 from a lane that ran after that short hold expired confirms the
|
|
24
|
+
* reset; a successful turn drops it. (Measured 2026-10-13: account a returned
|
|
25
|
+
* "You've hit your weekly limit · resets Oct 16 at 1am (America/Denver)"
|
|
26
|
+
* while the owner's /usage showed the week at 43%.)
|
|
19
27
|
*/
|
|
20
28
|
import * as fs from "node:fs";
|
|
21
29
|
import * as path from "node:path";
|
|
22
30
|
import { withLaneRecordFileLock } from "./store.js";
|
|
23
31
|
export const QUOTA_STATE_REL = path.join("personal", "lanes", "quota.json");
|
|
24
32
|
export const QUOTA_SCHEMA_VERSION = "hq-lanes-quota.v1";
|
|
33
|
+
/** A provider reset further away than this needs a second 429 before it holds. */
|
|
34
|
+
export const QUOTA_CONFIRM_HORIZON_MS = 24 * 60 * 60 * 1000;
|
|
25
35
|
/**
|
|
26
36
|
* A 429 whose message names no reset time and no credits wording would
|
|
27
37
|
* otherwise record no hold at all, and the next lane on the account would
|
|
@@ -127,6 +137,15 @@ export function loadQuotaState(hqRoot) {
|
|
|
127
137
|
else if (raw.reset_source === "default_hold" || raw.reset_source === "derived") {
|
|
128
138
|
account.reset_source = "derived";
|
|
129
139
|
}
|
|
140
|
+
else if (raw.reset_source === "unconfirmed") {
|
|
141
|
+
account.reset_source = "unconfirmed";
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
if (typeof raw.unconfirmed_reset_at === "string" && raw.unconfirmed_reset_at.length > 0) {
|
|
145
|
+
account.unconfirmed_reset_at = raw.unconfirmed_reset_at;
|
|
146
|
+
}
|
|
147
|
+
if (typeof raw.provider_text === "string" && raw.provider_text.length > 0) {
|
|
148
|
+
account.provider_text = raw.provider_text;
|
|
130
149
|
}
|
|
131
150
|
accounts[id] = account;
|
|
132
151
|
}
|
|
@@ -141,37 +160,113 @@ function writeQuotaState(hqRoot, state) {
|
|
|
141
160
|
fs.writeFileSync(tmp, `${JSON.stringify(state, null, 2)}\n`, "utf8");
|
|
142
161
|
fs.renameSync(tmp, filePath);
|
|
143
162
|
}
|
|
163
|
+
/** True when `zone` is a timezone name the runtime's Intl data knows. */
|
|
164
|
+
function isKnownTimeZone(zone) {
|
|
165
|
+
try {
|
|
166
|
+
new Intl.DateTimeFormat("en-US", { timeZone: zone });
|
|
167
|
+
return true;
|
|
168
|
+
}
|
|
169
|
+
catch (err) {
|
|
170
|
+
if (err instanceof RangeError)
|
|
171
|
+
return false;
|
|
172
|
+
throw err;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
/** Offset of `zone` from UTC at instant `ms`, in milliseconds. */
|
|
176
|
+
function zoneOffsetMs(zone, ms) {
|
|
177
|
+
const parts = new Intl.DateTimeFormat("en-US", {
|
|
178
|
+
timeZone: zone,
|
|
179
|
+
hourCycle: "h23",
|
|
180
|
+
year: "numeric",
|
|
181
|
+
month: "numeric",
|
|
182
|
+
day: "numeric",
|
|
183
|
+
hour: "numeric",
|
|
184
|
+
minute: "numeric",
|
|
185
|
+
second: "numeric",
|
|
186
|
+
}).formatToParts(new Date(ms));
|
|
187
|
+
const get = (type) => Number(parts.find((p) => p.type === type)?.value ?? 0);
|
|
188
|
+
const asUtc = Date.UTC(get("year"), get("month") - 1, get("day"), get("hour"), get("minute"), get("second"));
|
|
189
|
+
return asUtc - Math.floor(ms / 1000) * 1000;
|
|
190
|
+
}
|
|
144
191
|
/**
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
192
|
+
* The UTC instant of a wall-clock time in `zone`, or null when that wall time
|
|
193
|
+
* does not exist there (a spring-forward gap) or a field is out of range.
|
|
194
|
+
*/
|
|
195
|
+
function zonedWallTimeToUtc(zone, year, month, day, hour, minute) {
|
|
196
|
+
const wall = Date.UTC(year, month, day, hour, minute, 0, 0);
|
|
197
|
+
const check = new Date(wall);
|
|
198
|
+
if (check.getUTCFullYear() !== year ||
|
|
199
|
+
check.getUTCMonth() !== month ||
|
|
200
|
+
check.getUTCDate() !== day ||
|
|
201
|
+
check.getUTCHours() !== hour ||
|
|
202
|
+
check.getUTCMinutes() !== minute) {
|
|
203
|
+
return null;
|
|
204
|
+
}
|
|
205
|
+
if (zone.toUpperCase() === "UTC")
|
|
206
|
+
return wall;
|
|
207
|
+
let guess = wall - zoneOffsetMs(zone, wall);
|
|
208
|
+
guess = wall - zoneOffsetMs(zone, guess);
|
|
209
|
+
return guess + zoneOffsetMs(zone, guess) === wall ? guess : null;
|
|
210
|
+
}
|
|
211
|
+
function zonedDateParts(zone, ms) {
|
|
212
|
+
const offset = zone.toUpperCase() === "UTC" ? 0 : zoneOffsetMs(zone, ms);
|
|
213
|
+
const local = new Date(ms + offset);
|
|
214
|
+
return { year: local.getUTCFullYear(), month: local.getUTCMonth(), day: local.getUTCDate() };
|
|
215
|
+
}
|
|
216
|
+
function to24Hour(hourText, minuteText, meridiem) {
|
|
217
|
+
let hour = Number(hourText);
|
|
218
|
+
const minute = Number(minuteText ?? 0);
|
|
219
|
+
if (!Number.isInteger(hour) || hour < 1 || hour > 12 || !Number.isInteger(minute) || minute > 59)
|
|
220
|
+
return null;
|
|
221
|
+
const ap = meridiem.toLowerCase();
|
|
222
|
+
if (ap === "pm" && hour < 12)
|
|
223
|
+
hour += 12;
|
|
224
|
+
if (ap === "am" && hour === 12)
|
|
225
|
+
hour = 0;
|
|
226
|
+
return { hour, minute };
|
|
227
|
+
}
|
|
228
|
+
const ZONE_PATTERN = String.raw `\(\s*(UTC|[A-Za-z][A-Za-z0-9_+-]*(?:\/[A-Za-z0-9_+-]+)+)\s*\)`;
|
|
229
|
+
/**
|
|
230
|
+
* Parse provider reset times into an ISO timestamp. Handled forms:
|
|
231
|
+
*
|
|
232
|
+
* - "resets 2am (UTC)" and "resets 1am (America/Denver)": the next such time.
|
|
233
|
+
* - "resets Sep 24, 2am (UTC)", "resets Oct 16 at 1am (America/Denver)",
|
|
234
|
+
* "resets Sep 24, 2026 2am (UTC)": a dated reset.
|
|
235
|
+
* - Codex's "try again at Sep 14th, 2026 9:24 PM", shown in the provider
|
|
236
|
+
* CLI's local timezone unless it names `(UTC)`.
|
|
237
|
+
*
|
|
238
|
+
* The zone is `UTC` or an IANA name; an unknown zone name fails the parse. A
|
|
239
|
+
* dated provider reset with no zone is read as UTC. The year is the zone's
|
|
240
|
+
* current year for an unyear-tagged reset, rolled forward when it is already
|
|
241
|
+
* past. An explicit provider year is kept as stated.
|
|
151
242
|
*/
|
|
152
243
|
export function parseProviderResetTime(text, now) {
|
|
153
|
-
const timeOnlyMatch =
|
|
244
|
+
const timeOnlyMatch = new RegExp(String.raw `\bresets\s+(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm)\s*${ZONE_PATTERN}`, "i").exec(text);
|
|
154
245
|
if (timeOnlyMatch) {
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
const meridiem = (timeOnlyMatch[3] ?? "").toLowerCase();
|
|
158
|
-
if (!Number.isInteger(hour) || hour < 1 || hour > 12 || minute < 0 || minute > 59)
|
|
246
|
+
const zone = timeOnlyMatch[4] ?? "UTC";
|
|
247
|
+
if (!isKnownTimeZone(zone))
|
|
159
248
|
return null;
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
249
|
+
const time = to24Hour(timeOnlyMatch[1] ?? "", timeOnlyMatch[2], timeOnlyMatch[3] ?? "");
|
|
250
|
+
if (!time)
|
|
251
|
+
return null;
|
|
252
|
+
const today = zonedDateParts(zone, now.getTime());
|
|
253
|
+
for (let offsetDays = 0; offsetDays <= 2; offsetDays += 1) {
|
|
254
|
+
const date = new Date(Date.UTC(today.year, today.month, today.day + offsetDays));
|
|
255
|
+
const ts = zonedWallTimeToUtc(zone, date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate(), time.hour, time.minute);
|
|
256
|
+
if (ts !== null && ts > now.getTime())
|
|
257
|
+
return new Date(ts).toISOString();
|
|
258
|
+
}
|
|
259
|
+
return null;
|
|
168
260
|
}
|
|
169
|
-
const resetMatch =
|
|
261
|
+
const resetMatch = new RegExp(String.raw `\bresets\s+([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?(?:(?:,\s*(?:(\d{4})(?:,|\s+at)?\s+)?|\s+at\s+)(?:at\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm))?(?:\s*${ZONE_PATTERN})?`, "i").exec(text);
|
|
170
262
|
const codexMatch = /\btry\s+again\s+at\s+([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?,\s*(\d{4})\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)(?:\s*\((UTC)\))?/i.exec(text);
|
|
171
263
|
const match = resetMatch ?? codexMatch;
|
|
172
264
|
if (!match)
|
|
173
265
|
return null;
|
|
174
|
-
const codexLocalTime =
|
|
266
|
+
const codexLocalTime = resetMatch === null && (match[7] ?? "").toLowerCase() !== "utc";
|
|
267
|
+
const zone = match[7] ?? "UTC";
|
|
268
|
+
if (!codexLocalTime && !isKnownTimeZone(zone))
|
|
269
|
+
return null;
|
|
175
270
|
const month = MONTHS[(match[1] ?? "").slice(0, 3).toLowerCase()];
|
|
176
271
|
if (month === undefined)
|
|
177
272
|
return null;
|
|
@@ -179,37 +274,31 @@ export function parseProviderResetTime(text, now) {
|
|
|
179
274
|
if (!Number.isInteger(day) || day < 1 || day > 31)
|
|
180
275
|
return null;
|
|
181
276
|
const explicitYear = match[3] !== undefined;
|
|
182
|
-
const parsedYear = explicitYear
|
|
277
|
+
const parsedYear = explicitYear
|
|
278
|
+
? Number(match[3])
|
|
279
|
+
: codexLocalTime
|
|
280
|
+
? now.getFullYear()
|
|
281
|
+
: zonedDateParts(zone, now.getTime()).year;
|
|
183
282
|
if (!Number.isInteger(parsedYear) || parsedYear < 1970)
|
|
184
283
|
return null;
|
|
185
284
|
let hour = 0;
|
|
186
285
|
let minute = 0;
|
|
187
286
|
if (match[4] !== undefined) {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
const ap = (match[6] ?? "").toLowerCase();
|
|
191
|
-
if (ap === "pm" && hour < 12)
|
|
192
|
-
hour += 12;
|
|
193
|
-
if (ap === "am" && hour === 12)
|
|
194
|
-
hour = 0;
|
|
195
|
-
if (hour > 23 || minute > 59)
|
|
287
|
+
const time = to24Hour(match[4], match[5], match[6] ?? "");
|
|
288
|
+
if (!time)
|
|
196
289
|
return null;
|
|
290
|
+
hour = time.hour;
|
|
291
|
+
minute = time.minute;
|
|
197
292
|
}
|
|
198
293
|
const timestampForYear = (targetYear) => {
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
const valid =
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
candidate.getMinutes() === minute
|
|
208
|
-
: candidate.getUTCFullYear() === targetYear &&
|
|
209
|
-
candidate.getUTCMonth() === month &&
|
|
210
|
-
candidate.getUTCDate() === day &&
|
|
211
|
-
candidate.getUTCHours() === hour &&
|
|
212
|
-
candidate.getUTCMinutes() === minute;
|
|
294
|
+
if (!codexLocalTime)
|
|
295
|
+
return zonedWallTimeToUtc(zone, targetYear, month, day, hour, minute);
|
|
296
|
+
const candidate = new Date(targetYear, month, day, hour, minute, 0, 0);
|
|
297
|
+
const valid = candidate.getFullYear() === targetYear &&
|
|
298
|
+
candidate.getMonth() === month &&
|
|
299
|
+
candidate.getDate() === day &&
|
|
300
|
+
candidate.getHours() === hour &&
|
|
301
|
+
candidate.getMinutes() === minute;
|
|
213
302
|
return valid ? candidate.getTime() : null;
|
|
214
303
|
};
|
|
215
304
|
let year = parsedYear;
|
|
@@ -231,10 +320,47 @@ function isUsageLimitShape(text) {
|
|
|
231
320
|
return /weekly limit|usage limit|rate limit|quota|usage credits|hit your .+ limit/i.test(text);
|
|
232
321
|
}
|
|
233
322
|
/**
|
|
234
|
-
*
|
|
235
|
-
*
|
|
236
|
-
*
|
|
323
|
+
* A structured reset from a result object: `resets_at` / `resetsAt` at the
|
|
324
|
+
* top level, or `rate_limit_info.resetsAt` directly or under
|
|
325
|
+
* `api_error_params` / `apiErrorParams`. Undefined when none is present.
|
|
237
326
|
*/
|
|
327
|
+
function structuredResetFrom(rec) {
|
|
328
|
+
const direct = rec.resets_at ?? rec.resetsAt;
|
|
329
|
+
if (direct !== undefined)
|
|
330
|
+
return direct;
|
|
331
|
+
const holders = [rec, rec.api_error_params, rec.apiErrorParams];
|
|
332
|
+
for (const holder of holders) {
|
|
333
|
+
if (!isPlainObject(holder))
|
|
334
|
+
continue;
|
|
335
|
+
const info = holder.rate_limit_info ?? holder.rateLimitInfo;
|
|
336
|
+
if (isPlainObject(info) && info.resetsAt !== undefined)
|
|
337
|
+
return info.resetsAt;
|
|
338
|
+
}
|
|
339
|
+
return undefined;
|
|
340
|
+
}
|
|
341
|
+
/** Epoch seconds, epoch milliseconds, or an ISO string, to ISO; null otherwise. */
|
|
342
|
+
export function normalizeStructuredReset(value) {
|
|
343
|
+
let ms;
|
|
344
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
345
|
+
ms = value < 1e12 ? value * 1000 : value;
|
|
346
|
+
}
|
|
347
|
+
else if (typeof value === "string" && value.trim().length > 0) {
|
|
348
|
+
const trimmed = value.trim();
|
|
349
|
+
if (/^\d+(?:\.\d+)?$/.test(trimmed)) {
|
|
350
|
+
const n = Number(trimmed);
|
|
351
|
+
ms = n < 1e12 ? n * 1000 : n;
|
|
352
|
+
}
|
|
353
|
+
else {
|
|
354
|
+
ms = Date.parse(trimmed);
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
else {
|
|
358
|
+
return null;
|
|
359
|
+
}
|
|
360
|
+
if (!Number.isFinite(ms) || ms <= 0)
|
|
361
|
+
return null;
|
|
362
|
+
return new Date(ms).toISOString();
|
|
363
|
+
}
|
|
238
364
|
export function quotaFieldsFromOutput(text) {
|
|
239
365
|
const lines = text.split("\n");
|
|
240
366
|
for (let i = lines.length - 1; i >= 0; i -= 1) {
|
|
@@ -250,10 +376,12 @@ export function quotaFieldsFromOutput(text) {
|
|
|
250
376
|
}
|
|
251
377
|
if (!isPlainObject(parsed) || parsed.type !== "result")
|
|
252
378
|
continue;
|
|
379
|
+
const resets_at = structuredResetFrom(parsed);
|
|
253
380
|
return {
|
|
254
381
|
api_error_status: parsed.api_error_status,
|
|
255
382
|
is_error: parsed.is_error,
|
|
256
383
|
result: parsed.result,
|
|
384
|
+
...(resets_at !== undefined ? { resets_at } : {}),
|
|
257
385
|
};
|
|
258
386
|
}
|
|
259
387
|
return null;
|
|
@@ -273,12 +401,16 @@ export function quotaSignalFromFields(fields, model, now) {
|
|
|
273
401
|
marker,
|
|
274
402
|
};
|
|
275
403
|
}
|
|
276
|
-
const
|
|
404
|
+
const structured = normalizeStructuredReset(fields.resets_at) ?? undefined;
|
|
405
|
+
const quota_reset_at = structured ?? (text.length > 0 ? parseProviderResetTime(text, now) ?? undefined : undefined);
|
|
277
406
|
return {
|
|
278
407
|
reason: "account_quota_exhausted",
|
|
279
408
|
quota_reset_at,
|
|
280
409
|
model,
|
|
281
410
|
account_wide: true,
|
|
411
|
+
// Kept only when it reads as a provider quota message, so the senior can
|
|
412
|
+
// see the provider's own words next to the hold.
|
|
413
|
+
...(isUsageLimitShape(text) && text.length <= 300 ? { provider_text: text } : {}),
|
|
282
414
|
};
|
|
283
415
|
}
|
|
284
416
|
export function readQuotaSignal(runDir, model, now, workerOutOffset = 0) {
|
|
@@ -314,11 +446,46 @@ export function readQuotaSignal(runDir, model, now, workerOutOffset = 0) {
|
|
|
314
446
|
const observedAt = Number.isFinite(outputAt.getTime()) ? outputAt : now;
|
|
315
447
|
return quotaSignalFromFields(fields, model, observedAt);
|
|
316
448
|
}
|
|
317
|
-
|
|
449
|
+
/**
|
|
450
|
+
* Hold a provider reset more than a day away to a short hold until a second
|
|
451
|
+
* 429 confirms it. The second 429 counts only when its lane ran after the
|
|
452
|
+
* short hold expired, so lanes that failed together on one 429 burst do not
|
|
453
|
+
* confirm each other.
|
|
454
|
+
*/
|
|
455
|
+
function gateFarReset(current, signal, options) {
|
|
456
|
+
if (!signal.account_wide ||
|
|
457
|
+
!signal.quota_reset_at ||
|
|
458
|
+
signal.needs_credits ||
|
|
459
|
+
signal.reset_source === "derived") {
|
|
460
|
+
return { signal };
|
|
461
|
+
}
|
|
462
|
+
const resetMs = Date.parse(signal.quota_reset_at);
|
|
463
|
+
const nowMs = options.now.getTime();
|
|
464
|
+
if (!Number.isFinite(resetMs) || resetMs - nowMs <= QUOTA_CONFIRM_HORIZON_MS)
|
|
465
|
+
return { signal };
|
|
466
|
+
const pending = current.unconfirmed_reset_at;
|
|
467
|
+
const shortHoldActive = resetStillAhead(current.all_models_quota_reset_at, options.now);
|
|
468
|
+
if (pending !== undefined && !shortHoldActive) {
|
|
469
|
+
return { signal: { ...signal, reset_source: "parsed" } };
|
|
470
|
+
}
|
|
471
|
+
return {
|
|
472
|
+
signal: {
|
|
473
|
+
...signal,
|
|
474
|
+
quota_reset_at: new Date(nowMs + options.unconfirmedHoldMs).toISOString(),
|
|
475
|
+
reset_source: "unconfirmed",
|
|
476
|
+
},
|
|
477
|
+
unconfirmed_reset_at: pending !== undefined && !shouldReplaceReset(pending, signal.quota_reset_at)
|
|
478
|
+
? pending
|
|
479
|
+
: signal.quota_reset_at,
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
export function recordAccountQuota(hqRoot, provider, accountId, incoming, options) {
|
|
318
483
|
return withLaneRecordFileLock(quotaStatePath(hqRoot), () => {
|
|
319
484
|
const state = loadQuotaState(hqRoot);
|
|
320
485
|
const accounts = state.providers[provider] ?? {};
|
|
321
486
|
const current = accounts[accountId] ?? { models: {} };
|
|
487
|
+
const gated = options ? gateFarReset(current, incoming, options) : { signal: incoming };
|
|
488
|
+
const signal = gated.signal;
|
|
322
489
|
const models = { ...current.models };
|
|
323
490
|
if (signal.model) {
|
|
324
491
|
const prior = models[signal.model] ?? {};
|
|
@@ -340,6 +507,14 @@ export function recordAccountQuota(hqRoot, provider, accountId, signal) {
|
|
|
340
507
|
if (shouldReplaceReset(current.all_models_quota_reset_at, signal.quota_reset_at)) {
|
|
341
508
|
account.all_models_quota_reset_at = signal.quota_reset_at;
|
|
342
509
|
account.reset_source = signal.reset_source ?? "parsed";
|
|
510
|
+
if (signal.provider_text)
|
|
511
|
+
account.provider_text = signal.provider_text;
|
|
512
|
+
if (gated.unconfirmed_reset_at) {
|
|
513
|
+
account.unconfirmed_reset_at = gated.unconfirmed_reset_at;
|
|
514
|
+
}
|
|
515
|
+
else {
|
|
516
|
+
delete account.unconfirmed_reset_at;
|
|
517
|
+
}
|
|
343
518
|
}
|
|
344
519
|
else if (current.all_models_quota_reset_at) {
|
|
345
520
|
account.all_models_quota_reset_at = current.all_models_quota_reset_at;
|
|
@@ -367,11 +542,17 @@ export function clearAccountQuota(hqRoot, provider, accountId, options = {}) {
|
|
|
367
542
|
const shouldClear = (reset) => reset !== undefined &&
|
|
368
543
|
(!options.futureOnly || resetStillAhead(reset, now));
|
|
369
544
|
let changed = false;
|
|
545
|
+
// A successful turn or an operator clear disproves a pending far reset.
|
|
546
|
+
if (account.unconfirmed_reset_at !== undefined) {
|
|
547
|
+
delete account.unconfirmed_reset_at;
|
|
548
|
+
changed = true;
|
|
549
|
+
}
|
|
370
550
|
const accountWideReset = account.all_models_quota_reset_at;
|
|
371
551
|
if (shouldClear(accountWideReset)) {
|
|
372
552
|
cleared.all_models_quota_reset_at = accountWideReset;
|
|
373
553
|
delete account.all_models_quota_reset_at;
|
|
374
554
|
delete account.reset_source;
|
|
555
|
+
delete account.provider_text;
|
|
375
556
|
changed = true;
|
|
376
557
|
}
|
|
377
558
|
const modelNames = options.model === undefined
|
|
@@ -395,6 +576,7 @@ export function clearAccountQuota(hqRoot, provider, accountId, options = {}) {
|
|
|
395
576
|
return cleared;
|
|
396
577
|
if (account.all_models_quota_reset_at === undefined) {
|
|
397
578
|
delete account.reset_source;
|
|
579
|
+
delete account.provider_text;
|
|
398
580
|
}
|
|
399
581
|
const accounts = state.providers[provider];
|
|
400
582
|
if (account.all_models_quota_reset_at === undefined &&
|
|
@@ -34,6 +34,7 @@ import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LE
|
|
|
34
34
|
import { isFailedQuotaKeepAliveLane, providerSupportsSessionResume } from "./delivery.js";
|
|
35
35
|
import { LANE_PROVIDERS, isIsoDateTime, isTerminalLaneState } from "./types.js";
|
|
36
36
|
import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, loadQuotaState, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
37
|
+
import { deliverQuotaHoldNotice, quotaHoldNotice } from "./quota-notice.js";
|
|
37
38
|
import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
38
39
|
import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
39
40
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, laneMeshCompany, } from "./mesh-events.js";
|
|
@@ -1733,7 +1734,22 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1733
1734
|
classified.account &&
|
|
1734
1735
|
(classification.reason === "account_quota_exhausted" ||
|
|
1735
1736
|
classification.reason === "account_model_unavailable")) {
|
|
1736
|
-
recordAccountQuota(hqRoot, classified.provider, classified.account, quota
|
|
1737
|
+
const quotaState = recordAccountQuota(hqRoot, classified.provider, classified.account, quota, {
|
|
1738
|
+
now,
|
|
1739
|
+
unconfirmedHoldMs: providerQuotaDefaultHoldMs(loadLanesConfig(hqRoot).providers[classified.provider]),
|
|
1740
|
+
});
|
|
1741
|
+
if (classification.reason === "account_quota_exhausted") {
|
|
1742
|
+
const notice = quotaHoldNotice(quotaState, classified.provider, classified.account, lane.lane_id);
|
|
1743
|
+
if (notice) {
|
|
1744
|
+
try {
|
|
1745
|
+
deliverQuotaHoldNotice(hqRoot, lane, notice, now.toISOString());
|
|
1746
|
+
}
|
|
1747
|
+
catch (err) {
|
|
1748
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1749
|
+
process.stderr.write(`hq lanes reconcile: quota hold notice failed for ${lane.lane_id}: ${message}\n`);
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1737
1753
|
}
|
|
1738
1754
|
if (!deps.dryRun &&
|
|
1739
1755
|
accountRetryReason &&
|
|
@@ -12,6 +12,8 @@ export interface EnsureSessionHomeResult extends SessionCopyResult {
|
|
|
12
12
|
source_home: string;
|
|
13
13
|
transcript_path: string;
|
|
14
14
|
homes_searched: string[];
|
|
15
|
+
/** Set when the transcript was found outside the expected project directory. */
|
|
16
|
+
found_outside_cwd?: true;
|
|
15
17
|
}
|
|
16
18
|
/**
|
|
17
19
|
* Older Claude account homes can share a registered `account-N/.claude`
|