@indigoai-us/hq-cli 5.131.0 → 5.132.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -0
- package/dist/command-catalog.generated.d.ts +12 -0
- package/dist/command-catalog.generated.js +16 -0
- package/dist/commands/lanes.d.ts +13 -2
- package/dist/commands/lanes.js +394 -42
- package/dist/commands/mesh.js +20 -1
- package/dist/commands/update.js +1 -1
- package/dist/lib/bot/paths.d.ts +7 -5
- package/dist/lib/bot/paths.js +8 -1
- package/dist/lib/bot/prompt.d.ts +0 -2
- package/dist/lib/bot/prompt.js +5 -11
- package/dist/lib/bot/room-policy.d.ts +1 -24
- package/dist/lib/bot/room-policy.js +2 -35
- package/dist/lib/bot/run.d.ts +7 -8
- package/dist/lib/bot/run.js +21 -33
- package/dist/lib/bot/session.d.ts +1 -1
- package/dist/lib/bot/session.js +1 -1
- package/dist/lib/lanes/accounts.d.ts +6 -0
- package/dist/lib/lanes/accounts.js +2 -1
- package/dist/lib/lanes/config.d.ts +4 -0
- package/dist/lib/lanes/config.js +9 -0
- package/dist/lib/lanes/index.d.ts +10 -3
- package/dist/lib/lanes/index.js +5 -1
- package/dist/lib/lanes/mesh-events.js +4 -0
- package/dist/lib/lanes/models.d.ts +78 -0
- package/dist/lib/lanes/models.js +205 -0
- package/dist/lib/lanes/move-account.d.ts +56 -0
- package/dist/lib/lanes/move-account.js +193 -0
- package/dist/lib/lanes/provider-rejection.d.ts +86 -0
- package/dist/lib/lanes/provider-rejection.js +258 -0
- package/dist/lib/lanes/quota.d.ts +17 -0
- package/dist/lib/lanes/quota.js +24 -0
- package/dist/lib/lanes/reconcile-install.d.ts +24 -3
- package/dist/lib/lanes/reconcile-install.js +48 -12
- package/dist/lib/lanes/reconcile.js +41 -11
- package/dist/lib/lanes/spawn.js +26 -5
- package/dist/lib/lanes/store.js +10 -0
- package/dist/lib/lanes/types.d.ts +49 -2
- package/dist/lib/lanes/types.js +4 -3
- package/dist/lib/lanes/watcher.d.ts +27 -0
- package/dist/lib/lanes/watcher.js +68 -5
- package/dist/lib/mesh/live/backfill-held.d.ts +4 -3
- package/dist/lib/mesh/live/backfill-held.js +37 -14
- package/dist/lib/mesh/live/daemon/doctor.js +5 -2
- package/dist/lib/mesh/live/daemon/install.d.ts +5 -0
- package/dist/lib/mesh/live/daemon/install.js +84 -1
- package/dist/lib/mesh/live/daemon/run.d.ts +5 -0
- package/dist/lib/mesh/live/daemon/run.js +14 -4
- package/dist/lib/mesh/live/emit-evidence.d.ts +19 -0
- package/dist/lib/mesh/live/emit-evidence.js +20 -2
- package/dist/lib/work-context/company.d.ts +16 -0
- package/dist/lib/work-context/company.js +46 -14
- package/dist/utils/self-update.js +4 -1
- package/package.json +1 -1
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider-level turn failures read off the worker's own stdout.
|
|
3
|
+
*
|
|
4
|
+
* A worker whose turn the provider refused (a 400 for a model the account
|
|
5
|
+
* cannot use, a 402 for an exhausted balance, a 5xx) exits with partial
|
|
6
|
+
* output and no envelope. Without this module the watcher classifies that
|
|
7
|
+
* `interrupted` with `exited_with_partial_output`, and the operator has to
|
|
8
|
+
* read the raw log to learn why. Here the failure is named: the lane is
|
|
9
|
+
* `failed` with `terminal_reason: provider_rejected` and the provider's own
|
|
10
|
+
* message on `terminal_details`.
|
|
11
|
+
*
|
|
12
|
+
* A 429 is parsed here like any other status so that a codex or grok 429,
|
|
13
|
+
* which the claude-shaped quota reader cannot see, still reaches the quota
|
|
14
|
+
* path: the watcher routes a `status === QUOTA_STATUS` failure to quota.ts
|
|
15
|
+
* (reset time, account hold) and everything else to `provider_rejected`.
|
|
16
|
+
*
|
|
17
|
+
* Only the fields named below are read out of the payload. The prompt, the
|
|
18
|
+
* response body and everything else stay on disk. Nothing here logs.
|
|
19
|
+
*/
|
|
20
|
+
import * as fs from "node:fs";
|
|
21
|
+
import * as path from "node:path";
|
|
22
|
+
export const PROVIDER_REJECTED_REASON = "provider_rejected";
|
|
23
|
+
/** Bytes of the provider message kept on the record. The log keeps the rest. */
|
|
24
|
+
export const PROVIDER_REJECTION_MESSAGE_CAP = 2000;
|
|
25
|
+
/** Status the watcher routes to the quota path instead of `provider_rejected`. */
|
|
26
|
+
export const QUOTA_STATUS = 429;
|
|
27
|
+
function isPlainObject(value) {
|
|
28
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Walk the JSON lines of `text` from the last line backward, parsing one
|
|
32
|
+
* line at a time, and return the first value `pick` accepts. A long codex
|
|
33
|
+
* run writes thousands of events to worker.out; the failure, when there is
|
|
34
|
+
* one, is at the tail, so the scan stops there and never holds every parsed
|
|
35
|
+
* event at once. Lines that are not objects are skipped without parsing.
|
|
36
|
+
*/
|
|
37
|
+
function scanJsonLinesBackward(text, pick) {
|
|
38
|
+
let end = text.length;
|
|
39
|
+
while (end > 0) {
|
|
40
|
+
const start = text.lastIndexOf("\n", end - 1);
|
|
41
|
+
const line = text.slice(start + 1, end).trim();
|
|
42
|
+
end = start === -1 ? 0 : start;
|
|
43
|
+
if (!line.startsWith("{"))
|
|
44
|
+
continue;
|
|
45
|
+
let parsed;
|
|
46
|
+
try {
|
|
47
|
+
parsed = JSON.parse(line);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
if (!isPlainObject(parsed))
|
|
53
|
+
continue;
|
|
54
|
+
const picked = pick(parsed);
|
|
55
|
+
if (picked !== null)
|
|
56
|
+
return picked;
|
|
57
|
+
}
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
/** Parse the whole text as one JSON object (grok prints a single pretty-printed object). */
|
|
61
|
+
function wholeObject(text) {
|
|
62
|
+
const trimmed = text.trim();
|
|
63
|
+
if (!trimmed.startsWith("{"))
|
|
64
|
+
return null;
|
|
65
|
+
try {
|
|
66
|
+
const parsed = JSON.parse(trimmed);
|
|
67
|
+
return isPlainObject(parsed) ? parsed : null;
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const NOT_REJECTED = Symbol("not_rejected");
|
|
74
|
+
function statusOf(value) {
|
|
75
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 100 && value <= 599
|
|
76
|
+
? value
|
|
77
|
+
: null;
|
|
78
|
+
}
|
|
79
|
+
export function capMessage(message) {
|
|
80
|
+
return message.length > PROVIDER_REJECTION_MESSAGE_CAP
|
|
81
|
+
? `${message.slice(0, PROVIDER_REJECTION_MESSAGE_CAP)}…`
|
|
82
|
+
: message;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Whether a later run could succeed without the operator changing anything.
|
|
86
|
+
* A 400 model rejection cannot; a request timeout, a too-early, or a server
|
|
87
|
+
* error can. An unknown status is not retryable: guessing "try again" on a
|
|
88
|
+
* failure we cannot classify would loop the lane.
|
|
89
|
+
*/
|
|
90
|
+
export function retryableFromStatus(status) {
|
|
91
|
+
if (status === null)
|
|
92
|
+
return false;
|
|
93
|
+
if (status === 408 || status === 425)
|
|
94
|
+
return true;
|
|
95
|
+
return status >= 500 && status <= 599;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Codex `exec --json`: `{"type":"turn.failed","error":{"message":"<text>"}}`.
|
|
99
|
+
* Measured 2026-09-22 (lane 01M34HCCP1ZDN3H68ESCRZE7RZ): the message is
|
|
100
|
+
* itself a JSON string, `{"type":"error","status":400,"error":{"type":
|
|
101
|
+
* "invalid_request_error","message":"The 'terra' model is not supported when
|
|
102
|
+
* using Codex with a ChatGPT account."}}`. The inner message and status are
|
|
103
|
+
* read when it parses; otherwise the outer text is the message.
|
|
104
|
+
*/
|
|
105
|
+
export function parseCodexTurnFailed(text) {
|
|
106
|
+
return scanJsonLinesBackward(text, (rec) => {
|
|
107
|
+
if (rec.type !== "turn.failed")
|
|
108
|
+
return null;
|
|
109
|
+
const error = isPlainObject(rec.error) ? rec.error : {};
|
|
110
|
+
const outer = typeof error.message === "string" ? error.message : "";
|
|
111
|
+
let message = outer;
|
|
112
|
+
let status = null;
|
|
113
|
+
let errorType = null;
|
|
114
|
+
const inner = wholeObject(outer);
|
|
115
|
+
if (inner) {
|
|
116
|
+
status = statusOf(inner.status);
|
|
117
|
+
const innerError = isPlainObject(inner.error) ? inner.error : {};
|
|
118
|
+
if (typeof innerError.message === "string" && innerError.message.length > 0) {
|
|
119
|
+
message = innerError.message;
|
|
120
|
+
}
|
|
121
|
+
if (typeof innerError.type === "string")
|
|
122
|
+
errorType = innerError.type;
|
|
123
|
+
}
|
|
124
|
+
return {
|
|
125
|
+
reason: PROVIDER_REJECTED_REASON,
|
|
126
|
+
provider: "codex",
|
|
127
|
+
message: capMessage(message),
|
|
128
|
+
status,
|
|
129
|
+
error_type: errorType,
|
|
130
|
+
retryable: retryableFromStatus(status),
|
|
131
|
+
event: "codex.turn.failed",
|
|
132
|
+
};
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Claude `-p --output-format json`: the last `type: "result"` object with
|
|
137
|
+
* `is_error: true` and a numeric `api_error_status`. A result that is an
|
|
138
|
+
* error with no status is not a provider rejection: it is either the quota
|
|
139
|
+
* text shape quota.ts reads, or the worker's own failure, and keeps the
|
|
140
|
+
* existing classification.
|
|
141
|
+
*/
|
|
142
|
+
export function parseClaudeResultRejection(text) {
|
|
143
|
+
// The last `result` decides: a success or a status-less error there is
|
|
144
|
+
// final (NOT_REJECTED), so the scan does not continue to earlier results.
|
|
145
|
+
const found = scanJsonLinesBackward(text, (rec) => {
|
|
146
|
+
if (rec.type !== "result")
|
|
147
|
+
return null;
|
|
148
|
+
if (rec.is_error !== true)
|
|
149
|
+
return NOT_REJECTED;
|
|
150
|
+
const status = statusOf(rec.api_error_status);
|
|
151
|
+
if (status === null)
|
|
152
|
+
return NOT_REJECTED;
|
|
153
|
+
const message = typeof rec.result === "string" ? rec.result : "";
|
|
154
|
+
return {
|
|
155
|
+
reason: PROVIDER_REJECTED_REASON,
|
|
156
|
+
provider: "claude",
|
|
157
|
+
message: capMessage(message),
|
|
158
|
+
status,
|
|
159
|
+
error_type: typeof rec.subtype === "string" ? rec.subtype : null,
|
|
160
|
+
retryable: retryableFromStatus(status),
|
|
161
|
+
event: "claude.result.is_error",
|
|
162
|
+
};
|
|
163
|
+
});
|
|
164
|
+
return found === NOT_REJECTED ? null : found;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Grok `--output-format json` prints one object. Measured 2026-09-22 (lane
|
|
168
|
+
* 01M33X586J18J1B6GK0E6M4Y07): `{"type":"error","message":"Internal error:
|
|
169
|
+
* {\n \"message\": \"API error (status 402 Payment Required): Grok Build
|
|
170
|
+
* usage balance exhausted\",\n \"http_status\": 402, ...}", ...}`. The
|
|
171
|
+
* inner object carries `message` and `http_status`; when it does not parse,
|
|
172
|
+
* the status is read from `(status NNN` in the text.
|
|
173
|
+
*/
|
|
174
|
+
export function parseGrokErrorOutput(text) {
|
|
175
|
+
// One pretty-printed object is the measured shape; a line stream is the
|
|
176
|
+
// fallback, read from the tail like the other providers.
|
|
177
|
+
const whole = wholeObject(text);
|
|
178
|
+
if (whole !== null)
|
|
179
|
+
return grokRejectionFrom(whole);
|
|
180
|
+
return scanJsonLinesBackward(text, grokRejectionFrom);
|
|
181
|
+
}
|
|
182
|
+
function grokRejectionFrom(rec) {
|
|
183
|
+
if (rec.type !== "error")
|
|
184
|
+
return null;
|
|
185
|
+
const outer = typeof rec.message === "string" ? rec.message : "";
|
|
186
|
+
let message = outer;
|
|
187
|
+
let status = null;
|
|
188
|
+
const braceAt = outer.indexOf("{");
|
|
189
|
+
const inner = braceAt >= 0 ? wholeObject(outer.slice(braceAt)) : null;
|
|
190
|
+
if (inner) {
|
|
191
|
+
if (typeof inner.message === "string" && inner.message.length > 0) {
|
|
192
|
+
message = inner.message;
|
|
193
|
+
}
|
|
194
|
+
status = statusOf(inner.http_status);
|
|
195
|
+
}
|
|
196
|
+
if (status === null) {
|
|
197
|
+
const match = /\(status\s+(\d{3})\b/.exec(outer);
|
|
198
|
+
status = match ? statusOf(Number(match[1])) : null;
|
|
199
|
+
}
|
|
200
|
+
return {
|
|
201
|
+
reason: PROVIDER_REJECTED_REASON,
|
|
202
|
+
provider: "grok",
|
|
203
|
+
message: capMessage(message),
|
|
204
|
+
status,
|
|
205
|
+
error_type: null,
|
|
206
|
+
retryable: retryableFromStatus(status),
|
|
207
|
+
event: "grok.error",
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
/** The `terminal_details` record both the watcher and the reconcile pass write for a rejection. */
|
|
211
|
+
export function providerRejectionDetails(rejection, lane) {
|
|
212
|
+
return {
|
|
213
|
+
reason: rejection.reason,
|
|
214
|
+
provider: rejection.provider,
|
|
215
|
+
message: rejection.message,
|
|
216
|
+
status: rejection.status,
|
|
217
|
+
error_type: rejection.error_type,
|
|
218
|
+
retryable: rejection.retryable,
|
|
219
|
+
event: rejection.event,
|
|
220
|
+
model: lane.model ?? null,
|
|
221
|
+
account: lane.account ?? null,
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
export function parseProviderRejection(provider, text) {
|
|
225
|
+
switch (provider) {
|
|
226
|
+
case "codex":
|
|
227
|
+
return parseCodexTurnFailed(text);
|
|
228
|
+
case "claude":
|
|
229
|
+
return parseClaudeResultRejection(text);
|
|
230
|
+
case "grok":
|
|
231
|
+
return parseGrokErrorOutput(text);
|
|
232
|
+
default: {
|
|
233
|
+
const _exhaustive = provider;
|
|
234
|
+
return _exhaustive;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Read this run's bytes of `logs/worker.out` (past `workerOutOffset`, the
|
|
240
|
+
* same cut delivery.ts uses) and return the provider rejection they carry,
|
|
241
|
+
* or null. A missing file is null; any other read error propagates.
|
|
242
|
+
*/
|
|
243
|
+
export function readProviderRejection(runDir, provider, workerOutOffset = 0) {
|
|
244
|
+
const outPath = path.join(runDir, "logs", "worker.out");
|
|
245
|
+
let text;
|
|
246
|
+
try {
|
|
247
|
+
const bytes = fs.readFileSync(outPath);
|
|
248
|
+
text = bytes.subarray(Math.min(workerOutOffset, bytes.length)).toString("utf8");
|
|
249
|
+
}
|
|
250
|
+
catch (err) {
|
|
251
|
+
const code = err.code;
|
|
252
|
+
if (code === "ENOENT")
|
|
253
|
+
return null;
|
|
254
|
+
throw err;
|
|
255
|
+
}
|
|
256
|
+
return parseProviderRejection(provider, text);
|
|
257
|
+
}
|
|
258
|
+
//# sourceMappingURL=provider-rejection.js.map
|
|
@@ -26,9 +26,15 @@ export interface ModelQuota {
|
|
|
26
26
|
/** Operator-facing marker, e.g. "needs credits for claude-fable-5-1". */
|
|
27
27
|
marker?: string;
|
|
28
28
|
}
|
|
29
|
+
export type QuotaResetSource = "provider" | "default_hold";
|
|
29
30
|
export interface AccountQuota {
|
|
30
31
|
/** Set when the 429 is an account-wide usage limit with a reset time. */
|
|
31
32
|
all_models_quota_reset_at?: string;
|
|
33
|
+
/**
|
|
34
|
+
* Where `all_models_quota_reset_at` came from: the provider's own message,
|
|
35
|
+
* or the configured default hold applied to a 429 that named no reset.
|
|
36
|
+
*/
|
|
37
|
+
reset_source?: QuotaResetSource;
|
|
32
38
|
models: Record<string, ModelQuota>;
|
|
33
39
|
}
|
|
34
40
|
export interface QuotaState {
|
|
@@ -43,7 +49,18 @@ export interface QuotaSignal {
|
|
|
43
49
|
marker?: string;
|
|
44
50
|
/** True when the text is a usage limit with a reset, not a per-model credits refusal. */
|
|
45
51
|
account_wide?: boolean;
|
|
52
|
+
/** Set by applyDefaultQuotaHold: whether the reset is the provider's or the default hold. */
|
|
53
|
+
reset_source?: QuotaResetSource;
|
|
46
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* A 429 whose message names no reset time and no credits wording would
|
|
57
|
+
* otherwise record no hold at all, and the next lane on the account would
|
|
58
|
+
* hit the same 429. Give such a signal the provider's configured default
|
|
59
|
+
* hold (`quota_default_hold_ms`, fifteen minutes unless set) as its reset,
|
|
60
|
+
* marked `default_hold`; a signal that already carries a reset keeps it,
|
|
61
|
+
* marked `provider`. A per-model credits refusal is untouched.
|
|
62
|
+
*/
|
|
63
|
+
export declare function applyDefaultQuotaHold(signal: QuotaSignal, defaultHoldMs: number, now: Date): QuotaSignal;
|
|
47
64
|
export declare function quotaStatePath(hqRoot: string): string;
|
|
48
65
|
export declare function loadQuotaState(hqRoot: string): QuotaState;
|
|
49
66
|
/**
|
package/dist/lib/lanes/quota.js
CHANGED
|
@@ -21,6 +21,26 @@ import * as fs from "node:fs";
|
|
|
21
21
|
import * as path from "node:path";
|
|
22
22
|
export const QUOTA_STATE_REL = path.join("personal", "lanes", "quota.json");
|
|
23
23
|
export const QUOTA_SCHEMA_VERSION = "hq-lanes-quota.v1";
|
|
24
|
+
/**
|
|
25
|
+
* A 429 whose message names no reset time and no credits wording would
|
|
26
|
+
* otherwise record no hold at all, and the next lane on the account would
|
|
27
|
+
* hit the same 429. Give such a signal the provider's configured default
|
|
28
|
+
* hold (`quota_default_hold_ms`, fifteen minutes unless set) as its reset,
|
|
29
|
+
* marked `default_hold`; a signal that already carries a reset keeps it,
|
|
30
|
+
* marked `provider`. A per-model credits refusal is untouched.
|
|
31
|
+
*/
|
|
32
|
+
export function applyDefaultQuotaHold(signal, defaultHoldMs, now) {
|
|
33
|
+
if (signal.reason !== "account_quota_exhausted" || signal.needs_credits)
|
|
34
|
+
return signal;
|
|
35
|
+
if (signal.quota_reset_at)
|
|
36
|
+
return { ...signal, reset_source: "provider" };
|
|
37
|
+
return {
|
|
38
|
+
...signal,
|
|
39
|
+
quota_reset_at: new Date(now.getTime() + defaultHoldMs).toISOString(),
|
|
40
|
+
account_wide: true,
|
|
41
|
+
reset_source: "default_hold",
|
|
42
|
+
};
|
|
43
|
+
}
|
|
24
44
|
const MONTHS = {
|
|
25
45
|
jan: 0,
|
|
26
46
|
feb: 1,
|
|
@@ -98,6 +118,9 @@ export function loadQuotaState(hqRoot) {
|
|
|
98
118
|
if (typeof raw.all_models_quota_reset_at === "string" &&
|
|
99
119
|
raw.all_models_quota_reset_at.length > 0) {
|
|
100
120
|
account.all_models_quota_reset_at = raw.all_models_quota_reset_at;
|
|
121
|
+
if (raw.reset_source === "provider" || raw.reset_source === "default_hold") {
|
|
122
|
+
account.reset_source = raw.reset_source;
|
|
123
|
+
}
|
|
101
124
|
}
|
|
102
125
|
accounts[id] = account;
|
|
103
126
|
}
|
|
@@ -244,6 +267,7 @@ export function recordAccountQuota(hqRoot, provider, accountId, signal) {
|
|
|
244
267
|
const account = { ...current, models };
|
|
245
268
|
if (signal.account_wide && signal.quota_reset_at) {
|
|
246
269
|
account.all_models_quota_reset_at = signal.quota_reset_at;
|
|
270
|
+
account.reset_source = signal.reset_source ?? "provider";
|
|
247
271
|
}
|
|
248
272
|
accounts[accountId] = account;
|
|
249
273
|
state.providers[provider] = accounts;
|
|
@@ -8,8 +8,12 @@
|
|
|
8
8
|
*/
|
|
9
9
|
export declare const RECONCILE_TIMER_INTERVAL = "1min";
|
|
10
10
|
export interface ProviderPathEnv {
|
|
11
|
-
/**
|
|
12
|
-
|
|
11
|
+
/**
|
|
12
|
+
* For each resolved provider: the directory where the command name was
|
|
13
|
+
* found on the installing shell's PATH, then the realpath target's
|
|
14
|
+
* directory when it differs. Provider order, deduplicated.
|
|
15
|
+
*/
|
|
16
|
+
resolved_dirs: string[];
|
|
13
17
|
/** Providers whose binary resolved on the installing shell's PATH. */
|
|
14
18
|
resolved: string[];
|
|
15
19
|
/** Providers whose binary did not resolve. Reported, never fatal on their own. */
|
|
@@ -27,6 +31,22 @@ export interface ProviderPathEnv {
|
|
|
27
31
|
* run.
|
|
28
32
|
*/
|
|
29
33
|
export declare function providerPathEnv(providers: readonly string[], pathEnv?: string | undefined): ProviderPathEnv;
|
|
34
|
+
/**
|
|
35
|
+
* The unit's PATH: each resolved provider binary directory, the directory of
|
|
36
|
+
* the hq binary itself, the directory of the node binary that runs it (a
|
|
37
|
+
* provider installed as a `#!/usr/bin/env node` script needs `node` on the
|
|
38
|
+
* unit PATH, and nvm/asdf installs keep it outside the baseline), then the
|
|
39
|
+
* systemd baseline, deduplicated. Nothing
|
|
40
|
+
* from the installing shell's PATH is carried over: on a box whose profile
|
|
41
|
+
* is synced from another machine that PATH holds directories from the other
|
|
42
|
+
* machine (measured 2026-09-22: `/Users/…` and `/opt/homebrew/…` entries on
|
|
43
|
+
* a Linux outpost), and none of them belong in a systemd unit.
|
|
44
|
+
*/
|
|
45
|
+
export declare function unitPathEnv(input: {
|
|
46
|
+
providerDirs: readonly string[];
|
|
47
|
+
hqBinary: string;
|
|
48
|
+
nodeBinary: string;
|
|
49
|
+
}): string;
|
|
30
50
|
export interface RunCommandResult {
|
|
31
51
|
status: number;
|
|
32
52
|
stdout?: string;
|
|
@@ -38,7 +58,8 @@ export interface ReconcileInstallOpts {
|
|
|
38
58
|
hqRoot: string;
|
|
39
59
|
nodeBinary?: string;
|
|
40
60
|
hqBinary?: string;
|
|
41
|
-
|
|
61
|
+
/** Resolved provider binary directories (providerPathEnv().resolved_dirs). */
|
|
62
|
+
providerDirs?: readonly string[];
|
|
42
63
|
home?: string;
|
|
43
64
|
enable?: boolean;
|
|
44
65
|
runCommand?: RunCommand;
|
|
@@ -46,21 +46,57 @@ export function providerPathEnv(providers, pathEnv = process.env.PATH) {
|
|
|
46
46
|
continue;
|
|
47
47
|
}
|
|
48
48
|
resolved.push(provider);
|
|
49
|
-
//
|
|
50
|
-
//
|
|
51
|
-
//
|
|
52
|
-
|
|
49
|
+
// Both directories: the one holding the literal command name (an npm
|
|
50
|
+
// `codex` link may target `codex.js`, a versioned claude launcher has
|
|
51
|
+
// another basename; spawn.ts launches the literal name, so its directory
|
|
52
|
+
// must be on the unit PATH) and the real binary's directory (a wrapper
|
|
53
|
+
// that only resolves with the installing shell's environment is not
|
|
54
|
+
// enough for systemd). Unrelated shell PATH entries stay out.
|
|
55
|
+
const linkDir = path.resolve(found);
|
|
56
|
+
const realDir = path.dirname(fs.realpathSync(path.join(found, provider)));
|
|
57
|
+
for (const dir of [linkDir, realDir]) {
|
|
58
|
+
if (!resolvedDirs.includes(dir))
|
|
59
|
+
resolvedDirs.push(dir);
|
|
60
|
+
}
|
|
53
61
|
}
|
|
54
62
|
if (providers.length > 0 && resolved.length === 0) {
|
|
55
63
|
throw new LanesError("provider_binary_unresolved", `Cannot resolve any provider binary (${providers.join(", ")}) from PATH`, { providers: [...providers], searched_path: searched });
|
|
56
64
|
}
|
|
57
|
-
return {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
65
|
+
return { resolved_dirs: resolvedDirs, resolved, unresolved };
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Directory of the hq entry point as invoked, not its realpath: the `hq`
|
|
69
|
+
* name on PATH is a symlink in a bin directory, and that bin directory is
|
|
70
|
+
* what the reconcile pass needs on PATH to run `hq` subcommands.
|
|
71
|
+
*/
|
|
72
|
+
function hqBinaryDir(hqBinary) {
|
|
73
|
+
return path.dirname(path.resolve(hqBinary));
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* The unit's PATH: each resolved provider binary directory, the directory of
|
|
77
|
+
* the hq binary itself, the directory of the node binary that runs it (a
|
|
78
|
+
* provider installed as a `#!/usr/bin/env node` script needs `node` on the
|
|
79
|
+
* unit PATH, and nvm/asdf installs keep it outside the baseline), then the
|
|
80
|
+
* systemd baseline, deduplicated. Nothing
|
|
81
|
+
* from the installing shell's PATH is carried over: on a box whose profile
|
|
82
|
+
* is synced from another machine that PATH holds directories from the other
|
|
83
|
+
* machine (measured 2026-09-22: `/Users/…` and `/opt/homebrew/…` entries on
|
|
84
|
+
* a Linux outpost), and none of them belong in a systemd unit.
|
|
85
|
+
*/
|
|
86
|
+
export function unitPathEnv(input) {
|
|
87
|
+
const baseline = SYSTEMD_PATH_ENV.split(":");
|
|
88
|
+
const nodeDir = path.dirname(path.resolve(input.nodeBinary));
|
|
89
|
+
return [
|
|
90
|
+
...new Set([
|
|
91
|
+
...input.providerDirs,
|
|
92
|
+
hqBinaryDir(input.hqBinary),
|
|
93
|
+
// A baseline node (/usr/bin/node) keeps its baseline position.
|
|
94
|
+
...(baseline.includes(nodeDir) ? [] : [nodeDir]),
|
|
95
|
+
...baseline,
|
|
96
|
+
]),
|
|
97
|
+
]
|
|
98
|
+
.filter(Boolean)
|
|
99
|
+
.join(":");
|
|
64
100
|
}
|
|
65
101
|
function shellQuote(v) {
|
|
66
102
|
return /^[A-Za-z0-9_./:-]+$/.test(v) ? v : `'${v.replace(/'/g, `'\\''`)}'`;
|
|
@@ -167,7 +203,7 @@ export function installReconcileTimer(opts) {
|
|
|
167
203
|
hqBinary,
|
|
168
204
|
hqRoot: opts.hqRoot,
|
|
169
205
|
home: opts.home,
|
|
170
|
-
pathEnv: opts.
|
|
206
|
+
pathEnv: unitPathEnv({ providerDirs: opts.providerDirs ?? [], hqBinary, nodeBinary }),
|
|
171
207
|
}));
|
|
172
208
|
const timerChanged = writeIfChanged(timerPath, renderReconcileTimer());
|
|
173
209
|
let enabled = false;
|
|
@@ -8,17 +8,18 @@
|
|
|
8
8
|
import * as fs from "node:fs";
|
|
9
9
|
import * as path from "node:path";
|
|
10
10
|
import { parseUserYaml } from "../../utils/user-yaml-error.js";
|
|
11
|
-
import { accountMaxRunning, loadLanesConfig } from "./config.js";
|
|
11
|
+
import { accountMaxRunning, loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
|
|
12
12
|
import { LanesError, isLaneVersionConflict } from "./errors.js";
|
|
13
13
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
|
|
14
14
|
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
|
|
15
15
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, updateLane, updateQuestion, } from "./store.js";
|
|
16
16
|
import { applyThrottleDecision, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
17
|
-
import { readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
17
|
+
import { applyDefaultQuotaHold, readQuotaSignal, recordAccountQuota } from "./quota.js";
|
|
18
18
|
import { answeredQuestionEntries, raiseEnvelopeQuestions } from "./questions.js";
|
|
19
19
|
import { SILENCE_MS, classifyLaneExit, isPidAlive, readLatestEnvelope, readLatestEnvelopeFile, readLatestEnvelopeRecord, } from "./watcher.js";
|
|
20
20
|
import { createLaneMesh, emitLaneDecisionEvents } from "./mesh-events.js";
|
|
21
|
-
import { decideEnvelopeRepair, envelopeQuestionIds } from "./watcher.js";
|
|
21
|
+
import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from "./watcher.js";
|
|
22
|
+
import { providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
22
23
|
/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
|
|
23
24
|
export const QUESTION_AGE_MS = 60 * 60 * 1000;
|
|
24
25
|
/** Spec §6.6 + §9: v1 permits exactly one aged advance. */
|
|
@@ -431,13 +432,29 @@ function classifyDeadLocalLane(hqRoot, laneId) {
|
|
|
431
432
|
path.join(runDir, "logs", "last-message.txt"),
|
|
432
433
|
].some((file) => fileNonEmpty(file));
|
|
433
434
|
const lane = readLane(hqRoot, laneId);
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
435
|
+
// The same routing as the watcher's exit path (watcher.ts): a provider
|
|
436
|
+
// failure on the stream is read first, a 429 or quota-worded one becomes
|
|
437
|
+
// the quota signal, any other becomes provider_rejected, and only a stream
|
|
438
|
+
// with no failure object falls back to the quota reader.
|
|
439
|
+
const workerOutOffset = readSpawnManifest(runDir).worker_out_offset ?? 0;
|
|
440
|
+
const now = new Date();
|
|
441
|
+
const routed = envelope
|
|
442
|
+
? { quota: null, rejection: null }
|
|
443
|
+
: routeProviderFailure(readProviderRejection(runDir, lane.provider, workerOutOffset), () => readQuotaSignal(runDir, lane.model, now, workerOutOffset), lane.model, now);
|
|
444
|
+
// As in the watcher: a reset-less 429 gets the provider's default hold.
|
|
445
|
+
const quota = routed.quota
|
|
446
|
+
? applyDefaultQuotaHold(routed.quota, providerQuotaDefaultHoldMs(loadLanesConfig(hqRoot).providers[lane.provider]), now)
|
|
447
|
+
: null;
|
|
437
448
|
return {
|
|
438
|
-
classification: classifyLaneExit({
|
|
449
|
+
classification: classifyLaneExit({
|
|
450
|
+
envelope,
|
|
451
|
+
hasPartialOutput,
|
|
452
|
+
quota,
|
|
453
|
+
providerRejection: routed.rejection,
|
|
454
|
+
}),
|
|
439
455
|
latest,
|
|
440
456
|
quota,
|
|
457
|
+
providerRejection: routed.rejection,
|
|
441
458
|
lane,
|
|
442
459
|
};
|
|
443
460
|
}
|
|
@@ -722,12 +739,17 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
722
739
|
continue;
|
|
723
740
|
}
|
|
724
741
|
try {
|
|
725
|
-
const { classification, latest, quota, lane: classified } = classifyDeadLocalLane(hqRoot, lane.lane_id);
|
|
742
|
+
const { classification, latest, quota, providerRejection, lane: classified } = classifyDeadLocalLane(hqRoot, lane.lane_id);
|
|
726
743
|
// Match the watcher's one-repair decision before classifying an
|
|
727
|
-
// unparseable envelope after both processes have died.
|
|
744
|
+
// unparseable envelope after both processes have died. As in the
|
|
745
|
+
// watcher, a provider rejection or a quota signal skips the repair.
|
|
728
746
|
const rawLatest = readLatestEnvelopeFile(hqRoot, lane.lane_id);
|
|
729
747
|
let invalidEnvelope = null;
|
|
730
|
-
if (!deps.dryRun &&
|
|
748
|
+
if (!deps.dryRun &&
|
|
749
|
+
!providerRejection &&
|
|
750
|
+
!quota &&
|
|
751
|
+
rawLatest &&
|
|
752
|
+
rawLatest.envelope === null) {
|
|
731
753
|
const repair = decideEnvelopeRepair(hqRoot, lane.lane_id, rawLatest, nowMs, deps.spawnRepair ?? spawnEnvelopeRepair, () => undefined);
|
|
732
754
|
if (repair.action === "spawned") {
|
|
733
755
|
armLaneWatcher(hqRoot, lane.lane_id, deps.spawnLaneDeps);
|
|
@@ -757,6 +779,9 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
757
779
|
state: repairedClassification.state,
|
|
758
780
|
queued_reason: undefined,
|
|
759
781
|
terminal_reason: repairedClassification.reason ?? "host_lost",
|
|
782
|
+
...(providerRejection
|
|
783
|
+
? { terminal_details: providerRejectionDetails(providerRejection, current) }
|
|
784
|
+
: {}),
|
|
760
785
|
...(quota?.quota_reset_at ? { quota_reset_at: quota.quota_reset_at } : {}),
|
|
761
786
|
...(quota?.marker ? { model_quota_marker: quota.marker } : {}),
|
|
762
787
|
last_host_report_at: nowIso,
|
|
@@ -837,8 +862,13 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
837
862
|
}
|
|
838
863
|
}
|
|
839
864
|
async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
865
|
+
// A lane whose pickup probe is still out is not admitted: its record is
|
|
866
|
+
// about to be bound or failed by the pickup, and a worker spawned now
|
|
867
|
+
// would keep running behind that write.
|
|
840
868
|
const localQueued = lanes
|
|
841
|
-
.filter((lane) => lane.state === "queued" &&
|
|
869
|
+
.filter((lane) => lane.state === "queued" &&
|
|
870
|
+
lane.pickup_validation_pending !== true &&
|
|
871
|
+
laneRunsHere(lane, deps.host))
|
|
842
872
|
.sort(compareQueueOrder);
|
|
843
873
|
if (localQueued.length === 0)
|
|
844
874
|
return;
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -979,13 +979,19 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
979
979
|
}
|
|
980
980
|
return existing;
|
|
981
981
|
}
|
|
982
|
+
// A queued lane that a move parked at the target's cap carries the turn
|
|
983
|
+
// it was resumed with: the admit spawn continues that session instead of
|
|
984
|
+
// starting fresh from the brief.
|
|
985
|
+
const pendingResume = launch.trigger === undefined && existing.state === "queued" ? existing.pending_resume : undefined;
|
|
982
986
|
const trigger = launch.trigger ?? inferTrigger(existing);
|
|
983
987
|
// A message into a finished lane must not turn that lane `failed` when the
|
|
984
988
|
// account cannot be bound: nothing has started yet, so the record keeps its
|
|
985
989
|
// state and the caller sees the error. Admission and question resume keep
|
|
986
990
|
// today's behaviour of naming the terminal reason on the record.
|
|
987
991
|
const markPreSpawnFailure = trigger.kind !== "message" && trigger.kind !== "resume";
|
|
988
|
-
const plan =
|
|
992
|
+
const plan = pendingResume !== undefined && providerSupportsSessionResume(existing.provider)
|
|
993
|
+
? { mode: "resume", session_id: pendingResume.session_id }
|
|
994
|
+
: planLaneLaunch(existing, trigger);
|
|
989
995
|
if (existing.host.kind === "outpost") {
|
|
990
996
|
refuseOutpostSelfTarget(hqRoot, existing, deps);
|
|
991
997
|
}
|
|
@@ -1065,8 +1071,8 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1065
1071
|
? trigger.text
|
|
1066
1072
|
: trigger.kind === "resume"
|
|
1067
1073
|
? trigger.text
|
|
1068
|
-
:
|
|
1069
|
-
resumedFrom: trigger.kind === "resume" ? trigger.from :
|
|
1074
|
+
: pendingResume?.text,
|
|
1075
|
+
resumedFrom: trigger.kind === "resume" ? trigger.from : pendingResume?.resumed_from,
|
|
1070
1076
|
answersPath: trigger.kind === "answered_questions" && existsSync(answersPath)
|
|
1071
1077
|
? answersPath
|
|
1072
1078
|
: undefined,
|
|
@@ -1119,7 +1125,8 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1119
1125
|
// Resume-time compaction (compaction.ts). Runs before the claim so the
|
|
1120
1126
|
// lane is never `running` without a worker while `/compact` works.
|
|
1121
1127
|
let compaction;
|
|
1122
|
-
|
|
1128
|
+
const compactWanted = pendingResume?.compact ?? launch.compact;
|
|
1129
|
+
if (resume && compactWanted !== false) {
|
|
1123
1130
|
const nowFn = deps.now ?? (() => new Date());
|
|
1124
1131
|
const decision = compactionDue(existing, nowFn().getTime(), providerCompactAfterMs(config.providers[lane.provider]));
|
|
1125
1132
|
if (decision.due) {
|
|
@@ -1169,6 +1176,8 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1169
1176
|
...current,
|
|
1170
1177
|
state: "running",
|
|
1171
1178
|
queued_reason: undefined,
|
|
1179
|
+
// An account move's reservation ends with the claim.
|
|
1180
|
+
pending_resume: undefined,
|
|
1172
1181
|
account: resolved.id,
|
|
1173
1182
|
...(model !== undefined ? { model } : {}),
|
|
1174
1183
|
effort_source: effort.source,
|
|
@@ -1187,10 +1196,22 @@ export function spawnLaneWorker(hqRoot, lane, deps = {}, launch = {}) {
|
|
|
1187
1196
|
}
|
|
1188
1197
|
if (compaction)
|
|
1189
1198
|
next.last_compaction = { ...compaction, resume_at: nowIso };
|
|
1190
|
-
if (resume &&
|
|
1199
|
+
if (resume && pendingResume !== undefined) {
|
|
1191
1200
|
next.last_resume = {
|
|
1192
1201
|
session_id: resume.session_id,
|
|
1193
1202
|
at: nowIso,
|
|
1203
|
+
account: resolved.id,
|
|
1204
|
+
trigger: pendingResume.trigger,
|
|
1205
|
+
...(pendingResume.message_id !== undefined ? { message_id: pendingResume.message_id } : {}),
|
|
1206
|
+
...(pendingResume.resumed_from !== undefined ? { resumed_from: pendingResume.resumed_from } : {}),
|
|
1207
|
+
...(pendingResume.text !== undefined ? { text: pendingResume.text } : {}),
|
|
1208
|
+
};
|
|
1209
|
+
}
|
|
1210
|
+
else if (resume && trigger.kind !== "admit") {
|
|
1211
|
+
next.last_resume = {
|
|
1212
|
+
session_id: resume.session_id,
|
|
1213
|
+
at: nowIso,
|
|
1214
|
+
account: resolved.id,
|
|
1194
1215
|
trigger: trigger.kind,
|
|
1195
1216
|
...(trigger.kind === "message" ? { message_id: trigger.message_id } : {}),
|
|
1196
1217
|
...(trigger.kind === "resume"
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -325,6 +325,11 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
325
325
|
path: filePath,
|
|
326
326
|
});
|
|
327
327
|
}
|
|
328
|
+
if (raw.account_history !== undefined && !Array.isArray(raw.account_history)) {
|
|
329
|
+
throw new LaneSchemaError("account_history must be an array when present", {
|
|
330
|
+
path: filePath,
|
|
331
|
+
});
|
|
332
|
+
}
|
|
328
333
|
if (!Array.isArray(raw.attribution_history)) {
|
|
329
334
|
throw new LaneSchemaError("attribution_history must be an array", {
|
|
330
335
|
path: filePath,
|
|
@@ -466,6 +471,11 @@ export function mergeLaneRecords(live, sidecars) {
|
|
|
466
471
|
...base,
|
|
467
472
|
senior_history: unionHistory(all.map((record) => record.senior_history), entryIdentity, base.senior_history),
|
|
468
473
|
attribution_history: unionHistory(all.map((record) => record.attribution_history), attributionIdentity, base.attribution_history),
|
|
474
|
+
...(all.some((record) => record.account_history !== undefined)
|
|
475
|
+
? {
|
|
476
|
+
account_history: unionHistory(all.map((record) => record.account_history ?? []), entryIdentity, base.account_history ?? []),
|
|
477
|
+
}
|
|
478
|
+
: {}),
|
|
469
479
|
};
|
|
470
480
|
}
|
|
471
481
|
export function mergeQuestionRecords(live, sidecars) {
|