@indigoai-us/hq-cli 5.341.5 → 5.342.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/dist/commands/cloud.js +1 -0
- package/dist/commands/integrations-core.d.ts +1 -1
- package/dist/commands/integrations-core.js +97 -7
- package/dist/commands/integrations.js +2 -1
- package/dist/commands/lanes.js +24 -7
- package/dist/lib/lanes/envelope-sequence.d.ts +3 -0
- package/dist/lib/lanes/envelope-sequence.js +13 -0
- package/dist/lib/lanes/keep-alive-audit.d.ts +9 -0
- package/dist/lib/lanes/keep-alive-audit.js +8 -3
- package/dist/lib/lanes/reconcile.d.ts +1 -0
- package/dist/lib/lanes/reconcile.js +77 -9
- package/dist/lib/lanes/spawn.js +9 -4
- package/dist/lib/lanes/store.js +10 -0
- package/dist/lib/lanes/types.d.ts +5 -1
- package/dist/lib/lanes/types.js +1 -0
- package/dist/lib/lanes/watcher.js +3 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.342.1] — 2026-10-04
|
|
6
|
+
- `hq lanes` now continues envelope numbering beyond 999 and records unexpected reconcile admission failures as a distinct queued reason with error details.
|
|
7
|
+
- Reconcile skips configured Codex homes absent on the current host and reports a note; other filesystem errors remain visible as store errors. `--dry-run` no longer compresses rollouts or advances the retention schedule.
|
|
8
|
+
|
|
9
|
+
## [5.342.0] — 2026-10-04
|
|
10
|
+
|
|
11
|
+
- Keep `hq integrations call` alive through slow provider calls with bounded async-job polling, safe idempotent retry guidance on timeout, and validation of terminal responses.
|
|
12
|
+
|
|
13
|
+
- Let `hq sync now` skip unchanged server-owned cloud placeholders while keeping their warnings visible, so the sync can complete.
|
|
14
|
+
|
|
5
15
|
## [5.341.5] — 2026-10-04
|
|
6
16
|
- Reconcile can close an eligible done-story worker with a legacy missing `interrupted_at` when a matching watcher classification proves the interruption age.
|
|
7
17
|
- `hq doctor --verbose` now shows passing checks, including hqd runtime health details.
|
package/dist/commands/cloud.js
CHANGED
|
@@ -1751,6 +1751,7 @@ async function runNowSingle(hqRoot, company, personal, message, onConflict, mode
|
|
|
1751
1751
|
hqRoot,
|
|
1752
1752
|
progressRecorder: trackingProgressRecorder(planLimitTracker, pushProgressCompany),
|
|
1753
1753
|
skipUnchanged: true,
|
|
1754
|
+
serverOwnedPathPolicy: "skip",
|
|
1754
1755
|
propagateDeletes: true,
|
|
1755
1756
|
propagateDeletePolicy: resolveDeletePolicy(),
|
|
1756
1757
|
...(onConflict ? { onConflict } : {}),
|
|
@@ -248,7 +248,7 @@ export declare function resolveConnection(token: string, companyUid: string, app
|
|
|
248
248
|
}, recoveryOpts?: {
|
|
249
249
|
allowSingleRevoked?: boolean;
|
|
250
250
|
}): Promise<AdminConnection>;
|
|
251
|
-
export declare function callGateway(token: string, params: Record<string, unknown
|
|
251
|
+
export declare function callGateway(token: string, params: Record<string, unknown>, idempotencyKey?: string): Promise<GatewayMessage>;
|
|
252
252
|
/**
|
|
253
253
|
* Gateway results arrive MCP-style: `{ content: [{ type: "text", text }] }`
|
|
254
254
|
* where `text` is the provider's JSON. Unwrap to the inner payload; fall back
|
|
@@ -18,6 +18,8 @@ import { redactSensitiveUrls, redactUrlsInText } from "../utils/redact-url.js";
|
|
|
18
18
|
import { connectionHealthNotice } from "../lib/integrations/health.js";
|
|
19
19
|
import { bareProvider } from "../lib/integrations/provider-slug.js";
|
|
20
20
|
export { bareProvider } from "../lib/integrations/provider-slug.js";
|
|
21
|
+
const ASYNC_GATEWAY_POLL_INTERVAL_MS = 1_000;
|
|
22
|
+
const ASYNC_GATEWAY_DEADLINE_MS = 4 * 60_000 + 30_000;
|
|
21
23
|
export class IntegrationsCliError extends Error {
|
|
22
24
|
/**
|
|
23
25
|
* True when the error is the caller's request/state/permission (a client 4xx
|
|
@@ -397,7 +399,8 @@ export async function resolveConnection(token, companyUid, app, opts, recoveryOp
|
|
|
397
399
|
}
|
|
398
400
|
return selectConnection(connections, opts);
|
|
399
401
|
}
|
|
400
|
-
export async function callGateway(token, params) {
|
|
402
|
+
export async function callGateway(token, params, idempotencyKey) {
|
|
403
|
+
const deadline = Date.now() + ASYNC_GATEWAY_DEADLINE_MS;
|
|
401
404
|
const res = await vaultApiFetch({
|
|
402
405
|
token,
|
|
403
406
|
path: "/v1/integrations/mcp",
|
|
@@ -410,7 +413,23 @@ export async function callGateway(token, params) {
|
|
|
410
413
|
},
|
|
411
414
|
});
|
|
412
415
|
const message = (await res.json().catch(() => null));
|
|
413
|
-
|
|
416
|
+
let terminalMessage = message;
|
|
417
|
+
if (res.status === 202) {
|
|
418
|
+
const accepted = message;
|
|
419
|
+
const jobId = accepted && typeof accepted.jobId === "string" ? accepted.jobId : "";
|
|
420
|
+
if (accepted?.status !== "pending" || !/^imj_[A-Za-z0-9_-]+$/.test(jobId)) {
|
|
421
|
+
throw new IntegrationsCliError(`Integration gateway returned an invalid async job response (job id: ${jobId || "missing"}).`, { expected: false, status: 202 });
|
|
422
|
+
}
|
|
423
|
+
const companyUid = typeof params.companyUid === "string" ? params.companyUid : "";
|
|
424
|
+
if (!companyUid) {
|
|
425
|
+
throw new IntegrationsCliError(`Cannot poll integration job ${jobId}: the company id is missing.`, { expected: true });
|
|
426
|
+
}
|
|
427
|
+
if (!idempotencyKey) {
|
|
428
|
+
throw new IntegrationsCliError(`Integration gateway accepted async job ${jobId} without an idempotency key.`, { expected: false, status: 202 });
|
|
429
|
+
}
|
|
430
|
+
terminalMessage = await pollAsyncGatewayJob(token, companyUid, jobId, idempotencyKey, deadline);
|
|
431
|
+
}
|
|
432
|
+
if (res.status !== 202 && (!res.ok || !terminalMessage)) {
|
|
414
433
|
raiseIfUnauthorized(res);
|
|
415
434
|
raiseIfUpstreamUnavailable(res);
|
|
416
435
|
// Carry the HTTP status so a reportable transport failure (e.g. a 501 that
|
|
@@ -420,7 +439,7 @@ export async function callGateway(token, params) {
|
|
|
420
439
|
// `status`. This does not change the `expected` decision above.
|
|
421
440
|
throw new IntegrationsCliError(`Integration gateway request failed (HTTP ${res.status}).`, { expected: isClientError(res.status), status: res.status });
|
|
422
441
|
}
|
|
423
|
-
if (
|
|
442
|
+
if (terminalMessage?.error) {
|
|
424
443
|
// The gateway's error text is minted UPSTREAM (hq-pro, the integration
|
|
425
444
|
// factory, and beyond it the third-party provider), so it is untrusted:
|
|
426
445
|
// scrub credentials and bound the length here, at the throw site, because
|
|
@@ -429,16 +448,87 @@ export async function callGateway(token, params) {
|
|
|
429
448
|
// The chain is idempotent, so the unexpected path scrubbing again is a
|
|
430
449
|
// no-op. This preserves PR #298's user-visible diagnostic; it only makes
|
|
431
450
|
// it safe on the newly-expected path.
|
|
432
|
-
throw new IntegrationsCliError(redactErrorText(
|
|
451
|
+
throw new IntegrationsCliError(redactErrorText(terminalMessage.error.message ?? "") ||
|
|
433
452
|
"Integration gateway returned an error.", {
|
|
434
|
-
expected: isExpectedGatewayError(
|
|
453
|
+
expected: isExpectedGatewayError(terminalMessage.error.code),
|
|
435
454
|
// Carry the protocol code to the capture site as a bounded grouping
|
|
436
455
|
// discriminator (HQ-CLI collision, Sentry 7642756130). This does not
|
|
437
456
|
// alter the `expected` decision above; it only labels the error.
|
|
438
|
-
...(
|
|
457
|
+
...(terminalMessage.error.code !== undefined ? { rpcCode: terminalMessage.error.code } : {}),
|
|
439
458
|
});
|
|
440
459
|
}
|
|
441
|
-
return
|
|
460
|
+
return terminalMessage;
|
|
461
|
+
}
|
|
462
|
+
async function pollAsyncGatewayJob(token, companyUid, jobId, idempotencyKey, deadline) {
|
|
463
|
+
const jobPath = `/v1/integrations/mcp/jobs/${encodeURIComponent(jobId)}`;
|
|
464
|
+
while (Date.now() < deadline) {
|
|
465
|
+
const requestRemainingMs = deadline - Date.now();
|
|
466
|
+
if (requestRemainingMs <= 0)
|
|
467
|
+
break;
|
|
468
|
+
const controller = new AbortController();
|
|
469
|
+
const requestTimeout = setTimeout(() => controller.abort(), requestRemainingMs);
|
|
470
|
+
let res;
|
|
471
|
+
let poll;
|
|
472
|
+
try {
|
|
473
|
+
res = await vaultApiFetch({
|
|
474
|
+
token,
|
|
475
|
+
path: jobPath,
|
|
476
|
+
method: "GET",
|
|
477
|
+
query: { companyUid },
|
|
478
|
+
signal: controller.signal,
|
|
479
|
+
});
|
|
480
|
+
poll = (await res.json().catch(() => null));
|
|
481
|
+
}
|
|
482
|
+
catch (error) {
|
|
483
|
+
if (!controller.signal.aborted)
|
|
484
|
+
throw error;
|
|
485
|
+
break;
|
|
486
|
+
}
|
|
487
|
+
finally {
|
|
488
|
+
clearTimeout(requestTimeout);
|
|
489
|
+
}
|
|
490
|
+
if (!res.ok) {
|
|
491
|
+
raiseIfUnauthorized(res);
|
|
492
|
+
raiseIfUpstreamUnavailable(res);
|
|
493
|
+
if (res.status === 404) {
|
|
494
|
+
throw new IntegrationsCliError(`Integration job ${jobId} was not found or is no longer available.`, { expected: true, status: 404 });
|
|
495
|
+
}
|
|
496
|
+
throw new IntegrationsCliError(`Integration job ${jobId} polling failed (HTTP ${res.status}).`, { expected: isClientError(res.status), status: res.status });
|
|
497
|
+
}
|
|
498
|
+
if (!poll || typeof poll !== "object") {
|
|
499
|
+
throw new IntegrationsCliError(`Integration job ${jobId} returned an invalid poll response.`, { expected: true });
|
|
500
|
+
}
|
|
501
|
+
if (poll.status === "result" || poll.status === "error") {
|
|
502
|
+
const response = poll.response;
|
|
503
|
+
const isPlainObject = response !== null &&
|
|
504
|
+
typeof response === "object" &&
|
|
505
|
+
!Array.isArray(response) &&
|
|
506
|
+
(Object.getPrototypeOf(response) === Object.prototype || Object.getPrototypeOf(response) === null);
|
|
507
|
+
const responseRecord = isPlainObject ? response : undefined;
|
|
508
|
+
const hasResult = responseRecord ? Object.hasOwn(responseRecord, "result") : false;
|
|
509
|
+
const hasError = responseRecord ? Object.hasOwn(responseRecord, "error") : false;
|
|
510
|
+
const error = responseRecord?.error;
|
|
511
|
+
const validError = error !== null &&
|
|
512
|
+
typeof error === "object" &&
|
|
513
|
+
!Array.isArray(error) &&
|
|
514
|
+
(Object.getPrototypeOf(error) === Object.prototype || Object.getPrototypeOf(error) === null);
|
|
515
|
+
const hasJsonRpcOutcome = responseRecord?.jsonrpc === "2.0" &&
|
|
516
|
+
hasResult !== hasError &&
|
|
517
|
+
(!hasError || validError);
|
|
518
|
+
if (!hasJsonRpcOutcome) {
|
|
519
|
+
throw new IntegrationsCliError(`Integration job ${jobId} returned an invalid JSON-RPC response.`, { expected: false });
|
|
520
|
+
}
|
|
521
|
+
return response;
|
|
522
|
+
}
|
|
523
|
+
if (poll.status !== "pending") {
|
|
524
|
+
throw new IntegrationsCliError(`Integration job ${jobId} returned unknown status '${String(poll.status)}'.`, { expected: true });
|
|
525
|
+
}
|
|
526
|
+
const remainingMs = deadline - Date.now();
|
|
527
|
+
if (remainingMs <= 0)
|
|
528
|
+
break;
|
|
529
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(ASYNC_GATEWAY_POLL_INTERVAL_MS, remainingMs)));
|
|
530
|
+
}
|
|
531
|
+
throw new IntegrationsCliError(`Timed out waiting for integration job ${jobId}; the server job may still finish. To safely retry, rerun with --idempotency-key ${idempotencyKey}.`, { expected: true });
|
|
442
532
|
}
|
|
443
533
|
/**
|
|
444
534
|
* Gateway results arrive MCP-style: `{ content: [{ type: "text", text }] }`
|
|
@@ -338,6 +338,7 @@ export function registerIntegrationsCommand(program) {
|
|
|
338
338
|
const idempotencyKey = opts.idempotencyKey ?? `hq-cli-${randomUUID()}`;
|
|
339
339
|
const message = await callGateway(token, {
|
|
340
340
|
companyUid,
|
|
341
|
+
acceptAsync: true,
|
|
341
342
|
name: `${prefix}.mcp.tools.call`,
|
|
342
343
|
arguments: {
|
|
343
344
|
companyUid,
|
|
@@ -346,7 +347,7 @@ export function registerIntegrationsCommand(program) {
|
|
|
346
347
|
arguments: parsedArgs,
|
|
347
348
|
idempotencyKey,
|
|
348
349
|
},
|
|
349
|
-
});
|
|
350
|
+
}, idempotencyKey);
|
|
350
351
|
const payload = unwrapGatewayResult(message.result);
|
|
351
352
|
const queued = queuedOutcome(payload);
|
|
352
353
|
if (queued) {
|
package/dist/commands/lanes.js
CHANGED
|
@@ -5089,8 +5089,12 @@ async function runList(ctx, opts) {
|
|
|
5089
5089
|
? {
|
|
5090
5090
|
state: "remote",
|
|
5091
5091
|
remote_state: lane.state,
|
|
5092
|
-
queued_reason: lane.queued_reason === "low_disk"
|
|
5093
|
-
|
|
5092
|
+
queued_reason: lane.queued_reason === "low_disk" || lane.queued_reason === "admit_failed"
|
|
5093
|
+
? lane.queued_reason
|
|
5094
|
+
: undefined,
|
|
5095
|
+
queued_details: lane.queued_reason === "low_disk" || lane.queued_reason === "admit_failed"
|
|
5096
|
+
? lane.queued_details
|
|
5097
|
+
: undefined,
|
|
5094
5098
|
}
|
|
5095
5099
|
: {}),
|
|
5096
5100
|
overdue,
|
|
@@ -5142,7 +5146,9 @@ function formatLaneListHuman(lanes) {
|
|
|
5142
5146
|
: "no story: " + (lane.no_story_reason ?? "reason not recorded");
|
|
5143
5147
|
const queueLabel = (lane.state === "queued" || lane.remote_state === "queued") && lane.queued_reason === "low_disk"
|
|
5144
5148
|
? ` queued: low_disk (HQ ${lane.queued_details?.reason === "low_disk" ? lane.queued_details.hq_free_bytes : "?"} bytes; provider ${lane.queued_details?.reason === "low_disk" ? lane.queued_details.provider_free_bytes : "?"} bytes free)`
|
|
5145
|
-
: ""
|
|
5149
|
+
: (lane.state === "queued" || lane.remote_state === "queued") && lane.queued_reason === "admit_failed"
|
|
5150
|
+
? ` queued: admit_failed (${lane.queued_details?.reason === "admit_failed" ? `${lane.queued_details.error}: ${lane.queued_details.message}` : "details unavailable"})`
|
|
5151
|
+
: "";
|
|
5146
5152
|
return lane.lane_id + " " + lane.state + " " + lane.provider + " " + lane.project_id +
|
|
5147
5153
|
" " + storyLabel + queueLabel;
|
|
5148
5154
|
});
|
|
@@ -5199,9 +5205,11 @@ async function runStatus(ctx, opts) {
|
|
|
5199
5205
|
async function runCapacity(ctx) {
|
|
5200
5206
|
const hqRoot = ctx.hqRoot();
|
|
5201
5207
|
const host = laneHostFromCtx(ctx);
|
|
5208
|
+
const admissionHost = reconcileHostFromCtx(ctx);
|
|
5209
|
+
const lanes = loadAllLanes(hqRoot);
|
|
5202
5210
|
const snapshot = buildHostCapacitySnapshot({
|
|
5203
5211
|
host,
|
|
5204
|
-
lanes
|
|
5212
|
+
lanes,
|
|
5205
5213
|
config: loadLanesConfig(hqRoot),
|
|
5206
5214
|
existsSync: ctx.existsSync,
|
|
5207
5215
|
probe: ctx.probeProvider,
|
|
@@ -5209,7 +5217,16 @@ async function runCapacity(ctx) {
|
|
|
5209
5217
|
hostname: ctx.hostname(),
|
|
5210
5218
|
now: ctx.now(),
|
|
5211
5219
|
});
|
|
5212
|
-
printJson(
|
|
5220
|
+
printJson({
|
|
5221
|
+
...snapshot,
|
|
5222
|
+
queued_lanes: lanes
|
|
5223
|
+
.filter((lane) => lane.state === "queued" && laneRunsHere(lane, admissionHost))
|
|
5224
|
+
.map((lane) => ({
|
|
5225
|
+
lane_id: lane.lane_id,
|
|
5226
|
+
queued_reason: lane.queued_reason ?? null,
|
|
5227
|
+
...(lane.queued_details !== undefined ? { queued_details: lane.queued_details } : {}),
|
|
5228
|
+
})),
|
|
5229
|
+
});
|
|
5213
5230
|
}
|
|
5214
5231
|
async function runGcRollouts(ctx, opts) {
|
|
5215
5232
|
const hqRoot = ctx.hqRoot();
|
|
@@ -5395,8 +5412,8 @@ async function runShow(ctx, laneId, hasPendingQuestionForLane, resolveLaneTarget
|
|
|
5395
5412
|
...storedLane,
|
|
5396
5413
|
state: "remote",
|
|
5397
5414
|
remote_state: storedLane.state,
|
|
5398
|
-
queued_reason: storedLane.queued_reason
|
|
5399
|
-
queued_details: storedLane.
|
|
5415
|
+
queued_reason: storedLane.queued_reason,
|
|
5416
|
+
queued_details: storedLane.queued_details,
|
|
5400
5417
|
}
|
|
5401
5418
|
: storedLane;
|
|
5402
5419
|
const askDeliveryFailed = readJsonFile(askDeliveryFailedPath(hqRoot, laneId));
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Numeric ordering for numbered lane envelope filenames, including 1000+. */
|
|
2
|
+
export function compareEnvelopeNames(left, right) {
|
|
3
|
+
const leftMatch = /^(\d+)\.json$/.exec(left);
|
|
4
|
+
const rightMatch = /^(\d+)\.json$/.exec(right);
|
|
5
|
+
if (leftMatch && rightMatch) {
|
|
6
|
+
const leftNumber = BigInt(leftMatch[1]);
|
|
7
|
+
const rightNumber = BigInt(rightMatch[1]);
|
|
8
|
+
if (leftNumber !== rightNumber)
|
|
9
|
+
return leftNumber < rightNumber ? -1 : 1;
|
|
10
|
+
}
|
|
11
|
+
return left.localeCompare(right);
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=envelope-sequence.js.map
|
|
@@ -38,6 +38,15 @@ export interface KeepAliveAuditResult {
|
|
|
38
38
|
errors: KeepAliveAuditError[];
|
|
39
39
|
summary: KeepAliveAuditSummary;
|
|
40
40
|
}
|
|
41
|
+
export interface StopEnvelope {
|
|
42
|
+
name: string;
|
|
43
|
+
mtimeMs: number;
|
|
44
|
+
nextCheckAt?: string;
|
|
45
|
+
}
|
|
46
|
+
export declare function readStopEnvelopes(hqRoot: string, laneId: string): {
|
|
47
|
+
allMtimes: number[];
|
|
48
|
+
stops: StopEnvelope[];
|
|
49
|
+
};
|
|
41
50
|
export declare function buildKeepAliveAudit(input: {
|
|
42
51
|
hqRoot: string;
|
|
43
52
|
now: Date;
|
|
@@ -3,6 +3,7 @@ import * as path from "node:path";
|
|
|
3
3
|
import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS } from "./keep-alive.js";
|
|
4
4
|
import { LanesError } from "./errors.js";
|
|
5
5
|
import { laneRunDir } from "./spawn.js";
|
|
6
|
+
import { compareEnvelopeNames } from "./envelope-sequence.js";
|
|
6
7
|
import { listQuestionIds, readAllLanes, tryReadQuestion } from "./store.js";
|
|
7
8
|
function readDoneClassifications(hqRoot, laneId) {
|
|
8
9
|
const root = fs.realpathSync(hqRoot);
|
|
@@ -77,7 +78,7 @@ function withinRoot(root, candidate) {
|
|
|
77
78
|
const relative = path.relative(root, candidate);
|
|
78
79
|
return relative !== "" && relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
|
|
79
80
|
}
|
|
80
|
-
function readStopEnvelopes(hqRoot, laneId) {
|
|
81
|
+
export function readStopEnvelopes(hqRoot, laneId) {
|
|
81
82
|
const root = fs.realpathSync(hqRoot);
|
|
82
83
|
const envelopeDir = path.join(laneRunDir(root, laneId), "envelopes");
|
|
83
84
|
if (!withinRoot(root, envelopeDir)) {
|
|
@@ -89,7 +90,7 @@ function readStopEnvelopes(hqRoot, laneId) {
|
|
|
89
90
|
if (!withinRoot(root, realDir)) {
|
|
90
91
|
throw new LanesError("keep_alive_audit_path_invalid", `Lane envelope directory escapes the HQ root for ${laneId}`, { lane_id: laneId });
|
|
91
92
|
}
|
|
92
|
-
names = fs.readdirSync(realDir).filter((name) => /^\d+\.json$/.test(name)).sort();
|
|
93
|
+
names = fs.readdirSync(realDir).filter((name) => /^\d+\.json$/.test(name)).sort(compareEnvelopeNames);
|
|
93
94
|
}
|
|
94
95
|
catch (err) {
|
|
95
96
|
if (err.code === "ENOENT")
|
|
@@ -118,11 +119,15 @@ function readStopEnvelopes(hqRoot, laneId) {
|
|
|
118
119
|
if (raw.decision !== "done" || raw.story_complete !== false)
|
|
119
120
|
continue;
|
|
120
121
|
stops.push({
|
|
122
|
+
name,
|
|
121
123
|
mtimeMs: stat.mtimeMs,
|
|
122
124
|
...(typeof raw.next_check_at === "string" ? { nextCheckAt: raw.next_check_at } : {}),
|
|
123
125
|
});
|
|
124
126
|
}
|
|
125
|
-
return {
|
|
127
|
+
return {
|
|
128
|
+
allMtimes: allMtimes.sort((a, b) => a - b),
|
|
129
|
+
stops: stops.sort((a, b) => a.mtimeMs - b.mtimeMs),
|
|
130
|
+
};
|
|
126
131
|
}
|
|
127
132
|
function readKeepAliveResumeEvents(hqRoot, laneId) {
|
|
128
133
|
const root = fs.realpathSync(hqRoot);
|
|
@@ -18,7 +18,7 @@ import { AccountResolutionError, LanesError, isLaneVersionConflict } from "./err
|
|
|
18
18
|
import { loadWorkerProfile } from "./profile.js";
|
|
19
19
|
import { terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
|
|
20
20
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, unindexedClaudeSessionLiveness, } from "./liveness.js";
|
|
21
|
-
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, stopLaneProcesses, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
21
|
+
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, nextEnvelopePath, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, stopLaneProcesses, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
22
22
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
|
|
23
23
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
24
24
|
import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
|
|
@@ -87,6 +87,27 @@ function markDirty(summary, skip) {
|
|
|
87
87
|
summary.ok = false;
|
|
88
88
|
logSkip(skip);
|
|
89
89
|
}
|
|
90
|
+
function isMissingPathError(error) {
|
|
91
|
+
const code = error?.code;
|
|
92
|
+
return code === "ENOENT" || code === "ENOTDIR";
|
|
93
|
+
}
|
|
94
|
+
function recordMissingCodexHomeNote(summary, home) {
|
|
95
|
+
const note = `Codex rollout retention skipped home absent on this host: ${home}`;
|
|
96
|
+
summary.notes ??= [];
|
|
97
|
+
summary.notes.push(note);
|
|
98
|
+
process.stderr.write(`hq lanes reconcile: note: ${note}\n`);
|
|
99
|
+
}
|
|
100
|
+
function homeIsMissingAfterError(home, error) {
|
|
101
|
+
if (!isMissingPathError(error))
|
|
102
|
+
return false;
|
|
103
|
+
try {
|
|
104
|
+
fs.statSync(home);
|
|
105
|
+
return false;
|
|
106
|
+
}
|
|
107
|
+
catch (statError) {
|
|
108
|
+
return isMissingPathError(statError);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
90
111
|
function terminalizeModelUnsupported(hqRoot, lane, err) {
|
|
91
112
|
if (!(err instanceof LanesError) || err.error !== "model_unsupported")
|
|
92
113
|
return;
|
|
@@ -940,14 +961,28 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
940
961
|
companyRefs.set(company.uid ?? company.slug, company);
|
|
941
962
|
}
|
|
942
963
|
const readFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
943
|
-
const
|
|
964
|
+
const configuredHomes = configuredCodexRolloutHomes(hqRoot, config, deps.host.hostname);
|
|
944
965
|
let retentionEnabled = false;
|
|
945
|
-
for (const company of
|
|
966
|
+
for (const company of configuredHomes.length > 0 ? companyRefs.values() : []) {
|
|
946
967
|
retentionEnabled ||= await readFlag("lanes.codex-rollout-retention", { company });
|
|
947
968
|
if (retentionEnabled)
|
|
948
969
|
break;
|
|
949
970
|
}
|
|
950
971
|
if (retentionEnabled) {
|
|
972
|
+
const presentHomes = [];
|
|
973
|
+
for (const home of configuredHomes) {
|
|
974
|
+
try {
|
|
975
|
+
fs.statSync(home);
|
|
976
|
+
presentHomes.push(home);
|
|
977
|
+
}
|
|
978
|
+
catch (err) {
|
|
979
|
+
if (isMissingPathError(err)) {
|
|
980
|
+
recordMissingCodexHomeNote(summary, home);
|
|
981
|
+
continue;
|
|
982
|
+
}
|
|
983
|
+
throw err;
|
|
984
|
+
}
|
|
985
|
+
}
|
|
951
986
|
const statePath = path.join(hqRoot, "workspace", "lanes-runs", `.rollout-retention-${deps.host.hostname.replaceAll(/[^a-zA-Z0-9_-]/g, "_")}.json`);
|
|
952
987
|
let lastRunAt;
|
|
953
988
|
try {
|
|
@@ -961,20 +996,38 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
961
996
|
if (err.code !== "ENOENT")
|
|
962
997
|
throw err;
|
|
963
998
|
}
|
|
964
|
-
|
|
999
|
+
let lowDisk = false;
|
|
1000
|
+
const retentionHomes = [];
|
|
1001
|
+
for (const home of presentHomes) {
|
|
1002
|
+
try {
|
|
1003
|
+
const disk = checkLaneDiskSpace(hqRoot, home, deps.spawnLaneDeps?.statfs, config.low_disk_min_free_bytes);
|
|
1004
|
+
lowDisk = lowDisk || disk.low;
|
|
1005
|
+
retentionHomes.push(home);
|
|
1006
|
+
}
|
|
1007
|
+
catch (err) {
|
|
1008
|
+
if (homeIsMissingAfterError(home, err)) {
|
|
1009
|
+
recordMissingCodexHomeNote(summary, home);
|
|
1010
|
+
continue;
|
|
1011
|
+
}
|
|
1012
|
+
throw err;
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
965
1015
|
if (lowDisk || lastRunAt === undefined || nowMs - lastRunAt >= 60 * 60 * 1000) {
|
|
966
1016
|
const result = await retainCodexRollouts({
|
|
967
|
-
homes,
|
|
1017
|
+
homes: retentionHomes,
|
|
968
1018
|
lanes,
|
|
969
1019
|
nowMs,
|
|
1020
|
+
dryRun: deps.dryRun,
|
|
970
1021
|
compressAfterMs: config.codex_rollout_compress_after_hours * 60 * 60 * 1000,
|
|
971
1022
|
deleteAfterMs: config.codex_rollout_delete_after_days * 24 * 60 * 60 * 1000,
|
|
972
1023
|
});
|
|
973
1024
|
summary.rollout_retention = result;
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
1025
|
+
if (!deps.dryRun) {
|
|
1026
|
+
fs.mkdirSync(path.dirname(statePath), { recursive: true });
|
|
1027
|
+
const tempPath = `${statePath}.${process.pid}.tmp`;
|
|
1028
|
+
fs.writeFileSync(tempPath, `${JSON.stringify({ last_run_at: nowIso }, null, 2)}\n`, { flag: "wx" });
|
|
1029
|
+
fs.renameSync(tempPath, statePath);
|
|
1030
|
+
}
|
|
978
1031
|
}
|
|
979
1032
|
}
|
|
980
1033
|
}
|
|
@@ -1848,6 +1901,9 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
1848
1901
|
}
|
|
1849
1902
|
try {
|
|
1850
1903
|
if (deps.dryRun) {
|
|
1904
|
+
// Spawn reserves an envelope path. Probe it without writing so a dry
|
|
1905
|
+
// run does not promise admission when the run cannot start.
|
|
1906
|
+
nextEnvelopePath(laneRunDir(hqRoot, lane.lane_id));
|
|
1851
1907
|
summary.admitted.push({
|
|
1852
1908
|
id: lane.lane_id,
|
|
1853
1909
|
from: "queued",
|
|
@@ -1907,6 +1963,18 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
1907
1963
|
}
|
|
1908
1964
|
}
|
|
1909
1965
|
terminalizeModelUnsupported(hqRoot, lane, err);
|
|
1966
|
+
if (err instanceof LanesError && err.error !== "model_unsupported") {
|
|
1967
|
+
const queueRecorded = deps.dryRun || casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued", (current) => ({
|
|
1968
|
+
...current,
|
|
1969
|
+
queued_reason: "admit_failed",
|
|
1970
|
+
queued_details: { reason: "admit_failed", error: err.error, message: err.message },
|
|
1971
|
+
admission_reserved_at: undefined,
|
|
1972
|
+
}), false).changed;
|
|
1973
|
+
if (queueRecorded) {
|
|
1974
|
+
summary.skipped.push({ kind: "lane", id: lane.lane_id, why: `admit_failed:${err.error}` });
|
|
1975
|
+
continue;
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1910
1978
|
markDirty(summary, {
|
|
1911
1979
|
kind: "lane",
|
|
1912
1980
|
id: lane.lane_id,
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -983,16 +983,21 @@ function proveDetachedViaPs(pid) {
|
|
|
983
983
|
}
|
|
984
984
|
export function nextEnvelopePath(runDir) {
|
|
985
985
|
const dir = path.join(runDir, "envelopes");
|
|
986
|
+
let names = new Set();
|
|
987
|
+
try {
|
|
988
|
+
names = new Set(fs.readdirSync(dir));
|
|
989
|
+
}
|
|
990
|
+
catch (err) {
|
|
991
|
+
if (err.code !== "ENOENT")
|
|
992
|
+
throw err;
|
|
993
|
+
}
|
|
986
994
|
let n = 1;
|
|
987
995
|
while (true) {
|
|
988
996
|
const name = `${String(n).padStart(3, "0")}.json`;
|
|
989
997
|
const filePath = path.join(dir, name);
|
|
990
|
-
if (!
|
|
998
|
+
if (!names.has(name))
|
|
991
999
|
return filePath;
|
|
992
1000
|
n += 1;
|
|
993
|
-
if (n > 999) {
|
|
994
|
-
throw new LanesError("envelope_sequence_exhausted", `No free envelope slot under ${dir}`, { dir });
|
|
995
|
-
}
|
|
996
1001
|
}
|
|
997
1002
|
}
|
|
998
1003
|
export function envelopeInstruction(envelopePath) {
|
package/dist/lib/lanes/store.js
CHANGED
|
@@ -623,6 +623,16 @@ export function parseLaneRecord(raw, filePath) {
|
|
|
623
623
|
}
|
|
624
624
|
requireTimestamp(detailsRecord.measured_at, "queued_details.measured_at", filePath);
|
|
625
625
|
}
|
|
626
|
+
else if (detailsRecord.reason === "admit_failed") {
|
|
627
|
+
if (raw.state !== "queued" || raw.queued_reason !== "admit_failed" ||
|
|
628
|
+
typeof detailsRecord.error !== "string" || detailsRecord.error.trim() === "" ||
|
|
629
|
+
typeof detailsRecord.message !== "string" || detailsRecord.message.trim() === "") {
|
|
630
|
+
throw new LaneSchemaError("queued_details must describe a failed lane admission", {
|
|
631
|
+
path: filePath,
|
|
632
|
+
field: "queued_details",
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
}
|
|
626
636
|
else {
|
|
627
637
|
throw new LaneSchemaError("queued_details must describe a supported queue condition", {
|
|
628
638
|
path: filePath,
|
|
@@ -177,7 +177,7 @@ export declare const TERMINAL_LANE_STATES: readonly ["done", "failed", "interrup
|
|
|
177
177
|
export type TerminalLaneState = (typeof TERMINAL_LANE_STATES)[number];
|
|
178
178
|
export declare const LANE_PROVIDERS: readonly ["claude", "codex", "grok"];
|
|
179
179
|
export type LaneProvider = (typeof LANE_PROVIDERS)[number];
|
|
180
|
-
export declare const QUEUED_REASONS: readonly ["cap", "provider_cap", "account_cap", "profile_cap", "queue_position", "throttle", "provider_binary_missing", "account_retry", "worker_profile_changed", "low_disk"];
|
|
180
|
+
export declare const QUEUED_REASONS: readonly ["cap", "provider_cap", "account_cap", "profile_cap", "queue_position", "throttle", "provider_binary_missing", "account_retry", "worker_profile_changed", "low_disk", "admit_failed"];
|
|
181
181
|
export type QueuedReason = (typeof QUEUED_REASONS)[number];
|
|
182
182
|
export type QueuedDetails = {
|
|
183
183
|
reason: "worker_profile_changed";
|
|
@@ -193,6 +193,10 @@ export type QueuedDetails = {
|
|
|
193
193
|
hq_free_bytes: number;
|
|
194
194
|
provider_free_bytes: number;
|
|
195
195
|
measured_at: string;
|
|
196
|
+
} | {
|
|
197
|
+
reason: "admit_failed";
|
|
198
|
+
error: string;
|
|
199
|
+
message: string;
|
|
196
200
|
};
|
|
197
201
|
export declare function isQueuedReason(value: unknown): value is QueuedReason;
|
|
198
202
|
export declare const HOST_KINDS: readonly ["local", "outpost"];
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -34,6 +34,7 @@
|
|
|
34
34
|
import * as fs from "node:fs";
|
|
35
35
|
import * as path from "node:path";
|
|
36
36
|
import { LanesError } from "./errors.js";
|
|
37
|
+
import { compareEnvelopeNames } from "./envelope-sequence.js";
|
|
37
38
|
import { questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
38
39
|
import { isTerminalLaneState, ENVELOPE_INVALID_REASON, parseLaneEnvelope, STORY_STATUSES, LANE_PROVIDERS, } from "./types.js";
|
|
39
40
|
import { laneStories, terminalEnvelopeStoryStatus } from "./stories.js";
|
|
@@ -619,8 +620,8 @@ export function readLatestEnvelopeFile(hqRoot, laneId, minName) {
|
|
|
619
620
|
names = fs
|
|
620
621
|
.readdirSync(dir)
|
|
621
622
|
.filter((name) => name.endsWith(".json"))
|
|
622
|
-
.filter((name) => minName === undefined || name >=
|
|
623
|
-
.sort();
|
|
623
|
+
.filter((name) => minName === undefined || compareEnvelopeNames(name, minName) >= 0)
|
|
624
|
+
.sort(compareEnvelopeNames);
|
|
624
625
|
}
|
|
625
626
|
catch (err) {
|
|
626
627
|
const code = err.code;
|