@indigoai-us/hq-cli 5.132.0 → 5.133.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/dist/commands/lanes.d.ts +2 -2
- package/dist/commands/lanes.js +98 -20
- package/dist/commands/mesh.js +4 -1
- package/dist/lib/lanes/constants.d.ts +3 -0
- package/dist/lib/lanes/constants.js +3 -0
- package/dist/lib/lanes/index.d.ts +1 -0
- package/dist/lib/lanes/index.js +1 -0
- package/dist/lib/lanes/quota.d.ts +6 -3
- package/dist/lib/lanes/quota.js +65 -15
- package/dist/lib/lanes/reconcile.d.ts +5 -2
- package/dist/lib/lanes/reconcile.js +94 -2
- package/dist/lib/lanes/types.d.ts +2 -0
- package/dist/lib/lanes/types.js +2 -1
- package/dist/lib/mesh/live/daemon/doctor.d.ts +6 -0
- package/dist/lib/mesh/live/daemon/doctor.js +11 -0
- package/dist/lib/mesh/live/daemon/install.d.ts +7 -0
- package/dist/lib/mesh/live/daemon/install.js +16 -6
- package/dist/lib/mesh/live/daemon/legacy-listen.d.ts +43 -0
- package/dist/lib/mesh/live/daemon/legacy-listen.js +89 -0
- package/dist/lib/mesh/live/daemon/live-cache.d.ts +26 -0
- package/dist/lib/mesh/live/daemon/live-cache.js +116 -0
- package/dist/lib/mesh/live/daemon/presence.d.ts +7 -3
- package/dist/lib/mesh/live/daemon/presence.js +10 -9
- package/dist/lib/mesh/live/daemon/run.d.ts +13 -0
- package/dist/lib/mesh/live/daemon/run.js +51 -2
- package/dist/lib/mesh/live/daemon/work-subscribe.d.ts +31 -0
- package/dist/lib/mesh/live/daemon/work-subscribe.js +124 -0
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,32 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.133.0] — 2026-09-22
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Lane follow-ups now probe all enabled, logged-in accounts for an unpinned
|
|
10
|
+
model, preserve the later quota reset and its source, expire abandoned pickup
|
|
11
|
+
validation markers with a terminal reason, and parse Codex's ordinal
|
|
12
|
+
`try again at` reset time.
|
|
13
|
+
|
|
14
|
+
- `hq sync push` no longer fails every file when the target vault was
|
|
15
|
+
provisioned on or after 2026-09-19. Those vaults carry an S3 Object Lock
|
|
16
|
+
default retention rule, and S3 refuses a PutObject into them unless the
|
|
17
|
+
request carries a content checksum; the CLI's declared `@indigoai-us/hq-cloud`
|
|
18
|
+
range still allowed a client that sends none, so an affected push reported
|
|
19
|
+
`400 InvalidRequest: Content-MD5 OR x-amz-checksum- HTTP header is required
|
|
20
|
+
for Put Object requests with Object Lock parameters` on every file. The
|
|
21
|
+
dependency floor now starts at the release that attaches the checksum, so no
|
|
22
|
+
install can resolve a client that cannot write to a locked vault. A CI guard
|
|
23
|
+
holds the floor.
|
|
24
|
+
|
|
25
|
+
- Lane creation and re-parenting now allow depth 3. A depth-3 caller and any
|
|
26
|
+
derived depth 4 lane are refused with `max_lane_depth`.
|
|
27
|
+
|
|
28
|
+
- Question ageing now derives its two-advance cap from the depth-3 lane cap, so
|
|
29
|
+
an unanswered worker-lane question reaches the user after two hourly hops.
|
|
30
|
+
|
|
5
31
|
## [5.132.0] — 2026-09-22
|
|
6
32
|
|
|
7
33
|
### Added
|
|
@@ -104,6 +130,16 @@
|
|
|
104
130
|
|
|
105
131
|
## [Unreleased]
|
|
106
132
|
|
|
133
|
+
- `hq mesh daemon install` and `hq mesh daemon run` unload and remove the
|
|
134
|
+
legacy `ai.getindigo.hq-work-mesh-listen` unit (LaunchAgent, systemd user
|
|
135
|
+
unit, or Scheduled Task) so only the daemon subscribes. The daemon writes
|
|
136
|
+
`~/.hq/work-mesh/live-cache.json` and bumps its cursor once per project-view
|
|
137
|
+
push. MQTT reconnect logs once per transition into reconnecting instead of
|
|
138
|
+
on every close. `hq mesh daemon doctor` reports "legacy listener still
|
|
139
|
+
installed" while that unit remains (US-006).
|
|
140
|
+
|
|
141
|
+
## [Unreleased]
|
|
142
|
+
|
|
107
143
|
- Held-event repair no longer treats a later event timestamp as proof that
|
|
108
144
|
metadata belongs to the current session instance. A reused session id stays
|
|
109
145
|
unresolved when the held event's binding episode does not match the current
|
package/dist/commands/lanes.d.ts
CHANGED
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
import { Command } from "commander";
|
|
8
8
|
import { type OutpostDispatchResult } from "../lib/lanes/outpost-dispatch.js";
|
|
9
9
|
import { emitLaneDecisionEvents, type LaneMesh } from "../lib/lanes/mesh-events.js";
|
|
10
|
-
import { type LaneCompanyRef, type LaneRecord, type QuestionRecord, type ResolvedLaneWorker, type SeniorRef, type LivenessDeps, type MachineIdentity, type RunCommand, type ThrottleDeps } from "../lib/lanes/index.js";
|
|
10
|
+
import { MAX_LANE_DEPTH, type LaneCompanyRef, type LaneRecord, type QuestionRecord, type ResolvedLaneWorker, type SeniorRef, type LivenessDeps, type MachineIdentity, type RunCommand, type ThrottleDeps } from "../lib/lanes/index.js";
|
|
11
11
|
import { questionBodyAbsPath } from "../lib/lanes/questions.js";
|
|
12
12
|
import { type RunCompactionFn } from "../lib/lanes/compaction.js";
|
|
13
13
|
import { type KnownModelsDeps, type ProbeCommand } from "../lib/lanes/models.js";
|
|
14
14
|
import { spawnLaneWorker, type KillProcessFn, type ProveDetachedFn, type SpawnProcessFn } from "../lib/lanes/spawn.js";
|
|
15
15
|
export { spawnLaneWorker };
|
|
16
16
|
export { questionBodyAbsPath };
|
|
17
|
-
export
|
|
17
|
+
export { MAX_LANE_DEPTH };
|
|
18
18
|
export declare const OUTPOST_ID_PATH = "/etc/outpost/outpost-id";
|
|
19
19
|
export declare const OUTPOST_DIR = "/etc/outpost";
|
|
20
20
|
export declare const LANES_RUNS_REL: string;
|
package/dist/commands/lanes.js
CHANGED
|
@@ -17,7 +17,7 @@ import { vaultApiFetch } from "../utils/vault-api.js";
|
|
|
17
17
|
import { openMeshTransport } from "../lib/mesh/client.js";
|
|
18
18
|
import { createLaneMesh, emitLaneDecisionEvents, } from "../lib/lanes/mesh-events.js";
|
|
19
19
|
import { sessionIdFromEnv } from "../lib/mesh/live/session-identity.js";
|
|
20
|
-
import { AccountResolutionError, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountMaxRunning, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneState, parseLivenessState, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, readQuestion, resolveAccount, resolveLaneWorker, isLoggedOut, runReconcilePass, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasRecordedSessionId, providerSupportsSessionResume, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
|
|
20
|
+
import { AccountResolutionError, DEFAULT_ACCOUNT_HOMES, LANE_PROVIDERS, LANE_STATES, PROVIDER_HOME_ENV, QUESTION_STATUSES, SENIOR_KINDS, accountMaxRunning, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installDrainHook, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, listAccountStatuses, listLaneIds, listQuestionIds, listSessionLiveness, loadLanesConfig, MAX_LANE_DEPTH, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneState, parseLivenessState, parseQuestionStatus, parseSeniorKind, readDaemonPresenceConnected, readLane, retryUpdateLane, readQuestion, resolveAccount, resolveLaneWorker, isLoggedOut, runReconcilePass, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, writeLanesConfig, appendDropboxMessage, answerDropboxMessage, deliveryRefusalReason, resumeCommandRefusal, dropboxDir, dropboxInboxPath, laneAcceptsDropboxDelivery, laneAcceptsOperatorMessage, laneDeliveryMechanism, hasRecordedSessionId, providerSupportsSessionResume, laneIdForAnswerDelivery, operatorMessageRefusalReason, raiseLaneQuestion, sha256Text, writeQuestionBody, } from "../lib/lanes/index.js";
|
|
21
21
|
import { questionBodyAbsPath } from "../lib/lanes/questions.js";
|
|
22
22
|
import { compactionAfterUsage, } from "../lib/lanes/compaction.js";
|
|
23
23
|
import { providerCompactAfterMs } from "../lib/lanes/config.js";
|
|
@@ -29,7 +29,7 @@ import { defaultProbe, validateLaneModel, } from "../lib/lanes/models.js";
|
|
|
29
29
|
import { parseCliModel, refuseOutpostSelfTarget, resolveLaneModel, FOREGROUND_ONLY_RULE, STANDING_AUTHORIZATION, spawnLaneWorker, stopLaneProcesses, } from "../lib/lanes/spawn.js";
|
|
30
30
|
export { spawnLaneWorker };
|
|
31
31
|
export { questionBodyAbsPath };
|
|
32
|
-
export
|
|
32
|
+
export { MAX_LANE_DEPTH };
|
|
33
33
|
export const OUTPOST_ID_PATH = "/etc/outpost/outpost-id";
|
|
34
34
|
export const OUTPOST_DIR = "/etc/outpost";
|
|
35
35
|
export const LANES_RUNS_REL = path.join("workspace", "lanes-runs");
|
|
@@ -1092,6 +1092,7 @@ export function pickupFailureUpdater(reasonRaw, details) {
|
|
|
1092
1092
|
state: "failed",
|
|
1093
1093
|
queued_reason: undefined,
|
|
1094
1094
|
pickup_validation_pending: undefined,
|
|
1095
|
+
pickup_validation_pending_since: undefined,
|
|
1095
1096
|
terminal_reason: reason,
|
|
1096
1097
|
terminal_details: details,
|
|
1097
1098
|
};
|
|
@@ -1166,7 +1167,12 @@ export function applyOutpostAccountPickup(hqRoot, laneId, modelDeps = {}) {
|
|
|
1166
1167
|
if (current.account === resolved.id && current.pickup_validation_pending === undefined) {
|
|
1167
1168
|
return current;
|
|
1168
1169
|
}
|
|
1169
|
-
return {
|
|
1170
|
+
return {
|
|
1171
|
+
...current,
|
|
1172
|
+
account: resolved.id,
|
|
1173
|
+
pickup_validation_pending: undefined,
|
|
1174
|
+
pickup_validation_pending_since: undefined,
|
|
1175
|
+
};
|
|
1170
1176
|
});
|
|
1171
1177
|
}
|
|
1172
1178
|
catch (err) {
|
|
@@ -1315,8 +1321,8 @@ async function runCreate(ctx, opts) {
|
|
|
1315
1321
|
// creates wait on. A pinned create checks the pinned account's home. An
|
|
1316
1322
|
// unpinned create is not bound to an account here (persist re-resolves the
|
|
1317
1323
|
// balanced account under the lock, so overlapping creates still spread),
|
|
1318
|
-
// so it checks the balanced candidate first and then every other enabled
|
|
1319
|
-
// account the locked selection could land on (codex catalogs are per home
|
|
1324
|
+
// so it checks the balanced candidate first and then every other enabled,
|
|
1325
|
+
// logged-in account the locked selection could land on (codex catalogs are per home
|
|
1320
1326
|
// and can differ). `validatedAccounts` records which accepted the model
|
|
1321
1327
|
// or had no readable list, and the balanced pick under the lock is
|
|
1322
1328
|
// restricted to that set (resolveAccount's allowedAccounts): an account
|
|
@@ -1344,26 +1350,68 @@ async function runCreate(ctx, opts) {
|
|
|
1344
1350
|
: [
|
|
1345
1351
|
candidate,
|
|
1346
1352
|
...Object.keys(accounts)
|
|
1347
|
-
.filter((id) => id !== candidate &&
|
|
1353
|
+
.filter((id) => id !== candidate &&
|
|
1354
|
+
accounts[id]?.enabled &&
|
|
1355
|
+
accountStatus(provider, id, config, { existsSync: ctx.existsSync }).logged_in === true)
|
|
1348
1356
|
.sort(),
|
|
1349
1357
|
];
|
|
1358
|
+
const checkedAccounts = [];
|
|
1359
|
+
const unsupported = [];
|
|
1360
|
+
const quotaBlocked = [];
|
|
1361
|
+
const quota = loadQuotaState(hqRoot);
|
|
1350
1362
|
for (const id of toValidate) {
|
|
1351
1363
|
const home = accounts[id]?.home;
|
|
1352
1364
|
if (home === undefined)
|
|
1353
1365
|
continue;
|
|
1366
|
+
checkedAccounts.push(id);
|
|
1354
1367
|
try {
|
|
1355
1368
|
validateLaneModel(provider, cliModel, home, modelDeps);
|
|
1356
|
-
validatedAccounts.add(id);
|
|
1357
1369
|
}
|
|
1358
1370
|
catch (err) {
|
|
1359
1371
|
if (!(isLanesError(err) && err.error === "model_unsupported"))
|
|
1360
1372
|
throw err;
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1373
|
+
unsupported.push({ account: id, error: err });
|
|
1374
|
+
continue;
|
|
1375
|
+
}
|
|
1376
|
+
const block = quotaBlockFor(quota, provider, id, cliModel, now);
|
|
1377
|
+
if (block.blocked) {
|
|
1378
|
+
quotaBlocked.push({ account: id, block });
|
|
1379
|
+
continue;
|
|
1380
|
+
}
|
|
1381
|
+
validatedAccounts.add(id);
|
|
1382
|
+
}
|
|
1383
|
+
if (validatedAccounts.size === 0 && quotaBlocked.length > 0) {
|
|
1384
|
+
const blocked = quotaBlocked[0];
|
|
1385
|
+
const model = blocked.block.model ?? cliModel;
|
|
1386
|
+
const checkedSuffix = ` Checked accounts: ${checkedAccounts.join(", ")}`;
|
|
1387
|
+
if (blocked.block.reason === "account_model_unavailable") {
|
|
1388
|
+
throw new AccountResolutionError("account_model_unavailable", `Account ${JSON.stringify(blocked.account)} cannot run model ${model ?? "(no model)"}: ${blocked.block.marker ?? "needs credits"}${checkedSuffix}`, {
|
|
1389
|
+
provider,
|
|
1390
|
+
account: blocked.account,
|
|
1391
|
+
model: model ?? null,
|
|
1392
|
+
marker: blocked.block.marker ?? null,
|
|
1393
|
+
checked_accounts: checkedAccounts,
|
|
1394
|
+
});
|
|
1366
1395
|
}
|
|
1396
|
+
throw new AccountResolutionError("account_quota_exhausted", `Account ${JSON.stringify(blocked.account)} is out of quota${blocked.block.quota_reset_at ? ` until ${blocked.block.quota_reset_at}` : ""} for model ${model ?? "(no model)"}${checkedSuffix}`, {
|
|
1397
|
+
provider,
|
|
1398
|
+
account: blocked.account,
|
|
1399
|
+
model: model ?? null,
|
|
1400
|
+
quota_reset_at: blocked.block.quota_reset_at ?? null,
|
|
1401
|
+
checked_accounts: checkedAccounts,
|
|
1402
|
+
});
|
|
1403
|
+
}
|
|
1404
|
+
if (validatedAccounts.size === 0 && unsupported.length > 0) {
|
|
1405
|
+
const first = unsupported[0].error;
|
|
1406
|
+
throw new LanesError(first.error, `${first.message} Checked accounts: ${checkedAccounts.join(", ")}`, {
|
|
1407
|
+
...first.details,
|
|
1408
|
+
checked_accounts: checkedAccounts,
|
|
1409
|
+
account_errors: unsupported.map(({ account, error }) => ({
|
|
1410
|
+
account,
|
|
1411
|
+
accepted: error.details.accepted,
|
|
1412
|
+
source: error.details.source,
|
|
1413
|
+
})),
|
|
1414
|
+
});
|
|
1367
1415
|
}
|
|
1368
1416
|
}
|
|
1369
1417
|
const threadId = company
|
|
@@ -1637,7 +1685,7 @@ function parseRemotePickupJson(stdout) {
|
|
|
1637
1685
|
}
|
|
1638
1686
|
return null;
|
|
1639
1687
|
}
|
|
1640
|
-
function ingestPickupPayload(hqRoot, payload) {
|
|
1688
|
+
function ingestPickupPayload(hqRoot, payload, now) {
|
|
1641
1689
|
const laneId = payload.lane.lane_id;
|
|
1642
1690
|
if (!laneId) {
|
|
1643
1691
|
throw new LanesError("outpost_payload_invalid", "Pickup payload lane.lane_id is required", {});
|
|
@@ -1646,9 +1694,20 @@ function ingestPickupPayload(hqRoot, payload) {
|
|
|
1646
1694
|
// The lane is stored ineligible for admission until the account bind
|
|
1647
1695
|
// clears the marker (see LaneRecord.pickup_validation_pending).
|
|
1648
1696
|
const lane = existing === null
|
|
1649
|
-
? createLane(hqRoot, {
|
|
1650
|
-
|
|
1651
|
-
|
|
1697
|
+
? createLane(hqRoot, {
|
|
1698
|
+
...payload.lane,
|
|
1699
|
+
updated_at: now.toISOString(),
|
|
1700
|
+
pickup_validation_pending: true,
|
|
1701
|
+
pickup_validation_pending_since: now.toISOString(),
|
|
1702
|
+
})
|
|
1703
|
+
: existing.state === "queued"
|
|
1704
|
+
? retryUpdateLane(hqRoot, laneId, "command", (current) => current.state === "queued"
|
|
1705
|
+
? {
|
|
1706
|
+
...current,
|
|
1707
|
+
pickup_validation_pending: true,
|
|
1708
|
+
pickup_validation_pending_since: now.toISOString(),
|
|
1709
|
+
}
|
|
1710
|
+
: current)
|
|
1652
1711
|
: existing;
|
|
1653
1712
|
const dest = runDir(hqRoot, lane.lane_id);
|
|
1654
1713
|
ensureDir(path.join(dest, "logs"));
|
|
@@ -1692,7 +1751,7 @@ async function runPickup(ctx, opts) {
|
|
|
1692
1751
|
throw new LanesError("outpost_payload_invalid", `Pickup payload is not JSON: ${err instanceof Error ? err.message : String(err)}`, {});
|
|
1693
1752
|
}
|
|
1694
1753
|
const payload = parseOutpostPickupPayload(parsedJson);
|
|
1695
|
-
const ingested = ingestPickupPayload(hqRoot, payload);
|
|
1754
|
+
const ingested = ingestPickupPayload(hqRoot, payload, ctx.now());
|
|
1696
1755
|
if (ingested.host.kind === "outpost") {
|
|
1697
1756
|
try {
|
|
1698
1757
|
refuseOutpostSelfTarget(hqRoot, ingested, {
|
|
@@ -2373,15 +2432,26 @@ async function runReparent(ctx, laneId, opts) {
|
|
|
2373
2432
|
throw new LanesError("demotion_with_pending_questions", "Cannot increase a lane's depth while it has pending questions", { lane_id: laneId, from_depth: oldDepth, to_depth: newDepth });
|
|
2374
2433
|
}
|
|
2375
2434
|
const override = { laneId, senior: nextSenior, depth: newDepth };
|
|
2435
|
+
const descendants = [];
|
|
2376
2436
|
for (const other of loadAllLanes(hqRoot)) {
|
|
2377
2437
|
if (other.lane_id === laneId)
|
|
2378
2438
|
continue;
|
|
2379
2439
|
if (!walkHitsLane(hqRoot, other.senior, laneId, ctx))
|
|
2380
2440
|
continue;
|
|
2381
|
-
const
|
|
2382
|
-
|
|
2383
|
-
|
|
2441
|
+
const oldDescendantDepth = derivedLaneDepth(hqRoot, other.senior, ctx);
|
|
2442
|
+
const descendantDepth = derivedLaneDepth(hqRoot, other.senior, ctx, override);
|
|
2443
|
+
if (descendantDepth > MAX_LANE_DEPTH) {
|
|
2444
|
+
throw new LanesError("max_lane_depth", `Re-parent would put descendant ${other.lane_id} at depth ${descendantDepth}`, { lane_id: other.lane_id, depth: descendantDepth, max: MAX_LANE_DEPTH });
|
|
2384
2445
|
}
|
|
2446
|
+
if (descendantDepth > oldDescendantDepth &&
|
|
2447
|
+
pendingQuestionsForLane(hqRoot, other.lane_id).length > 0) {
|
|
2448
|
+
throw new LanesError("demotion_with_pending_questions", "Cannot increase a lane's depth while it has pending questions", {
|
|
2449
|
+
lane_id: other.lane_id,
|
|
2450
|
+
from_depth: oldDescendantDepth,
|
|
2451
|
+
to_depth: descendantDepth,
|
|
2452
|
+
});
|
|
2453
|
+
}
|
|
2454
|
+
descendants.push({ laneId: other.lane_id, depth: descendantDepth });
|
|
2385
2455
|
}
|
|
2386
2456
|
const at = ctx.now().toISOString();
|
|
2387
2457
|
const updated = casUpdateLane(hqRoot, laneId, (current) => {
|
|
@@ -2401,6 +2471,13 @@ async function runReparent(ctx, laneId, opts) {
|
|
|
2401
2471
|
}),
|
|
2402
2472
|
};
|
|
2403
2473
|
});
|
|
2474
|
+
for (const descendant of descendants) {
|
|
2475
|
+
casUpdateLane(hqRoot, descendant.laneId, (current) => {
|
|
2476
|
+
if (current.depth === descendant.depth)
|
|
2477
|
+
return current;
|
|
2478
|
+
return { ...current, depth: descendant.depth };
|
|
2479
|
+
});
|
|
2480
|
+
}
|
|
2404
2481
|
for (const question of pending) {
|
|
2405
2482
|
casUpdateQuestion(hqRoot, question.question_id, (current) => {
|
|
2406
2483
|
const already = current.routing_history.some((entry) => entry.reason === "reparented" && entry.at === at);
|
|
@@ -3019,6 +3096,7 @@ async function runReconcile(ctx, opts) {
|
|
|
3019
3096
|
throttleDeps: ctx.throttleDeps,
|
|
3020
3097
|
livenessDeps: livenessDepsFromCtx(ctx),
|
|
3021
3098
|
presenceConnected: ctx.presenceConnected,
|
|
3099
|
+
mesh: ctx.mesh,
|
|
3022
3100
|
env: ctx.env,
|
|
3023
3101
|
});
|
|
3024
3102
|
printJson(summary);
|
package/dist/commands/mesh.js
CHANGED
|
@@ -1545,7 +1545,10 @@ export function registerMeshCommand(program) {
|
|
|
1545
1545
|
}
|
|
1546
1546
|
async function runDaemonRun(opts = {}) {
|
|
1547
1547
|
const hqRoot = opts.hqRoot?.trim() || undefined;
|
|
1548
|
-
const handle = await runMeshDaemon(
|
|
1548
|
+
const handle = await runMeshDaemon({
|
|
1549
|
+
...(hqRoot ? { hqRoot } : {}),
|
|
1550
|
+
enableWorkCache: true,
|
|
1551
|
+
});
|
|
1549
1552
|
// Stay alive until signal handlers call stop + exit.
|
|
1550
1553
|
await new Promise(() => {
|
|
1551
1554
|
void handle;
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Library layer for `hq lanes`. The command module should import from here.
|
|
3
3
|
*/
|
|
4
4
|
export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
|
|
5
|
+
export { MAX_LANE_DEPTH } from "./constants.js";
|
|
5
6
|
export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
|
|
6
7
|
export type { AccountConfig, AttributionHistoryEntry, EnvelopeDecision, EnvelopeQuestion, HostKind, LaneCompanyRef, LaneEnvelope, LaneHost, LaneProvider, LaneRecord, PendingResume, AccountHistoryEntry, LaneState, LanesConfig, LastEnvelopeRef, LivenessState, LivenessWindows, ProviderConfig, QuestionAnswer, QuestionBodyRef, QuestionRecord, QuestionStatus, QueuedReason, RaisedVia, RoutingHistoryEntry, RoutingReason, SeniorHistoryEntry, SeniorKind, SeniorRef, StoryStatus, TerminalLaneState, ThrottleConfig, } from "./types.js";
|
|
7
8
|
export { DEFAULT_ACCOUNT_HOMES, DEFAULT_COMPACT_AFTER_MS, DEFAULT_CPU_BUSY_PAUSE, DEFAULT_CPU_BUSY_RESUME, DEFAULT_LIVE_WITHIN_MS, DEFAULT_LIVENESS_WINDOWS, DEFAULT_MAX_QUEUED, DEFAULT_MIN_AVAILABLE_MEM_MB, DEFAULT_PROVIDER_MAX_RUNNING, DEFAULT_SILENT_LOST_MS, DEFAULT_TURN_START_LOST_MS, LANES_CONFIG_REL, accountMaxRunning, canonicalizeHome, documentedDefaultConfig, expandHome, lanesConfigPath, loadLanesConfig, parseLanesConfig, providerCompactAfterMs, providerQuotaDefaultHoldMs, DEFAULT_QUOTA_HOLD_MS, writeLanesConfig, } from "./config.js";
|
package/dist/lib/lanes/index.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Library layer for `hq lanes`. The command module should import from here.
|
|
3
3
|
*/
|
|
4
4
|
export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
|
|
5
|
+
export { MAX_LANE_DEPTH } from "./constants.js";
|
|
5
6
|
export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
|
|
6
7
|
export { DEFAULT_ACCOUNT_HOMES, DEFAULT_COMPACT_AFTER_MS, DEFAULT_CPU_BUSY_PAUSE, DEFAULT_CPU_BUSY_RESUME, DEFAULT_LIVE_WITHIN_MS, DEFAULT_LIVENESS_WINDOWS, DEFAULT_MAX_QUEUED, DEFAULT_MIN_AVAILABLE_MEM_MB, DEFAULT_PROVIDER_MAX_RUNNING, DEFAULT_SILENT_LOST_MS, DEFAULT_TURN_START_LOST_MS, LANES_CONFIG_REL, accountMaxRunning, canonicalizeHome, documentedDefaultConfig, expandHome, lanesConfigPath, loadLanesConfig, parseLanesConfig, providerCompactAfterMs, providerQuotaDefaultHoldMs, DEFAULT_QUOTA_HOLD_MS, writeLanesConfig, } from "./config.js";
|
|
7
8
|
export { resolveLanesHqRoot } from "./hq-root.js";
|
|
@@ -64,9 +64,12 @@ export declare function applyDefaultQuotaHold(signal: QuotaSignal, defaultHoldMs
|
|
|
64
64
|
export declare function quotaStatePath(hqRoot: string): string;
|
|
65
65
|
export declare function loadQuotaState(hqRoot: string): QuotaState;
|
|
66
66
|
/**
|
|
67
|
-
* Parse "resets Sep 24, 2am (UTC)"
|
|
68
|
-
*
|
|
69
|
-
*
|
|
67
|
+
* Parse provider reset times such as "resets Sep 24, 2am (UTC)" and Codex's
|
|
68
|
+
* "try again at Sep 14th, 2026 9:24 PM" into an ISO timestamp. Codex's
|
|
69
|
+
* unzoned form is displayed in the provider CLI's local timezone, while the
|
|
70
|
+
* provider `resets` form is UTC when it names `(UTC)`. The year is the current
|
|
71
|
+
* UTC year for an unyear-tagged reset, rolled forward when it is already past.
|
|
72
|
+
* An explicit provider year is kept as stated.
|
|
70
73
|
*/
|
|
71
74
|
export declare function parseProviderResetTime(text: string, now: Date): string | null;
|
|
72
75
|
/**
|
package/dist/lib/lanes/quota.js
CHANGED
|
@@ -136,26 +136,36 @@ function writeQuotaState(hqRoot, state) {
|
|
|
136
136
|
fs.renameSync(tmp, filePath);
|
|
137
137
|
}
|
|
138
138
|
/**
|
|
139
|
-
* Parse "resets Sep 24, 2am (UTC)"
|
|
140
|
-
*
|
|
141
|
-
*
|
|
139
|
+
* Parse provider reset times such as "resets Sep 24, 2am (UTC)" and Codex's
|
|
140
|
+
* "try again at Sep 14th, 2026 9:24 PM" into an ISO timestamp. Codex's
|
|
141
|
+
* unzoned form is displayed in the provider CLI's local timezone, while the
|
|
142
|
+
* provider `resets` form is UTC when it names `(UTC)`. The year is the current
|
|
143
|
+
* UTC year for an unyear-tagged reset, rolled forward when it is already past.
|
|
144
|
+
* An explicit provider year is kept as stated.
|
|
142
145
|
*/
|
|
143
146
|
export function parseProviderResetTime(text, now) {
|
|
144
|
-
const
|
|
147
|
+
const resetMatch = /\bresets\s+([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?(?:,\s*(?:(\d{4})\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm))?(?:\s*\((UTC)\))?/i.exec(text);
|
|
148
|
+
const codexMatch = /\btry\s+again\s+at\s+([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?,\s*(\d{4})\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)(?:\s*\((UTC)\))?/i.exec(text);
|
|
149
|
+
const match = resetMatch ?? codexMatch;
|
|
145
150
|
if (!match)
|
|
146
151
|
return null;
|
|
152
|
+
const codexLocalTime = codexMatch !== null && (match[7] ?? "").toLowerCase() !== "utc";
|
|
147
153
|
const month = MONTHS[(match[1] ?? "").slice(0, 3).toLowerCase()];
|
|
148
154
|
if (month === undefined)
|
|
149
155
|
return null;
|
|
150
156
|
const day = Number(match[2]);
|
|
151
157
|
if (!Number.isInteger(day) || day < 1 || day > 31)
|
|
152
158
|
return null;
|
|
159
|
+
const explicitYear = match[3] !== undefined;
|
|
160
|
+
const parsedYear = explicitYear ? Number(match[3]) : now.getUTCFullYear();
|
|
161
|
+
if (!Number.isInteger(parsedYear) || parsedYear < 1970)
|
|
162
|
+
return null;
|
|
153
163
|
let hour = 0;
|
|
154
164
|
let minute = 0;
|
|
155
|
-
if (match[
|
|
156
|
-
hour = Number(match[
|
|
157
|
-
minute = match[
|
|
158
|
-
const ap = (match[
|
|
165
|
+
if (match[4] !== undefined) {
|
|
166
|
+
hour = Number(match[4]);
|
|
167
|
+
minute = match[5] !== undefined ? Number(match[5]) : 0;
|
|
168
|
+
const ap = (match[6] ?? "").toLowerCase();
|
|
159
169
|
if (ap === "pm" && hour < 12)
|
|
160
170
|
hour += 12;
|
|
161
171
|
if (ap === "am" && hour === 12)
|
|
@@ -163,11 +173,32 @@ export function parseProviderResetTime(text, now) {
|
|
|
163
173
|
if (hour > 23 || minute > 59)
|
|
164
174
|
return null;
|
|
165
175
|
}
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
176
|
+
const timestampForYear = (targetYear) => {
|
|
177
|
+
const candidate = codexLocalTime
|
|
178
|
+
? new Date(targetYear, month, day, hour, minute, 0, 0)
|
|
179
|
+
: new Date(Date.UTC(targetYear, month, day, hour, minute, 0, 0));
|
|
180
|
+
const valid = codexLocalTime
|
|
181
|
+
? candidate.getFullYear() === targetYear &&
|
|
182
|
+
candidate.getMonth() === month &&
|
|
183
|
+
candidate.getDate() === day &&
|
|
184
|
+
candidate.getHours() === hour &&
|
|
185
|
+
candidate.getMinutes() === minute
|
|
186
|
+
: candidate.getUTCFullYear() === targetYear &&
|
|
187
|
+
candidate.getUTCMonth() === month &&
|
|
188
|
+
candidate.getUTCDate() === day &&
|
|
189
|
+
candidate.getUTCHours() === hour &&
|
|
190
|
+
candidate.getUTCMinutes() === minute;
|
|
191
|
+
return valid ? candidate.getTime() : null;
|
|
192
|
+
};
|
|
193
|
+
let year = parsedYear;
|
|
194
|
+
let ts = timestampForYear(year);
|
|
195
|
+
if (ts === null)
|
|
196
|
+
return null;
|
|
197
|
+
if (!explicitYear && ts < now.getTime() - 36 * 60 * 60 * 1000) {
|
|
169
198
|
year += 1;
|
|
170
|
-
ts =
|
|
199
|
+
ts = timestampForYear(year);
|
|
200
|
+
if (ts === null)
|
|
201
|
+
return null;
|
|
171
202
|
}
|
|
172
203
|
return new Date(ts).toISOString();
|
|
173
204
|
}
|
|
@@ -259,21 +290,40 @@ export function recordAccountQuota(hqRoot, provider, accountId, signal) {
|
|
|
259
290
|
next.marker = signal.marker ?? `needs credits for ${signal.model}`;
|
|
260
291
|
delete next.quota_reset_at;
|
|
261
292
|
}
|
|
262
|
-
if (signal.quota_reset_at &&
|
|
293
|
+
if (signal.quota_reset_at &&
|
|
294
|
+
!signal.needs_credits &&
|
|
295
|
+
shouldReplaceReset(next.quota_reset_at, signal.quota_reset_at)) {
|
|
263
296
|
next.quota_reset_at = signal.quota_reset_at;
|
|
264
297
|
}
|
|
265
298
|
models[signal.model] = next;
|
|
266
299
|
}
|
|
267
300
|
const account = { ...current, models };
|
|
268
301
|
if (signal.account_wide && signal.quota_reset_at) {
|
|
269
|
-
|
|
270
|
-
|
|
302
|
+
if (shouldReplaceReset(current.all_models_quota_reset_at, signal.quota_reset_at)) {
|
|
303
|
+
account.all_models_quota_reset_at = signal.quota_reset_at;
|
|
304
|
+
account.reset_source = signal.reset_source ?? "provider";
|
|
305
|
+
}
|
|
306
|
+
else if (current.all_models_quota_reset_at) {
|
|
307
|
+
account.all_models_quota_reset_at = current.all_models_quota_reset_at;
|
|
308
|
+
account.reset_source = current.reset_source ?? "provider";
|
|
309
|
+
}
|
|
271
310
|
}
|
|
272
311
|
accounts[accountId] = account;
|
|
273
312
|
state.providers[provider] = accounts;
|
|
274
313
|
writeQuotaState(hqRoot, state);
|
|
275
314
|
return state;
|
|
276
315
|
}
|
|
316
|
+
function shouldReplaceReset(current, incoming) {
|
|
317
|
+
if (!current)
|
|
318
|
+
return true;
|
|
319
|
+
const currentMs = Date.parse(current);
|
|
320
|
+
const incomingMs = Date.parse(incoming);
|
|
321
|
+
if (!Number.isFinite(currentMs))
|
|
322
|
+
return Number.isFinite(incomingMs);
|
|
323
|
+
if (!Number.isFinite(incomingMs))
|
|
324
|
+
return false;
|
|
325
|
+
return incomingMs > currentMs;
|
|
326
|
+
}
|
|
277
327
|
function resetStillAhead(iso, now) {
|
|
278
328
|
if (!iso)
|
|
279
329
|
return false;
|
|
@@ -12,12 +12,15 @@ import type { LaneRecord, LivenessState, QuestionRecord, SeniorRef } from "./typ
|
|
|
12
12
|
import { type LaneMesh } from "./mesh-events.js";
|
|
13
13
|
/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
|
|
14
14
|
export declare const QUESTION_AGE_MS: number;
|
|
15
|
-
/** Spec §6.6 + §9:
|
|
16
|
-
export declare const MAX_AGED_ADVANCES
|
|
15
|
+
/** Spec §6.6 + §9: a depth-3 worker has two hops to age toward the user. */
|
|
16
|
+
export declare const MAX_AGED_ADVANCES: number;
|
|
17
17
|
/** Aligns with watcher silence: a host that has not reported for this long is dead. */
|
|
18
18
|
export declare const DEAD_HOST_MS: number;
|
|
19
19
|
export declare const RECONCILE_LOCK_STALE_MS = 30000;
|
|
20
20
|
export declare const RECONCILE_LOCK_TIMEOUT_MS = 5000;
|
|
21
|
+
/** A pickup probe that has not cleared its marker within this window is abandoned. */
|
|
22
|
+
export declare const PICKUP_VALIDATION_MAX_AGE_MS: number;
|
|
23
|
+
export declare const PICKUP_VALIDATION_TIMEOUT_REASON: "pickup_validation_timeout";
|
|
21
24
|
export declare const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
|
|
22
25
|
export declare const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
|
|
23
26
|
export interface ReconcileSkip {
|
|
@@ -9,6 +9,7 @@ import * as fs from "node:fs";
|
|
|
9
9
|
import * as path from "node:path";
|
|
10
10
|
import { parseUserYaml } from "../../utils/user-yaml-error.js";
|
|
11
11
|
import { accountMaxRunning, loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
|
|
12
|
+
import { MAX_LANE_DEPTH } from "./constants.js";
|
|
12
13
|
import { LanesError, isLaneVersionConflict } from "./errors.js";
|
|
13
14
|
import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
|
|
14
15
|
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
|
|
@@ -22,12 +23,15 @@ import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from
|
|
|
22
23
|
import { providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
23
24
|
/** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
|
|
24
25
|
export const QUESTION_AGE_MS = 60 * 60 * 1000;
|
|
25
|
-
/** Spec §6.6 + §9:
|
|
26
|
-
export const MAX_AGED_ADVANCES = 1;
|
|
26
|
+
/** Spec §6.6 + §9: a depth-3 worker has two hops to age toward the user. */
|
|
27
|
+
export const MAX_AGED_ADVANCES = MAX_LANE_DEPTH - 1;
|
|
27
28
|
/** Aligns with watcher silence: a host that has not reported for this long is dead. */
|
|
28
29
|
export const DEAD_HOST_MS = SILENCE_MS;
|
|
29
30
|
export const RECONCILE_LOCK_STALE_MS = 30_000;
|
|
30
31
|
export const RECONCILE_LOCK_TIMEOUT_MS = 5_000;
|
|
32
|
+
/** A pickup probe that has not cleared its marker within this window is abandoned. */
|
|
33
|
+
export const PICKUP_VALIDATION_MAX_AGE_MS = 15 * 60 * 1000;
|
|
34
|
+
export const PICKUP_VALIDATION_TIMEOUT_REASON = "pickup_validation_timeout";
|
|
31
35
|
export const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
|
|
32
36
|
export const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
|
|
33
37
|
function logSkip(skip) {
|
|
@@ -309,6 +313,89 @@ function casQuestionIf(hqRoot, questionId, shouldWrite, updater, dryRun) {
|
|
|
309
313
|
}
|
|
310
314
|
throw new LanesError("version_conflict", `Exhausted CAS retries for question ${questionId}`, { question_id: questionId });
|
|
311
315
|
}
|
|
316
|
+
async function emitAgedQuestionAtUser(summary, deps, lane, question) {
|
|
317
|
+
if (deps.dryRun)
|
|
318
|
+
return;
|
|
319
|
+
const mesh = deps.mesh === undefined ? createLaneMesh({ interactive: false }) : deps.mesh;
|
|
320
|
+
if (!mesh || !lane.company || !lane.thread_id)
|
|
321
|
+
return;
|
|
322
|
+
const timeoutMs = deps.meshEmitTimeoutMs ?? 10_000;
|
|
323
|
+
let timer;
|
|
324
|
+
try {
|
|
325
|
+
await Promise.race([
|
|
326
|
+
mesh.appendThreadEvent(lane.company, lane.thread_id, "lane_question_at_user", { lane_id: lane.lane_id, question_id: question.question_id }),
|
|
327
|
+
new Promise((_resolve, reject) => {
|
|
328
|
+
timer = setTimeout(() => reject(new Error(`question-at-user mesh emit timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
329
|
+
}),
|
|
330
|
+
]);
|
|
331
|
+
}
|
|
332
|
+
catch (err) {
|
|
333
|
+
markDirty(summary, {
|
|
334
|
+
kind: "question",
|
|
335
|
+
id: question.question_id,
|
|
336
|
+
why: `question-at-user mesh emit failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
337
|
+
});
|
|
338
|
+
}
|
|
339
|
+
finally {
|
|
340
|
+
if (timer)
|
|
341
|
+
clearTimeout(timer);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
function pickupValidationPendingAtMs(lane) {
|
|
345
|
+
const raw = lane.pickup_validation_pending_since ?? lane.updated_at;
|
|
346
|
+
const at = Date.parse(raw);
|
|
347
|
+
return Number.isFinite(at) ? at : null;
|
|
348
|
+
}
|
|
349
|
+
function pickupValidationExpired(lane, nowMs) {
|
|
350
|
+
const pendingAtMs = pickupValidationPendingAtMs(lane);
|
|
351
|
+
return pendingAtMs === null || nowMs - pendingAtMs > PICKUP_VALIDATION_MAX_AGE_MS;
|
|
352
|
+
}
|
|
353
|
+
function expireAbandonedPickupValidation(hqRoot, lanes, summary, deps, now) {
|
|
354
|
+
const nowMs = now.getTime();
|
|
355
|
+
for (const lane of lanes) {
|
|
356
|
+
if (lane.state !== "queued" ||
|
|
357
|
+
lane.pickup_validation_pending !== true ||
|
|
358
|
+
!laneRunsHere(lane, deps.host) ||
|
|
359
|
+
!pickupValidationExpired(lane, nowMs)) {
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
try {
|
|
363
|
+
const result = casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" &&
|
|
364
|
+
current.pickup_validation_pending === true &&
|
|
365
|
+
pickupValidationExpired(current, nowMs), (current) => {
|
|
366
|
+
const pendingAtMs = pickupValidationPendingAtMs(current);
|
|
367
|
+
return {
|
|
368
|
+
...current,
|
|
369
|
+
state: "failed",
|
|
370
|
+
queued_reason: undefined,
|
|
371
|
+
pickup_validation_pending: undefined,
|
|
372
|
+
pickup_validation_pending_since: undefined,
|
|
373
|
+
terminal_reason: PICKUP_VALIDATION_TIMEOUT_REASON,
|
|
374
|
+
terminal_details: {
|
|
375
|
+
pickup_validation_pending_since: current.pickup_validation_pending_since ?? current.updated_at,
|
|
376
|
+
age_ms: pendingAtMs === null ? null : Math.max(0, nowMs - pendingAtMs),
|
|
377
|
+
max_age_ms: PICKUP_VALIDATION_MAX_AGE_MS,
|
|
378
|
+
},
|
|
379
|
+
};
|
|
380
|
+
}, deps.dryRun);
|
|
381
|
+
if (result.changed) {
|
|
382
|
+
summary.aged.push({
|
|
383
|
+
id: lane.lane_id,
|
|
384
|
+
from: "queued",
|
|
385
|
+
to: "failed",
|
|
386
|
+
reason: PICKUP_VALIDATION_TIMEOUT_REASON,
|
|
387
|
+
});
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
catch (err) {
|
|
391
|
+
markDirty(summary, {
|
|
392
|
+
kind: "lane",
|
|
393
|
+
id: lane.lane_id,
|
|
394
|
+
why: `pickup validation expiry failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
}
|
|
312
399
|
function fileNonEmpty(filePath) {
|
|
313
400
|
try {
|
|
314
401
|
return fs.statSync(filePath).size > 0;
|
|
@@ -589,6 +676,10 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
589
676
|
to: String(nextHop),
|
|
590
677
|
reason: "unanswered_timeout",
|
|
591
678
|
});
|
|
679
|
+
if (nextAudience.kind === "user") {
|
|
680
|
+
const originLane = laneById.get(question.lane_id) ?? readLane(hqRoot, question.lane_id);
|
|
681
|
+
await emitAgedQuestionAtUser(summary, deps, originLane, result.record);
|
|
682
|
+
}
|
|
592
683
|
}
|
|
593
684
|
}
|
|
594
685
|
}
|
|
@@ -862,6 +953,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
862
953
|
}
|
|
863
954
|
}
|
|
864
955
|
async function admitQueued(hqRoot, lanes, summary, deps, now) {
|
|
956
|
+
expireAbandonedPickupValidation(hqRoot, lanes, summary, deps, now);
|
|
865
957
|
// A lane whose pickup probe is still out is not admitted: its record is
|
|
866
958
|
// about to be bound or failed by the pickup, and a worker spawned now
|
|
867
959
|
// would keep running behind that write.
|
|
@@ -257,6 +257,8 @@ export interface LaneRecord {
|
|
|
257
257
|
* leave a running worker behind a record the probe result then rewrites.
|
|
258
258
|
*/
|
|
259
259
|
pickup_validation_pending?: true;
|
|
260
|
+
/** When the pickup validation marker was written. */
|
|
261
|
+
pickup_validation_pending_since?: string;
|
|
260
262
|
priority: number;
|
|
261
263
|
worker?: string;
|
|
262
264
|
/** Registry-relative directory of the resolved worker, when --worker was set. */
|