@indigoai-us/hq-cli 5.132.0 → 5.134.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/CHANGELOG.md +44 -0
  2. package/dist/commands/agent-kit.d.ts +2 -0
  3. package/dist/commands/agent-kit.js +4 -1
  4. package/dist/commands/lanes.d.ts +2 -2
  5. package/dist/commands/lanes.js +98 -20
  6. package/dist/commands/mesh.js +4 -1
  7. package/dist/lib/agent-kit/run/heartbeat.d.ts +14 -2
  8. package/dist/lib/agent-kit/run/heartbeat.js +38 -6
  9. package/dist/lib/bot/api.d.ts +3 -1
  10. package/dist/lib/bot/api.js +7 -2
  11. package/dist/lib/bot/run.d.ts +4 -0
  12. package/dist/lib/bot/run.js +30 -1
  13. package/dist/lib/bot/status.d.ts +4 -0
  14. package/dist/lib/doctor/checks/bots.js +4 -1
  15. package/dist/lib/heartbeat-identity.d.ts +29 -0
  16. package/dist/lib/heartbeat-identity.js +71 -0
  17. package/dist/lib/lanes/constants.d.ts +3 -0
  18. package/dist/lib/lanes/constants.js +3 -0
  19. package/dist/lib/lanes/index.d.ts +1 -0
  20. package/dist/lib/lanes/index.js +1 -0
  21. package/dist/lib/lanes/quota.d.ts +6 -3
  22. package/dist/lib/lanes/quota.js +65 -15
  23. package/dist/lib/lanes/reconcile.d.ts +5 -2
  24. package/dist/lib/lanes/reconcile.js +94 -2
  25. package/dist/lib/lanes/types.d.ts +2 -0
  26. package/dist/lib/lanes/types.js +2 -1
  27. package/dist/lib/mesh/live/daemon/doctor.d.ts +6 -0
  28. package/dist/lib/mesh/live/daemon/doctor.js +11 -0
  29. package/dist/lib/mesh/live/daemon/install.d.ts +7 -0
  30. package/dist/lib/mesh/live/daemon/install.js +16 -6
  31. package/dist/lib/mesh/live/daemon/legacy-listen.d.ts +43 -0
  32. package/dist/lib/mesh/live/daemon/legacy-listen.js +89 -0
  33. package/dist/lib/mesh/live/daemon/live-cache.d.ts +26 -0
  34. package/dist/lib/mesh/live/daemon/live-cache.js +116 -0
  35. package/dist/lib/mesh/live/daemon/presence.d.ts +7 -3
  36. package/dist/lib/mesh/live/daemon/presence.js +10 -9
  37. package/dist/lib/mesh/live/daemon/run.d.ts +13 -0
  38. package/dist/lib/mesh/live/daemon/run.js +51 -2
  39. package/dist/lib/mesh/live/daemon/work-subscribe.d.ts +31 -0
  40. package/dist/lib/mesh/live/daemon/work-subscribe.js +124 -0
  41. package/package.json +2 -2
@@ -0,0 +1,29 @@
1
+ import type { CaptureContext } from "@sentry/node";
2
+ export declare const HEARTBEAT_IDENTITY_MISMATCH_CODE = "AGENT_IDENTITY_MISMATCH";
3
+ export type HeartbeatRunner = "agent-kit" | "bot";
4
+ export type HeartbeatAgentUidSource = "config_file" | "env" | "flag";
5
+ export type HeartbeatCredsSource = "explicit" | "default";
6
+ export interface HeartbeatFailure {
7
+ status: number;
8
+ code?: string;
9
+ message?: string;
10
+ }
11
+ export interface HeartbeatIdentityMismatchContext {
12
+ runner: HeartbeatRunner;
13
+ configuredAgentUid: string;
14
+ tokenAgentUid?: string;
15
+ agentUidSource: HeartbeatAgentUidSource;
16
+ credsSource: HeartbeatCredsSource;
17
+ }
18
+ export type HeartbeatIdentityMismatchReporter = (context: HeartbeatIdentityMismatchContext) => void;
19
+ /** Parse only the bounded fields the heartbeat route uses for classification. */
20
+ export declare function parseHeartbeatFailureBody(bodyText: string | undefined): {
21
+ code?: string;
22
+ message?: string;
23
+ };
24
+ /** Match the server's registered code/name, never an arbitrary 403. */
25
+ export declare function isHeartbeatIdentityMismatch(failure: HeartbeatFailure): boolean;
26
+ /** Read the agent uid claim for diagnostics only; this never authorizes a call. */
27
+ export declare function tokenAgentUid(token: string): string | undefined;
28
+ export declare function captureHeartbeatIdentityMismatch(context: HeartbeatIdentityMismatchContext, captureMessage?: (message: string, context: CaptureContext) => string): void;
29
+ //# sourceMappingURL=heartbeat-identity.d.ts.map
@@ -0,0 +1,71 @@
1
+ import { CLI_VERSION } from "../cli-version.js";
2
+ import { Sentry } from "../sentry.js";
3
+ import { peekIdToken } from "../utils/id-token.js";
4
+ export const HEARTBEAT_IDENTITY_MISMATCH_CODE = "AGENT_IDENTITY_MISMATCH";
5
+ const AGENT_UID_RE = /^agt_[A-Za-z0-9_-]{1,100}$/;
6
+ function normalize(value) {
7
+ return (value ?? "").replace(/[^A-Za-z0-9]/g, "").toLowerCase();
8
+ }
9
+ function safeAgentUid(value) {
10
+ return typeof value === "string" && AGENT_UID_RE.test(value) ? value : undefined;
11
+ }
12
+ /** Parse only the bounded fields the heartbeat route uses for classification. */
13
+ export function parseHeartbeatFailureBody(bodyText) {
14
+ if (!bodyText)
15
+ return {};
16
+ try {
17
+ const body = JSON.parse(bodyText);
18
+ const code = typeof body.code === "string" ? body.code.slice(0, 120) : undefined;
19
+ const message = typeof body.error === "string"
20
+ ? body.error.slice(0, 300)
21
+ : typeof body.message === "string"
22
+ ? body.message.slice(0, 300)
23
+ : undefined;
24
+ return { ...(code ? { code } : {}), ...(message ? { message } : {}) };
25
+ }
26
+ catch {
27
+ return {};
28
+ }
29
+ }
30
+ /** Match the server's registered code/name, never an arbitrary 403. */
31
+ export function isHeartbeatIdentityMismatch(failure) {
32
+ if (failure.status !== 403)
33
+ return false;
34
+ const code = normalize(failure.code);
35
+ if (code === normalize(HEARTBEAT_IDENTITY_MISMATCH_CODE) || code === "agentidentitymismatcherror")
36
+ return true;
37
+ const message = normalize(failure.message);
38
+ return (message.includes("heartbeatmustbereportedbytheagentsownmachineidentity") ||
39
+ message.includes("agentidentitymismatcherror"));
40
+ }
41
+ /** Read the agent uid claim for diagnostics only; this never authorizes a call. */
42
+ export function tokenAgentUid(token) {
43
+ const claims = peekIdToken(token);
44
+ return (safeAgentUid(claims["custom:entityUid"]) ??
45
+ safeAgentUid(claims.entityUid) ??
46
+ safeAgentUid(claims.sub));
47
+ }
48
+ function diagnosticUid(value) {
49
+ return safeAgentUid(value) ?? "unknown";
50
+ }
51
+ function captureHeartbeatIdentityMismatchMessage(message, context) {
52
+ return Sentry.captureMessage(message, context);
53
+ }
54
+ export function captureHeartbeatIdentityMismatch(context, captureMessage = captureHeartbeatIdentityMismatchMessage) {
55
+ const options = {
56
+ level: "warning",
57
+ fingerprint: ["heartbeat-identity-mismatch"],
58
+ tags: {
59
+ agent_uid_source: context.agentUidSource,
60
+ creds_source: context.credsSource,
61
+ runner: context.runner,
62
+ hq_cli_version: CLI_VERSION,
63
+ },
64
+ extra: {
65
+ configured_agent_uid: diagnosticUid(context.configuredAgentUid),
66
+ ...(context.tokenAgentUid ? { token_agent_uid: diagnosticUid(context.tokenAgentUid) } : {}),
67
+ },
68
+ };
69
+ captureMessage("heartbeat identity mismatch", options);
70
+ }
71
+ //# sourceMappingURL=heartbeat-identity.js.map
@@ -0,0 +1,3 @@
1
+ /** Maximum number of lane levels below the user. */
2
+ export declare const MAX_LANE_DEPTH = 3;
3
+ //# sourceMappingURL=constants.d.ts.map
@@ -0,0 +1,3 @@
1
+ /** Maximum number of lane levels below the user. */
2
+ export const MAX_LANE_DEPTH = 3;
3
+ //# sourceMappingURL=constants.js.map
@@ -2,6 +2,7 @@
2
2
  * Library layer for `hq lanes`. The command module should import from here.
3
3
  */
4
4
  export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
5
+ export { MAX_LANE_DEPTH } from "./constants.js";
5
6
  export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
6
7
  export type { AccountConfig, AttributionHistoryEntry, EnvelopeDecision, EnvelopeQuestion, HostKind, LaneCompanyRef, LaneEnvelope, LaneHost, LaneProvider, LaneRecord, PendingResume, AccountHistoryEntry, LaneState, LanesConfig, LastEnvelopeRef, LivenessState, LivenessWindows, ProviderConfig, QuestionAnswer, QuestionBodyRef, QuestionRecord, QuestionStatus, QueuedReason, RaisedVia, RoutingHistoryEntry, RoutingReason, SeniorHistoryEntry, SeniorKind, SeniorRef, StoryStatus, TerminalLaneState, ThrottleConfig, } from "./types.js";
7
8
  export { DEFAULT_ACCOUNT_HOMES, DEFAULT_COMPACT_AFTER_MS, DEFAULT_CPU_BUSY_PAUSE, DEFAULT_CPU_BUSY_RESUME, DEFAULT_LIVE_WITHIN_MS, DEFAULT_LIVENESS_WINDOWS, DEFAULT_MAX_QUEUED, DEFAULT_MIN_AVAILABLE_MEM_MB, DEFAULT_PROVIDER_MAX_RUNNING, DEFAULT_SILENT_LOST_MS, DEFAULT_TURN_START_LOST_MS, LANES_CONFIG_REL, accountMaxRunning, canonicalizeHome, documentedDefaultConfig, expandHome, lanesConfigPath, loadLanesConfig, parseLanesConfig, providerCompactAfterMs, providerQuotaDefaultHoldMs, DEFAULT_QUOTA_HOLD_MS, writeLanesConfig, } from "./config.js";
@@ -2,6 +2,7 @@
2
2
  * Library layer for `hq lanes`. The command module should import from here.
3
3
  */
4
4
  export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
5
+ export { MAX_LANE_DEPTH } from "./constants.js";
5
6
  export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
6
7
  export { DEFAULT_ACCOUNT_HOMES, DEFAULT_COMPACT_AFTER_MS, DEFAULT_CPU_BUSY_PAUSE, DEFAULT_CPU_BUSY_RESUME, DEFAULT_LIVE_WITHIN_MS, DEFAULT_LIVENESS_WINDOWS, DEFAULT_MAX_QUEUED, DEFAULT_MIN_AVAILABLE_MEM_MB, DEFAULT_PROVIDER_MAX_RUNNING, DEFAULT_SILENT_LOST_MS, DEFAULT_TURN_START_LOST_MS, LANES_CONFIG_REL, accountMaxRunning, canonicalizeHome, documentedDefaultConfig, expandHome, lanesConfigPath, loadLanesConfig, parseLanesConfig, providerCompactAfterMs, providerQuotaDefaultHoldMs, DEFAULT_QUOTA_HOLD_MS, writeLanesConfig, } from "./config.js";
7
8
  export { resolveLanesHqRoot } from "./hq-root.js";
@@ -64,9 +64,12 @@ export declare function applyDefaultQuotaHold(signal: QuotaSignal, defaultHoldMs
64
64
  export declare function quotaStatePath(hqRoot: string): string;
65
65
  export declare function loadQuotaState(hqRoot: string): QuotaState;
66
66
  /**
67
- * Parse "resets Sep 24, 2am (UTC)" into an ISO timestamp. The year is the
68
- * current UTC year, rolled forward when that instant is already past.
69
- * Returns null when the text names no reset.
67
+ * Parse provider reset times such as "resets Sep 24, 2am (UTC)" and Codex's
68
+ * "try again at Sep 14th, 2026 9:24 PM" into an ISO timestamp. Codex's
69
+ * unzoned form is displayed in the provider CLI's local timezone, while the
70
+ * provider `resets` form is UTC when it names `(UTC)`. The year is the current
71
+ * UTC year for an unyear-tagged reset, rolled forward when it is already past.
72
+ * An explicit provider year is kept as stated.
70
73
  */
71
74
  export declare function parseProviderResetTime(text: string, now: Date): string | null;
72
75
  /**
@@ -136,26 +136,36 @@ function writeQuotaState(hqRoot, state) {
136
136
  fs.renameSync(tmp, filePath);
137
137
  }
138
138
  /**
139
- * Parse "resets Sep 24, 2am (UTC)" into an ISO timestamp. The year is the
140
- * current UTC year, rolled forward when that instant is already past.
141
- * Returns null when the text names no reset.
139
+ * Parse provider reset times such as "resets Sep 24, 2am (UTC)" and Codex's
140
+ * "try again at Sep 14th, 2026 9:24 PM" into an ISO timestamp. Codex's
141
+ * unzoned form is displayed in the provider CLI's local timezone, while the
142
+ * provider `resets` form is UTC when it names `(UTC)`. The year is the current
143
+ * UTC year for an unyear-tagged reset, rolled forward when it is already past.
144
+ * An explicit provider year is kept as stated.
142
145
  */
143
146
  export function parseProviderResetTime(text, now) {
144
- const match = /resets\s+([A-Za-z]+)\s+(\d{1,2})(?:,\s*(\d{1,2})(?::(\d{2}))?\s*(am|pm))?(?:\s*\(UTC\))?/i.exec(text);
147
+ const resetMatch = /\bresets\s+([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?(?:,\s*(?:(\d{4})\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm))?(?:\s*\((UTC)\))?/i.exec(text);
148
+ const codexMatch = /\btry\s+again\s+at\s+([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?,\s*(\d{4})\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)(?:\s*\((UTC)\))?/i.exec(text);
149
+ const match = resetMatch ?? codexMatch;
145
150
  if (!match)
146
151
  return null;
152
+ const codexLocalTime = codexMatch !== null && (match[7] ?? "").toLowerCase() !== "utc";
147
153
  const month = MONTHS[(match[1] ?? "").slice(0, 3).toLowerCase()];
148
154
  if (month === undefined)
149
155
  return null;
150
156
  const day = Number(match[2]);
151
157
  if (!Number.isInteger(day) || day < 1 || day > 31)
152
158
  return null;
159
+ const explicitYear = match[3] !== undefined;
160
+ const parsedYear = explicitYear ? Number(match[3]) : now.getUTCFullYear();
161
+ if (!Number.isInteger(parsedYear) || parsedYear < 1970)
162
+ return null;
153
163
  let hour = 0;
154
164
  let minute = 0;
155
- if (match[3] !== undefined) {
156
- hour = Number(match[3]);
157
- minute = match[4] !== undefined ? Number(match[4]) : 0;
158
- const ap = (match[5] ?? "").toLowerCase();
165
+ if (match[4] !== undefined) {
166
+ hour = Number(match[4]);
167
+ minute = match[5] !== undefined ? Number(match[5]) : 0;
168
+ const ap = (match[6] ?? "").toLowerCase();
159
169
  if (ap === "pm" && hour < 12)
160
170
  hour += 12;
161
171
  if (ap === "am" && hour === 12)
@@ -163,11 +173,32 @@ export function parseProviderResetTime(text, now) {
163
173
  if (hour > 23 || minute > 59)
164
174
  return null;
165
175
  }
166
- let year = now.getUTCFullYear();
167
- let ts = Date.UTC(year, month, day, hour, minute, 0, 0);
168
- if (ts < now.getTime() - 36 * 60 * 60 * 1000) {
176
+ const timestampForYear = (targetYear) => {
177
+ const candidate = codexLocalTime
178
+ ? new Date(targetYear, month, day, hour, minute, 0, 0)
179
+ : new Date(Date.UTC(targetYear, month, day, hour, minute, 0, 0));
180
+ const valid = codexLocalTime
181
+ ? candidate.getFullYear() === targetYear &&
182
+ candidate.getMonth() === month &&
183
+ candidate.getDate() === day &&
184
+ candidate.getHours() === hour &&
185
+ candidate.getMinutes() === minute
186
+ : candidate.getUTCFullYear() === targetYear &&
187
+ candidate.getUTCMonth() === month &&
188
+ candidate.getUTCDate() === day &&
189
+ candidate.getUTCHours() === hour &&
190
+ candidate.getUTCMinutes() === minute;
191
+ return valid ? candidate.getTime() : null;
192
+ };
193
+ let year = parsedYear;
194
+ let ts = timestampForYear(year);
195
+ if (ts === null)
196
+ return null;
197
+ if (!explicitYear && ts < now.getTime() - 36 * 60 * 60 * 1000) {
169
198
  year += 1;
170
- ts = Date.UTC(year, month, day, hour, minute, 0, 0);
199
+ ts = timestampForYear(year);
200
+ if (ts === null)
201
+ return null;
171
202
  }
172
203
  return new Date(ts).toISOString();
173
204
  }
@@ -259,21 +290,40 @@ export function recordAccountQuota(hqRoot, provider, accountId, signal) {
259
290
  next.marker = signal.marker ?? `needs credits for ${signal.model}`;
260
291
  delete next.quota_reset_at;
261
292
  }
262
- if (signal.quota_reset_at && !signal.needs_credits) {
293
+ if (signal.quota_reset_at &&
294
+ !signal.needs_credits &&
295
+ shouldReplaceReset(next.quota_reset_at, signal.quota_reset_at)) {
263
296
  next.quota_reset_at = signal.quota_reset_at;
264
297
  }
265
298
  models[signal.model] = next;
266
299
  }
267
300
  const account = { ...current, models };
268
301
  if (signal.account_wide && signal.quota_reset_at) {
269
- account.all_models_quota_reset_at = signal.quota_reset_at;
270
- account.reset_source = signal.reset_source ?? "provider";
302
+ if (shouldReplaceReset(current.all_models_quota_reset_at, signal.quota_reset_at)) {
303
+ account.all_models_quota_reset_at = signal.quota_reset_at;
304
+ account.reset_source = signal.reset_source ?? "provider";
305
+ }
306
+ else if (current.all_models_quota_reset_at) {
307
+ account.all_models_quota_reset_at = current.all_models_quota_reset_at;
308
+ account.reset_source = current.reset_source ?? "provider";
309
+ }
271
310
  }
272
311
  accounts[accountId] = account;
273
312
  state.providers[provider] = accounts;
274
313
  writeQuotaState(hqRoot, state);
275
314
  return state;
276
315
  }
316
+ function shouldReplaceReset(current, incoming) {
317
+ if (!current)
318
+ return true;
319
+ const currentMs = Date.parse(current);
320
+ const incomingMs = Date.parse(incoming);
321
+ if (!Number.isFinite(currentMs))
322
+ return Number.isFinite(incomingMs);
323
+ if (!Number.isFinite(incomingMs))
324
+ return false;
325
+ return incomingMs > currentMs;
326
+ }
277
327
  function resetStillAhead(iso, now) {
278
328
  if (!iso)
279
329
  return false;
@@ -12,12 +12,15 @@ import type { LaneRecord, LivenessState, QuestionRecord, SeniorRef } from "./typ
12
12
  import { type LaneMesh } from "./mesh-events.js";
13
13
  /** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
14
14
  export declare const QUESTION_AGE_MS: number;
15
- /** Spec §6.6 + §9: v1 permits exactly one aged advance. */
16
- export declare const MAX_AGED_ADVANCES = 1;
15
+ /** Spec §6.6 + §9: a depth-3 worker has two hops to age toward the user. */
16
+ export declare const MAX_AGED_ADVANCES: number;
17
17
  /** Aligns with watcher silence: a host that has not reported for this long is dead. */
18
18
  export declare const DEAD_HOST_MS: number;
19
19
  export declare const RECONCILE_LOCK_STALE_MS = 30000;
20
20
  export declare const RECONCILE_LOCK_TIMEOUT_MS = 5000;
21
+ /** A pickup probe that has not cleared its marker within this window is abandoned. */
22
+ export declare const PICKUP_VALIDATION_MAX_AGE_MS: number;
23
+ export declare const PICKUP_VALIDATION_TIMEOUT_REASON: "pickup_validation_timeout";
21
24
  export declare const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
22
25
  export declare const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
23
26
  export interface ReconcileSkip {
@@ -9,6 +9,7 @@ import * as fs from "node:fs";
9
9
  import * as path from "node:path";
10
10
  import { parseUserYaml } from "../../utils/user-yaml-error.js";
11
11
  import { accountMaxRunning, loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
12
+ import { MAX_LANE_DEPTH } from "./constants.js";
12
13
  import { LanesError, isLaneVersionConflict } from "./errors.js";
13
14
  import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDaemonPresenceConnected, } from "./liveness.js";
14
15
  import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, isPidRecordAlive, laneRunDir, readPidRecord, readSpawnManifest, spawnEnvelopeRepair, spawnLaneWorker, } from "./spawn.js";
@@ -22,12 +23,15 @@ import { decideEnvelopeRepair, envelopeQuestionIds, routeProviderFailure } from
22
23
  import { providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
23
24
  /** Spec §9: unanswered at the current hop for 60 minutes → one hop. */
24
25
  export const QUESTION_AGE_MS = 60 * 60 * 1000;
25
- /** Spec §6.6 + §9: v1 permits exactly one aged advance. */
26
- export const MAX_AGED_ADVANCES = 1;
26
+ /** Spec §6.6 + §9: a depth-3 worker has two hops to age toward the user. */
27
+ export const MAX_AGED_ADVANCES = MAX_LANE_DEPTH - 1;
27
28
  /** Aligns with watcher silence: a host that has not reported for this long is dead. */
28
29
  export const DEAD_HOST_MS = SILENCE_MS;
29
30
  export const RECONCILE_LOCK_STALE_MS = 30_000;
30
31
  export const RECONCILE_LOCK_TIMEOUT_MS = 5_000;
32
+ /** A pickup probe that has not cleared its marker within this window is abandoned. */
33
+ export const PICKUP_VALIDATION_MAX_AGE_MS = 15 * 60 * 1000;
34
+ export const PICKUP_VALIDATION_TIMEOUT_REASON = "pickup_validation_timeout";
31
35
  export const RECONCILE_UNIT_NAME = "hq-lanes-reconcile.service";
32
36
  export const RECONCILE_TIMER_NAME = "hq-lanes-reconcile.timer";
33
37
  function logSkip(skip) {
@@ -309,6 +313,89 @@ function casQuestionIf(hqRoot, questionId, shouldWrite, updater, dryRun) {
309
313
  }
310
314
  throw new LanesError("version_conflict", `Exhausted CAS retries for question ${questionId}`, { question_id: questionId });
311
315
  }
316
+ async function emitAgedQuestionAtUser(summary, deps, lane, question) {
317
+ if (deps.dryRun)
318
+ return;
319
+ const mesh = deps.mesh === undefined ? createLaneMesh({ interactive: false }) : deps.mesh;
320
+ if (!mesh || !lane.company || !lane.thread_id)
321
+ return;
322
+ const timeoutMs = deps.meshEmitTimeoutMs ?? 10_000;
323
+ let timer;
324
+ try {
325
+ await Promise.race([
326
+ mesh.appendThreadEvent(lane.company, lane.thread_id, "lane_question_at_user", { lane_id: lane.lane_id, question_id: question.question_id }),
327
+ new Promise((_resolve, reject) => {
328
+ timer = setTimeout(() => reject(new Error(`question-at-user mesh emit timed out after ${timeoutMs}ms`)), timeoutMs);
329
+ }),
330
+ ]);
331
+ }
332
+ catch (err) {
333
+ markDirty(summary, {
334
+ kind: "question",
335
+ id: question.question_id,
336
+ why: `question-at-user mesh emit failed: ${err instanceof Error ? err.message : String(err)}`,
337
+ });
338
+ }
339
+ finally {
340
+ if (timer)
341
+ clearTimeout(timer);
342
+ }
343
+ }
344
+ function pickupValidationPendingAtMs(lane) {
345
+ const raw = lane.pickup_validation_pending_since ?? lane.updated_at;
346
+ const at = Date.parse(raw);
347
+ return Number.isFinite(at) ? at : null;
348
+ }
349
+ function pickupValidationExpired(lane, nowMs) {
350
+ const pendingAtMs = pickupValidationPendingAtMs(lane);
351
+ return pendingAtMs === null || nowMs - pendingAtMs > PICKUP_VALIDATION_MAX_AGE_MS;
352
+ }
353
+ function expireAbandonedPickupValidation(hqRoot, lanes, summary, deps, now) {
354
+ const nowMs = now.getTime();
355
+ for (const lane of lanes) {
356
+ if (lane.state !== "queued" ||
357
+ lane.pickup_validation_pending !== true ||
358
+ !laneRunsHere(lane, deps.host) ||
359
+ !pickupValidationExpired(lane, nowMs)) {
360
+ continue;
361
+ }
362
+ try {
363
+ const result = casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "queued" &&
364
+ current.pickup_validation_pending === true &&
365
+ pickupValidationExpired(current, nowMs), (current) => {
366
+ const pendingAtMs = pickupValidationPendingAtMs(current);
367
+ return {
368
+ ...current,
369
+ state: "failed",
370
+ queued_reason: undefined,
371
+ pickup_validation_pending: undefined,
372
+ pickup_validation_pending_since: undefined,
373
+ terminal_reason: PICKUP_VALIDATION_TIMEOUT_REASON,
374
+ terminal_details: {
375
+ pickup_validation_pending_since: current.pickup_validation_pending_since ?? current.updated_at,
376
+ age_ms: pendingAtMs === null ? null : Math.max(0, nowMs - pendingAtMs),
377
+ max_age_ms: PICKUP_VALIDATION_MAX_AGE_MS,
378
+ },
379
+ };
380
+ }, deps.dryRun);
381
+ if (result.changed) {
382
+ summary.aged.push({
383
+ id: lane.lane_id,
384
+ from: "queued",
385
+ to: "failed",
386
+ reason: PICKUP_VALIDATION_TIMEOUT_REASON,
387
+ });
388
+ }
389
+ }
390
+ catch (err) {
391
+ markDirty(summary, {
392
+ kind: "lane",
393
+ id: lane.lane_id,
394
+ why: `pickup validation expiry failed: ${err instanceof Error ? err.message : String(err)}`,
395
+ });
396
+ }
397
+ }
398
+ }
312
399
  function fileNonEmpty(filePath) {
313
400
  try {
314
401
  return fs.statSync(filePath).size > 0;
@@ -589,6 +676,10 @@ export async function runPassBody(hqRoot, deps) {
589
676
  to: String(nextHop),
590
677
  reason: "unanswered_timeout",
591
678
  });
679
+ if (nextAudience.kind === "user") {
680
+ const originLane = laneById.get(question.lane_id) ?? readLane(hqRoot, question.lane_id);
681
+ await emitAgedQuestionAtUser(summary, deps, originLane, result.record);
682
+ }
592
683
  }
593
684
  }
594
685
  }
@@ -862,6 +953,7 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
862
953
  }
863
954
  }
864
955
  async function admitQueued(hqRoot, lanes, summary, deps, now) {
956
+ expireAbandonedPickupValidation(hqRoot, lanes, summary, deps, now);
865
957
  // A lane whose pickup probe is still out is not admitted: its record is
866
958
  // about to be bound or failed by the pickup, and a worker spawned now
867
959
  // would keep running behind that write.
@@ -257,6 +257,8 @@ export interface LaneRecord {
257
257
  * leave a running worker behind a record the probe result then rewrites.
258
258
  */
259
259
  pickup_validation_pending?: true;
260
+ /** When the pickup validation marker was written. */
261
+ pickup_validation_pending_since?: string;
260
262
  priority: number;
261
263
  worker?: string;
262
264
  /** Registry-relative directory of the resolved worker, when --worker was set. */
@@ -198,7 +198,8 @@ export const LANE_FIELD_WRITERS = {
198
198
  project_id: ["command"], story_id: ["command"], thread_id: ["creator", "command"],
199
199
  depth: ["command"], senior: ["command"], senior_history: ["command"],
200
200
  state: ["creator", "command", "watcher", "reconcile", "spawn"], queued_reason: ["creator", "command", "spawn", "reconcile"],
201
- pickup_validation_pending: ["creator", "command", "spawn"],
201
+ pickup_validation_pending: ["creator", "command", "spawn", "reconcile"],
202
+ pickup_validation_pending_since: ["creator", "command", "spawn", "reconcile"],
202
203
  priority: ["command"], worker: ["creator"], worker_path: ["creator"], worker_pack: ["creator"],
203
204
  provider: ["creator"], account: ["creator", "spawn", "command"], account_history: ["command"], pending_resume: ["command", "spawn"], host: ["creator"], origin_host: ["creator"], origin_host_source: ["creator"],
204
205
  version: ["creator"], created_at: ["creator"], updated_at: ["creator"], attribution_history: ["command"],
@@ -5,6 +5,7 @@
5
5
  import { type CognitoActorKind, type CognitoTokenSource } from "../../../../utils/cognito-session.js";
6
6
  import { type UnattributedEventCounts } from "../spool.js";
7
7
  import { type DaemonStateFile } from "./state.js";
8
+ import { type LegacyListenPlatform } from "./legacy-listen.js";
8
9
  export declare const UNHEALTHY_SPOOL_AGE_MS = 60000;
9
10
  export interface DaemonDoctorReport {
10
11
  running: boolean;
@@ -54,6 +55,8 @@ export interface DaemonDoctorReport {
54
55
  };
55
56
  /** Work held or quarantined after default-derived membership was denied. */
56
57
  deviceDefaultAuthorizationDenials: number;
58
+ /** True when the pack-era listen unit is still on disk. */
59
+ legacyListenerInstalled: boolean;
57
60
  }
58
61
  export interface DaemonDoctorDeps {
59
62
  home?: string;
@@ -64,6 +67,9 @@ export interface DaemonDoctorDeps {
64
67
  env?: NodeJS.ProcessEnv;
65
68
  isProcessAlive?: (pid: number) => boolean;
66
69
  now?: () => number;
70
+ /** Override the legacy-listen probe (tests). */
71
+ legacyListenerInstalled?: boolean;
72
+ platform?: LegacyListenPlatform;
67
73
  }
68
74
  export declare function collectDaemonDoctor(deps?: DaemonDoctorDeps): DaemonDoctorReport;
69
75
  export declare function formatDaemonDoctor(report: DaemonDoctorReport): string[];
@@ -18,6 +18,7 @@ import { hasCurrentAcknowledgedRegistration, readHeldSessions, refWithLocalCompa
18
18
  import { readDaemonConfig } from "./config.js";
19
19
  import { pidLockStatus, readPidLock } from "./pid-lock.js";
20
20
  import { readDaemonState } from "./state.js";
21
+ import { LEGACY_LISTENER_DOCTOR_LINE, legacyListenInstalled, } from "./legacy-listen.js";
21
22
  export const UNHEALTHY_SPOOL_AGE_MS = 60_000;
22
23
  function oldestJsonlAgeMs(filePath, nowMs) {
23
24
  if (!fs.existsSync(filePath))
@@ -144,6 +145,14 @@ export function collectDaemonDoctor(deps = {}) {
144
145
  if (presenceRefusal) {
145
146
  unhealthyReasons.push(`presence credential refused (${presenceRefusal.code}); next retry ${presenceRefusal.nextRetryAt}`);
146
147
  }
148
+ const platform = deps.platform ?? process.platform;
149
+ const legacyListener = deps.legacyListenerInstalled ??
150
+ (platform === "darwin" || platform === "linux"
151
+ ? legacyListenInstalled({ home, platform })
152
+ : false);
153
+ if (legacyListener) {
154
+ unhealthyReasons.push(LEGACY_LISTENER_DOCTOR_LINE);
155
+ }
147
156
  const auth = describeCognitoTokenSource({ home, env });
148
157
  const whoamiTokens = readPersonTokenFile(home, env);
149
158
  const daemonTokens = auth.tokenSource === "machine"
@@ -192,6 +201,7 @@ export function collectDaemonDoctor(deps = {}) {
192
201
  }
193
202
  : undefined,
194
203
  deviceDefaultAuthorizationDenials,
204
+ legacyListenerInstalled: legacyListener,
195
205
  };
196
206
  }
197
207
  export function formatDaemonDoctor(report) {
@@ -200,6 +210,7 @@ export function formatDaemonDoctor(report) {
200
210
  `mode: ${report.emitMode}`,
201
211
  `mqtt: ${report.mqttState}`,
202
212
  `subscription: ${report.subscribed ? "connected" : "disconnected"}`,
213
+ ...(report.legacyListenerInstalled ? [LEGACY_LISTENER_DOCTOR_LINE] : []),
203
214
  `companies online: ${report.companiesOnline.length ? report.companiesOnline.join(", ") : "(none)"}`,
204
215
  ];
205
216
  if (report.emitMode === "legacy") {
@@ -235,6 +235,13 @@ export declare function installDaemonService(opts: {
235
235
  schtasks?: SchtasksRunner;
236
236
  /** Windows: terminate the recorded daemon pid when replacing a task (tests inject). */
237
237
  kill?: (pid: number) => void;
238
+ /** Service-manager runner for retiring the legacy listen unit (tests inject). */
239
+ runCommand?: (cmd: string, args: string[]) => {
240
+ status: number | null;
241
+ stdout: string;
242
+ stderr: string;
243
+ };
244
+ windowsLegacyTaskPresent?: boolean;
238
245
  }): InstallResult;
239
246
  export declare function uninstallDaemonService(opts: {
240
247
  platform?: DaemonInstallPlatform;
@@ -19,6 +19,7 @@ import { resolveDaemonAssetDir } from "./log.js";
19
19
  import { LAUNCHD_LABEL, SYSTEMD_UNIT_NAME, WINDOWS_TASK_NAME, daemonDir, daemonLogPath, daemonPidPath, } from "./paths.js";
20
20
  import { isInstallableHqRoot, readDaemonConfig, registerTrustedHqRoot, verifyHqRoot } from "./config.js";
21
21
  import { defaultPidLockDeps, pidLockStatus, readPidLock } from "./pid-lock.js";
22
+ import { retireLegacyListen } from "./legacy-listen.js";
22
23
  /**
23
24
  * `schtasks /End` terminates the task's root process only. The daemon is
24
25
  * launched through `conhost.exe --headless`, so ending the task kills conhost
@@ -651,6 +652,15 @@ export function installDaemonService(opts) {
651
652
  const mkdirSync = opts.mkdirSync ?? fs.mkdirSync;
652
653
  mkdirSync(paths.daemonDir, { recursive: true, mode: 0o700 });
653
654
  registerTrustedHqRoot(paths.daemonDir, paths.hqRoot);
655
+ const retired = retireLegacyListen({
656
+ home: paths.home,
657
+ platform,
658
+ run: opts.runCommand,
659
+ windowsTaskPresent: opts.windowsLegacyTaskPresent,
660
+ });
661
+ const legacyNote = retired.removed
662
+ ? " Retired legacy listener ai.getindigo.hq-work-mesh-listen."
663
+ : "";
654
664
  if (platform === "darwin") {
655
665
  const rendered = renderLaunchdPlist(paths);
656
666
  mkdirSync(paths.launchAgentsDir, { recursive: true });
@@ -662,7 +672,7 @@ export function installDaemonService(opts) {
662
672
  paths,
663
673
  rendered,
664
674
  dest: paths.plistPath,
665
- message: `Installed LaunchAgent ${LAUNCHD_LABEL} → ${paths.plistPath}`,
675
+ message: `Installed LaunchAgent ${LAUNCHD_LABEL} → ${paths.plistPath}${legacyNote}`,
666
676
  };
667
677
  }
668
678
  if (platform === "linux") {
@@ -682,7 +692,7 @@ export function installDaemonService(opts) {
682
692
  dest,
683
693
  message: `Wrote system unit template for user ${opts.systemUser ?? paths.systemUser} at ${dest}. ` +
684
694
  `Install with: sudo cp ${dest} /etc/systemd/system/${SYSTEMD_UNIT_NAME} && ` +
685
- `sudo systemctl daemon-reload && sudo systemctl enable --now ${SYSTEMD_UNIT_NAME}`,
695
+ `sudo systemctl daemon-reload && sudo systemctl enable --now ${SYSTEMD_UNIT_NAME}${legacyNote}`,
686
696
  };
687
697
  }
688
698
  const rendered = renderSystemdUserUnit(paths);
@@ -696,7 +706,7 @@ export function installDaemonService(opts) {
696
706
  paths,
697
707
  rendered,
698
708
  dest: paths.systemdUnitPath,
699
- message: `Installed systemd user unit ${SYSTEMD_UNIT_NAME} → ${paths.systemdUnitPath}`,
709
+ message: `Installed systemd user unit ${SYSTEMD_UNIT_NAME} → ${paths.systemdUnitPath}${legacyNote}`,
700
710
  };
701
711
  }
702
712
  if (platform === "win32") {
@@ -728,9 +738,9 @@ export function installDaemonService(opts) {
728
738
  dest: paths.windowsTaskXmlPath,
729
739
  running,
730
740
  message: running
731
- ? `${rootLine}Installed Scheduled Task ${WINDOWS_TASK_NAME} → ${paths.windowsTaskXmlPath} (started; ${mode})`
741
+ ? `${rootLine}Installed Scheduled Task ${WINDOWS_TASK_NAME} → ${paths.windowsTaskXmlPath} (started; ${mode})${legacyNote}`
732
742
  : `${rootLine}Installed Scheduled Task ${WINDOWS_TASK_NAME} → ${paths.windowsTaskXmlPath} (${mode}); ` +
733
- `start failed: ${(started.stderr || started.stdout).trim()}. It starts at next logon.`,
743
+ `start failed: ${(started.stderr || started.stdout).trim()}. It starts at next logon.${legacyNote}`,
734
744
  notice: windowsLogonNotice(paths),
735
745
  };
736
746
  }
@@ -738,7 +748,7 @@ export function installDaemonService(opts) {
738
748
  platform,
739
749
  action: "install",
740
750
  paths,
741
- message: `Unsupported platform for service install; run: ${paths.nodeBinary} ${paths.hqBinary} mesh daemon run`,
751
+ message: `Unsupported platform for service install; run: ${paths.nodeBinary} ${paths.hqBinary} mesh daemon run${legacyNote}`,
742
752
  };
743
753
  }
744
754
  export function uninstallDaemonService(opts) {
@@ -0,0 +1,43 @@
1
+ /**
2
+ * Retire the pack-era work-mesh listen unit so only `hq mesh daemon` subscribes.
3
+ *
4
+ * macOS: ~/Library/LaunchAgents/ai.getindigo.hq-work-mesh-listen.plist
5
+ * Linux: ~/.config/systemd/user/hq-work-mesh-listen.service
6
+ * Windows: Scheduled Task ai.getindigo.hq-work-mesh-listen
7
+ */
8
+ export type LegacyListenPlatform = "darwin" | "linux" | "win32" | "other";
9
+ export declare const LEGACY_LISTEN_LABEL = "ai.getindigo.hq-work-mesh-listen";
10
+ export declare const LEGACY_LISTEN_SYSTEMD_UNIT = "hq-work-mesh-listen.service";
11
+ export declare const LEGACY_LISTEN_TASK = "ai.getindigo.hq-work-mesh-listen";
12
+ export declare const LEGACY_LISTENER_DOCTOR_LINE = "legacy listener still installed";
13
+ export interface CommandResult {
14
+ status: number | null;
15
+ stdout: string;
16
+ stderr: string;
17
+ }
18
+ export type CommandRunner = (cmd: string, args: string[]) => CommandResult;
19
+ export declare function legacyListenPlistPath(home: string): string;
20
+ export declare function legacyListenSystemdPath(home: string): string;
21
+ export declare function legacyListenInstalled(opts: {
22
+ home: string;
23
+ platform: LegacyListenPlatform;
24
+ existsSync?: (p: string) => boolean;
25
+ }): boolean;
26
+ /**
27
+ * Unload and delete the legacy listen unit when it is present.
28
+ * On-disk unit files are removed on every host. launchctl, systemctl, and
29
+ * schtasks run only for the matching platform, and only through `run` when
30
+ * the caller injected one (tests never hit a real service manager).
31
+ * Missing unit is a no-op.
32
+ */
33
+ export declare function retireLegacyListen(opts: {
34
+ home: string;
35
+ platform: LegacyListenPlatform;
36
+ run?: CommandRunner;
37
+ uid?: number;
38
+ /** Windows task probe: true when the scheduled task is registered. */
39
+ windowsTaskPresent?: boolean;
40
+ }): {
41
+ removed: boolean;
42
+ };
43
+ //# sourceMappingURL=legacy-listen.d.ts.map