@indigoai-us/hq-cli 5.225.0 → 5.226.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/CHANGELOG.md +38 -1
  2. package/dist/command-catalog.generated.d.ts +94 -0
  3. package/dist/command-catalog.generated.js +120 -0
  4. package/dist/command-registration-plan.d.ts +18 -0
  5. package/dist/command-registration-plan.js +6 -0
  6. package/dist/commands/agent-enroll.d.ts +2 -0
  7. package/dist/commands/agent-enroll.js +17 -2
  8. package/dist/commands/cloud.d.ts +10 -0
  9. package/dist/commands/cloud.js +52 -47
  10. package/dist/commands/lanes.d.ts +5 -0
  11. package/dist/commands/lanes.js +498 -312
  12. package/dist/commands/staff/reset-test-account.d.ts +110 -0
  13. package/dist/commands/staff/reset-test-account.js +268 -0
  14. package/dist/commands/staff/test-account.d.ts +51 -0
  15. package/dist/commands/staff/test-account.js +196 -0
  16. package/dist/lib/doctor/client-health-flag-token.d.ts +15 -0
  17. package/dist/lib/doctor/client-health-flag-token.js +66 -0
  18. package/dist/lib/doctor/client-health-reason-codes-flag.d.ts +1 -0
  19. package/dist/lib/doctor/client-health-reason-codes-flag.js +16 -5
  20. package/dist/lib/doctor/client-health-sentry-flag.d.ts +1 -0
  21. package/dist/lib/doctor/client-health-sentry-flag.js +17 -6
  22. package/dist/lib/lanes/dropbox.d.ts +9 -0
  23. package/dist/lib/lanes/dropbox.js +70 -1
  24. package/dist/lib/lanes/reconcile.d.ts +2 -0
  25. package/dist/lib/lanes/reconcile.js +12 -0
  26. package/dist/lib/lanes/watcher-main.js +16 -0
  27. package/dist/lib/lanes/watcher.d.ts +4 -1
  28. package/dist/lib/lanes/watcher.js +77 -3
  29. package/dist/utils/cognito-session.d.ts +8 -0
  30. package/dist/utils/cognito-session.js +198 -3
  31. package/dist/utils/upload-author.d.ts +15 -0
  32. package/dist/utils/upload-author.js +39 -0
  33. package/package.json +1 -1
@@ -0,0 +1,66 @@
1
+ import { ensureCognitoIdToken } from "../../utils/cognito-session.js";
2
+ export const CLIENT_HEALTH_FLAG_TOKEN_TIMEOUT_MS = 1_000;
3
+ export class MissingFlagTokenError extends Error {
4
+ constructor() {
5
+ super("No cached HQ identity token is available");
6
+ this.name = "MissingFlagTokenError";
7
+ }
8
+ }
9
+ export class ClientHealthFlagTokenTimeoutError extends Error {
10
+ constructor(timeoutMs) {
11
+ super(`Client-health flag token acquisition exceeded its ${timeoutMs} ms bound`);
12
+ this.name = "ClientHealthFlagTokenTimeoutError";
13
+ }
14
+ }
15
+ /**
16
+ * Obtain a non-interactive ID token, separately bounded from the flag read.
17
+ * ensureCognitoIdToken refreshes expired person sessions and selects machine
18
+ * identity credentials when the current installation is a machine.
19
+ */
20
+ export async function getClientHealthFlagToken(provider, timeoutMs = CLIENT_HEALTH_FLAG_TOKEN_TIMEOUT_MS, callerSignal) {
21
+ if (callerSignal?.aborted) {
22
+ throw callerSignal.reason ?? new Error("Token acquisition was aborted");
23
+ }
24
+ const controller = new AbortController();
25
+ const signal = callerSignal ? AbortSignal.any([callerSignal, controller.signal]) : controller.signal;
26
+ const timeout = new ClientHealthFlagTokenTimeoutError(timeoutMs);
27
+ const timer = setTimeout(() => controller.abort(timeout), timeoutMs);
28
+ let onAbort;
29
+ const aborted = new Promise((_resolve, reject) => {
30
+ onAbort = () => reject(signal.reason ?? new Error("Token acquisition was aborted"));
31
+ if (signal.aborted)
32
+ onAbort();
33
+ else
34
+ signal.addEventListener("abort", onAbort, { once: true });
35
+ });
36
+ try {
37
+ const acquire = provider ?? ((requestSignal) => ensureCognitoIdToken({ interactive: false, signal: requestSignal }));
38
+ const acquisition = Promise.resolve().then(() => {
39
+ if (signal.aborted) {
40
+ throw signal.reason ?? new Error("Token acquisition was aborted");
41
+ }
42
+ return acquire(signal);
43
+ });
44
+ const token = await Promise.race([acquisition, aborted]);
45
+ if (signal.aborted) {
46
+ throw signal.reason ?? new Error("Token acquisition was aborted");
47
+ }
48
+ if (typeof token !== "string" || token.trim().length === 0) {
49
+ throw new MissingFlagTokenError();
50
+ }
51
+ return token;
52
+ }
53
+ catch (error) {
54
+ if (error instanceof Error &&
55
+ error.message.startsWith("No valid HQ session and interactive login is disabled")) {
56
+ throw new MissingFlagTokenError();
57
+ }
58
+ throw error;
59
+ }
60
+ finally {
61
+ clearTimeout(timer);
62
+ if (onAbort)
63
+ signal.removeEventListener("abort", onAbort);
64
+ }
65
+ }
66
+ //# sourceMappingURL=client-health-flag-token.js.map
@@ -7,6 +7,7 @@ export interface ClientHealthReasonCodesFlagDependencies {
7
7
  env?: NodeJS.ProcessEnv;
8
8
  createClient?: (options: FlagClientOptions) => ClientHealthFlagClient;
9
9
  getToken?: FlagClientOptions["getToken"];
10
+ tokenTimeoutMs?: number;
10
11
  fetch?: typeof fetch;
11
12
  reportFailure?: (error: unknown) => void;
12
13
  }
@@ -1,6 +1,6 @@
1
- import { loadCachedTokens } from "@indigoai-us/hq-cloud";
2
1
  import { createFlagClient, } from "@indigoai-us/hq-flags-client";
3
2
  import { resolveFlagRegistryEndpoint } from "../flag-registry-endpoint.js";
3
+ import { getClientHealthFlagToken } from "./client-health-flag-token.js";
4
4
  export const CLIENT_HEALTH_REASON_CODES_FLAG = "client-health-sync-reason-codes";
5
5
  export const CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK = false;
6
6
  export const CLIENT_HEALTH_REASON_CODES_FLAG_TIMEOUT_MS = 150;
@@ -39,9 +39,8 @@ export async function clientHealthReasonCodesEnabledForCompany(company, dependen
39
39
  return CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK;
40
40
  }
41
41
  const companySlug = company.companySlug?.trim() || "";
42
- const deadline = AbortSignal.timeout(CLIENT_HEALTH_REASON_CODES_FLAG_TIMEOUT_MS);
43
- const signal = callerSignal ? AbortSignal.any([callerSignal, deadline]) : deadline;
44
- const doFetch = dependencies.fetch ?? globalThis.fetch;
42
+ if (callerSignal?.aborted)
43
+ return CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK;
45
44
  let client;
46
45
  let failed = false;
47
46
  let failureReported = false;
@@ -53,6 +52,18 @@ export async function clientHealthReasonCodesEnabledForCompany(company, dependen
53
52
  failureReported = true;
54
53
  (dependencies.reportFailure ?? reportFlagFailure)(error);
55
54
  };
55
+ let token;
56
+ try {
57
+ token = await getClientHealthFlagToken(dependencies.getToken, dependencies.tokenTimeoutMs, callerSignal);
58
+ }
59
+ catch (error) {
60
+ if (!callerSignal?.aborted)
61
+ reportFailureOnce(error);
62
+ return CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK;
63
+ }
64
+ const deadline = AbortSignal.timeout(CLIENT_HEALTH_REASON_CODES_FLAG_TIMEOUT_MS);
65
+ const signal = callerSignal ? AbortSignal.any([callerSignal, deadline]) : deadline;
66
+ const doFetch = dependencies.fetch ?? globalThis.fetch;
56
67
  try {
57
68
  client = (dependencies.createClient ?? createFlagClient)({
58
69
  endpoint,
@@ -61,7 +72,7 @@ export async function clientHealthReasonCodesEnabledForCompany(company, dependen
61
72
  ? { companyIdentifiers: [companySlug] }
62
73
  : {}),
63
74
  env: {},
64
- getToken: dependencies.getToken ?? (() => loadCachedTokens()?.idToken ?? ""),
75
+ getToken: () => token,
65
76
  fetch: (input, init) => doFetch(input, {
66
77
  ...init,
67
78
  signal: combineSignals(init?.signal ?? undefined, signal),
@@ -7,6 +7,7 @@ export interface ClientHealthSentryFlagDependencies {
7
7
  env?: NodeJS.ProcessEnv;
8
8
  createClient?: (options: FlagClientOptions) => ClientHealthFlagClient;
9
9
  getToken?: FlagClientOptions["getToken"];
10
+ tokenTimeoutMs?: number;
10
11
  fetch?: typeof fetch;
11
12
  reportFailure?: (error: unknown) => void;
12
13
  }
@@ -1,6 +1,6 @@
1
- import { loadCachedTokens } from "@indigoai-us/hq-cloud";
2
1
  import { createFlagClient, } from "@indigoai-us/hq-flags-client";
3
2
  import { resolveFlagRegistryEndpoint } from "../flag-registry-endpoint.js";
3
+ import { getClientHealthFlagToken } from "./client-health-flag-token.js";
4
4
  export const CLIENT_HEALTH_SENTRY_DIRECT_FLAG = "client-health.sentry-direct";
5
5
  export const CLIENT_HEALTH_SENTRY_DIRECT_FALLBACK = false;
6
6
  export const CLIENT_HEALTH_SENTRY_DIRECT_TIMEOUT_MS = 150;
@@ -31,10 +31,8 @@ export async function clientHealthSentryDirectEnabledForCompany(company, depende
31
31
  if (!/^cmp_[A-Za-z0-9]{3,128}$/.test(companyUid)) {
32
32
  return CLIENT_HEALTH_SENTRY_DIRECT_FALLBACK;
33
33
  }
34
- const companySlug = company.companySlug?.trim() || "";
35
- const deadline = AbortSignal.timeout(CLIENT_HEALTH_SENTRY_DIRECT_TIMEOUT_MS);
36
- const signal = callerSignal ? AbortSignal.any([callerSignal, deadline]) : deadline;
37
- const doFetch = dependencies.fetch ?? globalThis.fetch;
34
+ if (callerSignal?.aborted)
35
+ return CLIENT_HEALTH_SENTRY_DIRECT_FALLBACK;
38
36
  let client;
39
37
  let failed = false;
40
38
  let failureReported = false;
@@ -46,6 +44,19 @@ export async function clientHealthSentryDirectEnabledForCompany(company, depende
46
44
  failureReported = true;
47
45
  (dependencies.reportFailure ?? reportFlagFailure)(error);
48
46
  };
47
+ let token;
48
+ try {
49
+ token = await getClientHealthFlagToken(dependencies.getToken, dependencies.tokenTimeoutMs, callerSignal);
50
+ }
51
+ catch (error) {
52
+ if (!callerSignal?.aborted)
53
+ reportFailureOnce(error);
54
+ return CLIENT_HEALTH_SENTRY_DIRECT_FALLBACK;
55
+ }
56
+ const companySlug = company.companySlug?.trim() || "";
57
+ const deadline = AbortSignal.timeout(CLIENT_HEALTH_SENTRY_DIRECT_TIMEOUT_MS);
58
+ const signal = callerSignal ? AbortSignal.any([callerSignal, deadline]) : deadline;
59
+ const doFetch = dependencies.fetch ?? globalThis.fetch;
49
60
  try {
50
61
  client = (dependencies.createClient ?? createFlagClient)({
51
62
  endpoint: resolveFlagRegistryEndpoint(undefined, env),
@@ -54,7 +65,7 @@ export async function clientHealthSentryDirectEnabledForCompany(company, depende
54
65
  ? { companyIdentifiers: [companySlug] }
55
66
  : {}),
56
67
  env: {},
57
- getToken: dependencies.getToken ?? (() => loadCachedTokens()?.idToken ?? ""),
68
+ getToken: () => token,
58
69
  fetch: (input, init) => doFetch(input, {
59
70
  ...init,
60
71
  signal: combineSignals(init?.signal ?? undefined, signal),
@@ -50,6 +50,8 @@ export interface DrainDeps {
50
50
  afterRead?: () => void;
51
51
  /** Prefix marks archives that belong to one provider turn. */
52
52
  archivePrefix?: string;
53
+ /** Terminal messages with no permitted delivery path are kept separately. */
54
+ archiveDirectory?: "delivered" | "undeliverable";
53
55
  }
54
56
  export interface LaneDrainHookInput {
55
57
  transcript_path?: string | null;
@@ -68,6 +70,13 @@ export declare function mintClaimId(): string;
68
70
  export declare function assertSafeLaneId(laneId: string): void;
69
71
  export declare function dropboxDir(hqRoot: string, laneId: string): string;
70
72
  export declare function dropboxInboxPath(hqRoot: string, laneId: string): string;
73
+ /** A recovery drain can claim the inbox, draining files, or dead inflight files. */
74
+ export declare function dropboxInboxHasMessages(hqRoot: string, laneId: string): boolean;
75
+ /**
76
+ * Return a failed recovery claim to the append-only inbox without replacing
77
+ * messages appended while resume validation or spawn was in progress.
78
+ */
79
+ export declare function requeueDropboxMessages(hqRoot: string, laneId: string, messages: readonly DropboxMessage[]): void;
71
80
  /**
72
81
  * Worker tool-call liveness (watcher.ts reads it). A Claude worker run with
73
82
  * `--output-format json` writes nothing to worker.out until it exits, so the
@@ -65,6 +65,75 @@ export function dropboxDir(hqRoot, laneId) {
65
65
  export function dropboxInboxPath(hqRoot, laneId) {
66
66
  return path.join(dropboxDir(hqRoot, laneId), INBOX_NAME);
67
67
  }
68
+ function directoryHasRecoverableFiles(dir, nestedInboxOnly = false) {
69
+ let names;
70
+ try {
71
+ names = fs.readdirSync(dir).filter((name) => !name.startsWith("."));
72
+ }
73
+ catch (err) {
74
+ if (err.code === "ENOENT")
75
+ return false;
76
+ throw err;
77
+ }
78
+ for (const name of names) {
79
+ const candidate = path.join(dir, name);
80
+ let stat;
81
+ try {
82
+ stat = fs.statSync(candidate);
83
+ }
84
+ catch (err) {
85
+ if (err.code === "ENOENT")
86
+ continue;
87
+ throw err;
88
+ }
89
+ if (nestedInboxOnly) {
90
+ if (!stat.isDirectory())
91
+ continue;
92
+ try {
93
+ const inbox = fs.statSync(path.join(candidate, INBOX_NAME));
94
+ if (inbox.isFile() && inbox.size > 0)
95
+ return true;
96
+ }
97
+ catch (err) {
98
+ if (err.code !== "ENOENT")
99
+ throw err;
100
+ }
101
+ }
102
+ else if (stat.isFile() && stat.size > 0) {
103
+ return true;
104
+ }
105
+ }
106
+ return false;
107
+ }
108
+ function fileHasBytes(filePath) {
109
+ try {
110
+ const stat = fs.statSync(filePath);
111
+ return stat.isFile() && stat.size > 0;
112
+ }
113
+ catch (err) {
114
+ if (err.code === "ENOENT")
115
+ return false;
116
+ throw err;
117
+ }
118
+ }
119
+ /** A recovery drain can claim the inbox, draining files, or dead inflight files. */
120
+ export function dropboxInboxHasMessages(hqRoot, laneId) {
121
+ const box = dropboxDir(hqRoot, laneId);
122
+ return fileHasBytes(path.join(box, INBOX_NAME)) ||
123
+ directoryHasRecoverableFiles(path.join(box, DRAINING_DIR)) ||
124
+ directoryHasRecoverableFiles(path.join(box, INFLIGHT_DIR), true);
125
+ }
126
+ /**
127
+ * Return a failed recovery claim to the append-only inbox without replacing
128
+ * messages appended while resume validation or spawn was in progress.
129
+ */
130
+ export function requeueDropboxMessages(hqRoot, laneId, messages) {
131
+ if (messages.length === 0)
132
+ return;
133
+ const inbox = dropboxInboxPath(hqRoot, laneId);
134
+ fs.mkdirSync(path.dirname(inbox), { recursive: true });
135
+ fs.appendFileSync(inbox, messages.map((message) => `${JSON.stringify(message)}\n`).join(""), "utf8");
136
+ }
68
137
  /**
69
138
  * Worker tool-call liveness (watcher.ts reads it). A Claude worker run with
70
139
  * `--output-format json` writes nothing to worker.out until it exits, so the
@@ -220,7 +289,7 @@ export function drainDropboxAtPath(box, depsInput = {}) {
220
289
  return [];
221
290
  const inbox = path.join(box, INBOX_NAME);
222
291
  const draining = path.join(box, DRAINING_DIR);
223
- const delivered = path.join(box, DELIVERED_DIR);
292
+ const delivered = path.join(box, deps.archiveDirectory ?? DELIVERED_DIR);
224
293
  const inflightRoot = path.join(box, INFLIGHT_DIR);
225
294
  try {
226
295
  deps.mkdirSync(box, { recursive: true });
@@ -81,6 +81,8 @@ export interface ReconcileDeps {
81
81
  readRegistryFlag?: typeof readLanesRegistryFlag;
82
82
  /** Active-company fallback for questions raised from an unscoped lane. */
83
83
  resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
84
+ /** Resume queued messages after this pass wins a local dead-worker CAS. */
85
+ recoverQueuedMessages?: (lane: LaneRecord) => Promise<void>;
84
86
  /** Detached mesh seam for dead-watcher recovery. */
85
87
  mesh?: LaneMesh | null;
86
88
  /** Bound on one terminal mesh emit during dead-host recovery. Default 10 s. */
@@ -1172,6 +1172,18 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
1172
1172
  }
1173
1173
  }
1174
1174
  }
1175
+ if (!deps.dryRun && deps.recoverQueuedMessages) {
1176
+ try {
1177
+ await deps.recoverQueuedMessages(result.record);
1178
+ }
1179
+ catch (err) {
1180
+ markDirty(summary, {
1181
+ kind: "lane",
1182
+ id: lane.lane_id,
1183
+ why: `queued message recovery failed: ${err instanceof Error ? err.message : String(err)}`,
1184
+ });
1185
+ }
1186
+ }
1175
1187
  }
1176
1188
  }
1177
1189
  catch (err) {
@@ -5,6 +5,9 @@
5
5
  * Usage: node watcher-main.js <lane-id>
6
6
  * Requires HQ_ROOT in the environment.
7
7
  */
8
+ import { spawnSync } from "node:child_process";
9
+ import * as path from "node:path";
10
+ import { fileURLToPath } from "node:url";
8
11
  import { runWatcherLoop } from "./watcher.js";
9
12
  import { LanesError } from "./errors.js";
10
13
  import { openMeshTransport } from "../mesh/client.js";
@@ -27,8 +30,21 @@ function main() {
27
30
  process.stderr.write("hq lanes watcher: requires lane id argv and HQ_ROOT in the environment\n");
28
31
  process.exit(1);
29
32
  }
33
+ const cliEntry = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../index.js");
30
34
  void runWatcherLoop(hqRoot, laneId, {
31
35
  resolveQuestionActiveCompany,
36
+ recoverQueuedMessages: async () => {
37
+ const result = spawnSync(process.execPath, [cliEntry, "lanes", "_recover-queued-messages", laneId], {
38
+ env: process.env,
39
+ encoding: "utf8",
40
+ timeout: 120_000,
41
+ });
42
+ if (result.error)
43
+ throw result.error;
44
+ if (result.status !== 0) {
45
+ throw new Error(`recovery command exited ${result.status ?? result.signal ?? "without status"}: ${(result.stderr ?? "").trim()}`);
46
+ }
47
+ },
32
48
  }).catch((err) => {
33
49
  process.stderr.write(`hq lanes watcher: ${err instanceof Error ? err.message : String(err)}${err instanceof Error && err.stack ? `\n${err.stack}` : ""}\n`);
34
50
  process.exit(1);
@@ -48,7 +48,7 @@ export declare const WATCHER_HEARTBEAT_MS: number;
48
48
  * once the file would exceed this, the oldest complete lines are dropped.
49
49
  */
50
50
  export declare const WATCHER_OUT_MAX_BYTES: number;
51
- export declare const WATCHER_DECISION_EVENTS: readonly ["liveness_transition", "silence_deadline", "envelope_observed", "envelope_invalid", "envelope_repair", "classification", "terminal_write", "mesh_event", "mesh_event_failed"];
51
+ export declare const WATCHER_DECISION_EVENTS: readonly ["liveness_transition", "silence_deadline", "envelope_observed", "envelope_invalid", "envelope_repair", "classification", "terminal_write", "message_recovery", "message_recovery_failed", "message_undeliverable", "mesh_event", "mesh_event_failed"];
52
52
  /** Terminal reason when the envelope failed to parse after the repair turn (spec §4). */
53
53
  export { ENVELOPE_INVALID_REASON };
54
54
  export type WatcherDecisionEvent = (typeof WATCHER_DECISION_EVENTS)[number];
@@ -222,6 +222,8 @@ export interface WatcherLoopDeps {
222
222
  retrySpawnDeps?: Omit<SpawnLaneDeps, "admissionHost">;
223
223
  /** Active-company fallback when an unscoped lane asks from its envelope. */
224
224
  resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
225
+ /** Resume an atomically queued inbox after this watcher wins terminal CAS. */
226
+ recoverQueuedMessages?: (lane: LaneRecord) => Promise<void>;
225
227
  }
226
228
  /**
227
229
  * Wrap one mesh call. A failed emit is logged as `mesh_event_failed` with
@@ -231,6 +233,7 @@ export interface WatcherLoopDeps {
231
233
  export declare function emitMeshSoft(event: string, logDecision: (record: Record<string, unknown>) => void, at: string, fn: (signal: AbortSignal) => Promise<void>, timeoutMs?: number): Promise<boolean>;
232
234
  /** Question ids an ask envelope raised, as `raiseEnvelopeQuestions` mints them. */
233
235
  export declare function envelopeQuestionIds(laneId: string, envelopeRef: string, envelope: LaneEnvelope, questionIndexes?: readonly number[]): string[];
236
+ /** Detached and synchronous classifiers share one process-identity claim. */
234
237
  export declare function runWatcherLoop(hqRoot: string, laneId: string, deps?: WatcherLoopDeps): Promise<void>;
235
238
  export type EnvelopeRepairDecision = {
236
239
  action: "spawned";
@@ -37,16 +37,16 @@ import { LanesError } from "./errors.js";
37
37
  import { questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
38
38
  import { isTerminalLaneState, ENVELOPE_INVALID_REASON, parseLaneEnvelope, STORY_STATUSES, } from "./types.js";
39
39
  import { laneStories, storyStatusForEnvelope } from "./stories.js";
40
- import { readLane, retryUpdateLane } from "./store.js";
40
+ import { readLane, retryUpdateLane, withLaneHireLocks } from "./store.js";
41
41
  import { readWorkerRunResult } from "./delivery.js";
42
42
  import { applyDefaultQuotaHold, quotaSignalFromFields, readQuotaSignal, recordAccountQuota, } from "./quota.js";
43
43
  import { PROVIDER_REJECTED_REASON, QUOTA_STATUS, isProviderLoginFailure, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
44
- import { readToolActivity, toolActivityPath } from "./dropbox.js";
44
+ import { dropboxInboxHasMessages, readToolActivity, toolActivityPath } from "./dropbox.js";
45
45
  import { loadLanesConfig, providerProgressEventMinIntervalMs, providerQuotaDefaultHoldMs } from "./config.js";
46
46
  import { resolveAdmissionHost as resolveProcessAdmissionHost } from "./admission-host.js";
47
47
  import { createLaneMesh, emitLaneDecisionEvents, emitLaneProgressEvent, emitLaneTerminalStateEvent, laneStoryStatusSuppressionKey, progressEventDue, } from "./mesh-events.js";
48
48
  import { envelopeQuestionId } from "./questions.js";
49
- import { laneRunDir, missingPidProbe, probePidRecord, readPidRecord, readProcStarttime, readSpawnManifest, envelopeRepairMode, spawnEnvelopeRepair, retryLaneOnAlternateAccount, } from "./spawn.js";
49
+ import { laneRunDir, missingPidProbe, probePidRecord, readPidRecord, readProcStarttime, readSpawnManifest, envelopeRepairMode, spawnEnvelopeRepair, writePidRecord, retryLaneOnAlternateAccount, } from "./spawn.js";
50
50
  /** Spec §5.7: no signal for 20 minutes while the process is still alive. */
51
51
  export const SILENCE_MS = 20 * 60 * 1000;
52
52
  export const WATCHER_POLL_MS = 5_000;
@@ -66,6 +66,9 @@ export const WATCHER_DECISION_EVENTS = [
66
66
  "envelope_repair",
67
67
  "classification",
68
68
  "terminal_write",
69
+ "message_recovery",
70
+ "message_recovery_failed",
71
+ "message_undeliverable",
69
72
  "mesh_event",
70
73
  "mesh_event_failed",
71
74
  ];
@@ -670,7 +673,56 @@ function defaultSleep(ms) {
670
673
  setTimeout(resolve, ms);
671
674
  });
672
675
  }
676
+ const WATCHER_LOOP_OWNER_FILE = "watcher-loop.pid";
677
+ function claimWatcherLoop(hqRoot, laneId, deps) {
678
+ const runDir = laneRunDir(hqRoot, laneId);
679
+ const ownerPath = path.join(runDir, WATCHER_LOOP_OWNER_FILE);
680
+ const isAlive = deps.isAlive ?? isPidAlive;
681
+ const readStarttime = deps.readStarttime ?? readProcStarttime;
682
+ const owner = withLaneHireLocks(hqRoot, [laneId], () => {
683
+ fs.mkdirSync(runDir, { recursive: true });
684
+ const existing = readPidRecord(ownerPath);
685
+ if (existing !== null && probePidRecord(existing, isAlive, readStarttime).alive) {
686
+ return null;
687
+ }
688
+ if (existing !== null)
689
+ fs.unlinkSync(ownerPath);
690
+ writePidRecord(ownerPath, {
691
+ pid: process.pid,
692
+ starttime: readStarttime(process.pid) ?? readProcStarttime(process.pid) ?? undefined,
693
+ });
694
+ const stat = fs.statSync(ownerPath);
695
+ return { dev: stat.dev, ino: stat.ino };
696
+ });
697
+ if (owner === null)
698
+ return null;
699
+ return () => {
700
+ withLaneHireLocks(hqRoot, [laneId], () => {
701
+ try {
702
+ const current = fs.statSync(ownerPath);
703
+ if (current.dev === owner.dev && current.ino === owner.ino)
704
+ fs.unlinkSync(ownerPath);
705
+ }
706
+ catch (err) {
707
+ if (err.code !== "ENOENT")
708
+ throw err;
709
+ }
710
+ });
711
+ };
712
+ }
713
+ /** Detached and synchronous classifiers share one process-identity claim. */
673
714
  export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
715
+ const release = claimWatcherLoop(hqRoot, laneId, deps);
716
+ if (release === null)
717
+ return;
718
+ try {
719
+ await runWatcherLoopOwned(hqRoot, laneId, deps);
720
+ }
721
+ finally {
722
+ release();
723
+ }
724
+ }
725
+ async function runWatcherLoopOwned(hqRoot, laneId, deps) {
674
726
  const sleep = deps.sleep ?? defaultSleep;
675
727
  const now = deps.now ?? (() => Date.now());
676
728
  const isAlive = deps.isAlive ?? isPidAlive;
@@ -1118,6 +1170,28 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
1118
1170
  await emitMeshSoft(`lane_${updated.state}`, logDecision, tick.emit.at, (signal) => emitLaneTerminalStateEvent(mesh, updated, updated.state, updated.terminal_reason ?? classification.reason, signal), meshRequestTimeoutMs);
1119
1171
  }
1120
1172
  }
1173
+ if (terminalWrite.changed && deps.recoverQueuedMessages) {
1174
+ try {
1175
+ if (dropboxInboxHasMessages(hqRoot, laneId)) {
1176
+ await deps.recoverQueuedMessages(updated);
1177
+ logDecision({
1178
+ event: "message_recovery",
1179
+ at: tick.emit.at,
1180
+ state: updated.state,
1181
+ });
1182
+ }
1183
+ }
1184
+ catch (err) {
1185
+ const message = err instanceof Error ? err.message : String(err);
1186
+ process.stderr.write(`hq lanes watcher: queued message recovery failed for ${laneId}: ${message}\n`);
1187
+ logDecision({
1188
+ event: "message_recovery_failed",
1189
+ at: tick.emit.at,
1190
+ state: updated.state,
1191
+ error: message,
1192
+ });
1193
+ }
1194
+ }
1121
1195
  }
1122
1196
  }
1123
1197
  lastSignalMs = tick.nextLastSignalMs;
@@ -114,6 +114,8 @@ export type EnsureCognitoTokenOptions = {
114
114
  * cached. Explicit `"person"` never uses machine creds.
115
115
  */
116
116
  tokenSource?: CognitoTokenSource;
117
+ /** Cancels bounded, non-interactive person-session refreshes. */
118
+ signal?: AbortSignal;
117
119
  };
118
120
  /**
119
121
  * On-disk person Cognito cache path for `env` (`HQ_STATE_DIR` or `~/.hq`).
@@ -196,6 +198,12 @@ export declare function readPersonTokenFile(home?: string, env?: NodeJS.ProcessE
196
198
  * the response subject differs, put the bytes back and leave them there.
197
199
  */
198
200
  export declare function refreshActivePersonTokens(cached: CognitoTokens, env?: NodeJS.ProcessEnv): Promise<CognitoTokens>;
201
+ /**
202
+ * Client-health's short-lived flag read cannot use hq-cloud's unbounded refresh
203
+ * helper. Reuse its lockfile protocol, attach the caller signal to the refresh
204
+ * request, and persist only after the request has completed before the deadline.
205
+ */
206
+ export declare function refreshActivePersonTokensForClientHealth(cached: CognitoTokens, signal: AbortSignal): Promise<CognitoTokens>;
199
207
  /** Derive actor kind from an ID token's custom:entity* claims (no verify). */
200
208
  export declare function actorKindFromIdToken(idToken: string | undefined): CognitoActorKind;
201
209
  /**