@indigoai-us/hq-cli 5.345.86 → 5.345.88

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,313 @@
1
+ /**
2
+ * One writer per bot at a time.
3
+ *
4
+ * `hq bot create`, `adopt`, `restore`, `rm` and `set` all change the same
5
+ * bot's folder, and an adopt also rotates the bot's machine secret. Two of
6
+ * them running at once for the same bot (the desktop's automatic restore and
7
+ * its first-run create, a minute after sign-in) used to race on a
8
+ * check-then-write of bot.json: the loser either threw "already set up on this
9
+ * computer" or, worse, wrote a credential the winner had already rotated away.
10
+ *
11
+ * The lock is a file under the bots folder, `~/.hq/bots/.locks/<name>.lock`,
12
+ * created with O_CREAT|O_EXCL. It sits beside the bot's folder rather than in
13
+ * it: a folder that exists only to hold a lock would read as "a different bot
14
+ * already has this name" to the code that picks a restored bot's folder, and
15
+ * as "something is left here" to `hq bot rm`. Its body names the holder (pid,
16
+ * host, start time, a random token), so:
17
+ * - a holder whose process is gone, or that has held it for longer than
18
+ * {@link BOT_LOCK_STALE_MS}, is reclaimed;
19
+ * - release only removes the lock it took (the token must match), so a
20
+ * reclaim by another process never makes this process remove someone
21
+ * else's lock.
22
+ *
23
+ * The lock is re-entrant within one async call chain (AsyncLocalStorage):
24
+ * create holds it and calls the shared adopt path, which takes it again
25
+ * without waiting on itself. Two separate chains in one process still
26
+ * serialize, exactly like two processes.
27
+ */
28
+ import { AsyncLocalStorage } from "node:async_hooks";
29
+ import { randomUUID } from "node:crypto";
30
+ import * as fs from "node:fs";
31
+ import * as os from "node:os";
32
+ import * as path from "node:path";
33
+ import { botsRoot } from "./paths.js";
34
+ /** Not a valid bot name (it starts with a dot), so no bot can collide with it. */
35
+ export const BOT_LOCKS_DIRNAME = ".locks";
36
+ /** A holder older than this is treated as crashed, whatever its pid says. */
37
+ export const BOT_LOCK_STALE_MS = 10 * 60 * 1000;
38
+ /** How long a second writer waits for the first before giving up. */
39
+ export const BOT_LOCK_WAIT_MS = 3 * 60 * 1000;
40
+ /** A lock file with no readable body yet: its creator may still be writing it. */
41
+ const UNREADABLE_GRACE_MS = 5_000;
42
+ export class BotLockBusyError extends Error {
43
+ botName;
44
+ holder;
45
+ expected = true;
46
+ constructor(botName, holder) {
47
+ super(`Another hq bot command is changing "${botName}" right now${holder?.command ? ` (${holder.command})` : ""}. Try again in a moment.`);
48
+ this.botName = botName;
49
+ this.holder = holder;
50
+ this.name = "BotLockBusyError";
51
+ }
52
+ }
53
+ export function botLockPath(name, root) {
54
+ return path.join(root ?? botsRoot(), BOT_LOCKS_DIRNAME, `${name}.lock`);
55
+ }
56
+ /** Alive unless the OS says there is no such process. EPERM is a live process we may not signal. */
57
+ function defaultIsPidAlive(pid) {
58
+ if (!Number.isInteger(pid) || pid <= 0)
59
+ return false;
60
+ try {
61
+ process.kill(pid, 0);
62
+ return true;
63
+ }
64
+ catch (err) {
65
+ return err.code === "EPERM";
66
+ }
67
+ }
68
+ function parseHolder(raw) {
69
+ try {
70
+ const v = JSON.parse(raw);
71
+ if (typeof v.pid !== "number" || typeof v.startedAt !== "string" || typeof v.token !== "string")
72
+ return null;
73
+ return {
74
+ pid: v.pid,
75
+ host: typeof v.host === "string" ? v.host : "",
76
+ startedAt: v.startedAt,
77
+ token: v.token,
78
+ command: typeof v.command === "string" ? v.command : "",
79
+ };
80
+ }
81
+ catch {
82
+ return null;
83
+ }
84
+ }
85
+ /**
86
+ * Why an existing lock may be taken over, or null while its holder is live.
87
+ * Exported for tests.
88
+ */
89
+ export function botLockStaleReason(raw, mtimeMs, opts) {
90
+ const holder = parseHolder(raw);
91
+ if (!holder) {
92
+ // Created but not yet written, or torn by a crash. Give a live creator a
93
+ // moment to finish writing before calling it abandoned.
94
+ return opts.now - mtimeMs > UNREADABLE_GRACE_MS ? "unreadable lock" : null;
95
+ }
96
+ const started = Date.parse(holder.startedAt);
97
+ if (Number.isNaN(started))
98
+ return "unreadable start time";
99
+ if (opts.now - started > opts.staleMs)
100
+ return `held since ${holder.startedAt}`;
101
+ // ~/.hq/bots is per machine, so the pid is this machine's. The host name is
102
+ // not compared: macOS changes it with the network, which would leave a
103
+ // crashed holder blocking for the whole stale age.
104
+ if (!opts.isPidAlive(holder.pid))
105
+ return `holder pid ${holder.pid} has exited`;
106
+ return null;
107
+ }
108
+ const heldStorage = new AsyncLocalStorage();
109
+ /** Locks this process holds, so a hard exit (process.exit in a failure path) still lets go. */
110
+ const heldByProcess = new Map();
111
+ let exitHookInstalled = false;
112
+ function installExitHook() {
113
+ if (exitHookInstalled)
114
+ return;
115
+ exitHookInstalled = true;
116
+ process.on("exit", releaseAllHeldBotLocks);
117
+ }
118
+ /**
119
+ * Let go of every lock this process holds. Runs on process exit, so a command
120
+ * that ends through process.exit (every `hq bot` failure does) frees its bot
121
+ * at once instead of leaving it to the stale-holder reclaim.
122
+ */
123
+ export function releaseAllHeldBotLocks() {
124
+ for (const [file, token] of [...heldByProcess])
125
+ releaseFile(file, token);
126
+ }
127
+ function releaseFile(file, token) {
128
+ // Moved aside before it is judged, so a lock someone else took after
129
+ // reclaiming ours (a very long hold) is put back rather than deleted.
130
+ const aside = `${file}.release-${process.pid}-${randomUUID()}`;
131
+ try {
132
+ fs.renameSync(file, aside);
133
+ const raw = fs.readFileSync(aside, "utf8");
134
+ if (parseHolder(raw)?.token !== token) {
135
+ try {
136
+ fs.linkSync(aside, file);
137
+ }
138
+ catch {
139
+ /* a newer holder is already in place */
140
+ }
141
+ }
142
+ }
143
+ catch {
144
+ /* already gone, or unreadable: nothing of ours to remove */
145
+ }
146
+ fs.rmSync(aside, { force: true });
147
+ heldByProcess.delete(file);
148
+ // Leave the bots folder as it was when no lock is held. A concurrent
149
+ // acquirer that loses its folder here gets ENOENT and makes it again.
150
+ try {
151
+ fs.rmdirSync(path.dirname(file));
152
+ }
153
+ catch {
154
+ /* other locks are held, or it is already gone */
155
+ }
156
+ }
157
+ function tryCreate(file, body) {
158
+ let fd;
159
+ try {
160
+ fd = fs.openSync(file, "wx", 0o600);
161
+ }
162
+ catch (err) {
163
+ const code = err.code;
164
+ if (code === "EEXIST")
165
+ return false;
166
+ // Windows refuses to open a file that is pending deletion (a release in
167
+ // progress) with EPERM/EACCES rather than EEXIST: that is contention too.
168
+ if (process.platform === "win32" && (code === "EPERM" || code === "EACCES"))
169
+ return false;
170
+ if (code === "ENOENT") {
171
+ // The locks folder went away between mkdir and open (a wiped ~/.hq).
172
+ // Make it again and let the caller retry.
173
+ fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
174
+ return false;
175
+ }
176
+ throw err;
177
+ }
178
+ try {
179
+ fs.writeSync(fd, body);
180
+ }
181
+ finally {
182
+ fs.closeSync(fd);
183
+ }
184
+ return true;
185
+ }
186
+ /**
187
+ * Remove a stale lock, but only the one that was judged stale: it is renamed
188
+ * aside first, and if what was renamed is not what was read (a new holder got
189
+ * in between), it is put back.
190
+ */
191
+ function reclaim(file, staleRaw) {
192
+ // Another waiter may have reclaimed it and taken its own lock since we read
193
+ // it: only the lock judged stale is ever moved. Either way the file changed,
194
+ // so the caller may try again at once.
195
+ try {
196
+ if (fs.readFileSync(file, "utf8") !== staleRaw)
197
+ return true;
198
+ }
199
+ catch {
200
+ return true;
201
+ }
202
+ const aside = `${file}.stale-${process.pid}-${randomUUID()}`;
203
+ try {
204
+ fs.renameSync(file, aside);
205
+ }
206
+ catch (err) {
207
+ // Gone already (someone else reclaimed or released it): try again. Any
208
+ // other refusal leaves the stale lock in place: wait before trying again.
209
+ return err.code === "ENOENT";
210
+ }
211
+ let moved = "";
212
+ try {
213
+ moved = fs.readFileSync(aside, "utf8");
214
+ }
215
+ catch {
216
+ /* treat as different */
217
+ }
218
+ if (moved !== staleRaw) {
219
+ // A live holder's lock: restore it. link fails if yet another holder now
220
+ // exists, which still leaves exactly one lock in place.
221
+ try {
222
+ fs.linkSync(aside, file);
223
+ }
224
+ catch {
225
+ try {
226
+ fs.writeFileSync(file, moved, { flag: "wx", mode: 0o600 });
227
+ }
228
+ catch {
229
+ /* another holder is in place */
230
+ }
231
+ }
232
+ }
233
+ fs.rmSync(aside, { force: true });
234
+ return true;
235
+ }
236
+ /** Take the bot's lock (waiting for a live holder), run `fn`, and let go. */
237
+ export async function withBotLock(name, fn, opts = {}) {
238
+ const current = heldStorage.getStore();
239
+ const depth = current?.names.get(name) ?? 0;
240
+ if (depth > 0) {
241
+ current.names.set(name, depth + 1);
242
+ try {
243
+ return await fn();
244
+ }
245
+ finally {
246
+ current.names.set(name, depth);
247
+ }
248
+ }
249
+ const file = botLockPath(name, opts.root);
250
+ const now = opts.now ?? Date.now;
251
+ const isPidAlive = opts.isPidAlive ?? defaultIsPidAlive;
252
+ const staleMs = opts.staleMs ?? BOT_LOCK_STALE_MS;
253
+ const waitMs = opts.waitMs ?? BOT_LOCK_WAIT_MS;
254
+ const pollMs = opts.pollMs ?? 150;
255
+ const host = os.hostname();
256
+ const token = randomUUID();
257
+ const body = JSON.stringify({
258
+ pid: process.pid,
259
+ host,
260
+ startedAt: new Date(now()).toISOString(),
261
+ token,
262
+ command: opts.command ?? "",
263
+ });
264
+ fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
265
+ const deadline = now() + waitMs;
266
+ let waited = false;
267
+ let lastHolder = null;
268
+ for (;;) {
269
+ if (tryCreate(file, body))
270
+ break;
271
+ let raw;
272
+ let mtimeMs = now();
273
+ try {
274
+ raw = fs.readFileSync(file, "utf8");
275
+ mtimeMs = fs.statSync(file).mtimeMs;
276
+ }
277
+ catch {
278
+ raw = null; // released between our create and our read: just retry
279
+ }
280
+ if (raw !== null) {
281
+ lastHolder = parseHolder(raw);
282
+ const why = botLockStaleReason(raw, mtimeMs, { now: now(), staleMs, isPidAlive, host });
283
+ // Retry at once only when the stale lock is really gone. A reclaim that
284
+ // could not move it (Windows refusing a rename, a read-only folder) falls
285
+ // through to the deadline and the poll wait, so this never spins.
286
+ if (why) {
287
+ opts.onReclaim?.(lastHolder, why);
288
+ if (reclaim(file, raw))
289
+ continue;
290
+ }
291
+ if (!waited) {
292
+ waited = true;
293
+ opts.onWait?.(lastHolder);
294
+ }
295
+ }
296
+ if (now() >= deadline)
297
+ throw new BotLockBusyError(name, lastHolder);
298
+ await new Promise((r) => setTimeout(r, pollMs));
299
+ }
300
+ installExitHook();
301
+ heldByProcess.set(file, token);
302
+ const held = { names: new Map(current?.names ?? []) };
303
+ held.names.set(name, 1);
304
+ try {
305
+ return await heldStorage.run(held, fn);
306
+ }
307
+ finally {
308
+ // Work that `fn` started and left running no longer holds the lock.
309
+ held.names.set(name, 0);
310
+ releaseFile(file, token);
311
+ }
312
+ }
313
+ //# sourceMappingURL=lock.js.map
@@ -101,6 +101,12 @@ export interface PresenceClientOptions {
101
101
  * backoff counters.
102
102
  */
103
103
  export declare function backoffDelayMs(attempt: number, baseMs: number, maxMs: number, random: () => number, previousMs?: number, minMs?: number): number;
104
+ /**
105
+ * Stable episode key for transport failures so a network drop logs once
106
+ * instead of once per reconnect/error event (those Error objects otherwise
107
+ * pin the heap while the daemon crash-loops).
108
+ */
109
+ export declare function networkErrorEpisodeKey(err: unknown): string | null;
104
110
  export declare function buildPresencePayload(input: {
105
111
  status: "online" | "offline";
106
112
  actorUid: string;
@@ -129,6 +135,10 @@ export declare class PresenceClient {
129
135
  private publishedTopics;
130
136
  /** Last refusal code we logged via onError (episode dedupe). */
131
137
  private loggedRefusalKey;
138
+ /** Last transport-failure episode logged via onError (network-drop dedupe). */
139
+ private loggedNetworkErrorKey;
140
+ /** Serialize connect/vend so a drop cannot stack MQTT clients or fetches. */
141
+ private connectPromise;
132
142
  private refusal;
133
143
  private preferredHandle;
134
144
  private preferredSnapshot;
@@ -163,6 +173,9 @@ export declare class PresenceClient {
163
173
  /** Force a reconnect cycle (network change / sleep-wake). */
164
174
  reconnectNow(): void;
165
175
  private teardownClient;
176
+ private emitError;
177
+ private clearNetworkErrorEpisode;
178
+ private enqueueConnect;
166
179
  private onCredentialsRenewed;
167
180
  private handleRenewalError;
168
181
  private refusalKey;
@@ -54,6 +54,27 @@ export function backoffDelayMs(attempt, baseMs, maxMs, random, previousMs = 0, m
54
54
  const raw = Math.max(0, random() * cap);
55
55
  return Math.min(maxMs, Math.max(previousMs, raw, minMs));
56
56
  }
57
+ const NETWORK_ERROR_RE = /fetch failed|ETIMEDOUT|ENOTFOUND|ECONNREFUSED|EAI_AGAIN|ENETUNREACH|ECONNRESET|ENETDOWN|EHOSTUNREACH/i;
58
+ /**
59
+ * Stable episode key for transport failures so a network drop logs once
60
+ * instead of once per reconnect/error event (those Error objects otherwise
61
+ * pin the heap while the daemon crash-loops).
62
+ */
63
+ export function networkErrorEpisodeKey(err) {
64
+ const code = err !== null && typeof err === "object" && "code" in err &&
65
+ typeof err.code === "string"
66
+ ? err.code
67
+ : "";
68
+ const msg = err instanceof Error ? err.message : String(err);
69
+ if (code && NETWORK_ERROR_RE.test(code))
70
+ return code;
71
+ if (!NETWORK_ERROR_RE.test(msg))
72
+ return null;
73
+ if (/fetch failed/i.test(msg))
74
+ return "fetch_failed";
75
+ const match = msg.match(/ETIMEDOUT|ENOTFOUND|ECONNREFUSED|EAI_AGAIN|ENETUNREACH|ECONNRESET|ENETDOWN|EHOSTUNREACH/i);
76
+ return match ? match[0].toUpperCase() : "network";
77
+ }
57
78
  export function buildPresencePayload(input) {
58
79
  return {
59
80
  v: 1,
@@ -92,6 +113,10 @@ export class PresenceClient {
92
113
  publishedTopics = [];
93
114
  /** Last refusal code we logged via onError (episode dedupe). */
94
115
  loggedRefusalKey = null;
116
+ /** Last transport-failure episode logged via onError (network-drop dedupe). */
117
+ loggedNetworkErrorKey = null;
118
+ /** Serialize connect/vend so a drop cannot stack MQTT clients or fetches. */
119
+ connectPromise = null;
95
120
  refusal = null;
96
121
  preferredHandle = null;
97
122
  preferredSnapshot = [];
@@ -202,7 +227,7 @@ export class PresenceClient {
202
227
  if (this.stopped)
203
228
  throw new Error("PresenceClient already stopped");
204
229
  this.setState("connecting");
205
- await this.connectOnce();
230
+ await this.enqueueConnect();
206
231
  }
207
232
  /**
208
233
  * Graceful stop: disconnect cleanly (MQTT DISCONNECT so the broker emits
@@ -234,13 +259,20 @@ export class PresenceClient {
234
259
  }
235
260
  // Backoff counters reset only after a stable connection — not here.
236
261
  this.clearReconnect();
237
- void this.connectOnce();
262
+ void this.enqueueConnect();
238
263
  }
239
264
  teardownClient(force) {
240
265
  const client = this.client;
241
266
  if (!client)
242
267
  return;
243
268
  this.client = null;
269
+ try {
270
+ const removable = client;
271
+ removable.removeAllListeners?.();
272
+ }
273
+ catch {
274
+ /* best-effort */
275
+ }
244
276
  try {
245
277
  if (force) {
246
278
  client.end(true);
@@ -253,22 +285,43 @@ export class PresenceClient {
253
285
  /* best-effort */
254
286
  }
255
287
  }
288
+ emitError(err) {
289
+ const key = networkErrorEpisodeKey(err);
290
+ if (key) {
291
+ if (this.loggedNetworkErrorKey === key)
292
+ return;
293
+ this.loggedNetworkErrorKey = key;
294
+ }
295
+ this.options.onError?.(err);
296
+ }
297
+ clearNetworkErrorEpisode() {
298
+ this.loggedNetworkErrorKey = null;
299
+ }
300
+ enqueueConnect() {
301
+ if (this.connectPromise)
302
+ return this.connectPromise;
303
+ this.connectPromise = this.connectOnce().finally(() => {
304
+ this.connectPromise = null;
305
+ });
306
+ return this.connectPromise;
307
+ }
256
308
  async onCredentialsRenewed(bundle) {
257
309
  this.bundle = bundle;
258
310
  this.notePreferredVend();
259
311
  // Successful vend ends a refusal episode (renewal skips connectOnce's vend path).
260
312
  this.clearRefusal(true);
313
+ this.clearNetworkErrorEpisode();
261
314
  if (this.stopped)
262
315
  return;
263
316
  // Reconnect with the new presigned URL; backoff resets only after stable connect.
264
- await this.connectOnce();
317
+ await this.enqueueConnect();
265
318
  }
266
319
  handleRenewalError(err, info) {
267
320
  if (err instanceof CredentialVendError && err.kind === "refused") {
268
321
  this.noteRefusal(err, info?.nextRetryAt);
269
322
  return;
270
323
  }
271
- this.options.onError?.(err);
324
+ this.emitError(err);
272
325
  }
273
326
  refusalKey(err) {
274
327
  return err.code ?? `HTTP_${err.status}`;
@@ -314,6 +367,7 @@ export class PresenceClient {
314
367
  this.bundle = await this.options.fetchCredentials();
315
368
  // Successful vend ends a refusal episode (even before MQTT connects).
316
369
  this.clearRefusal(true);
370
+ this.clearNetworkErrorEpisode();
317
371
  this.renewal.schedule(this.bundle);
318
372
  this.notePreferredVend();
319
373
  this.armPreferredWatch();
@@ -334,7 +388,7 @@ export class PresenceClient {
334
388
  });
335
389
  }
336
390
  catch (err) {
337
- this.options.onError?.(err);
391
+ this.emitError(err);
338
392
  this.scheduleReconnect();
339
393
  return;
340
394
  }
@@ -342,6 +396,7 @@ export class PresenceClient {
342
396
  client.on("connect", () => {
343
397
  if (this.stopped || generation !== this.generation)
344
398
  return;
399
+ this.clearNetworkErrorEpisode();
345
400
  // Do not reset attempt here — broker may accept then close immediately
346
401
  // (denied publish / policy). Reset only after CONNECT_STABLE_GRACE_MS.
347
402
  this.connectedAtMs = this.timers.now();
@@ -362,7 +417,7 @@ export class PresenceClient {
362
417
  client.on("error", (err) => {
363
418
  if (this.stopped || generation !== this.generation)
364
419
  return;
365
- this.options.onError?.(err);
420
+ this.emitError(err);
366
421
  });
367
422
  client.on("close", () => {
368
423
  if (this.stopped || generation !== this.generation)
@@ -381,7 +436,7 @@ export class PresenceClient {
381
436
  this.scheduleReconnect({ refused: err });
382
437
  return;
383
438
  }
384
- this.options.onError?.(err);
439
+ this.emitError(err);
385
440
  this.scheduleReconnect();
386
441
  }
387
442
  }
@@ -401,7 +456,7 @@ export class PresenceClient {
401
456
  for (const company of bundle.companies) {
402
457
  const topic = company.presenceTopic;
403
458
  if (!isOwnPresenceTopic(topic, bundle.actorUid, companyUids)) {
404
- this.options.onError?.(new Error(`refusing to publish to non-presence topic`));
459
+ this.emitError(new Error(`refusing to publish to non-presence topic`));
405
460
  continue;
406
461
  }
407
462
  await new Promise((resolve) => {
@@ -412,7 +467,7 @@ export class PresenceClient {
412
467
  }
413
468
  }
414
469
  catch (err) {
415
- this.options.onError?.(err);
470
+ this.emitError(err);
416
471
  resolve();
417
472
  }
418
473
  });
@@ -451,7 +506,7 @@ export class PresenceClient {
451
506
  if (opts?.refused) {
452
507
  this.bundle = null;
453
508
  }
454
- void this.connectOnce();
509
+ void this.enqueueConnect();
455
510
  }, delay);
456
511
  return delay;
457
512
  }
@@ -24,6 +24,9 @@ export declare const FLUSH_INTERVAL_MS = 10000;
24
24
  /** Keep startup recovery bounded: the next daemon start continues the backlog. */
25
25
  export declare const HELD_BACKFILL_STARTUP_BATCH_SIZE = 100;
26
26
  export declare const DEVICE_DEFAULT_MEMBERSHIP_TIMEOUT_MS = 2000;
27
+ /** Skip further membership probes after a transport failure so a network drop
28
+ * cannot fan out one hanging lookup per held session. */
29
+ export declare const DEVICE_DEFAULT_MEMBERSHIP_UNAVAILABLE_COOLDOWN_MS = 30000;
27
30
  /** Three sequential async shutdown steps must finish within six seconds. */
28
31
  export declare const SHUTDOWN_STEP_TIMEOUT_MS = 2000;
29
32
  /** Non-zero exit reserved for CLI version drift so legacy Restart=on-failure units restart. */
@@ -45,6 +45,9 @@ export const FLUSH_INTERVAL_MS = 10_000;
45
45
  /** Keep startup recovery bounded: the next daemon start continues the backlog. */
46
46
  export const HELD_BACKFILL_STARTUP_BATCH_SIZE = 100;
47
47
  export const DEVICE_DEFAULT_MEMBERSHIP_TIMEOUT_MS = 2_000;
48
+ /** Skip further membership probes after a transport failure so a network drop
49
+ * cannot fan out one hanging lookup per held session. */
50
+ export const DEVICE_DEFAULT_MEMBERSHIP_UNAVAILABLE_COOLDOWN_MS = 30_000;
48
51
  /** Three sequential async shutdown steps must finish within six seconds. */
49
52
  export const SHUTDOWN_STEP_TIMEOUT_MS = 2_000;
50
53
  /** Non-zero exit reserved for CLI version drift so legacy Restart=on-failure units restart. */
@@ -172,6 +175,9 @@ export async function runMeshDaemon(deps = {}) {
172
175
  const resolveMembership = deps.resolveMembership ?? resolveActiveMembershipCompany;
173
176
  const machineIdentity = deps.machineIdentity ?? isMachineIdentity();
174
177
  /** The sole authorization seam for daemon tenant I/O derived from a device default. */
178
+ let membershipUnavailableUntilMs = 0;
179
+ let membershipUnavailableError = "membership lookup unavailable";
180
+ let membershipUnavailableLogged = false;
175
181
  const tenantGate = {
176
182
  authorize: async (input) => {
177
183
  const state = input.state ?? (input.sessionId
@@ -190,6 +196,9 @@ export async function runMeshDaemon(deps = {}) {
190
196
  const lookupKey = slug || state?.companyUid || (usingDefaultFallback
191
197
  ? (defaultCompany?.slug || defaultCompany?.uid)
192
198
  : undefined);
199
+ if (lookupKey && now().getTime() < membershipUnavailableUntilMs) {
200
+ return { status: "unavailable", error: membershipUnavailableError };
201
+ }
193
202
  let authorization;
194
203
  if (!lookupKey) {
195
204
  authorization = { status: "denied" };
@@ -240,14 +249,21 @@ export async function runMeshDaemon(deps = {}) {
240
249
  }
241
250
  }
242
251
  if (authorization.status === "unavailable") {
243
- const prior = readDaemonState(dir)?.deviceDefaultMembershipLookupFailures ?? 0;
244
- patchDaemonState(dir, {
245
- lastErrorCode: "DEVICE_DEFAULT_MEMBERSHIP_LOOKUP_FAILED",
246
- deviceDefaultMembershipLookupFailures: prior + 1,
247
- }, now);
248
- log(dir, `device-default membership lookup unavailable${slug ? ` company=${slug}` : ""}: ${authorization.error.slice(0, 160)}`);
252
+ membershipUnavailableUntilMs = now().getTime() + DEVICE_DEFAULT_MEMBERSHIP_UNAVAILABLE_COOLDOWN_MS;
253
+ membershipUnavailableError = authorization.error;
254
+ if (!membershipUnavailableLogged) {
255
+ membershipUnavailableLogged = true;
256
+ const prior = readDaemonState(dir)?.deviceDefaultMembershipLookupFailures ?? 0;
257
+ patchDaemonState(dir, {
258
+ lastErrorCode: "DEVICE_DEFAULT_MEMBERSHIP_LOOKUP_FAILED",
259
+ deviceDefaultMembershipLookupFailures: prior + 1,
260
+ }, now);
261
+ log(dir, `device-default membership lookup unavailable${slug ? ` company=${slug}` : ""}: ${authorization.error.slice(0, 160)}`);
262
+ }
249
263
  return authorization;
250
264
  }
265
+ membershipUnavailableUntilMs = 0;
266
+ membershipUnavailableLogged = false;
251
267
  if (!input.deviceDefault)
252
268
  return authorization;
253
269
  if (authorization.status === "authorized" &&
@@ -351,12 +367,17 @@ export async function runMeshDaemon(deps = {}) {
351
367
  lastHeldRetryAtMs = atMs;
352
368
  },
353
369
  log: (message) => log(dir, message),
354
- authorizeDeviceDefaultTenant: async (state) => (await tenantGate.authorize({
355
- state,
356
- companySlug: state.companySlug,
357
- companyUid: state.companyUid,
358
- deviceDefault: true,
359
- })).status === "authorized",
370
+ authorizeDeviceDefaultTenant: async (state) => {
371
+ const authorization = await tenantGate.authorize({
372
+ state,
373
+ companySlug: state.companySlug,
374
+ companyUid: state.companyUid,
375
+ deviceDefault: true,
376
+ });
377
+ if (authorization.status === "unavailable")
378
+ return "unavailable";
379
+ return authorization.status === "authorized";
380
+ },
360
381
  });
361
382
  });
362
383
  const replayFn = deps.replayOutbox ??
@@ -45,8 +45,11 @@ export interface FlushDeps {
45
45
  heldMaxLines?: number;
46
46
  /** Optional logger (daemon wires this); used once per overflow trim. */
47
47
  log?: (message: string) => void;
48
- /** Authoritative gate for state derived from a device default. */
49
- authorizeDeviceDefaultTenant?: (state: SessionStateFile) => Promise<boolean>;
48
+ /** Authoritative gate for state derived from a device default.
49
+ * `true` allow, `false` deny (hold NEEDS_COMPANY), `"unavailable"` skip the
50
+ * network this cycle and keep the original held envelope.
51
+ */
52
+ authorizeDeviceDefaultTenant?: (state: SessionStateFile) => Promise<boolean | "unavailable">;
50
53
  /** Passed through to bind-best (catalog cache root and HQ_WORK_MESH_CACHE). */
51
54
  env?: NodeJS.ProcessEnv;
52
55
  home?: string;