@indigoai-us/hq-cli 5.345.86 → 5.345.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/dist/command-catalog.generated.d.ts +5 -1
- package/dist/command-catalog.generated.js +6 -1
- package/dist/commands/bot.d.ts +60 -12
- package/dist/commands/bot.js +349 -98
- package/dist/commands/storage-core.d.ts +2 -0
- package/dist/commands/storage-core.js +12 -1
- package/dist/commands/storage-offload.d.ts +10 -0
- package/dist/commands/storage-offload.js +4 -0
- package/dist/commands/storage.d.ts +6 -0
- package/dist/commands/storage.js +25 -6
- package/dist/lib/bot/api.d.ts +15 -0
- package/dist/lib/bot/api.js +11 -0
- package/dist/lib/bot/config.d.ts +6 -0
- package/dist/lib/bot/display-name.d.ts +64 -0
- package/dist/lib/bot/display-name.js +117 -0
- package/dist/lib/bot/index.d.ts +2 -0
- package/dist/lib/bot/index.js +2 -0
- package/dist/lib/bot/lock.d.ts +84 -0
- package/dist/lib/bot/lock.js +313 -0
- package/dist/lib/mesh/live/daemon/presence.d.ts +13 -0
- package/dist/lib/mesh/live/daemon/presence.js +65 -10
- package/dist/lib/mesh/live/daemon/run.d.ts +3 -0
- package/dist/lib/mesh/live/daemon/run.js +33 -12
- package/dist/lib/mesh/live/flush.d.ts +5 -2
- package/dist/lib/mesh/live/flush.js +74 -1
- package/package.json +1 -1
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One writer per bot at a time.
|
|
3
|
+
*
|
|
4
|
+
* `hq bot create`, `adopt`, `restore`, `rm` and `set` all change the same
|
|
5
|
+
* bot's folder, and an adopt also rotates the bot's machine secret. Two of
|
|
6
|
+
* them running at once for the same bot (the desktop's automatic restore and
|
|
7
|
+
* its first-run create, a minute after sign-in) used to race on a
|
|
8
|
+
* check-then-write of bot.json: the loser either threw "already set up on this
|
|
9
|
+
* computer" or, worse, wrote a credential the winner had already rotated away.
|
|
10
|
+
*
|
|
11
|
+
* The lock is a file under the bots folder, `~/.hq/bots/.locks/<name>.lock`,
|
|
12
|
+
* created with O_CREAT|O_EXCL. It sits beside the bot's folder rather than in
|
|
13
|
+
* it: a folder that exists only to hold a lock would read as "a different bot
|
|
14
|
+
* already has this name" to the code that picks a restored bot's folder, and
|
|
15
|
+
* as "something is left here" to `hq bot rm`. Its body names the holder (pid,
|
|
16
|
+
* host, start time, a random token), so:
|
|
17
|
+
* - a holder whose process is gone, or that has held it for longer than
|
|
18
|
+
* {@link BOT_LOCK_STALE_MS}, is reclaimed;
|
|
19
|
+
* - release only removes the lock it took (the token must match), so a
|
|
20
|
+
* reclaim by another process never makes this process remove someone
|
|
21
|
+
* else's lock.
|
|
22
|
+
*
|
|
23
|
+
* The lock is re-entrant within one async call chain (AsyncLocalStorage):
|
|
24
|
+
* create holds it and calls the shared adopt path, which takes it again
|
|
25
|
+
* without waiting on itself. Two separate chains in one process still
|
|
26
|
+
* serialize, exactly like two processes.
|
|
27
|
+
*/
|
|
28
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
29
|
+
import { randomUUID } from "node:crypto";
|
|
30
|
+
import * as fs from "node:fs";
|
|
31
|
+
import * as os from "node:os";
|
|
32
|
+
import * as path from "node:path";
|
|
33
|
+
import { botsRoot } from "./paths.js";
|
|
34
|
+
/** Not a valid bot name (it starts with a dot), so no bot can collide with it. */
|
|
35
|
+
export const BOT_LOCKS_DIRNAME = ".locks";
|
|
36
|
+
/** A holder older than this is treated as crashed, whatever its pid says. */
|
|
37
|
+
export const BOT_LOCK_STALE_MS = 10 * 60 * 1000;
|
|
38
|
+
/** How long a second writer waits for the first before giving up. */
|
|
39
|
+
export const BOT_LOCK_WAIT_MS = 3 * 60 * 1000;
|
|
40
|
+
/** A lock file with no readable body yet: its creator may still be writing it. */
|
|
41
|
+
const UNREADABLE_GRACE_MS = 5_000;
|
|
42
|
+
export class BotLockBusyError extends Error {
|
|
43
|
+
botName;
|
|
44
|
+
holder;
|
|
45
|
+
expected = true;
|
|
46
|
+
constructor(botName, holder) {
|
|
47
|
+
super(`Another hq bot command is changing "${botName}" right now${holder?.command ? ` (${holder.command})` : ""}. Try again in a moment.`);
|
|
48
|
+
this.botName = botName;
|
|
49
|
+
this.holder = holder;
|
|
50
|
+
this.name = "BotLockBusyError";
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
export function botLockPath(name, root) {
|
|
54
|
+
return path.join(root ?? botsRoot(), BOT_LOCKS_DIRNAME, `${name}.lock`);
|
|
55
|
+
}
|
|
56
|
+
/** Alive unless the OS says there is no such process. EPERM is a live process we may not signal. */
|
|
57
|
+
function defaultIsPidAlive(pid) {
|
|
58
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
59
|
+
return false;
|
|
60
|
+
try {
|
|
61
|
+
process.kill(pid, 0);
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
catch (err) {
|
|
65
|
+
return err.code === "EPERM";
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function parseHolder(raw) {
|
|
69
|
+
try {
|
|
70
|
+
const v = JSON.parse(raw);
|
|
71
|
+
if (typeof v.pid !== "number" || typeof v.startedAt !== "string" || typeof v.token !== "string")
|
|
72
|
+
return null;
|
|
73
|
+
return {
|
|
74
|
+
pid: v.pid,
|
|
75
|
+
host: typeof v.host === "string" ? v.host : "",
|
|
76
|
+
startedAt: v.startedAt,
|
|
77
|
+
token: v.token,
|
|
78
|
+
command: typeof v.command === "string" ? v.command : "",
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Why an existing lock may be taken over, or null while its holder is live.
|
|
87
|
+
* Exported for tests.
|
|
88
|
+
*/
|
|
89
|
+
export function botLockStaleReason(raw, mtimeMs, opts) {
|
|
90
|
+
const holder = parseHolder(raw);
|
|
91
|
+
if (!holder) {
|
|
92
|
+
// Created but not yet written, or torn by a crash. Give a live creator a
|
|
93
|
+
// moment to finish writing before calling it abandoned.
|
|
94
|
+
return opts.now - mtimeMs > UNREADABLE_GRACE_MS ? "unreadable lock" : null;
|
|
95
|
+
}
|
|
96
|
+
const started = Date.parse(holder.startedAt);
|
|
97
|
+
if (Number.isNaN(started))
|
|
98
|
+
return "unreadable start time";
|
|
99
|
+
if (opts.now - started > opts.staleMs)
|
|
100
|
+
return `held since ${holder.startedAt}`;
|
|
101
|
+
// ~/.hq/bots is per machine, so the pid is this machine's. The host name is
|
|
102
|
+
// not compared: macOS changes it with the network, which would leave a
|
|
103
|
+
// crashed holder blocking for the whole stale age.
|
|
104
|
+
if (!opts.isPidAlive(holder.pid))
|
|
105
|
+
return `holder pid ${holder.pid} has exited`;
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
const heldStorage = new AsyncLocalStorage();
|
|
109
|
+
/** Locks this process holds, so a hard exit (process.exit in a failure path) still lets go. */
|
|
110
|
+
const heldByProcess = new Map();
|
|
111
|
+
let exitHookInstalled = false;
|
|
112
|
+
function installExitHook() {
|
|
113
|
+
if (exitHookInstalled)
|
|
114
|
+
return;
|
|
115
|
+
exitHookInstalled = true;
|
|
116
|
+
process.on("exit", releaseAllHeldBotLocks);
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Let go of every lock this process holds. Runs on process exit, so a command
|
|
120
|
+
* that ends through process.exit (every `hq bot` failure does) frees its bot
|
|
121
|
+
* at once instead of leaving it to the stale-holder reclaim.
|
|
122
|
+
*/
|
|
123
|
+
export function releaseAllHeldBotLocks() {
|
|
124
|
+
for (const [file, token] of [...heldByProcess])
|
|
125
|
+
releaseFile(file, token);
|
|
126
|
+
}
|
|
127
|
+
function releaseFile(file, token) {
|
|
128
|
+
// Moved aside before it is judged, so a lock someone else took after
|
|
129
|
+
// reclaiming ours (a very long hold) is put back rather than deleted.
|
|
130
|
+
const aside = `${file}.release-${process.pid}-${randomUUID()}`;
|
|
131
|
+
try {
|
|
132
|
+
fs.renameSync(file, aside);
|
|
133
|
+
const raw = fs.readFileSync(aside, "utf8");
|
|
134
|
+
if (parseHolder(raw)?.token !== token) {
|
|
135
|
+
try {
|
|
136
|
+
fs.linkSync(aside, file);
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
/* a newer holder is already in place */
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
/* already gone, or unreadable: nothing of ours to remove */
|
|
145
|
+
}
|
|
146
|
+
fs.rmSync(aside, { force: true });
|
|
147
|
+
heldByProcess.delete(file);
|
|
148
|
+
// Leave the bots folder as it was when no lock is held. A concurrent
|
|
149
|
+
// acquirer that loses its folder here gets ENOENT and makes it again.
|
|
150
|
+
try {
|
|
151
|
+
fs.rmdirSync(path.dirname(file));
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
/* other locks are held, or it is already gone */
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
function tryCreate(file, body) {
|
|
158
|
+
let fd;
|
|
159
|
+
try {
|
|
160
|
+
fd = fs.openSync(file, "wx", 0o600);
|
|
161
|
+
}
|
|
162
|
+
catch (err) {
|
|
163
|
+
const code = err.code;
|
|
164
|
+
if (code === "EEXIST")
|
|
165
|
+
return false;
|
|
166
|
+
// Windows refuses to open a file that is pending deletion (a release in
|
|
167
|
+
// progress) with EPERM/EACCES rather than EEXIST: that is contention too.
|
|
168
|
+
if (process.platform === "win32" && (code === "EPERM" || code === "EACCES"))
|
|
169
|
+
return false;
|
|
170
|
+
if (code === "ENOENT") {
|
|
171
|
+
// The locks folder went away between mkdir and open (a wiped ~/.hq).
|
|
172
|
+
// Make it again and let the caller retry.
|
|
173
|
+
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
|
174
|
+
return false;
|
|
175
|
+
}
|
|
176
|
+
throw err;
|
|
177
|
+
}
|
|
178
|
+
try {
|
|
179
|
+
fs.writeSync(fd, body);
|
|
180
|
+
}
|
|
181
|
+
finally {
|
|
182
|
+
fs.closeSync(fd);
|
|
183
|
+
}
|
|
184
|
+
return true;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Remove a stale lock, but only the one that was judged stale: it is renamed
|
|
188
|
+
* aside first, and if what was renamed is not what was read (a new holder got
|
|
189
|
+
* in between), it is put back.
|
|
190
|
+
*/
|
|
191
|
+
function reclaim(file, staleRaw) {
|
|
192
|
+
// Another waiter may have reclaimed it and taken its own lock since we read
|
|
193
|
+
// it: only the lock judged stale is ever moved. Either way the file changed,
|
|
194
|
+
// so the caller may try again at once.
|
|
195
|
+
try {
|
|
196
|
+
if (fs.readFileSync(file, "utf8") !== staleRaw)
|
|
197
|
+
return true;
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
return true;
|
|
201
|
+
}
|
|
202
|
+
const aside = `${file}.stale-${process.pid}-${randomUUID()}`;
|
|
203
|
+
try {
|
|
204
|
+
fs.renameSync(file, aside);
|
|
205
|
+
}
|
|
206
|
+
catch (err) {
|
|
207
|
+
// Gone already (someone else reclaimed or released it): try again. Any
|
|
208
|
+
// other refusal leaves the stale lock in place: wait before trying again.
|
|
209
|
+
return err.code === "ENOENT";
|
|
210
|
+
}
|
|
211
|
+
let moved = "";
|
|
212
|
+
try {
|
|
213
|
+
moved = fs.readFileSync(aside, "utf8");
|
|
214
|
+
}
|
|
215
|
+
catch {
|
|
216
|
+
/* treat as different */
|
|
217
|
+
}
|
|
218
|
+
if (moved !== staleRaw) {
|
|
219
|
+
// A live holder's lock: restore it. link fails if yet another holder now
|
|
220
|
+
// exists, which still leaves exactly one lock in place.
|
|
221
|
+
try {
|
|
222
|
+
fs.linkSync(aside, file);
|
|
223
|
+
}
|
|
224
|
+
catch {
|
|
225
|
+
try {
|
|
226
|
+
fs.writeFileSync(file, moved, { flag: "wx", mode: 0o600 });
|
|
227
|
+
}
|
|
228
|
+
catch {
|
|
229
|
+
/* another holder is in place */
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
fs.rmSync(aside, { force: true });
|
|
234
|
+
return true;
|
|
235
|
+
}
|
|
236
|
+
/** Take the bot's lock (waiting for a live holder), run `fn`, and let go. */
|
|
237
|
+
export async function withBotLock(name, fn, opts = {}) {
|
|
238
|
+
const current = heldStorage.getStore();
|
|
239
|
+
const depth = current?.names.get(name) ?? 0;
|
|
240
|
+
if (depth > 0) {
|
|
241
|
+
current.names.set(name, depth + 1);
|
|
242
|
+
try {
|
|
243
|
+
return await fn();
|
|
244
|
+
}
|
|
245
|
+
finally {
|
|
246
|
+
current.names.set(name, depth);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
const file = botLockPath(name, opts.root);
|
|
250
|
+
const now = opts.now ?? Date.now;
|
|
251
|
+
const isPidAlive = opts.isPidAlive ?? defaultIsPidAlive;
|
|
252
|
+
const staleMs = opts.staleMs ?? BOT_LOCK_STALE_MS;
|
|
253
|
+
const waitMs = opts.waitMs ?? BOT_LOCK_WAIT_MS;
|
|
254
|
+
const pollMs = opts.pollMs ?? 150;
|
|
255
|
+
const host = os.hostname();
|
|
256
|
+
const token = randomUUID();
|
|
257
|
+
const body = JSON.stringify({
|
|
258
|
+
pid: process.pid,
|
|
259
|
+
host,
|
|
260
|
+
startedAt: new Date(now()).toISOString(),
|
|
261
|
+
token,
|
|
262
|
+
command: opts.command ?? "",
|
|
263
|
+
});
|
|
264
|
+
fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
|
|
265
|
+
const deadline = now() + waitMs;
|
|
266
|
+
let waited = false;
|
|
267
|
+
let lastHolder = null;
|
|
268
|
+
for (;;) {
|
|
269
|
+
if (tryCreate(file, body))
|
|
270
|
+
break;
|
|
271
|
+
let raw;
|
|
272
|
+
let mtimeMs = now();
|
|
273
|
+
try {
|
|
274
|
+
raw = fs.readFileSync(file, "utf8");
|
|
275
|
+
mtimeMs = fs.statSync(file).mtimeMs;
|
|
276
|
+
}
|
|
277
|
+
catch {
|
|
278
|
+
raw = null; // released between our create and our read: just retry
|
|
279
|
+
}
|
|
280
|
+
if (raw !== null) {
|
|
281
|
+
lastHolder = parseHolder(raw);
|
|
282
|
+
const why = botLockStaleReason(raw, mtimeMs, { now: now(), staleMs, isPidAlive, host });
|
|
283
|
+
// Retry at once only when the stale lock is really gone. A reclaim that
|
|
284
|
+
// could not move it (Windows refusing a rename, a read-only folder) falls
|
|
285
|
+
// through to the deadline and the poll wait, so this never spins.
|
|
286
|
+
if (why) {
|
|
287
|
+
opts.onReclaim?.(lastHolder, why);
|
|
288
|
+
if (reclaim(file, raw))
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
if (!waited) {
|
|
292
|
+
waited = true;
|
|
293
|
+
opts.onWait?.(lastHolder);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
if (now() >= deadline)
|
|
297
|
+
throw new BotLockBusyError(name, lastHolder);
|
|
298
|
+
await new Promise((r) => setTimeout(r, pollMs));
|
|
299
|
+
}
|
|
300
|
+
installExitHook();
|
|
301
|
+
heldByProcess.set(file, token);
|
|
302
|
+
const held = { names: new Map(current?.names ?? []) };
|
|
303
|
+
held.names.set(name, 1);
|
|
304
|
+
try {
|
|
305
|
+
return await heldStorage.run(held, fn);
|
|
306
|
+
}
|
|
307
|
+
finally {
|
|
308
|
+
// Work that `fn` started and left running no longer holds the lock.
|
|
309
|
+
held.names.set(name, 0);
|
|
310
|
+
releaseFile(file, token);
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
//# sourceMappingURL=lock.js.map
|
|
@@ -101,6 +101,12 @@ export interface PresenceClientOptions {
|
|
|
101
101
|
* backoff counters.
|
|
102
102
|
*/
|
|
103
103
|
export declare function backoffDelayMs(attempt: number, baseMs: number, maxMs: number, random: () => number, previousMs?: number, minMs?: number): number;
|
|
104
|
+
/**
|
|
105
|
+
* Stable episode key for transport failures so a network drop logs once
|
|
106
|
+
* instead of once per reconnect/error event (those Error objects otherwise
|
|
107
|
+
* pin the heap while the daemon crash-loops).
|
|
108
|
+
*/
|
|
109
|
+
export declare function networkErrorEpisodeKey(err: unknown): string | null;
|
|
104
110
|
export declare function buildPresencePayload(input: {
|
|
105
111
|
status: "online" | "offline";
|
|
106
112
|
actorUid: string;
|
|
@@ -129,6 +135,10 @@ export declare class PresenceClient {
|
|
|
129
135
|
private publishedTopics;
|
|
130
136
|
/** Last refusal code we logged via onError (episode dedupe). */
|
|
131
137
|
private loggedRefusalKey;
|
|
138
|
+
/** Last transport-failure episode logged via onError (network-drop dedupe). */
|
|
139
|
+
private loggedNetworkErrorKey;
|
|
140
|
+
/** Serialize connect/vend so a drop cannot stack MQTT clients or fetches. */
|
|
141
|
+
private connectPromise;
|
|
132
142
|
private refusal;
|
|
133
143
|
private preferredHandle;
|
|
134
144
|
private preferredSnapshot;
|
|
@@ -163,6 +173,9 @@ export declare class PresenceClient {
|
|
|
163
173
|
/** Force a reconnect cycle (network change / sleep-wake). */
|
|
164
174
|
reconnectNow(): void;
|
|
165
175
|
private teardownClient;
|
|
176
|
+
private emitError;
|
|
177
|
+
private clearNetworkErrorEpisode;
|
|
178
|
+
private enqueueConnect;
|
|
166
179
|
private onCredentialsRenewed;
|
|
167
180
|
private handleRenewalError;
|
|
168
181
|
private refusalKey;
|
|
@@ -54,6 +54,27 @@ export function backoffDelayMs(attempt, baseMs, maxMs, random, previousMs = 0, m
|
|
|
54
54
|
const raw = Math.max(0, random() * cap);
|
|
55
55
|
return Math.min(maxMs, Math.max(previousMs, raw, minMs));
|
|
56
56
|
}
|
|
57
|
+
const NETWORK_ERROR_RE = /fetch failed|ETIMEDOUT|ENOTFOUND|ECONNREFUSED|EAI_AGAIN|ENETUNREACH|ECONNRESET|ENETDOWN|EHOSTUNREACH/i;
|
|
58
|
+
/**
|
|
59
|
+
* Stable episode key for transport failures so a network drop logs once
|
|
60
|
+
* instead of once per reconnect/error event (those Error objects otherwise
|
|
61
|
+
* pin the heap while the daemon crash-loops).
|
|
62
|
+
*/
|
|
63
|
+
export function networkErrorEpisodeKey(err) {
|
|
64
|
+
const code = err !== null && typeof err === "object" && "code" in err &&
|
|
65
|
+
typeof err.code === "string"
|
|
66
|
+
? err.code
|
|
67
|
+
: "";
|
|
68
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
69
|
+
if (code && NETWORK_ERROR_RE.test(code))
|
|
70
|
+
return code;
|
|
71
|
+
if (!NETWORK_ERROR_RE.test(msg))
|
|
72
|
+
return null;
|
|
73
|
+
if (/fetch failed/i.test(msg))
|
|
74
|
+
return "fetch_failed";
|
|
75
|
+
const match = msg.match(/ETIMEDOUT|ENOTFOUND|ECONNREFUSED|EAI_AGAIN|ENETUNREACH|ECONNRESET|ENETDOWN|EHOSTUNREACH/i);
|
|
76
|
+
return match ? match[0].toUpperCase() : "network";
|
|
77
|
+
}
|
|
57
78
|
export function buildPresencePayload(input) {
|
|
58
79
|
return {
|
|
59
80
|
v: 1,
|
|
@@ -92,6 +113,10 @@ export class PresenceClient {
|
|
|
92
113
|
publishedTopics = [];
|
|
93
114
|
/** Last refusal code we logged via onError (episode dedupe). */
|
|
94
115
|
loggedRefusalKey = null;
|
|
116
|
+
/** Last transport-failure episode logged via onError (network-drop dedupe). */
|
|
117
|
+
loggedNetworkErrorKey = null;
|
|
118
|
+
/** Serialize connect/vend so a drop cannot stack MQTT clients or fetches. */
|
|
119
|
+
connectPromise = null;
|
|
95
120
|
refusal = null;
|
|
96
121
|
preferredHandle = null;
|
|
97
122
|
preferredSnapshot = [];
|
|
@@ -202,7 +227,7 @@ export class PresenceClient {
|
|
|
202
227
|
if (this.stopped)
|
|
203
228
|
throw new Error("PresenceClient already stopped");
|
|
204
229
|
this.setState("connecting");
|
|
205
|
-
await this.
|
|
230
|
+
await this.enqueueConnect();
|
|
206
231
|
}
|
|
207
232
|
/**
|
|
208
233
|
* Graceful stop: disconnect cleanly (MQTT DISCONNECT so the broker emits
|
|
@@ -234,13 +259,20 @@ export class PresenceClient {
|
|
|
234
259
|
}
|
|
235
260
|
// Backoff counters reset only after a stable connection — not here.
|
|
236
261
|
this.clearReconnect();
|
|
237
|
-
void this.
|
|
262
|
+
void this.enqueueConnect();
|
|
238
263
|
}
|
|
239
264
|
teardownClient(force) {
|
|
240
265
|
const client = this.client;
|
|
241
266
|
if (!client)
|
|
242
267
|
return;
|
|
243
268
|
this.client = null;
|
|
269
|
+
try {
|
|
270
|
+
const removable = client;
|
|
271
|
+
removable.removeAllListeners?.();
|
|
272
|
+
}
|
|
273
|
+
catch {
|
|
274
|
+
/* best-effort */
|
|
275
|
+
}
|
|
244
276
|
try {
|
|
245
277
|
if (force) {
|
|
246
278
|
client.end(true);
|
|
@@ -253,22 +285,43 @@ export class PresenceClient {
|
|
|
253
285
|
/* best-effort */
|
|
254
286
|
}
|
|
255
287
|
}
|
|
288
|
+
emitError(err) {
|
|
289
|
+
const key = networkErrorEpisodeKey(err);
|
|
290
|
+
if (key) {
|
|
291
|
+
if (this.loggedNetworkErrorKey === key)
|
|
292
|
+
return;
|
|
293
|
+
this.loggedNetworkErrorKey = key;
|
|
294
|
+
}
|
|
295
|
+
this.options.onError?.(err);
|
|
296
|
+
}
|
|
297
|
+
clearNetworkErrorEpisode() {
|
|
298
|
+
this.loggedNetworkErrorKey = null;
|
|
299
|
+
}
|
|
300
|
+
enqueueConnect() {
|
|
301
|
+
if (this.connectPromise)
|
|
302
|
+
return this.connectPromise;
|
|
303
|
+
this.connectPromise = this.connectOnce().finally(() => {
|
|
304
|
+
this.connectPromise = null;
|
|
305
|
+
});
|
|
306
|
+
return this.connectPromise;
|
|
307
|
+
}
|
|
256
308
|
async onCredentialsRenewed(bundle) {
|
|
257
309
|
this.bundle = bundle;
|
|
258
310
|
this.notePreferredVend();
|
|
259
311
|
// Successful vend ends a refusal episode (renewal skips connectOnce's vend path).
|
|
260
312
|
this.clearRefusal(true);
|
|
313
|
+
this.clearNetworkErrorEpisode();
|
|
261
314
|
if (this.stopped)
|
|
262
315
|
return;
|
|
263
316
|
// Reconnect with the new presigned URL; backoff resets only after stable connect.
|
|
264
|
-
await this.
|
|
317
|
+
await this.enqueueConnect();
|
|
265
318
|
}
|
|
266
319
|
handleRenewalError(err, info) {
|
|
267
320
|
if (err instanceof CredentialVendError && err.kind === "refused") {
|
|
268
321
|
this.noteRefusal(err, info?.nextRetryAt);
|
|
269
322
|
return;
|
|
270
323
|
}
|
|
271
|
-
this.
|
|
324
|
+
this.emitError(err);
|
|
272
325
|
}
|
|
273
326
|
refusalKey(err) {
|
|
274
327
|
return err.code ?? `HTTP_${err.status}`;
|
|
@@ -314,6 +367,7 @@ export class PresenceClient {
|
|
|
314
367
|
this.bundle = await this.options.fetchCredentials();
|
|
315
368
|
// Successful vend ends a refusal episode (even before MQTT connects).
|
|
316
369
|
this.clearRefusal(true);
|
|
370
|
+
this.clearNetworkErrorEpisode();
|
|
317
371
|
this.renewal.schedule(this.bundle);
|
|
318
372
|
this.notePreferredVend();
|
|
319
373
|
this.armPreferredWatch();
|
|
@@ -334,7 +388,7 @@ export class PresenceClient {
|
|
|
334
388
|
});
|
|
335
389
|
}
|
|
336
390
|
catch (err) {
|
|
337
|
-
this.
|
|
391
|
+
this.emitError(err);
|
|
338
392
|
this.scheduleReconnect();
|
|
339
393
|
return;
|
|
340
394
|
}
|
|
@@ -342,6 +396,7 @@ export class PresenceClient {
|
|
|
342
396
|
client.on("connect", () => {
|
|
343
397
|
if (this.stopped || generation !== this.generation)
|
|
344
398
|
return;
|
|
399
|
+
this.clearNetworkErrorEpisode();
|
|
345
400
|
// Do not reset attempt here — broker may accept then close immediately
|
|
346
401
|
// (denied publish / policy). Reset only after CONNECT_STABLE_GRACE_MS.
|
|
347
402
|
this.connectedAtMs = this.timers.now();
|
|
@@ -362,7 +417,7 @@ export class PresenceClient {
|
|
|
362
417
|
client.on("error", (err) => {
|
|
363
418
|
if (this.stopped || generation !== this.generation)
|
|
364
419
|
return;
|
|
365
|
-
this.
|
|
420
|
+
this.emitError(err);
|
|
366
421
|
});
|
|
367
422
|
client.on("close", () => {
|
|
368
423
|
if (this.stopped || generation !== this.generation)
|
|
@@ -381,7 +436,7 @@ export class PresenceClient {
|
|
|
381
436
|
this.scheduleReconnect({ refused: err });
|
|
382
437
|
return;
|
|
383
438
|
}
|
|
384
|
-
this.
|
|
439
|
+
this.emitError(err);
|
|
385
440
|
this.scheduleReconnect();
|
|
386
441
|
}
|
|
387
442
|
}
|
|
@@ -401,7 +456,7 @@ export class PresenceClient {
|
|
|
401
456
|
for (const company of bundle.companies) {
|
|
402
457
|
const topic = company.presenceTopic;
|
|
403
458
|
if (!isOwnPresenceTopic(topic, bundle.actorUid, companyUids)) {
|
|
404
|
-
this.
|
|
459
|
+
this.emitError(new Error(`refusing to publish to non-presence topic`));
|
|
405
460
|
continue;
|
|
406
461
|
}
|
|
407
462
|
await new Promise((resolve) => {
|
|
@@ -412,7 +467,7 @@ export class PresenceClient {
|
|
|
412
467
|
}
|
|
413
468
|
}
|
|
414
469
|
catch (err) {
|
|
415
|
-
this.
|
|
470
|
+
this.emitError(err);
|
|
416
471
|
resolve();
|
|
417
472
|
}
|
|
418
473
|
});
|
|
@@ -451,7 +506,7 @@ export class PresenceClient {
|
|
|
451
506
|
if (opts?.refused) {
|
|
452
507
|
this.bundle = null;
|
|
453
508
|
}
|
|
454
|
-
void this.
|
|
509
|
+
void this.enqueueConnect();
|
|
455
510
|
}, delay);
|
|
456
511
|
return delay;
|
|
457
512
|
}
|
|
@@ -24,6 +24,9 @@ export declare const FLUSH_INTERVAL_MS = 10000;
|
|
|
24
24
|
/** Keep startup recovery bounded: the next daemon start continues the backlog. */
|
|
25
25
|
export declare const HELD_BACKFILL_STARTUP_BATCH_SIZE = 100;
|
|
26
26
|
export declare const DEVICE_DEFAULT_MEMBERSHIP_TIMEOUT_MS = 2000;
|
|
27
|
+
/** Skip further membership probes after a transport failure so a network drop
|
|
28
|
+
* cannot fan out one hanging lookup per held session. */
|
|
29
|
+
export declare const DEVICE_DEFAULT_MEMBERSHIP_UNAVAILABLE_COOLDOWN_MS = 30000;
|
|
27
30
|
/** Three sequential async shutdown steps must finish within six seconds. */
|
|
28
31
|
export declare const SHUTDOWN_STEP_TIMEOUT_MS = 2000;
|
|
29
32
|
/** Non-zero exit reserved for CLI version drift so legacy Restart=on-failure units restart. */
|
|
@@ -45,6 +45,9 @@ export const FLUSH_INTERVAL_MS = 10_000;
|
|
|
45
45
|
/** Keep startup recovery bounded: the next daemon start continues the backlog. */
|
|
46
46
|
export const HELD_BACKFILL_STARTUP_BATCH_SIZE = 100;
|
|
47
47
|
export const DEVICE_DEFAULT_MEMBERSHIP_TIMEOUT_MS = 2_000;
|
|
48
|
+
/** Skip further membership probes after a transport failure so a network drop
|
|
49
|
+
* cannot fan out one hanging lookup per held session. */
|
|
50
|
+
export const DEVICE_DEFAULT_MEMBERSHIP_UNAVAILABLE_COOLDOWN_MS = 30_000;
|
|
48
51
|
/** Three sequential async shutdown steps must finish within six seconds. */
|
|
49
52
|
export const SHUTDOWN_STEP_TIMEOUT_MS = 2_000;
|
|
50
53
|
/** Non-zero exit reserved for CLI version drift so legacy Restart=on-failure units restart. */
|
|
@@ -172,6 +175,9 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
172
175
|
const resolveMembership = deps.resolveMembership ?? resolveActiveMembershipCompany;
|
|
173
176
|
const machineIdentity = deps.machineIdentity ?? isMachineIdentity();
|
|
174
177
|
/** The sole authorization seam for daemon tenant I/O derived from a device default. */
|
|
178
|
+
let membershipUnavailableUntilMs = 0;
|
|
179
|
+
let membershipUnavailableError = "membership lookup unavailable";
|
|
180
|
+
let membershipUnavailableLogged = false;
|
|
175
181
|
const tenantGate = {
|
|
176
182
|
authorize: async (input) => {
|
|
177
183
|
const state = input.state ?? (input.sessionId
|
|
@@ -190,6 +196,9 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
190
196
|
const lookupKey = slug || state?.companyUid || (usingDefaultFallback
|
|
191
197
|
? (defaultCompany?.slug || defaultCompany?.uid)
|
|
192
198
|
: undefined);
|
|
199
|
+
if (lookupKey && now().getTime() < membershipUnavailableUntilMs) {
|
|
200
|
+
return { status: "unavailable", error: membershipUnavailableError };
|
|
201
|
+
}
|
|
193
202
|
let authorization;
|
|
194
203
|
if (!lookupKey) {
|
|
195
204
|
authorization = { status: "denied" };
|
|
@@ -240,14 +249,21 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
240
249
|
}
|
|
241
250
|
}
|
|
242
251
|
if (authorization.status === "unavailable") {
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
252
|
+
membershipUnavailableUntilMs = now().getTime() + DEVICE_DEFAULT_MEMBERSHIP_UNAVAILABLE_COOLDOWN_MS;
|
|
253
|
+
membershipUnavailableError = authorization.error;
|
|
254
|
+
if (!membershipUnavailableLogged) {
|
|
255
|
+
membershipUnavailableLogged = true;
|
|
256
|
+
const prior = readDaemonState(dir)?.deviceDefaultMembershipLookupFailures ?? 0;
|
|
257
|
+
patchDaemonState(dir, {
|
|
258
|
+
lastErrorCode: "DEVICE_DEFAULT_MEMBERSHIP_LOOKUP_FAILED",
|
|
259
|
+
deviceDefaultMembershipLookupFailures: prior + 1,
|
|
260
|
+
}, now);
|
|
261
|
+
log(dir, `device-default membership lookup unavailable${slug ? ` company=${slug}` : ""}: ${authorization.error.slice(0, 160)}`);
|
|
262
|
+
}
|
|
249
263
|
return authorization;
|
|
250
264
|
}
|
|
265
|
+
membershipUnavailableUntilMs = 0;
|
|
266
|
+
membershipUnavailableLogged = false;
|
|
251
267
|
if (!input.deviceDefault)
|
|
252
268
|
return authorization;
|
|
253
269
|
if (authorization.status === "authorized" &&
|
|
@@ -351,12 +367,17 @@ export async function runMeshDaemon(deps = {}) {
|
|
|
351
367
|
lastHeldRetryAtMs = atMs;
|
|
352
368
|
},
|
|
353
369
|
log: (message) => log(dir, message),
|
|
354
|
-
authorizeDeviceDefaultTenant: async (state) =>
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
370
|
+
authorizeDeviceDefaultTenant: async (state) => {
|
|
371
|
+
const authorization = await tenantGate.authorize({
|
|
372
|
+
state,
|
|
373
|
+
companySlug: state.companySlug,
|
|
374
|
+
companyUid: state.companyUid,
|
|
375
|
+
deviceDefault: true,
|
|
376
|
+
});
|
|
377
|
+
if (authorization.status === "unavailable")
|
|
378
|
+
return "unavailable";
|
|
379
|
+
return authorization.status === "authorized";
|
|
380
|
+
},
|
|
360
381
|
});
|
|
361
382
|
});
|
|
362
383
|
const replayFn = deps.replayOutbox ??
|
|
@@ -45,8 +45,11 @@ export interface FlushDeps {
|
|
|
45
45
|
heldMaxLines?: number;
|
|
46
46
|
/** Optional logger (daemon wires this); used once per overflow trim. */
|
|
47
47
|
log?: (message: string) => void;
|
|
48
|
-
/** Authoritative gate for state derived from a device default.
|
|
49
|
-
|
|
48
|
+
/** Authoritative gate for state derived from a device default.
|
|
49
|
+
* `true` allow, `false` deny (hold NEEDS_COMPANY), `"unavailable"` skip the
|
|
50
|
+
* network this cycle and keep the original held envelope.
|
|
51
|
+
*/
|
|
52
|
+
authorizeDeviceDefaultTenant?: (state: SessionStateFile) => Promise<boolean | "unavailable">;
|
|
50
53
|
/** Passed through to bind-best (catalog cache root and HQ_WORK_MESH_CACHE). */
|
|
51
54
|
env?: NodeJS.ProcessEnv;
|
|
52
55
|
home?: string;
|