@indigoai-us/hq-cli 5.108.2 → 5.108.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/dist/commands/index-cmd.js +13 -0
- package/dist/commands/mesh.js +11 -1
- package/dist/lib/mesh/live/daemon/credentials.d.ts +68 -5
- package/dist/lib/mesh/live/daemon/credentials.js +168 -7
- package/dist/lib/mesh/live/daemon/doctor.d.ts +2 -0
- package/dist/lib/mesh/live/daemon/doctor.js +8 -0
- package/dist/lib/mesh/live/daemon/index.d.ts +4 -4
- package/dist/lib/mesh/live/daemon/index.js +1 -1
- package/dist/lib/mesh/live/daemon/presence.d.ts +27 -3
- package/dist/lib/mesh/live/daemon/presence.js +105 -14
- package/dist/lib/mesh/live/daemon/run.js +10 -0
- package/dist/lib/mesh/live/daemon/state.d.ts +7 -0
- package/dist/lib/search-index/index.d.ts +53 -0
- package/dist/lib/search-index/index.js +98 -2
- package/dist/main.js +56 -4
- package/dist/utils/qmd-workdir-missing-error.d.ts +18 -0
- package/dist/utils/qmd-workdir-missing-error.js +78 -0
- package/dist/utils/sentry-fingerprint.js +5 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,28 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.108.4] — 2026-09-04
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Mesh daemon presence reconnect backoff no longer shrinks between failures:
|
|
10
|
+
consecutive errors grow the delay monotonically (full jitter floored at the
|
|
11
|
+
previous delay) up to the 60 s cap, and the attempt counter resets only after
|
|
12
|
+
a successful MQTT connect. Server refusals (`404 FEATURE_DISABLED`,
|
|
13
|
+
`409 REALTIME_CONTRACT_UNSUPPORTED`, and `403`) use a long retry of 10
|
|
14
|
+
minutes ±20% jitter (override with `HQ_MESH_DAEMON_REFUSED_RETRY_MS`; a
|
|
15
|
+
`Retry-After` header wins), log one line per refusal episode, and surface the
|
|
16
|
+
refused state plus next retry time on `hq mesh daemon doctor` and
|
|
17
|
+
`hq mesh daemon status`.
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
|
|
21
|
+
- Clarification: hq-core hooks from the matching Work Mesh Live release require
|
|
22
|
+
hq-cli **5.108.2** or newer (spool enqueue format and flush semantics). The
|
|
23
|
+
5.108.2 release notes incorrectly said 5.109.0.
|
|
24
|
+
|
|
25
|
+
## [5.108.3] — 2026-09-04
|
|
26
|
+
|
|
5
27
|
## [5.108.2] — 2026-09-04
|
|
6
28
|
|
|
7
29
|
### Fixed
|
|
@@ -5,6 +5,7 @@ import { findHqRoot } from '../utils/manifest.js';
|
|
|
5
5
|
import { QMD_NATIVE_BINDING_REMEDY, isQmdNativeBindingError, } from '../utils/qmd-native-binding-error.js';
|
|
6
6
|
import { isQmdStoreMissingError, qmdStoreMissingMessage, } from '../utils/qmd-store-missing-error.js';
|
|
7
7
|
import { isQmdStoreUnopenableError, qmdStoreUnopenableMessage, } from '../utils/qmd-store-unopenable-error.js';
|
|
8
|
+
import { isQmdWorkdirMissingError, qmdWorkdirMissingMessage, } from '../utils/qmd-workdir-missing-error.js';
|
|
8
9
|
const defaults = {
|
|
9
10
|
reconcileCollections,
|
|
10
11
|
deriveCollections,
|
|
@@ -147,6 +148,18 @@ export function registerIndexCommand(program, dependencies = defaults) {
|
|
|
147
148
|
process.stderr.write(`qmd: unusable — ${remedy}\n`);
|
|
148
149
|
process.exitCode = 1;
|
|
149
150
|
}
|
|
151
|
+
else if (isQmdWorkdirMissingError(error)) {
|
|
152
|
+
// The working directory hq handed qmd (the resolved hq root) does not
|
|
153
|
+
// exist, so the spawn failed with ENOENT (HQ-CLI-1A). Like the
|
|
154
|
+
// store-missing/unopenable cases, this diagnostic command should
|
|
155
|
+
// DESCRIBE the broken working directory, not crash on it or send the
|
|
156
|
+
// user to reinstall qmd: print the classified reason + remedy naming
|
|
157
|
+
// the directory and exit 1 without rethrowing. Every OTHER qmd failure
|
|
158
|
+
// keeps propagating.
|
|
159
|
+
const remedy = qmdWorkdirMissingMessage(error) ?? 'its working directory does not exist';
|
|
160
|
+
process.stderr.write(`qmd: unusable — ${remedy}\n`);
|
|
161
|
+
process.exitCode = 1;
|
|
162
|
+
}
|
|
150
163
|
else {
|
|
151
164
|
throw error;
|
|
152
165
|
}
|
package/dist/commands/mesh.js
CHANGED
|
@@ -18,7 +18,7 @@ import { CLI_KIND_TO_SCHEMA, EnqueueValidationError, enqueueSessionEvent, } from
|
|
|
18
18
|
import { flushSessionEvents } from "../lib/mesh/live/flush.js";
|
|
19
19
|
import { createSessionEventsPoster, resolveVaultApiBase, } from "../lib/mesh/live/session-events-client.js";
|
|
20
20
|
import { workMeshRoot } from "../lib/mesh/live/paths.js";
|
|
21
|
-
import { buildInstallPaths, collectDaemonDoctor, daemonServiceStatus, detectPlatform, formatDaemonDoctor, installDaemonService, runMeshDaemon, uninstallDaemonService, } from "../lib/mesh/live/daemon/index.js";
|
|
21
|
+
import { buildInstallPaths, collectDaemonDoctor, daemonServiceStatus, detectPlatform, formatDaemonDoctor, installDaemonService, readDaemonState, runMeshDaemon, uninstallDaemonService, daemonDir, } from "../lib/mesh/live/daemon/index.js";
|
|
22
22
|
import { workContextRoot } from "../lib/work-context/paths.js";
|
|
23
23
|
import { formatMigrateConfirmation, submitSessionMigration, } from "../lib/work-context/migrate.js";
|
|
24
24
|
import { formatOrganizeList, prepareOrganizeDecision, settleOrganizeAskWithoutBind, submitOrganizeDecision, } from "../lib/work-context/organize.js";
|
|
@@ -873,6 +873,8 @@ async function runDaemonStatus(opts) {
|
|
|
873
873
|
platform: detectPlatform(),
|
|
874
874
|
paths: buildInstallPaths({}),
|
|
875
875
|
});
|
|
876
|
+
const state = readDaemonState(daemonDir());
|
|
877
|
+
const presenceRefusal = state?.presenceRefusal ?? null;
|
|
876
878
|
if (opts.json) {
|
|
877
879
|
console.log(JSON.stringify({
|
|
878
880
|
ok: true,
|
|
@@ -881,10 +883,18 @@ async function runDaemonStatus(opts) {
|
|
|
881
883
|
running: result.running ?? false,
|
|
882
884
|
dest: result.dest,
|
|
883
885
|
message: result.message,
|
|
886
|
+
mqttState: state?.mqttState,
|
|
887
|
+
presenceRefusal,
|
|
884
888
|
}, null, 2));
|
|
885
889
|
return;
|
|
886
890
|
}
|
|
887
891
|
console.log(result.message);
|
|
892
|
+
if (presenceRefusal) {
|
|
893
|
+
console.log(`presence refusal: ${presenceRefusal.code} (HTTP ${presenceRefusal.status}); next retry at ${presenceRefusal.nextRetryAt}`);
|
|
894
|
+
}
|
|
895
|
+
else if (state?.mqttState) {
|
|
896
|
+
console.log(`mqtt: ${state.mqttState}`);
|
|
897
|
+
}
|
|
888
898
|
}
|
|
889
899
|
async function runDaemonDoctorCmd(opts) {
|
|
890
900
|
const report = collectDaemonDoctor();
|
|
@@ -8,6 +8,43 @@ export declare const MQTT_KEEPALIVE_SECONDS = 15;
|
|
|
8
8
|
/** Renew at 80% of credential lifetime. */
|
|
9
9
|
export declare const CREDENTIAL_RENEWAL_FRACTION = 0.8;
|
|
10
10
|
export declare const MIN_RENEWAL_DELAY_MS = 5000;
|
|
11
|
+
/** Long retry when the server refuses credentials (disabled / unsupported). */
|
|
12
|
+
export declare const DEFAULT_REFUSED_RETRY_MS: number;
|
|
13
|
+
export declare const REFUSED_RETRY_ENV = "HQ_MESH_DAEMON_REFUSED_RETRY_MS";
|
|
14
|
+
/**
|
|
15
|
+
* Node's setTimeout treats delays above this as ~1ms (signed 32-bit overflow).
|
|
16
|
+
* Always schedule long waits via {@link scheduleBoundedTimeout}.
|
|
17
|
+
*/
|
|
18
|
+
export declare const MAX_TIMER_DELAY_MS = 2147483647;
|
|
19
|
+
/** Cap any single retry so a bogus Retry-After cannot park the daemon. */
|
|
20
|
+
export declare const MAX_RETRY_DELAY_MS: number;
|
|
21
|
+
export type CredentialVendFailureKind = "refused" | "transient";
|
|
22
|
+
/**
|
|
23
|
+
* Typed vend failure. `refused` = server will not vend right now
|
|
24
|
+
* (FEATURE_DISABLED / REALTIME_CONTRACT_UNSUPPORTED / 403); daemon should
|
|
25
|
+
* wait a long interval instead of the short network backoff.
|
|
26
|
+
*/
|
|
27
|
+
export declare class CredentialVendError extends Error {
|
|
28
|
+
readonly status: number;
|
|
29
|
+
readonly code: string | undefined;
|
|
30
|
+
readonly kind: CredentialVendFailureKind;
|
|
31
|
+
readonly retryAfterMs: number | undefined;
|
|
32
|
+
constructor(input: {
|
|
33
|
+
status: number;
|
|
34
|
+
code?: string;
|
|
35
|
+
kind: CredentialVendFailureKind;
|
|
36
|
+
retryAfterMs?: number;
|
|
37
|
+
message?: string;
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
export declare function parseRetryAfterMs(value: string | null | undefined, nowMs?: number): number | undefined;
|
|
41
|
+
export declare function extractVendErrorCode(body: unknown): string | undefined;
|
|
42
|
+
/** True for server "not now" responses that should use the long retry. */
|
|
43
|
+
export declare function isCredentialVendRefused(status: number, code: string | undefined): boolean;
|
|
44
|
+
export declare function classifyCredentialVendFailure(status: number, body: unknown, retryAfterHeader?: string | null, nowMs?: number): CredentialVendError;
|
|
45
|
+
export declare function defaultRefusedRetryMs(env?: NodeJS.ProcessEnv): number;
|
|
46
|
+
/** ±20% jitter around the refused-retry base interval. */
|
|
47
|
+
export declare function refusedRetryDelayMs(baseMs: number, random: () => number): number;
|
|
11
48
|
export interface PresenceCompany {
|
|
12
49
|
companyUid: string;
|
|
13
50
|
presenceTopic: string;
|
|
@@ -31,6 +68,22 @@ export interface TimerHost {
|
|
|
31
68
|
now(): number;
|
|
32
69
|
}
|
|
33
70
|
export declare const realTimerHost: TimerHost;
|
|
71
|
+
/** Clamp a retry delay to [0, MAX_RETRY_DELAY_MS]. */
|
|
72
|
+
export declare function clampRetryDelayMs(delayMs: number): number;
|
|
73
|
+
export interface BoundedTimeoutHandle {
|
|
74
|
+
clear(): void;
|
|
75
|
+
/** Absolute epoch-ms deadline this schedule is aiming for. */
|
|
76
|
+
deadlineMs: number;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Schedule `fn` after `delayMs`, chaining timers in chunks of at most
|
|
80
|
+
* `maxChunkMs` so the absolute deadline is preserved under Node's timeout cap.
|
|
81
|
+
*
|
|
82
|
+
* Remaining time is tracked by subtracting each armed chunk (so frozen test
|
|
83
|
+
* clocks still advance the chain) and also clamped to the wall-clock deadline
|
|
84
|
+
* when `timers.now()` moves forward.
|
|
85
|
+
*/
|
|
86
|
+
export declare function scheduleBoundedTimeout(timers: TimerHost, fn: () => void, delayMs: number, maxChunkMs?: number): BoundedTimeoutHandle;
|
|
34
87
|
export declare function renewalDelayMs(nowMs: number, expirationIso: string): number;
|
|
35
88
|
export declare function normalizeContract3Bundle(raw: unknown): Contract3Bundle;
|
|
36
89
|
/**
|
|
@@ -40,16 +93,24 @@ export declare function normalizeContract3Bundle(raw: unknown): Contract3Bundle;
|
|
|
40
93
|
* No identity file → return the bundle unchanged (person laptop).
|
|
41
94
|
*/
|
|
42
95
|
export declare function scopeBundleToAgentIdentity(bundle: Contract3Bundle, env?: NodeJS.ProcessEnv): Contract3Bundle;
|
|
96
|
+
export type CredentialVendPostResult = {
|
|
97
|
+
status: number;
|
|
98
|
+
body: unknown;
|
|
99
|
+
/** Optional header map or Fetch Headers; used for Retry-After. */
|
|
100
|
+
headers?: Headers | Record<string, string | undefined | null>;
|
|
101
|
+
};
|
|
43
102
|
export declare function createContract3Fetcher(opts: {
|
|
44
103
|
token: string;
|
|
45
104
|
baseUrl?: string;
|
|
46
|
-
post?: (path: string, body: unknown) => Promise<
|
|
47
|
-
status: number;
|
|
48
|
-
body: unknown;
|
|
49
|
-
}>;
|
|
105
|
+
post?: (path: string, body: unknown) => Promise<CredentialVendPostResult>;
|
|
50
106
|
/** Process env for identity-file scoping (tests). */
|
|
51
107
|
env?: NodeJS.ProcessEnv;
|
|
52
108
|
}): CredentialsFetcher;
|
|
109
|
+
export interface CredentialRenewalErrorInfo {
|
|
110
|
+
/** Absolute ISO timestamp when the renewal retry is scheduled to fire. */
|
|
111
|
+
nextRetryAt: string;
|
|
112
|
+
delayMs: number;
|
|
113
|
+
}
|
|
53
114
|
/**
|
|
54
115
|
* Proactive renewal: at 80% of lifetime, fetch fresh creds and call onRenewed.
|
|
55
116
|
* Does not drop the connection itself — caller reconnects with the new URL.
|
|
@@ -60,9 +121,11 @@ export declare class CredentialRenewalManager {
|
|
|
60
121
|
private readonly onError;
|
|
61
122
|
private readonly timers;
|
|
62
123
|
private readonly retryDelayMs;
|
|
124
|
+
private readonly refusedRetryMs;
|
|
125
|
+
private readonly random;
|
|
63
126
|
private handle;
|
|
64
127
|
private stopped;
|
|
65
|
-
constructor(fetchCredentials: CredentialsFetcher, onRenewed: (bundle: Contract3Bundle) => void, onError?: (err: unknown) => void, timers?: TimerHost, retryDelayMs?: number);
|
|
128
|
+
constructor(fetchCredentials: CredentialsFetcher, onRenewed: (bundle: Contract3Bundle) => void, onError?: (err: unknown, info?: CredentialRenewalErrorInfo) => void, timers?: TimerHost, retryDelayMs?: number, refusedRetryMs?: number, random?: () => number);
|
|
66
129
|
schedule(bundle: Contract3Bundle): void;
|
|
67
130
|
private renew;
|
|
68
131
|
stop(): void;
|
|
@@ -10,11 +10,146 @@ export const MQTT_KEEPALIVE_SECONDS = 15;
|
|
|
10
10
|
/** Renew at 80% of credential lifetime. */
|
|
11
11
|
export const CREDENTIAL_RENEWAL_FRACTION = 0.8;
|
|
12
12
|
export const MIN_RENEWAL_DELAY_MS = 5_000;
|
|
13
|
+
/** Long retry when the server refuses credentials (disabled / unsupported). */
|
|
14
|
+
export const DEFAULT_REFUSED_RETRY_MS = 10 * 60_000;
|
|
15
|
+
export const REFUSED_RETRY_ENV = "HQ_MESH_DAEMON_REFUSED_RETRY_MS";
|
|
16
|
+
/**
|
|
17
|
+
* Node's setTimeout treats delays above this as ~1ms (signed 32-bit overflow).
|
|
18
|
+
* Always schedule long waits via {@link scheduleBoundedTimeout}.
|
|
19
|
+
*/
|
|
20
|
+
export const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
21
|
+
/** Cap any single retry so a bogus Retry-After cannot park the daemon. */
|
|
22
|
+
export const MAX_RETRY_DELAY_MS = 24 * 60 * 60 * 1000;
|
|
23
|
+
/**
|
|
24
|
+
* Typed vend failure. `refused` = server will not vend right now
|
|
25
|
+
* (FEATURE_DISABLED / REALTIME_CONTRACT_UNSUPPORTED / 403); daemon should
|
|
26
|
+
* wait a long interval instead of the short network backoff.
|
|
27
|
+
*/
|
|
28
|
+
export class CredentialVendError extends Error {
|
|
29
|
+
status;
|
|
30
|
+
code;
|
|
31
|
+
kind;
|
|
32
|
+
retryAfterMs;
|
|
33
|
+
constructor(input) {
|
|
34
|
+
const codePart = input.code ? ` ${input.code}` : "";
|
|
35
|
+
super(input.message ?? `credential vend failed: HTTP ${input.status}${codePart}`);
|
|
36
|
+
this.name = "CredentialVendError";
|
|
37
|
+
this.status = input.status;
|
|
38
|
+
this.code = input.code;
|
|
39
|
+
this.kind = input.kind;
|
|
40
|
+
this.retryAfterMs = input.retryAfterMs;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
export function parseRetryAfterMs(value, nowMs = Date.now()) {
|
|
44
|
+
if (!value)
|
|
45
|
+
return undefined;
|
|
46
|
+
const trimmed = value.trim();
|
|
47
|
+
if (!trimmed)
|
|
48
|
+
return undefined;
|
|
49
|
+
const seconds = Number(trimmed);
|
|
50
|
+
if (Number.isFinite(seconds) && seconds >= 0) {
|
|
51
|
+
return Math.round(seconds * 1000);
|
|
52
|
+
}
|
|
53
|
+
const dateMs = Date.parse(trimmed);
|
|
54
|
+
if (!Number.isNaN(dateMs)) {
|
|
55
|
+
return Math.max(0, dateMs - nowMs);
|
|
56
|
+
}
|
|
57
|
+
return undefined;
|
|
58
|
+
}
|
|
59
|
+
export function extractVendErrorCode(body) {
|
|
60
|
+
if (!body || typeof body !== "object" || Array.isArray(body))
|
|
61
|
+
return undefined;
|
|
62
|
+
const code = body.code;
|
|
63
|
+
return typeof code === "string" && code.trim() ? code.trim() : undefined;
|
|
64
|
+
}
|
|
65
|
+
/** True for server "not now" responses that should use the long retry. */
|
|
66
|
+
export function isCredentialVendRefused(status, code) {
|
|
67
|
+
if (status === 403)
|
|
68
|
+
return true;
|
|
69
|
+
if (status === 404 && code === "FEATURE_DISABLED")
|
|
70
|
+
return true;
|
|
71
|
+
if (status === 409 && code === "REALTIME_CONTRACT_UNSUPPORTED")
|
|
72
|
+
return true;
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
export function classifyCredentialVendFailure(status, body, retryAfterHeader, nowMs) {
|
|
76
|
+
const code = extractVendErrorCode(body);
|
|
77
|
+
const retryAfterMs = parseRetryAfterMs(retryAfterHeader, nowMs);
|
|
78
|
+
const kind = isCredentialVendRefused(status, code)
|
|
79
|
+
? "refused"
|
|
80
|
+
: "transient";
|
|
81
|
+
return new CredentialVendError({ status, code, kind, retryAfterMs });
|
|
82
|
+
}
|
|
83
|
+
export function defaultRefusedRetryMs(env = process.env) {
|
|
84
|
+
const raw = env[REFUSED_RETRY_ENV]?.trim();
|
|
85
|
+
if (raw) {
|
|
86
|
+
const n = Number(raw);
|
|
87
|
+
if (Number.isFinite(n) && n >= 0)
|
|
88
|
+
return n;
|
|
89
|
+
}
|
|
90
|
+
return DEFAULT_REFUSED_RETRY_MS;
|
|
91
|
+
}
|
|
92
|
+
/** ±20% jitter around the refused-retry base interval. */
|
|
93
|
+
export function refusedRetryDelayMs(baseMs, random) {
|
|
94
|
+
const unit = Math.max(0, baseMs);
|
|
95
|
+
return Math.round(unit * (0.8 + random() * 0.4));
|
|
96
|
+
}
|
|
13
97
|
export const realTimerHost = {
|
|
14
98
|
setTimeout: (fn, ms) => setTimeout(fn, ms),
|
|
15
99
|
clearTimeout: (h) => clearTimeout(h),
|
|
16
100
|
now: () => Date.now(),
|
|
17
101
|
};
|
|
102
|
+
/** Clamp a retry delay to [0, MAX_RETRY_DELAY_MS]. */
|
|
103
|
+
export function clampRetryDelayMs(delayMs) {
|
|
104
|
+
if (!Number.isFinite(delayMs) || delayMs <= 0)
|
|
105
|
+
return 0;
|
|
106
|
+
return Math.min(Math.floor(delayMs), MAX_RETRY_DELAY_MS);
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Schedule `fn` after `delayMs`, chaining timers in chunks of at most
|
|
110
|
+
* `maxChunkMs` so the absolute deadline is preserved under Node's timeout cap.
|
|
111
|
+
*
|
|
112
|
+
* Remaining time is tracked by subtracting each armed chunk (so frozen test
|
|
113
|
+
* clocks still advance the chain) and also clamped to the wall-clock deadline
|
|
114
|
+
* when `timers.now()` moves forward.
|
|
115
|
+
*/
|
|
116
|
+
export function scheduleBoundedTimeout(timers, fn, delayMs, maxChunkMs = MAX_TIMER_DELAY_MS) {
|
|
117
|
+
let handle = null;
|
|
118
|
+
let cancelled = false;
|
|
119
|
+
const safeDelay = Math.max(0, Number.isFinite(delayMs) ? delayMs : 0);
|
|
120
|
+
const chunkLimit = Math.max(1, Math.min(maxChunkMs, MAX_TIMER_DELAY_MS));
|
|
121
|
+
const deadlineMs = timers.now() + safeDelay;
|
|
122
|
+
let remainingMs = safeDelay;
|
|
123
|
+
const arm = () => {
|
|
124
|
+
const chunk = Math.min(remainingMs, chunkLimit);
|
|
125
|
+
handle = timers.setTimeout(() => {
|
|
126
|
+
handle = null;
|
|
127
|
+
if (cancelled)
|
|
128
|
+
return;
|
|
129
|
+
remainingMs -= chunk;
|
|
130
|
+
const byClock = deadlineMs - timers.now();
|
|
131
|
+
if (byClock < remainingMs) {
|
|
132
|
+
remainingMs = Math.max(0, byClock);
|
|
133
|
+
}
|
|
134
|
+
if (remainingMs <= 0) {
|
|
135
|
+
fn();
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
arm();
|
|
139
|
+
}, chunk);
|
|
140
|
+
};
|
|
141
|
+
arm();
|
|
142
|
+
return {
|
|
143
|
+
deadlineMs,
|
|
144
|
+
clear() {
|
|
145
|
+
cancelled = true;
|
|
146
|
+
if (handle !== null) {
|
|
147
|
+
timers.clearTimeout(handle);
|
|
148
|
+
handle = null;
|
|
149
|
+
}
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
}
|
|
18
153
|
export function renewalDelayMs(nowMs, expirationIso) {
|
|
19
154
|
const expMs = Date.parse(expirationIso);
|
|
20
155
|
if (Number.isNaN(expMs))
|
|
@@ -131,6 +266,16 @@ export function scopeBundleToAgentIdentity(bundle, env = process.env) {
|
|
|
131
266
|
}
|
|
132
267
|
return { ...bundle, companies: matched };
|
|
133
268
|
}
|
|
269
|
+
function headerGet(headers, name) {
|
|
270
|
+
if (!headers)
|
|
271
|
+
return null;
|
|
272
|
+
if (typeof headers.get === "function") {
|
|
273
|
+
return headers.get(name);
|
|
274
|
+
}
|
|
275
|
+
const rec = headers;
|
|
276
|
+
const direct = rec[name] ?? rec[name.toLowerCase()];
|
|
277
|
+
return direct == null ? null : String(direct);
|
|
278
|
+
}
|
|
134
279
|
export function createContract3Fetcher(opts) {
|
|
135
280
|
return async () => {
|
|
136
281
|
let bundle;
|
|
@@ -139,7 +284,7 @@ export function createContract3Fetcher(opts) {
|
|
|
139
284
|
contractVersion: 3,
|
|
140
285
|
});
|
|
141
286
|
if (res.status < 200 || res.status >= 300) {
|
|
142
|
-
throw
|
|
287
|
+
throw classifyCredentialVendFailure(res.status, res.body, headerGet(res.headers, "Retry-After"));
|
|
143
288
|
}
|
|
144
289
|
bundle = normalizeContract3Bundle(res.body);
|
|
145
290
|
}
|
|
@@ -162,7 +307,7 @@ export function createContract3Fetcher(opts) {
|
|
|
162
307
|
}
|
|
163
308
|
}
|
|
164
309
|
if (!res.ok) {
|
|
165
|
-
throw
|
|
310
|
+
throw classifyCredentialVendFailure(res.status, body, res.headers.get("Retry-After"));
|
|
166
311
|
}
|
|
167
312
|
bundle = normalizeContract3Bundle(body);
|
|
168
313
|
}
|
|
@@ -179,21 +324,25 @@ export class CredentialRenewalManager {
|
|
|
179
324
|
onError;
|
|
180
325
|
timers;
|
|
181
326
|
retryDelayMs;
|
|
327
|
+
refusedRetryMs;
|
|
328
|
+
random;
|
|
182
329
|
handle = null;
|
|
183
330
|
stopped = false;
|
|
184
|
-
constructor(fetchCredentials, onRenewed, onError = () => { }, timers = realTimerHost, retryDelayMs = 30_000) {
|
|
331
|
+
constructor(fetchCredentials, onRenewed, onError = () => { }, timers = realTimerHost, retryDelayMs = 30_000, refusedRetryMs = DEFAULT_REFUSED_RETRY_MS, random = Math.random) {
|
|
185
332
|
this.fetchCredentials = fetchCredentials;
|
|
186
333
|
this.onRenewed = onRenewed;
|
|
187
334
|
this.onError = onError;
|
|
188
335
|
this.timers = timers;
|
|
189
336
|
this.retryDelayMs = retryDelayMs;
|
|
337
|
+
this.refusedRetryMs = refusedRetryMs;
|
|
338
|
+
this.random = random;
|
|
190
339
|
}
|
|
191
340
|
schedule(bundle) {
|
|
192
341
|
if (this.stopped)
|
|
193
342
|
return;
|
|
194
343
|
this.clear();
|
|
195
344
|
const delay = renewalDelayMs(this.timers.now(), bundle.expiresAt);
|
|
196
|
-
this.handle = this.timers
|
|
345
|
+
this.handle = scheduleBoundedTimeout(this.timers, () => void this.renew(), delay);
|
|
197
346
|
}
|
|
198
347
|
async renew() {
|
|
199
348
|
if (this.stopped)
|
|
@@ -206,11 +355,23 @@ export class CredentialRenewalManager {
|
|
|
206
355
|
this.schedule(bundle);
|
|
207
356
|
}
|
|
208
357
|
catch (err) {
|
|
209
|
-
this.onError(err);
|
|
210
358
|
if (this.stopped)
|
|
211
359
|
return;
|
|
212
360
|
this.clear();
|
|
213
|
-
|
|
361
|
+
let delay = this.retryDelayMs;
|
|
362
|
+
if (err instanceof CredentialVendError && err.kind === "refused") {
|
|
363
|
+
delay =
|
|
364
|
+
err.retryAfterMs !== undefined
|
|
365
|
+
? Math.max(0, err.retryAfterMs)
|
|
366
|
+
: refusedRetryDelayMs(this.refusedRetryMs, this.random);
|
|
367
|
+
delay = clampRetryDelayMs(delay);
|
|
368
|
+
}
|
|
369
|
+
const nextRetryAt = new Date(this.timers.now() + delay).toISOString();
|
|
370
|
+
// Notify with the real deadline before arming so noteRefusal records it.
|
|
371
|
+
this.onError(err, { nextRetryAt, delayMs: delay });
|
|
372
|
+
if (this.stopped)
|
|
373
|
+
return;
|
|
374
|
+
this.handle = scheduleBoundedTimeout(this.timers, () => void this.renew(), delay);
|
|
214
375
|
}
|
|
215
376
|
}
|
|
216
377
|
stop() {
|
|
@@ -219,7 +380,7 @@ export class CredentialRenewalManager {
|
|
|
219
380
|
}
|
|
220
381
|
clear() {
|
|
221
382
|
if (this.handle !== null) {
|
|
222
|
-
this.
|
|
383
|
+
this.handle.clear();
|
|
223
384
|
this.handle = null;
|
|
224
385
|
}
|
|
225
386
|
}
|
|
@@ -23,6 +23,8 @@ export interface DaemonDoctorReport {
|
|
|
23
23
|
tokenSource: CognitoTokenSource;
|
|
24
24
|
/** human | agent | unknown — from token claims / machine signal. */
|
|
25
25
|
actorKind: CognitoActorKind;
|
|
26
|
+
/** Credential vend refused (disabled / unsupported / 403). */
|
|
27
|
+
presenceRefusal?: DaemonStateFile["presenceRefusal"];
|
|
26
28
|
}
|
|
27
29
|
export interface DaemonDoctorDeps {
|
|
28
30
|
home?: string;
|
|
@@ -91,6 +91,10 @@ export function collectDaemonDoctor(deps = {}) {
|
|
|
91
91
|
if (deadLetterCount > 0) {
|
|
92
92
|
unhealthyReasons.push(`dead-letter count=${deadLetterCount}`);
|
|
93
93
|
}
|
|
94
|
+
const presenceRefusal = state?.presenceRefusal ?? null;
|
|
95
|
+
if (presenceRefusal) {
|
|
96
|
+
unhealthyReasons.push(`presence credential refused (${presenceRefusal.code}); next retry ${presenceRefusal.nextRetryAt}`);
|
|
97
|
+
}
|
|
94
98
|
const auth = describeCognitoTokenSource({ home, env });
|
|
95
99
|
return {
|
|
96
100
|
running,
|
|
@@ -108,6 +112,7 @@ export function collectDaemonDoctor(deps = {}) {
|
|
|
108
112
|
oldestSpoolAgeMs,
|
|
109
113
|
tokenSource: auth.tokenSource,
|
|
110
114
|
actorKind: auth.actorKind,
|
|
115
|
+
presenceRefusal,
|
|
111
116
|
};
|
|
112
117
|
}
|
|
113
118
|
export function formatDaemonDoctor(report) {
|
|
@@ -125,6 +130,9 @@ export function formatDaemonDoctor(report) {
|
|
|
125
130
|
? ` ok=${report.lastFlushResult.ok} posted=${report.lastFlushResult.posted}`
|
|
126
131
|
: ""}`,
|
|
127
132
|
];
|
|
133
|
+
if (report.presenceRefusal) {
|
|
134
|
+
lines.push(`presence refusal: ${report.presenceRefusal.code} (HTTP ${report.presenceRefusal.status}); next retry at ${report.presenceRefusal.nextRetryAt}`);
|
|
135
|
+
}
|
|
128
136
|
if (report.unhealthy) {
|
|
129
137
|
lines.push(`health: UNHEALTHY`);
|
|
130
138
|
for (const reason of report.unhealthyReasons) {
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
export { DAEMON_DIRNAME, DAEMON_LOG_MAX_BYTES, DAEMON_LOG_NAME, DAEMON_PID_NAME, DAEMON_STATE_NAME, LAUNCHD_LABEL, SYSTEMD_UNIT_NAME, daemonDir, daemonLogPath, daemonPidPath, daemonStatePath, sessionBoardPath, } from "./paths.js";
|
|
2
2
|
export { acquirePidLock, defaultPidLockDeps, parsePidLock, pidLockStatus, readPidLock, releasePidLock, resolveDaemonDir, } from "./pid-lock.js";
|
|
3
3
|
export type { PidLockDeps, PidLockRecord, PidLockResult } from "./pid-lock.js";
|
|
4
|
-
export { CREDENTIAL_RENEWAL_FRACTION, MQTT_KEEPALIVE_SECONDS, REALTIME_CREDENTIALS_PATH, CredentialRenewalManager, createContract3Fetcher, normalizeContract3Bundle, realTimerHost, renewalDelayMs, } from "./credentials.js";
|
|
5
|
-
export type { Contract3Bundle, CredentialsFetcher, PresenceCompany, TimerHost, } from "./credentials.js";
|
|
4
|
+
export { CREDENTIAL_RENEWAL_FRACTION, DEFAULT_REFUSED_RETRY_MS, MAX_RETRY_DELAY_MS, MAX_TIMER_DELAY_MS, MQTT_KEEPALIVE_SECONDS, REALTIME_CREDENTIALS_PATH, REFUSED_RETRY_ENV, CredentialRenewalManager, CredentialVendError, classifyCredentialVendFailure, clampRetryDelayMs, createContract3Fetcher, defaultRefusedRetryMs, extractVendErrorCode, isCredentialVendRefused, normalizeContract3Bundle, parseRetryAfterMs, realTimerHost, refusedRetryDelayMs, renewalDelayMs, scheduleBoundedTimeout, } from "./credentials.js";
|
|
5
|
+
export type { BoundedTimeoutHandle, Contract3Bundle, CredentialRenewalErrorInfo, CredentialVendFailureKind, CredentialVendPostResult, CredentialsFetcher, PresenceCompany, TimerHost, } from "./credentials.js";
|
|
6
6
|
export { amzDateOf, hex, presignIotWssUrl, rfc3986Encode } from "./presign.js";
|
|
7
7
|
export type { IotCredentials } from "./presign.js";
|
|
8
8
|
export { PresenceClient, backoffDelayMs, buildPresencePayload, defaultMqttConnect, isOwnPresenceTopic, } from "./presence.js";
|
|
9
|
-
export type { MeshMqttClientLike, MqttConnectFn, MqttConnectionState, PresenceClientOptions, PresencePayload, } from "./presence.js";
|
|
9
|
+
export type { MeshMqttClientLike, MqttConnectFn, MqttConnectionState, PresenceClientOptions, PresencePayload, PresenceRefusal, } from "./presence.js";
|
|
10
10
|
export { defaultDaemonState, patchDaemonState, readDaemonState, writeDaemonState, } from "./state.js";
|
|
11
|
-
export type { DaemonStateFile } from "./state.js";
|
|
11
|
+
export type { DaemonStateFile, PresenceRefusalState } from "./state.js";
|
|
12
12
|
export { appendDaemonLog, daemonLogLine, ensureDaemonLog, resolveDaemonAssetDir, rotateDaemonLogIfNeeded, } from "./log.js";
|
|
13
13
|
export { BOARD_REFRESH_INTERVAL_MS, createVaultBoardReader, formatBoardMarkdown, refreshBoundSessionBoards, writeBoardMarkdown, } from "./board-refresh.js";
|
|
14
14
|
export type { BoardReader, BoardRefreshDeps, BoardStorySnapshot } from "./board-refresh.js";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { DAEMON_DIRNAME, DAEMON_LOG_MAX_BYTES, DAEMON_LOG_NAME, DAEMON_PID_NAME, DAEMON_STATE_NAME, LAUNCHD_LABEL, SYSTEMD_UNIT_NAME, daemonDir, daemonLogPath, daemonPidPath, daemonStatePath, sessionBoardPath, } from "./paths.js";
|
|
2
2
|
export { acquirePidLock, defaultPidLockDeps, parsePidLock, pidLockStatus, readPidLock, releasePidLock, resolveDaemonDir, } from "./pid-lock.js";
|
|
3
|
-
export { CREDENTIAL_RENEWAL_FRACTION, MQTT_KEEPALIVE_SECONDS, REALTIME_CREDENTIALS_PATH, CredentialRenewalManager, createContract3Fetcher, normalizeContract3Bundle, realTimerHost, renewalDelayMs, } from "./credentials.js";
|
|
3
|
+
export { CREDENTIAL_RENEWAL_FRACTION, DEFAULT_REFUSED_RETRY_MS, MAX_RETRY_DELAY_MS, MAX_TIMER_DELAY_MS, MQTT_KEEPALIVE_SECONDS, REALTIME_CREDENTIALS_PATH, REFUSED_RETRY_ENV, CredentialRenewalManager, CredentialVendError, classifyCredentialVendFailure, clampRetryDelayMs, createContract3Fetcher, defaultRefusedRetryMs, extractVendErrorCode, isCredentialVendRefused, normalizeContract3Bundle, parseRetryAfterMs, realTimerHost, refusedRetryDelayMs, renewalDelayMs, scheduleBoundedTimeout, } from "./credentials.js";
|
|
4
4
|
export { amzDateOf, hex, presignIotWssUrl, rfc3986Encode } from "./presign.js";
|
|
5
5
|
export { PresenceClient, backoffDelayMs, buildPresencePayload, defaultMqttConnect, isOwnPresenceTopic, } from "./presence.js";
|
|
6
6
|
export { defaultDaemonState, patchDaemonState, readDaemonState, writeDaemonState, } from "./state.js";
|
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
* - Publishes retained online ONLY to own presence topics
|
|
7
7
|
* - Offline is server-only (IoT lifecycle → PresenceIngestFunction)
|
|
8
8
|
* - Never subscribes to thread topics
|
|
9
|
-
* - Reconnects with
|
|
9
|
+
* - Reconnects with monotonic full-jitter backoff 1s–60s on close / network
|
|
10
|
+
* - Server credential refusals use a long retry (default 10m ±20%)
|
|
10
11
|
*/
|
|
11
12
|
import { type IClientOptions, type MqttClient } from "mqtt";
|
|
12
13
|
import { type Contract3Bundle, type CredentialsFetcher, type TimerHost } from "./credentials.js";
|
|
@@ -30,6 +31,11 @@ export interface MeshMqttClientLike {
|
|
|
30
31
|
subscribe?: unknown;
|
|
31
32
|
}
|
|
32
33
|
export type MqttConnectFn = (url: string, opts: IClientOptions) => MeshMqttClientLike;
|
|
34
|
+
export interface PresenceRefusal {
|
|
35
|
+
code: string;
|
|
36
|
+
status: number;
|
|
37
|
+
nextRetryAt: string;
|
|
38
|
+
}
|
|
33
39
|
export interface PresenceClientOptions {
|
|
34
40
|
fetchCredentials: CredentialsFetcher;
|
|
35
41
|
mqttConnect?: MqttConnectFn;
|
|
@@ -37,14 +43,22 @@ export interface PresenceClientOptions {
|
|
|
37
43
|
random?: () => number;
|
|
38
44
|
baseBackoffMs?: number;
|
|
39
45
|
maxBackoffMs?: number;
|
|
46
|
+
/** Base refused-retry interval (tests / HQ_MESH_DAEMON_REFUSED_RETRY_MS). */
|
|
47
|
+
refusedRetryMs?: number;
|
|
40
48
|
/** Invoked when online is (re)published after connect. */
|
|
41
49
|
onOnline?: (companies: string[]) => void;
|
|
42
50
|
onState?: (state: MqttConnectionState) => void;
|
|
43
51
|
onError?: (err: unknown) => void;
|
|
52
|
+
/** Fired when refusal appears, code changes, clears, or nextRetryAt updates. */
|
|
53
|
+
onRefusal?: (refusal: PresenceRefusal | null) => void;
|
|
44
54
|
now?: () => Date;
|
|
55
|
+
env?: NodeJS.ProcessEnv;
|
|
45
56
|
}
|
|
46
|
-
/**
|
|
47
|
-
|
|
57
|
+
/**
|
|
58
|
+
* Full-jitter capped exponential backoff (1s base → 60s cap by default),
|
|
59
|
+
* floored at `previousMs` so consecutive failures never shrink the delay.
|
|
60
|
+
*/
|
|
61
|
+
export declare function backoffDelayMs(attempt: number, baseMs: number, maxMs: number, random: () => number, previousMs?: number): number;
|
|
48
62
|
export declare function buildPresencePayload(input: {
|
|
49
63
|
status: "online" | "offline";
|
|
50
64
|
actorUid: string;
|
|
@@ -62,20 +76,26 @@ export declare class PresenceClient {
|
|
|
62
76
|
private bundle;
|
|
63
77
|
private state;
|
|
64
78
|
private attempt;
|
|
79
|
+
private lastBackoffMs;
|
|
65
80
|
private reconnectHandle;
|
|
66
81
|
private stopped;
|
|
67
82
|
private generation;
|
|
68
83
|
private publishedTopics;
|
|
84
|
+
/** Last refusal code we logged via onError (episode dedupe). */
|
|
85
|
+
private loggedRefusalKey;
|
|
86
|
+
private refusal;
|
|
69
87
|
private readonly mqttConnect;
|
|
70
88
|
private readonly timers;
|
|
71
89
|
private readonly random;
|
|
72
90
|
private readonly baseBackoffMs;
|
|
73
91
|
private readonly maxBackoffMs;
|
|
92
|
+
private readonly refusedRetryMs;
|
|
74
93
|
private readonly renewal;
|
|
75
94
|
private readonly now;
|
|
76
95
|
constructor(options: PresenceClientOptions);
|
|
77
96
|
getConnectionState(): MqttConnectionState;
|
|
78
97
|
getBundle(): Contract3Bundle | null;
|
|
98
|
+
getRefusal(): PresenceRefusal | null;
|
|
79
99
|
/** Topics this client has published to (for doctor / tests). */
|
|
80
100
|
getPublishedTopics(): readonly string[];
|
|
81
101
|
private setState;
|
|
@@ -90,6 +110,10 @@ export declare class PresenceClient {
|
|
|
90
110
|
reconnectNow(): void;
|
|
91
111
|
private teardownClient;
|
|
92
112
|
private onCredentialsRenewed;
|
|
113
|
+
private handleRenewalError;
|
|
114
|
+
private refusalKey;
|
|
115
|
+
private noteRefusal;
|
|
116
|
+
private clearRefusal;
|
|
93
117
|
private connectOnce;
|
|
94
118
|
/** Publish retained online to every own presence topic (connect / renew). */
|
|
95
119
|
private publishAll;
|