@indigoai-us/hq-cli 5.345.163 → 5.345.164
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/dist/commands/billing-usage.d.ts +16 -2
- package/dist/commands/billing-usage.js +63 -9
- package/dist/commands/billing.d.ts +34 -0
- package/dist/commands/billing.js +108 -1
- package/dist/commands/deploy-domains.d.ts +2 -0
- package/dist/commands/deploy-domains.js +18 -3
- package/dist/lib/billing/billing-status-lines.d.ts +14 -0
- package/dist/lib/billing/billing-status-lines.js +43 -0
- package/dist/lib/billing/usage-only.d.ts +18 -0
- package/dist/lib/billing/usage-only.js +18 -1
- package/dist/lib/doctor/checks/sync-runtime.d.ts +34 -0
- package/dist/lib/doctor/checks/sync-runtime.js +217 -8
- package/dist/lib/doctor/fix/apply.js +1 -0
- package/dist/lib/doctor/fix/refresh-sync-outcome.d.ts +4 -1
- package/dist/lib/doctor/fix/refresh-sync-outcome.js +15 -0
- package/dist/lib/doctor/fix/refresh-sync-progress.d.ts +42 -0
- package/dist/lib/doctor/fix/refresh-sync-progress.js +100 -0
- package/dist/lib/doctor/json-output.d.ts +7 -0
- package/dist/lib/doctor/json-output.js +8 -1
- package/dist/lib/doctor/types.d.ts +1 -1
- package/dist/lib/doctor/types.js +1 -0
- package/dist/lib/mesh/live/daemon/work-subscribe.d.ts +1 -1
- package/dist/lib/mesh/live/daemon/work-subscribe.js +121 -38
- package/dist/lib/mesh/live/paths.d.ts +1 -0
- package/dist/lib/mesh/live/paths.js +1 -0
- package/package.json +7 -7
|
@@ -3,13 +3,27 @@
|
|
|
3
3
|
* (realtime lag, receiver permission errors, restart loops, device-limit
|
|
4
4
|
* backoff) plus the conflict counter the desktop app keeps in client health.
|
|
5
5
|
*
|
|
6
|
-
* Results carry
|
|
7
|
-
*
|
|
6
|
+
* Results carry a closed reason code and counts. Log content (tenant ids, AWS
|
|
7
|
+
* error prose, queue URLs, absolute local paths) never leaves this module.
|
|
8
|
+
*
|
|
9
|
+
* Two exceptions, both bounded and both local-only (the client-health report
|
|
10
|
+
* forwards check ids, never `details`):
|
|
11
|
+
* - `sync.runtime.receiver-permission` carries `details.byCall`, a count per
|
|
12
|
+
* closed call class (which cloud call was refused), so a refusal on the
|
|
13
|
+
* per-device queue credentials can be told apart from a refusal on the
|
|
14
|
+
* object pull that follows a change event. Before this, 28 automated
|
|
15
|
+
* client-health reports carried only the count (feedback_cd56a501).
|
|
16
|
+
* - `sync.runtime.pull-access-denied` names the vault keys (never absolute
|
|
17
|
+
* local paths) the receiver was refused on pull, with the runner's bounded
|
|
18
|
+
* `reason` tag, capped at {@link PULL_DENIED_KEY_CAP} keys, so the stored
|
|
19
|
+
* grant can be compared with the key the pull looked up (policy
|
|
20
|
+
* hq-authorization-path-canonicalization).
|
|
8
21
|
*/
|
|
9
22
|
import * as fs from "node:fs";
|
|
10
23
|
import * as os from "node:os";
|
|
11
24
|
import * as path from "node:path";
|
|
12
25
|
import { currentSyncLogFamily, SYNC_LOG_GENERATIONS, syncLogBasePaths, syncLogGenerationPath, } from "../../../utils/sync-log-sources.js";
|
|
26
|
+
import { readSessionCompanyLock } from "../../session-company-lock.js";
|
|
13
27
|
export const SYNC_RUNTIME_WINDOW_MS = 60 * 60_000;
|
|
14
28
|
/** Ignore timing and restart signals until a newly started receiver has settled. */
|
|
15
29
|
export const SYNC_RUNTIME_RESTART_WARMUP_MS = 20 * 60_000;
|
|
@@ -18,9 +32,93 @@ const LAG_MIN_RECEIPTS = 3;
|
|
|
18
32
|
const RESTART_MIN_STARTS = 4;
|
|
19
33
|
const CONFLICT_MIN_COUNT = 5;
|
|
20
34
|
const LOG_TAIL_BYTES = 8 * 1024 * 1024;
|
|
35
|
+
/** Most refused vault keys named by `sync.runtime.pull-access-denied`. */
|
|
36
|
+
export const PULL_DENIED_KEY_CAP = 10;
|
|
21
37
|
const TIMESTAMP = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s/;
|
|
22
38
|
const PERMISSION = /AccessDenied|not authorized to perform|\b403\b/;
|
|
23
39
|
const DEVICE_LIMIT = /sync device limit exceeded|"sync_device_limit"/;
|
|
40
|
+
const SUBSCRIBE_ROUTE = /\/v1\/sync\/subscribe/;
|
|
41
|
+
/** A vault key is relative; anything that looks like a local absolute path is dropped. */
|
|
42
|
+
const ABSOLUTE_LOCAL_PATH = /^(?:\/|[A-Za-z]:[\\/]|\\\\)/;
|
|
43
|
+
/**
|
|
44
|
+
* Vault keys this check will name. Printable, no control characters, no
|
|
45
|
+
* backslashes, no shell metacharacters other than space, `@`, `+`, `(`, `)`.
|
|
46
|
+
* The key is also single-quoted when embedded in the remediation command, so
|
|
47
|
+
* a `'` inside a key cannot end the quote. Keys outside this set are counted
|
|
48
|
+
* but not named.
|
|
49
|
+
*/
|
|
50
|
+
const NAMEABLE_VAULT_KEY = /^[A-Za-z0-9._\/ @()+'-]{1,512}$/;
|
|
51
|
+
/** A key refused on at least this many pulls in the hour is a standing condition, not a one-off. */
|
|
52
|
+
export const PULL_DENIED_MIN_REPEATS = 2;
|
|
53
|
+
/** The runner's bounded machine tag on a `skip-access-denied` event. */
|
|
54
|
+
const REASON_TAG = /^[A-Za-z0-9_-]{1,40}$/;
|
|
55
|
+
/** Placeholder scope for a refusal whose runner event carried no usable company label. */
|
|
56
|
+
const UNKNOWN_COMPANY = "unknown";
|
|
57
|
+
const PERSONAL_VAULT_SLUGS = new Set(["personal", "__hq_personal_vault__"]);
|
|
58
|
+
const COMPANY_SLUG = /^[A-Za-z0-9][A-Za-z0-9._-]{0,80}$/;
|
|
59
|
+
const RECEIVER_EVENT_CALLS = {
|
|
60
|
+
"receiver.receive.failed": "sqs-receive",
|
|
61
|
+
"receiver.retry.failed": "sqs-visibility",
|
|
62
|
+
"receiver.lease.extend.failed": "sqs-visibility",
|
|
63
|
+
"receiver.release.failed": "sqs-visibility",
|
|
64
|
+
"receiver.delete.failed": "sqs-delete",
|
|
65
|
+
"receiver.metric.failed": "cloudwatch-metric",
|
|
66
|
+
"receiver.sync.failed": "targeted-pull",
|
|
67
|
+
"receiver.drain.failed": "targeted-pull",
|
|
68
|
+
};
|
|
69
|
+
function classifyRefusal(record, errText) {
|
|
70
|
+
const event = typeof record.event === "string" ? record.event : "";
|
|
71
|
+
if (SUBSCRIBE_ROUTE.test(errText))
|
|
72
|
+
return "subscribe";
|
|
73
|
+
const byEvent = RECEIVER_EVENT_CALLS[event];
|
|
74
|
+
if (byEvent)
|
|
75
|
+
return byEvent;
|
|
76
|
+
if (record.type === "error")
|
|
77
|
+
return "runner-error";
|
|
78
|
+
return "other";
|
|
79
|
+
}
|
|
80
|
+
const QUEUE_LEG = new Set([
|
|
81
|
+
"subscribe", "sqs-receive", "sqs-visibility", "sqs-delete", "cloudwatch-metric",
|
|
82
|
+
]);
|
|
83
|
+
function receiverPermissionRemediation(byCall) {
|
|
84
|
+
const classes = Object.keys(byCall);
|
|
85
|
+
const queueLeg = classes.some((call) => QUEUE_LEG.has(call));
|
|
86
|
+
const pullLeg = classes.includes("targeted-pull");
|
|
87
|
+
const parts = [];
|
|
88
|
+
if (queueLeg) {
|
|
89
|
+
parts.push("The queue credentials vended by POST /v1/sync/subscribe were refused on the per-device queue leg; " +
|
|
90
|
+
"restart the sync runner so it re-subscribes, then run `hq sync status`. If it repeats, the device roster entry for this machine needs attention (hq-pro device slots).");
|
|
91
|
+
}
|
|
92
|
+
if (pullLeg) {
|
|
93
|
+
parts.push("The object pull after a change event was refused; see sync.runtime.pull-access-denied for the refused keys and the grant to compare.");
|
|
94
|
+
}
|
|
95
|
+
if (parts.length === 0) {
|
|
96
|
+
parts.push("Run `hq sync status` and inspect the selected sync log under ~/.hq/logs/, ~/.hq/daemon/logs/, or the configured HQ_DAEMON_DIR/logs/.");
|
|
97
|
+
}
|
|
98
|
+
return parts.join(" ");
|
|
99
|
+
}
|
|
100
|
+
/** POSIX single-quote: safe to paste into a shell whatever the key holds. */
|
|
101
|
+
function shellQuote(value) {
|
|
102
|
+
return `'${value.replace(/'/g, "'\\''")}'`;
|
|
103
|
+
}
|
|
104
|
+
function pullDeniedRemediation(latest) {
|
|
105
|
+
let lookup;
|
|
106
|
+
if (!latest) {
|
|
107
|
+
lookup = "Run `hq doctor` in a session locked to the company that owns the refused keys to see them named, then compare the stored grant with `hq files acl <key> --company <company>`. ";
|
|
108
|
+
}
|
|
109
|
+
else if (latest.company === UNKNOWN_COMPANY) {
|
|
110
|
+
lookup = "The runner did not record which company owns this key, so no lookup command is suggested; find the company that holds it and run `hq files acl <key> --company <company>`. ";
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
const scope = latest.company === "personal" ? "" : ` --company ${latest.company}`;
|
|
114
|
+
lookup = `Compare the stored grant with the key the pull looked up: hq files acl ${shellQuote(latest.path)}${scope}. `;
|
|
115
|
+
}
|
|
116
|
+
return ("The receiver was addressed this change, so the publisher's grant evaluation allowed it, while the pull resolver refused it. " +
|
|
117
|
+
lookup +
|
|
118
|
+
"A read grant must cover the key via '*', the key itself, or an ancestor as 'A', 'A/' or 'A/*', naming you, one of your groups, or company-wide; " +
|
|
119
|
+
"the owner/admin bypass needs viewOthersFiles (viewOthersMeetings for meetings/, sources/, signals/ and knowledge/), and sessions/<other person> is creator-or-owner only. " +
|
|
120
|
+
"The key stays unjournaled and is retried each pass until the grant covers it.");
|
|
121
|
+
}
|
|
24
122
|
function parseRecord(rest) {
|
|
25
123
|
const start = rest.indexOf("{");
|
|
26
124
|
if (start < 0)
|
|
@@ -68,6 +166,9 @@ export function analyzeSyncRuntimeLog(lines, options) {
|
|
|
68
166
|
const slowReceiptTimes = [];
|
|
69
167
|
const receiverStartTimes = [];
|
|
70
168
|
let denied = 0;
|
|
169
|
+
const deniedByCall = {};
|
|
170
|
+
let pullsDenied = 0;
|
|
171
|
+
const deniedKeys = new Map();
|
|
71
172
|
let starts = 0;
|
|
72
173
|
let deviceLimited = 0;
|
|
73
174
|
for (const line of lines) {
|
|
@@ -84,6 +185,37 @@ export function analyzeSyncRuntimeLog(lines, options) {
|
|
|
84
185
|
if (!record)
|
|
85
186
|
continue;
|
|
86
187
|
const event = record.event;
|
|
188
|
+
if (record.type === "skip-access-denied" && time >= since) {
|
|
189
|
+
// A deliberate per-object pull skip from the runner: the publisher
|
|
190
|
+
// addressed this device a change the pull resolver then refused. The
|
|
191
|
+
// event carries the vault key (relative by construction) and a bounded
|
|
192
|
+
// reason tag; the free-text `message` is never copied.
|
|
193
|
+
pullsDenied += 1;
|
|
194
|
+
const pathValue = typeof record.path === "string" ? record.path : "";
|
|
195
|
+
const rawCompany = typeof record.company === "string" ? record.company : "";
|
|
196
|
+
const company = PERSONAL_VAULT_SLUGS.has(rawCompany)
|
|
197
|
+
? "personal"
|
|
198
|
+
: COMPANY_SLUG.test(rawCompany) ? rawCompany : UNKNOWN_COMPANY;
|
|
199
|
+
const reason = typeof record.reason === "string" && REASON_TAG.test(record.reason) ? record.reason : "denied";
|
|
200
|
+
// Refusals are grouped by (company, key) whatever the reason tag, and
|
|
201
|
+
// the latest tag is kept. A key that cannot be named (absolute or
|
|
202
|
+
// non-printable) is still grouped so it counts toward the repeat
|
|
203
|
+
// threshold, but it is never emitted.
|
|
204
|
+
const nameable = pathValue !== "" && !ABSOLUTE_LOCAL_PATH.test(pathValue) && NAMEABLE_VAULT_KEY.test(pathValue);
|
|
205
|
+
const key = `${company}\u0000${pathValue}`;
|
|
206
|
+
const existing = deniedKeys.get(key);
|
|
207
|
+
if (existing) {
|
|
208
|
+
existing.count += 1;
|
|
209
|
+
if (time >= existing.lastSeen) {
|
|
210
|
+
existing.lastSeen = time;
|
|
211
|
+
existing.reason = reason;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
else {
|
|
215
|
+
deniedKeys.set(key, { company, path: pathValue, reason, count: 1, lastSeen: time, nameable });
|
|
216
|
+
}
|
|
217
|
+
continue;
|
|
218
|
+
}
|
|
87
219
|
if (event === "receiver.start") {
|
|
88
220
|
receiverStartTimes.push(time);
|
|
89
221
|
const settledAt = time + SYNC_RUNTIME_RESTART_WARMUP_MS;
|
|
@@ -99,11 +231,36 @@ export function analyzeSyncRuntimeLog(lines, options) {
|
|
|
99
231
|
}
|
|
100
232
|
if (time >= since &&
|
|
101
233
|
(record.type === "receiver" || record.type === "error" ||
|
|
102
|
-
(typeof event === "string" && event.startsWith("receiver.")))
|
|
103
|
-
|
|
104
|
-
|
|
234
|
+
(typeof event === "string" && event.startsWith("receiver.")))) {
|
|
235
|
+
const errText = JSON.stringify(record.err ?? record.message ?? "");
|
|
236
|
+
if (PERMISSION.test(errText)) {
|
|
237
|
+
denied += 1;
|
|
238
|
+
const call = classifyRefusal(record, errText);
|
|
239
|
+
deniedByCall[call] = (deniedByCall[call] ?? 0) + 1;
|
|
240
|
+
}
|
|
105
241
|
}
|
|
106
242
|
}
|
|
243
|
+
const byCallSummary = Object.entries(deniedByCall)
|
|
244
|
+
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
245
|
+
.map(([call, count]) => `${call} ${count}`)
|
|
246
|
+
.join(", ");
|
|
247
|
+
// The runner retries a refused key on every pass by design (the object
|
|
248
|
+
// stays unjournaled until the grant covers it). One refusal is a race with
|
|
249
|
+
// a share in flight; the same key refused again within the hour is a grant
|
|
250
|
+
// that does not cover what the publisher addressed. Only repeated keys are
|
|
251
|
+
// reported, named or counted.
|
|
252
|
+
const repeatedKeys = [...deniedKeys.values()]
|
|
253
|
+
.filter((key) => key.count >= PULL_DENIED_MIN_REPEATS)
|
|
254
|
+
.sort((a, b) => b.lastSeen - a.lastSeen || b.count - a.count);
|
|
255
|
+
const repeatedDenials = repeatedKeys.length > 0;
|
|
256
|
+
const nameableScopes = options.nameableCompanies === undefined ? null : new Set(options.nameableCompanies);
|
|
257
|
+
const namedKeys = repeatedKeys.filter((key) => key.nameable && (nameableScopes === null || nameableScopes.has(key.company)));
|
|
258
|
+
const unnamedRepeated = repeatedKeys.length - namedKeys.length;
|
|
259
|
+
const sortedDeniedKeys = namedKeys.slice(0, PULL_DENIED_KEY_CAP);
|
|
260
|
+
const latestDenied = sortedDeniedKeys[0];
|
|
261
|
+
const unnamedNote = unnamedRepeated > 0
|
|
262
|
+
? ` ${unnamedRepeated} of them ${unnamedRepeated === 1 ? "is" : "are"} not named here (outside this session's companies, or not printable).`
|
|
263
|
+
: "";
|
|
107
264
|
// Startup can temporarily expose queued receive latency and restart events
|
|
108
265
|
// while the receiver is warming its backlog. Only judge receipts outside the
|
|
109
266
|
// warm-up interval following the most recent preceding start.
|
|
@@ -111,7 +268,41 @@ export function analyzeSyncRuntimeLog(lines, options) {
|
|
|
111
268
|
const conflicts = options.conflictCount ?? 0;
|
|
112
269
|
const checks = [
|
|
113
270
|
{ id: "realtime-lag", reasonCode: "realtime-lag-stuck", failing: slowReceipts >= LAG_MIN_RECEIPTS, count: slowReceipts, label: "incoming changes delayed over 15 minutes" },
|
|
114
|
-
{
|
|
271
|
+
{
|
|
272
|
+
id: "receiver-permission",
|
|
273
|
+
reasonCode: "receiver-permission-denied",
|
|
274
|
+
failing: denied > 0,
|
|
275
|
+
count: denied,
|
|
276
|
+
label: "cloud permission refusals",
|
|
277
|
+
...(denied > 0
|
|
278
|
+
? {
|
|
279
|
+
message: `${denied} cloud permission refusals in the last hour (${byCallSummary}).`,
|
|
280
|
+
details: { byCall: deniedByCall },
|
|
281
|
+
remediation: receiverPermissionRemediation(deniedByCall),
|
|
282
|
+
}
|
|
283
|
+
: {}),
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
id: "pull-access-denied",
|
|
287
|
+
reasonCode: "pull-access-denied",
|
|
288
|
+
failing: repeatedDenials,
|
|
289
|
+
count: pullsDenied,
|
|
290
|
+
label: "pulls refused",
|
|
291
|
+
...(repeatedDenials
|
|
292
|
+
? {
|
|
293
|
+
message: `${pullsDenied} pulls refused in the last hour; ${repeatedKeys.length} vault keys refused ${PULL_DENIED_MIN_REPEATS} or more times` +
|
|
294
|
+
(latestDenied ? `; most recent: ${latestDenied.company} ${latestDenied.path} (${latestDenied.reason}).` : ".") +
|
|
295
|
+
unnamedNote,
|
|
296
|
+
details: {
|
|
297
|
+
count: pullsDenied,
|
|
298
|
+
repeatedKeys: repeatedKeys.length,
|
|
299
|
+
unnamedKeys: unnamedRepeated,
|
|
300
|
+
keys: sortedDeniedKeys.map(({ company, path: keyPath, reason, count }) => ({ company, path: keyPath, reason, count })),
|
|
301
|
+
},
|
|
302
|
+
remediation: pullDeniedRemediation(latestDenied),
|
|
303
|
+
}
|
|
304
|
+
: {}),
|
|
305
|
+
},
|
|
115
306
|
{ id: "runner-restarts", reasonCode: "runner-restart-loop", failing: starts >= RESTART_MIN_STARTS, count: starts, label: "settled sync runner starts" },
|
|
116
307
|
{ id: "device-limit", reasonCode: "device-limit-backoff", failing: deviceLimited > 0, count: deviceLimited, label: "device-limit backoffs" },
|
|
117
308
|
{ id: "conflicts", reasonCode: "repeated-conflicts", failing: conflicts >= CONFLICT_MIN_COUNT, count: conflicts, label: "unresolved conflicts" },
|
|
@@ -121,8 +312,10 @@ export function analyzeSyncRuntimeLog(lines, options) {
|
|
|
121
312
|
status: "WARN",
|
|
122
313
|
checkId: `sync.runtime.${check.id}`,
|
|
123
314
|
reasonCode: check.reasonCode,
|
|
124
|
-
message: `${check.count} ${check.label} in the last hour.`,
|
|
125
|
-
|
|
315
|
+
message: check.message ?? `${check.count} ${check.label} in the last hour.`,
|
|
316
|
+
...(check.details ? { details: check.details } : {}),
|
|
317
|
+
remediation: check.remediation ??
|
|
318
|
+
"Run `hq sync status` and inspect the selected sync log under ~/.hq/logs/, ~/.hq/daemon/logs/, or the configured HQ_DAEMON_DIR/logs/.",
|
|
126
319
|
}
|
|
127
320
|
: {
|
|
128
321
|
status: "PASS",
|
|
@@ -185,11 +378,27 @@ function readConflictCount(file) {
|
|
|
185
378
|
return 0;
|
|
186
379
|
}
|
|
187
380
|
}
|
|
381
|
+
/**
|
|
382
|
+
* Company scopes this session may see named: `undefined` (all) when there is
|
|
383
|
+
* no session or no company lock; nothing when the lock cannot be verified.
|
|
384
|
+
*/
|
|
385
|
+
export function sessionNameableCompanies(env, readLock = readSessionCompanyLock) {
|
|
386
|
+
try {
|
|
387
|
+
const lock = readLock(env);
|
|
388
|
+
if (!lock || lock.companies.length === 0)
|
|
389
|
+
return undefined;
|
|
390
|
+
return lock.companies;
|
|
391
|
+
}
|
|
392
|
+
catch {
|
|
393
|
+
return [];
|
|
394
|
+
}
|
|
395
|
+
}
|
|
188
396
|
/** Read the machine's sync logs and client health file. */
|
|
189
397
|
export function defaultSyncRuntimeResults(now, hqHome = path.join(os.homedir(), ".hq"), env = process.env) {
|
|
190
398
|
const options = {
|
|
191
399
|
now: now.getTime(),
|
|
192
400
|
conflictCount: readConflictCount(path.join(hqHome, "client-health.json")),
|
|
401
|
+
nameableCompanies: sessionNameableCompanies(env),
|
|
193
402
|
};
|
|
194
403
|
const paths = syncLogBasePaths(hqHome, env);
|
|
195
404
|
const familyModifiedMs = (family) => {
|
|
@@ -380,6 +380,7 @@ export async function applyFixes(options) {
|
|
|
380
380
|
writeRefreshSyncLastRun(hqRoot, {
|
|
381
381
|
lastAttemptedSlug: lastAttemptedRefreshSyncSlug ?? refreshSyncApplied.at(-1).target,
|
|
382
382
|
lastRunAttemptedCheckIds: refreshSyncApplied.map((fix) => fix.checkId),
|
|
383
|
+
lastRunOutcomes: Object.fromEntries(refreshSyncApplied.flatMap((fix) => fix.refreshSyncOutcome ? [[fix.checkId, fix.refreshSyncOutcome]] : [])),
|
|
383
384
|
lastRunAtMs: backupClock().getTime(),
|
|
384
385
|
});
|
|
385
386
|
}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
/** Privacy-safe outcomes for a targeted refresh-sync attempt. */
|
|
2
|
-
export
|
|
2
|
+
export declare const REFRESH_SYNC_OUTCOMES: readonly ["pull-succeeded", "pull-timeout", "pull-failed:4xx", "pull-failed:5xx", "pull-failed:network", "pull-failed:other"];
|
|
3
|
+
export type RefreshSyncOutcome = (typeof REFRESH_SYNC_OUTCOMES)[number];
|
|
4
|
+
/** Outcomes that say nothing about the journal itself: offline, timed out, or the service failed. */
|
|
5
|
+
export declare const TRANSIENT_REFRESH_SYNC_OUTCOMES: ReadonlySet<RefreshSyncOutcome>;
|
|
3
6
|
export interface RefreshSyncObservation {
|
|
4
7
|
errorCode?: string;
|
|
5
8
|
signal?: string | null;
|
|
@@ -1,3 +1,18 @@
|
|
|
1
|
+
/** Privacy-safe outcomes for a targeted refresh-sync attempt. */
|
|
2
|
+
export const REFRESH_SYNC_OUTCOMES = [
|
|
3
|
+
"pull-succeeded",
|
|
4
|
+
"pull-timeout",
|
|
5
|
+
"pull-failed:4xx",
|
|
6
|
+
"pull-failed:5xx",
|
|
7
|
+
"pull-failed:network",
|
|
8
|
+
"pull-failed:other",
|
|
9
|
+
];
|
|
10
|
+
/** Outcomes that say nothing about the journal itself: offline, timed out, or the service failed. */
|
|
11
|
+
export const TRANSIENT_REFRESH_SYNC_OUTCOMES = new Set([
|
|
12
|
+
"pull-timeout",
|
|
13
|
+
"pull-failed:network",
|
|
14
|
+
"pull-failed:5xx",
|
|
15
|
+
]);
|
|
1
16
|
/** Classify process/HTTP results without retaining their free-text details. */
|
|
2
17
|
export function classifyRefreshSyncOutcome(observation) {
|
|
3
18
|
const details = String(observation.stdout ?? "") + "\n" + String(observation.stderr ?? "");
|
|
@@ -8,16 +8,45 @@
|
|
|
8
8
|
* pass did not try — SessionStart hooks, including older ones that only
|
|
9
9
|
* call `hq doctor --json`, otherwise file those leftover rows as unrepaired.
|
|
10
10
|
*/
|
|
11
|
+
import { type RefreshSyncOutcome } from "./refresh-sync-outcome.js";
|
|
11
12
|
import type { CheckResult } from "../types.js";
|
|
12
13
|
export declare const REFRESH_SYNC_PROGRESS_RELATIVE_PATH: string;
|
|
13
14
|
/** Hook re-verify runs immediately after `--fix`; 15 minutes covers a slow box. */
|
|
14
15
|
export declare const CLIENT_HEALTH_REFRESH_SYNC_FRESH_MS: number;
|
|
15
16
|
export interface RefreshSyncProgress {
|
|
17
|
+
/**
|
|
18
|
+
* Stays at 2. `lastRunOutcomes` and `transientSinceMs` are additive optional
|
|
19
|
+
* fields: an older CLI sharing this HQ root (desktop-bundled vs npm global,
|
|
20
|
+
* a pinned bot binary) accepts only versions 1 and 2 and throws on anything
|
|
21
|
+
* else from `--fix`'s own cursor read, so bumping the number would break its
|
|
22
|
+
* repair pass. Unknown fields are ignored by every reader.
|
|
23
|
+
*/
|
|
16
24
|
version: 1 | 2;
|
|
17
25
|
lastAttemptedSlug: string;
|
|
18
26
|
lastRunAttemptedCheckIds?: string[];
|
|
27
|
+
/**
|
|
28
|
+
* Per-check outcome of the last completed pass. Lets doctor JSON tell a
|
|
29
|
+
* pull that failed because the machine was offline or the service hiccuped
|
|
30
|
+
* (transient; retried next SessionStart) from one the service refused (4xx)
|
|
31
|
+
* or that failed for an unknown reason, so the client-health hook does not
|
|
32
|
+
* file an "unrepaired" report for a laptop that simply had no network at
|
|
33
|
+
* SessionStart (feedback_cd56a501 cluster: 19 of 28 reports were stale
|
|
34
|
+
* journals, most on installs whose pull did not complete).
|
|
35
|
+
*/
|
|
36
|
+
lastRunOutcomes?: Record<string, RefreshSyncOutcome>;
|
|
37
|
+
/**
|
|
38
|
+
* When each check's current run of consecutive transient outcomes began.
|
|
39
|
+
* Cleared the moment a pass ends with any other outcome. Bounds the hiding:
|
|
40
|
+
* a journal transiently unrefreshed for longer than
|
|
41
|
+
* {@link TRANSIENT_REFRESH_SYNC_MAX_MS} is reported again, so a machine
|
|
42
|
+
* that can never reach the service (wrong endpoint, blocked network) still
|
|
43
|
+
* files a report instead of hiding forever.
|
|
44
|
+
*/
|
|
45
|
+
transientSinceMs?: Record<string, number>;
|
|
19
46
|
lastRunAtMs?: number;
|
|
20
47
|
}
|
|
48
|
+
/** A journal transiently unrefreshed for longer than this is reported again. */
|
|
49
|
+
export declare const TRANSIENT_REFRESH_SYNC_MAX_MS: number;
|
|
21
50
|
export declare function readRefreshSyncProgress(hqRoot: string): RefreshSyncProgress | null;
|
|
22
51
|
/**
|
|
23
52
|
* `--fix` manifest-upload cursor. Kept in its own file so the refresh-sync
|
|
@@ -39,8 +68,21 @@ export declare function writeRefreshSyncCursor(hqRoot: string, slug: string): vo
|
|
|
39
68
|
export declare function writeRefreshSyncLastRun(hqRoot: string, input: {
|
|
40
69
|
lastAttemptedSlug: string;
|
|
41
70
|
lastRunAttemptedCheckIds: string[];
|
|
71
|
+
lastRunOutcomes?: Record<string, RefreshSyncOutcome>;
|
|
42
72
|
lastRunAtMs: number;
|
|
43
73
|
}): void;
|
|
74
|
+
/**
|
|
75
|
+
* Journal check ids whose refresh pull in the last completed `--fix` pass, if
|
|
76
|
+
* that pass is still fresh, ended in a transient outcome (offline, timeout,
|
|
77
|
+
* service 5xx). Null when there is no fresh pass or no outcomes were recorded.
|
|
78
|
+
*/
|
|
79
|
+
export declare function freshTransientRefreshSyncCheckIds(hqRoot: string, nowMs: number): ReadonlySet<string> | null;
|
|
80
|
+
/**
|
|
81
|
+
* True when doctor JSON should hide this stale journal because the fresh
|
|
82
|
+
* `--fix` pull for it failed transiently. Applies to company and personal
|
|
83
|
+
* journals alike: an offline laptop cannot refresh either.
|
|
84
|
+
*/
|
|
85
|
+
export declare function isTransientlyUnrefreshedJournalForClientHealth(result: CheckResult, transientCheckIds: ReadonlySet<string> | null): boolean;
|
|
44
86
|
export declare function freshAttemptedCompanyJournalCheckIds(hqRoot: string, nowMs: number): ReadonlySet<string> | null;
|
|
45
87
|
/** True when doctor JSON should hide this leftover company journal. */
|
|
46
88
|
export declare function isUnreachedCompanyJournalForClientHealth(result: CheckResult, attemptedCheckIds: ReadonlySet<string> | null): boolean;
|
|
@@ -12,15 +12,41 @@ import * as fs from "node:fs";
|
|
|
12
12
|
import * as path from "node:path";
|
|
13
13
|
import { isCompanyJournalSlug, journalSlugFromCheckId, } from "../checks/sync-health.js";
|
|
14
14
|
import { deriveRemediation } from "./remediation.js";
|
|
15
|
+
import { REFRESH_SYNC_OUTCOMES, TRANSIENT_REFRESH_SYNC_OUTCOMES, } from "./refresh-sync-outcome.js";
|
|
15
16
|
export const REFRESH_SYNC_PROGRESS_RELATIVE_PATH = path.join("workspace", "doctor", "refresh-sync-progress.json");
|
|
16
17
|
/** Hook re-verify runs immediately after `--fix`; 15 minutes covers a slow box. */
|
|
17
18
|
export const CLIENT_HEALTH_REFRESH_SYNC_FRESH_MS = 15 * 60 * 1000;
|
|
19
|
+
/** A journal transiently unrefreshed for longer than this is reported again. */
|
|
20
|
+
export const TRANSIENT_REFRESH_SYNC_MAX_MS = 24 * 60 * 60 * 1000;
|
|
18
21
|
function progressPath(hqRoot) {
|
|
19
22
|
return path.join(hqRoot, REFRESH_SYNC_PROGRESS_RELATIVE_PATH);
|
|
20
23
|
}
|
|
21
24
|
function isCheckIdList(value) {
|
|
22
25
|
return Array.isArray(value) && value.every((entry) => typeof entry === "string");
|
|
23
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* Keep only outcome entries this CLI understands. A newer CLI may write an
|
|
29
|
+
* outcome this one does not know; dropping that entry (it then counts as
|
|
30
|
+
* "no recorded outcome") keeps `--fix` working instead of rejecting the
|
|
31
|
+
* whole cursor. Returns null when the value is not an object at all.
|
|
32
|
+
*/
|
|
33
|
+
function knownOutcomes(value) {
|
|
34
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
35
|
+
return null;
|
|
36
|
+
const known = {};
|
|
37
|
+
for (const [checkId, outcome] of Object.entries(value)) {
|
|
38
|
+
if (typeof outcome === "string" && REFRESH_SYNC_OUTCOMES.includes(outcome)) {
|
|
39
|
+
known[checkId] = outcome;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return known;
|
|
43
|
+
}
|
|
44
|
+
function isTimestampMap(value) {
|
|
45
|
+
return (typeof value === "object" &&
|
|
46
|
+
value !== null &&
|
|
47
|
+
!Array.isArray(value) &&
|
|
48
|
+
Object.values(value).every((ms) => typeof ms === "number" && Number.isFinite(ms)));
|
|
49
|
+
}
|
|
24
50
|
export function readRefreshSyncProgress(hqRoot) {
|
|
25
51
|
let contents;
|
|
26
52
|
try {
|
|
@@ -47,6 +73,16 @@ export function readRefreshSyncProgress(hqRoot) {
|
|
|
47
73
|
if (record.lastRunAtMs !== undefined && !Number.isFinite(record.lastRunAtMs)) {
|
|
48
74
|
throw new Error(`Invalid refresh-sync progress cursor at ${REFRESH_SYNC_PROGRESS_RELATIVE_PATH}`);
|
|
49
75
|
}
|
|
76
|
+
if (record.lastRunOutcomes !== undefined) {
|
|
77
|
+
const known = knownOutcomes(record.lastRunOutcomes);
|
|
78
|
+
if (known === null) {
|
|
79
|
+
throw new Error(`Invalid refresh-sync progress cursor at ${REFRESH_SYNC_PROGRESS_RELATIVE_PATH}`);
|
|
80
|
+
}
|
|
81
|
+
record.lastRunOutcomes = known;
|
|
82
|
+
}
|
|
83
|
+
if (record.transientSinceMs !== undefined && !isTimestampMap(record.transientSinceMs)) {
|
|
84
|
+
throw new Error(`Invalid refresh-sync progress cursor at ${REFRESH_SYNC_PROGRESS_RELATIVE_PATH}`);
|
|
85
|
+
}
|
|
50
86
|
return record;
|
|
51
87
|
}
|
|
52
88
|
function writeProgressFile(hqRoot, progress, relativePath = REFRESH_SYNC_PROGRESS_RELATIVE_PATH) {
|
|
@@ -93,18 +129,82 @@ export function writeRefreshSyncCursor(hqRoot, slug) {
|
|
|
93
129
|
...(existing?.lastRunAttemptedCheckIds
|
|
94
130
|
? { lastRunAttemptedCheckIds: existing.lastRunAttemptedCheckIds }
|
|
95
131
|
: {}),
|
|
132
|
+
...(existing?.lastRunOutcomes ? { lastRunOutcomes: existing.lastRunOutcomes } : {}),
|
|
133
|
+
...(existing?.transientSinceMs ? { transientSinceMs: existing.transientSinceMs } : {}),
|
|
96
134
|
...(existing?.lastRunAtMs !== undefined ? { lastRunAtMs: existing.lastRunAtMs } : {}),
|
|
97
135
|
});
|
|
98
136
|
}
|
|
99
137
|
/** Record the check ids this completed `--fix` pass actually tried to refresh. */
|
|
100
138
|
export function writeRefreshSyncLastRun(hqRoot, input) {
|
|
139
|
+
// Carry the start of each check's transient streak across runs; any other
|
|
140
|
+
// outcome (or no attempt this run) ends the streak.
|
|
141
|
+
let previous;
|
|
142
|
+
try {
|
|
143
|
+
previous = readRefreshSyncProgress(hqRoot);
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
previous = null;
|
|
147
|
+
}
|
|
148
|
+
const transientSinceMs = {};
|
|
149
|
+
for (const [checkId, outcome] of Object.entries(input.lastRunOutcomes ?? {})) {
|
|
150
|
+
if (!TRANSIENT_REFRESH_SYNC_OUTCOMES.has(outcome))
|
|
151
|
+
continue;
|
|
152
|
+
const since = previous?.transientSinceMs?.[checkId];
|
|
153
|
+
transientSinceMs[checkId] = typeof since === "number" ? Math.min(since, input.lastRunAtMs) : input.lastRunAtMs;
|
|
154
|
+
}
|
|
101
155
|
writeProgressFile(hqRoot, {
|
|
102
156
|
version: 2,
|
|
103
157
|
lastAttemptedSlug: input.lastAttemptedSlug,
|
|
104
158
|
lastRunAttemptedCheckIds: input.lastRunAttemptedCheckIds,
|
|
159
|
+
...(input.lastRunOutcomes ? { lastRunOutcomes: input.lastRunOutcomes } : {}),
|
|
160
|
+
...(Object.keys(transientSinceMs).length > 0 ? { transientSinceMs } : {}),
|
|
105
161
|
lastRunAtMs: input.lastRunAtMs,
|
|
106
162
|
});
|
|
107
163
|
}
|
|
164
|
+
/**
|
|
165
|
+
* Journal check ids whose refresh pull in the last completed `--fix` pass, if
|
|
166
|
+
* that pass is still fresh, ended in a transient outcome (offline, timeout,
|
|
167
|
+
* service 5xx). Null when there is no fresh pass or no outcomes were recorded.
|
|
168
|
+
*/
|
|
169
|
+
export function freshTransientRefreshSyncCheckIds(hqRoot, nowMs) {
|
|
170
|
+
let progress;
|
|
171
|
+
try {
|
|
172
|
+
progress = readRefreshSyncProgress(hqRoot);
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
return null;
|
|
176
|
+
}
|
|
177
|
+
if (!progress ||
|
|
178
|
+
progress.lastRunAtMs === undefined ||
|
|
179
|
+
!progress.lastRunOutcomes ||
|
|
180
|
+
nowMs - progress.lastRunAtMs < 0 ||
|
|
181
|
+
nowMs - progress.lastRunAtMs > CLIENT_HEALTH_REFRESH_SYNC_FRESH_MS) {
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
const outcomes = progress.lastRunOutcomes;
|
|
185
|
+
const since = progress.transientSinceMs ?? {};
|
|
186
|
+
return new Set(Object.entries(outcomes)
|
|
187
|
+
.filter(([checkId, outcome]) => TRANSIENT_REFRESH_SYNC_OUTCOMES.has(outcome) &&
|
|
188
|
+
// A streak older than the ceiling is no longer "just offline": report it.
|
|
189
|
+
nowMs - (since[checkId] ?? progress.lastRunAtMs) <= TRANSIENT_REFRESH_SYNC_MAX_MS)
|
|
190
|
+
.map(([checkId]) => checkId));
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* True when doctor JSON should hide this stale journal because the fresh
|
|
194
|
+
* `--fix` pull for it failed transiently. Applies to company and personal
|
|
195
|
+
* journals alike: an offline laptop cannot refresh either.
|
|
196
|
+
*/
|
|
197
|
+
export function isTransientlyUnrefreshedJournalForClientHealth(result, transientCheckIds) {
|
|
198
|
+
if (!transientCheckIds)
|
|
199
|
+
return false;
|
|
200
|
+
if (result.status !== "FAIL" && result.status !== "WARN")
|
|
201
|
+
return false;
|
|
202
|
+
if (!journalSlugFromCheckId(result.checkId))
|
|
203
|
+
return false;
|
|
204
|
+
if (!transientCheckIds.has(result.checkId))
|
|
205
|
+
return false;
|
|
206
|
+
return deriveRemediation(result)?.autoFixable === true;
|
|
207
|
+
}
|
|
108
208
|
export function freshAttemptedCompanyJournalCheckIds(hqRoot, nowMs) {
|
|
109
209
|
let progress;
|
|
110
210
|
try {
|
|
@@ -89,6 +89,13 @@ export interface DoctorJsonDocument {
|
|
|
89
89
|
scope?: string[];
|
|
90
90
|
/** The full, flattened result list in family/registration order. */
|
|
91
91
|
results: DoctorJsonResult[];
|
|
92
|
+
/**
|
|
93
|
+
* Stale sync journals left out of `results` because the fresh `--fix` pull
|
|
94
|
+
* for them ended offline, timed out, or hit a service 5xx. They are retried
|
|
95
|
+
* at the next SessionStart; a client-health report must not count them as
|
|
96
|
+
* unrepaired. Absent when there are none.
|
|
97
|
+
*/
|
|
98
|
+
transientRefreshSyncCheckIds?: string[];
|
|
92
99
|
}
|
|
93
100
|
/** Input to {@link buildDoctorJson}. */
|
|
94
101
|
export interface BuildDoctorJsonInput {
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
*/
|
|
20
20
|
import { computeExitCode, emptyStatusCounts, UNKNOWN_PLATFORM, } from "./report.js";
|
|
21
21
|
import { deriveRemediation } from "./fix/remediation.js";
|
|
22
|
-
import { freshAttemptedCompanyJournalCheckIds, isUnreachedCompanyJournalForClientHealth, } from "./fix/refresh-sync-progress.js";
|
|
22
|
+
import { freshAttemptedCompanyJournalCheckIds, freshTransientRefreshSyncCheckIds, isTransientlyUnrefreshedJournalForClientHealth, isUnreachedCompanyJournalForClientHealth, } from "./fix/refresh-sync-progress.js";
|
|
23
23
|
/**
|
|
24
24
|
* The `--json` schema version. Bump on any breaking change to the document
|
|
25
25
|
* shape; the snapshot test in report.test.ts fails until the bump is
|
|
@@ -75,6 +75,8 @@ function summarizeJsonResults(results) {
|
|
|
75
75
|
/** Assemble the machine-readable document from a completed run. */
|
|
76
76
|
export function buildDoctorJson(input) {
|
|
77
77
|
const attemptedRefreshSyncCheckIds = freshAttemptedCompanyJournalCheckIds(input.hqRoot, (input.now ?? (() => new Date()))().getTime());
|
|
78
|
+
const transientRefreshSyncCheckIds = freshTransientRefreshSyncCheckIds(input.hqRoot, (input.now ?? (() => new Date()))().getTime());
|
|
79
|
+
const transientHidden = [];
|
|
78
80
|
const results = [];
|
|
79
81
|
for (const run of input.families) {
|
|
80
82
|
for (const result of run.results) {
|
|
@@ -84,6 +86,10 @@ export function buildDoctorJson(input) {
|
|
|
84
86
|
if (isUnreachedCompanyJournalForClientHealth(result, attemptedRefreshSyncCheckIds)) {
|
|
85
87
|
continue;
|
|
86
88
|
}
|
|
89
|
+
if (isTransientlyUnrefreshedJournalForClientHealth(result, transientRefreshSyncCheckIds)) {
|
|
90
|
+
transientHidden.push(result.checkId);
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
87
93
|
results.push(toJsonResult(run.family.id, result, input.clientHealthReasonCodesEnabledCheckIds, input.clientHealthCheckStatusEnabledCheckIds));
|
|
88
94
|
}
|
|
89
95
|
}
|
|
@@ -95,6 +101,7 @@ export function buildDoctorJson(input) {
|
|
|
95
101
|
exitCode: computeExitCode(input.families),
|
|
96
102
|
...(input.scope && input.scope.length > 0 ? { scope: [...input.scope] } : {}),
|
|
97
103
|
results,
|
|
104
|
+
...(transientHidden.length > 0 ? { transientRefreshSyncCheckIds: transientHidden } : {}),
|
|
98
105
|
};
|
|
99
106
|
}
|
|
100
107
|
/** Convert one {@link CheckResult} into its JSON form, omitting empty optionals. */
|
|
@@ -19,7 +19,7 @@ export declare const DOCTOR_STATUSES: readonly ["PASS", "FAIL", "WARN", "UNTESTE
|
|
|
19
19
|
*/
|
|
20
20
|
export type DoctorStatus = (typeof DOCTOR_STATUSES)[number];
|
|
21
21
|
/** Stable, privacy-safe diagnostics emitted by the sync health family. */
|
|
22
|
-
export declare const SYNC_REASON_CODES: readonly ["core-version-unavailable", "core-update-available", "no-local-tree", "company-not-resolved", "legacy-personal-journal", "never-synced", "invalid-last-sync", "stale-threshold", "manifest-unavailable", "manifest-state-unavailable", "no-manifest-scopes", "never-uploaded", "invalid-upload-time", "unresolved-company-uid", "realtime-lag-stuck", "receiver-permission-denied", "runner-restart-loop", "device-limit-backoff", "repeated-conflicts"];
|
|
22
|
+
export declare const SYNC_REASON_CODES: readonly ["core-version-unavailable", "core-update-available", "no-local-tree", "company-not-resolved", "legacy-personal-journal", "never-synced", "invalid-last-sync", "stale-threshold", "manifest-unavailable", "manifest-state-unavailable", "no-manifest-scopes", "never-uploaded", "invalid-upload-time", "unresolved-company-uid", "realtime-lag-stuck", "receiver-permission-denied", "pull-access-denied", "runner-restart-loop", "device-limit-backoff", "repeated-conflicts"];
|
|
23
23
|
export type SyncReasonCode = (typeof SYNC_REASON_CODES)[number];
|
|
24
24
|
/** Runtime type guard: true only for a member of the fixed vocabulary. */
|
|
25
25
|
export declare function isDoctorStatus(value: unknown): value is DoctorStatus;
|
package/dist/lib/doctor/types.js
CHANGED
|
@@ -14,7 +14,7 @@ export interface WorkCacheMqtt extends MeshMqttEventClient {
|
|
|
14
14
|
export interface WorkCacheSubscriber {
|
|
15
15
|
stop: () => Promise<void>;
|
|
16
16
|
}
|
|
17
|
-
export { AGENT_INBOX_PUSH_STATE_BASENAME } from "../paths.js";
|
|
17
|
+
export { AGENT_INBOX_PUSH_STATE_BASENAME, SYNC_WAKE_STATE_BASENAME } from "../paths.js";
|
|
18
18
|
export declare function startWorkCacheSubscriber(opts: {
|
|
19
19
|
meshRoot: string;
|
|
20
20
|
fetchCredentials?: PersonalCredentialsFetcher;
|