@indigoai-us/hq-cli 5.224.0 → 5.226.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -1
- package/dist/command-catalog.generated.d.ts +94 -0
- package/dist/command-catalog.generated.js +120 -0
- package/dist/command-registration-plan.d.ts +18 -0
- package/dist/command-registration-plan.js +6 -0
- package/dist/commands/agent-enroll.d.ts +2 -0
- package/dist/commands/agent-enroll.js +17 -2
- package/dist/commands/cloud.d.ts +10 -0
- package/dist/commands/cloud.js +52 -47
- package/dist/commands/lanes.d.ts +5 -0
- package/dist/commands/lanes.js +498 -312
- package/dist/commands/staff/reset-test-account.d.ts +110 -0
- package/dist/commands/staff/reset-test-account.js +268 -0
- package/dist/commands/staff/test-account.d.ts +51 -0
- package/dist/commands/staff/test-account.js +196 -0
- package/dist/lib/bot/inbox-poll-backoff-flag.d.ts +24 -0
- package/dist/lib/bot/inbox-poll-backoff-flag.js +132 -0
- package/dist/lib/bot/run.d.ts +3 -1
- package/dist/lib/bot/run.js +89 -11
- package/dist/lib/doctor/client-health-flag-token.d.ts +15 -0
- package/dist/lib/doctor/client-health-flag-token.js +66 -0
- package/dist/lib/doctor/client-health-reason-codes-flag.d.ts +1 -0
- package/dist/lib/doctor/client-health-reason-codes-flag.js +16 -5
- package/dist/lib/doctor/client-health-sentry-flag.d.ts +1 -0
- package/dist/lib/doctor/client-health-sentry-flag.js +17 -6
- package/dist/lib/lanes/dropbox.d.ts +9 -0
- package/dist/lib/lanes/dropbox.js +70 -1
- package/dist/lib/lanes/reconcile.d.ts +2 -0
- package/dist/lib/lanes/reconcile.js +12 -0
- package/dist/lib/lanes/watcher-main.js +16 -0
- package/dist/lib/lanes/watcher.d.ts +4 -1
- package/dist/lib/lanes/watcher.js +77 -3
- package/dist/utils/cognito-session.d.ts +8 -0
- package/dist/utils/cognito-session.js +198 -3
- package/dist/utils/upload-author.d.ts +15 -0
- package/dist/utils/upload-author.js +39 -0
- package/package.json +1 -1
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { loadCachedTokens } from "@indigoai-us/hq-cloud";
|
|
2
|
+
import { createFlagClient, } from "@indigoai-us/hq-flags-client";
|
|
3
|
+
import { resolveFlagGate, resolveProcessFlagCompanyContext } from "../flag-registry.js";
|
|
4
|
+
import { resolveFlagRegistryEndpoint } from "../flag-registry-endpoint.js";
|
|
5
|
+
import { effectiveBotCompanies, effectiveBotKind } from "./config.js";
|
|
6
|
+
export const INBOX_POLL_BACKOFF_FLAG = "agents.inbox-poll-backoff-local-cli";
|
|
7
|
+
export const INBOX_POLL_BACKOFF_FLAG_FALLBACK = false;
|
|
8
|
+
export const INBOX_POLL_BACKOFF_FLAG_TIMEOUT_MS = 150;
|
|
9
|
+
export const INBOX_POLL_BACKOFF_FLAG_CACHE_MS = 3 * 60_000;
|
|
10
|
+
const COMPANY_UID_RE = /^cmp_[A-Za-z0-9]{3,128}$/;
|
|
11
|
+
export function safeErrorClass(error) {
|
|
12
|
+
const name = error instanceof Error ? error.name : "UnknownError";
|
|
13
|
+
return /^[A-Za-z][A-Za-z0-9]{0,63}$/.test(name) ? name : "UnknownError";
|
|
14
|
+
}
|
|
15
|
+
function reportFlagFailure(error) {
|
|
16
|
+
process.stderr.write(`HQ local bot inbox poll backoff flag lookup failed (${safeErrorClass(error)}); using default off.\n`);
|
|
17
|
+
}
|
|
18
|
+
function companyContexts(config, env) {
|
|
19
|
+
if (effectiveBotKind(config) !== "company")
|
|
20
|
+
return [{}];
|
|
21
|
+
const slugs = effectiveBotCompanies(config);
|
|
22
|
+
const fallbackSlug = config.companySlug?.trim();
|
|
23
|
+
if (slugs.length === 0 && fallbackSlug)
|
|
24
|
+
slugs.push(fallbackSlug);
|
|
25
|
+
if (slugs.length === 0)
|
|
26
|
+
return null;
|
|
27
|
+
const contexts = [];
|
|
28
|
+
for (const slug of slugs) {
|
|
29
|
+
const company = resolveProcessFlagCompanyContext({
|
|
30
|
+
...env,
|
|
31
|
+
HQ_COMPANY_UID: "",
|
|
32
|
+
HQ_COMPANY_SLUG: slug,
|
|
33
|
+
HQ_ROOT: config.hqRoot,
|
|
34
|
+
});
|
|
35
|
+
if (!company.companyUid || !COMPANY_UID_RE.test(company.companyUid))
|
|
36
|
+
return null;
|
|
37
|
+
contexts.push({ companyUid: company.companyUid, companyIdentifiers: [slug] });
|
|
38
|
+
}
|
|
39
|
+
return contexts;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Create a per-bot reader. The flag client is opened at most once per three
|
|
43
|
+
* minutes; a missing identity, failed request, or malformed snapshot keeps
|
|
44
|
+
* idle polling at its existing two-second interval.
|
|
45
|
+
*/
|
|
46
|
+
export function createInboxPollBackoffFlagReader(config, dependencies = {}) {
|
|
47
|
+
const env = dependencies.env ?? process.env;
|
|
48
|
+
const contexts = companyContexts(config, env);
|
|
49
|
+
const endpoint = resolveFlagRegistryEndpoint(undefined, env);
|
|
50
|
+
const now = dependencies.now ?? Date.now;
|
|
51
|
+
const createClient = dependencies.createClient ?? createFlagClient;
|
|
52
|
+
const getToken = dependencies.getToken ?? (() => loadCachedTokens()?.idToken ?? "");
|
|
53
|
+
const doFetch = dependencies.fetch ?? globalThis.fetch;
|
|
54
|
+
const report = dependencies.reportFailure ?? reportFlagFailure;
|
|
55
|
+
let cachedValue = INBOX_POLL_BACKOFF_FLAG_FALLBACK;
|
|
56
|
+
let lastReadAt = Number.NEGATIVE_INFINITY;
|
|
57
|
+
let pending = null;
|
|
58
|
+
const readContext = async (context) => {
|
|
59
|
+
let client;
|
|
60
|
+
let failed = false;
|
|
61
|
+
let failureReported = false;
|
|
62
|
+
let enabled = INBOX_POLL_BACKOFF_FLAG_FALLBACK;
|
|
63
|
+
const reportFailureOnce = (error) => {
|
|
64
|
+
failed = true;
|
|
65
|
+
if (failureReported)
|
|
66
|
+
return;
|
|
67
|
+
failureReported = true;
|
|
68
|
+
report(error);
|
|
69
|
+
};
|
|
70
|
+
try {
|
|
71
|
+
client = createClient({
|
|
72
|
+
endpoint,
|
|
73
|
+
...(context.companyUid ? { companyUid: context.companyUid } : {}),
|
|
74
|
+
...(context.companyIdentifiers ? { companyIdentifiers: context.companyIdentifiers } : {}),
|
|
75
|
+
env: {},
|
|
76
|
+
getToken,
|
|
77
|
+
...(doFetch ? { fetch: doFetch } : {}),
|
|
78
|
+
refreshIntervalMs: 0,
|
|
79
|
+
requestTimeoutMs: INBOX_POLL_BACKOFF_FLAG_TIMEOUT_MS,
|
|
80
|
+
onError: reportFailureOnce,
|
|
81
|
+
});
|
|
82
|
+
await client.ready();
|
|
83
|
+
if (!failed) {
|
|
84
|
+
enabled = resolveFlagGate(client, INBOX_POLL_BACKOFF_FLAG, { fallback: INBOX_POLL_BACKOFF_FLAG_FALLBACK }, () => INBOX_POLL_BACKOFF_FLAG_FALLBACK);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
reportFailureOnce(error);
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
try {
|
|
92
|
+
client?.close();
|
|
93
|
+
}
|
|
94
|
+
catch (error) {
|
|
95
|
+
reportFailureOnce(error);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return failed ? INBOX_POLL_BACKOFF_FLAG_FALLBACK : enabled;
|
|
99
|
+
};
|
|
100
|
+
const read = async () => {
|
|
101
|
+
if (!contexts || contexts.length === 0) {
|
|
102
|
+
report(new Error("local bot inbox poll backoff flag has no resolvable company identity"));
|
|
103
|
+
return INBOX_POLL_BACKOFF_FLAG_FALLBACK;
|
|
104
|
+
}
|
|
105
|
+
const values = await Promise.all(contexts.map(readContext));
|
|
106
|
+
return values.every(Boolean);
|
|
107
|
+
};
|
|
108
|
+
return () => {
|
|
109
|
+
const currentTime = now();
|
|
110
|
+
if (pending)
|
|
111
|
+
return pending;
|
|
112
|
+
if (currentTime - lastReadAt < INBOX_POLL_BACKOFF_FLAG_CACHE_MS) {
|
|
113
|
+
return Promise.resolve(cachedValue);
|
|
114
|
+
}
|
|
115
|
+
lastReadAt = currentTime;
|
|
116
|
+
pending = read()
|
|
117
|
+
.then((value) => {
|
|
118
|
+
cachedValue = value;
|
|
119
|
+
return value;
|
|
120
|
+
})
|
|
121
|
+
.catch((error) => {
|
|
122
|
+
report(error);
|
|
123
|
+
cachedValue = INBOX_POLL_BACKOFF_FLAG_FALLBACK;
|
|
124
|
+
return cachedValue;
|
|
125
|
+
})
|
|
126
|
+
.finally(() => {
|
|
127
|
+
pending = null;
|
|
128
|
+
});
|
|
129
|
+
return pending;
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
//# sourceMappingURL=inbox-poll-backoff-flag.js.map
|
package/dist/lib/bot/run.d.ts
CHANGED
|
@@ -104,7 +104,7 @@ export declare const UNDELIVERABLE_JOB_REPORT_TEXT = "This scheduled run finishe
|
|
|
104
104
|
* instead of waiting, with no explanation, behind two long jobs.
|
|
105
105
|
*/
|
|
106
106
|
export declare const MAX_PARALLEL_JOB_TURNS = 1;
|
|
107
|
-
export type SleepFn = (ms: number) => Promise<void>;
|
|
107
|
+
export type SleepFn = (ms: number, signal?: AbortSignal) => Promise<void>;
|
|
108
108
|
export interface BotRunDeps {
|
|
109
109
|
dir: string;
|
|
110
110
|
config: BotConfig;
|
|
@@ -118,6 +118,8 @@ export interface BotRunDeps {
|
|
|
118
118
|
pid?: number;
|
|
119
119
|
isProcessAlive?: (pid: number) => boolean;
|
|
120
120
|
inboxIntervalMs?: number;
|
|
121
|
+
/** Cached, default-off reader for the local inbox polling gate. */
|
|
122
|
+
inboxPollBackoffEnabled?: () => Promise<boolean>;
|
|
121
123
|
heartbeatIntervalMs?: number;
|
|
122
124
|
/** Override the rendered system prompt (tests). */
|
|
123
125
|
systemPromptOverride?: string;
|
package/dist/lib/bot/run.js
CHANGED
|
@@ -76,6 +76,7 @@ import { clearBotSession, readBotSession, writeBotSession } from "./session.js";
|
|
|
76
76
|
import { defaultBotStatus, patchBotStatus, readBotStatus } from "./status.js";
|
|
77
77
|
import { CLI_VERSION } from "../../cli-version.js";
|
|
78
78
|
import { CLI_UPDATE_RESTART_EXIT_CODE } from "./cli-update.js";
|
|
79
|
+
import { createInboxPollBackoffFlagReader, safeErrorClass } from "./inbox-poll-backoff-flag.js";
|
|
79
80
|
export const INBOX_POLL_INTERVAL_MS = 2_000;
|
|
80
81
|
export const HEARTBEAT_INTERVAL_MS = 30_000;
|
|
81
82
|
/** First 429/5xx wait when Retry-After is absent (inbox polls at 2s). */
|
|
@@ -224,8 +225,20 @@ class FailureWindow {
|
|
|
224
225
|
return this.count(nowMs) >= this.limit;
|
|
225
226
|
}
|
|
226
227
|
}
|
|
227
|
-
function defaultSleep(ms) {
|
|
228
|
-
return new Promise((resolve) =>
|
|
228
|
+
function defaultSleep(ms, signal) {
|
|
229
|
+
return new Promise((resolve) => {
|
|
230
|
+
if (signal?.aborted) {
|
|
231
|
+
resolve();
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
const timer = setTimeout(done, ms);
|
|
235
|
+
function done() {
|
|
236
|
+
clearTimeout(timer);
|
|
237
|
+
signal?.removeEventListener("abort", done);
|
|
238
|
+
resolve();
|
|
239
|
+
}
|
|
240
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
241
|
+
});
|
|
229
242
|
}
|
|
230
243
|
/**
|
|
231
244
|
* A body posted as several parts where an early part got through and a later
|
|
@@ -262,6 +275,7 @@ export async function runBot(deps) {
|
|
|
262
275
|
const { dir, config, api, runtime } = deps;
|
|
263
276
|
const runTurn = deps.runTurn ?? ((rt, input) => runRuntimeTurn(rt, input));
|
|
264
277
|
const inboxIntervalMs = deps.inboxIntervalMs ?? INBOX_POLL_INTERVAL_MS;
|
|
278
|
+
const inboxPollBackoffEnabled = deps.inboxPollBackoffEnabled ?? createInboxPollBackoffFlagReader(config);
|
|
265
279
|
const heartbeatIntervalMs = deps.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;
|
|
266
280
|
const exit = deps.exit ?? ((code) => process.exit(code));
|
|
267
281
|
const fallback = () => defaultBotStatus(pid, config.runtime, now);
|
|
@@ -312,6 +326,16 @@ export async function runBot(deps) {
|
|
|
312
326
|
const cliVersion = deps.cliUpdate?.runningVersion ?? CLI_VERSION;
|
|
313
327
|
/** Set once an update restart is due: no new turns, then exit for the supervisor. */
|
|
314
328
|
let cliUpdatePending = null;
|
|
329
|
+
const pollForCliUpdate = () => {
|
|
330
|
+
const update = deps.cliUpdate?.poll(log) ?? null;
|
|
331
|
+
if (!update)
|
|
332
|
+
return false;
|
|
333
|
+
cliUpdatePending = update;
|
|
334
|
+
stopping = true;
|
|
335
|
+
const busy = running.size;
|
|
336
|
+
log("info", `hq-cli updated ${update.from} -> ${update.to}; restarting to load it${busy ? ` after ${busy} running turn${busy === 1 ? "" : "s"} finish${busy === 1 ? "es" : ""}` : ""}`);
|
|
337
|
+
return true;
|
|
338
|
+
};
|
|
315
339
|
// ── Preflight: creds, pid lock, owner verification ─────────────────────────
|
|
316
340
|
const credsProblem = preflightCreds(dir, config);
|
|
317
341
|
if (credsProblem) {
|
|
@@ -506,6 +530,45 @@ export async function runBot(deps) {
|
|
|
506
530
|
})();
|
|
507
531
|
// ── Inbox loop ─────────────────────────────────────────────────────────────
|
|
508
532
|
const attempts = new Map();
|
|
533
|
+
const inboxBackoffSteps = [inboxIntervalMs, 10_000, 20_000, 30_000];
|
|
534
|
+
let inboxBackoffIndex = 0;
|
|
535
|
+
let inboxWaitGeneration = 0;
|
|
536
|
+
let activeInboxWait = null;
|
|
537
|
+
const resetInboxBackoff = () => {
|
|
538
|
+
inboxBackoffIndex = 0;
|
|
539
|
+
inboxWaitGeneration += 1;
|
|
540
|
+
activeInboxWait?.abort();
|
|
541
|
+
};
|
|
542
|
+
const waitForInbox = async (delayMs, adaptive) => {
|
|
543
|
+
if (!adaptive) {
|
|
544
|
+
await sleep(delayMs);
|
|
545
|
+
return;
|
|
546
|
+
}
|
|
547
|
+
let remainingMs = delayMs;
|
|
548
|
+
let observedGeneration = inboxWaitGeneration;
|
|
549
|
+
while (remainingMs > 0 && !stopping && state === "running") {
|
|
550
|
+
const controller = new AbortController();
|
|
551
|
+
activeInboxWait = controller;
|
|
552
|
+
const chunkMs = Math.min(inboxIntervalMs, remainingMs);
|
|
553
|
+
await sleep(chunkMs, controller.signal);
|
|
554
|
+
if (activeInboxWait === controller)
|
|
555
|
+
activeInboxWait = null;
|
|
556
|
+
if (stopping || state !== "running")
|
|
557
|
+
break;
|
|
558
|
+
if (hasPromotionHold(dir))
|
|
559
|
+
break;
|
|
560
|
+
if (observedGeneration !== inboxWaitGeneration || controller.signal.aborted) {
|
|
561
|
+
observedGeneration = inboxWaitGeneration;
|
|
562
|
+
remainingMs = inboxIntervalMs;
|
|
563
|
+
continue;
|
|
564
|
+
}
|
|
565
|
+
remainingMs -= chunkMs;
|
|
566
|
+
// Keep the install watcher at its original two-second cadence even when
|
|
567
|
+
// the inbox itself is waiting longer between empty polls.
|
|
568
|
+
if (remainingMs > 0 && pollForCliUpdate())
|
|
569
|
+
break;
|
|
570
|
+
}
|
|
571
|
+
};
|
|
509
572
|
/** Build the user turn for a room trigger: header + recent transcript + the message. */
|
|
510
573
|
const buildRoomTurn = async (item, target, text) => {
|
|
511
574
|
const name = (item.channelName ?? "").trim() || target.channelId;
|
|
@@ -1551,6 +1614,7 @@ export async function runBot(deps) {
|
|
|
1551
1614
|
log("warn", `inbox item ${item.messageId} failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1552
1615
|
})
|
|
1553
1616
|
.finally(() => {
|
|
1617
|
+
resetInboxBackoff();
|
|
1554
1618
|
busyLanes.delete(lane);
|
|
1555
1619
|
pending.delete(item.messageId);
|
|
1556
1620
|
running.delete(task);
|
|
@@ -1588,14 +1652,8 @@ export async function runBot(deps) {
|
|
|
1588
1652
|
// The install under this process was updated: take nothing new, let the
|
|
1589
1653
|
// turns already running finish, then exit so the supervisor loads it.
|
|
1590
1654
|
// Queued items were never started or acked; the next process gets them.
|
|
1591
|
-
|
|
1592
|
-
if (update) {
|
|
1593
|
-
cliUpdatePending = update;
|
|
1594
|
-
stopping = true;
|
|
1595
|
-
const busy = running.size;
|
|
1596
|
-
log("info", `hq-cli updated ${update.from} -> ${update.to}; restarting to load it${busy ? ` after ${busy} running turn${busy === 1 ? "" : "s"} finish${busy === 1 ? "es" : ""}` : ""}`);
|
|
1655
|
+
if (pollForCliUpdate())
|
|
1597
1656
|
break;
|
|
1598
|
-
}
|
|
1599
1657
|
// A kickoff held for a sign-in retries on the same timer as messages.
|
|
1600
1658
|
if (busyLanes.size === 0 &&
|
|
1601
1659
|
(!signInIssue || now().getTime() >= nextSignInRetryAt) &&
|
|
@@ -1609,17 +1667,26 @@ export async function runBot(deps) {
|
|
|
1609
1667
|
// Completions the cloud has not accepted yet: re-send the report (never
|
|
1610
1668
|
// the work), alongside the poll rather than in front of it.
|
|
1611
1669
|
kickCompletionRetries();
|
|
1670
|
+
const useInboxBackoff = await inboxPollBackoffEnabled().catch((error) => {
|
|
1671
|
+
log("warn", `inbox poll backoff flag lookup failed (${safeErrorClass(error)}); using default-off`);
|
|
1672
|
+
return false;
|
|
1673
|
+
});
|
|
1674
|
+
let items;
|
|
1612
1675
|
try {
|
|
1613
|
-
|
|
1676
|
+
items = await api.pullInbox(config.agentUid);
|
|
1614
1677
|
inboxFails = 0;
|
|
1615
1678
|
setStatus({ lastInboxPollAt: now().toISOString(), lastInboxPollOk: true, lastInboxError: undefined });
|
|
1616
1679
|
// One catch-up per job per wake: when several runs of the same job are
|
|
1617
1680
|
// pending, the newest runs and the older ones are reported skipped.
|
|
1618
1681
|
for (const id of planJobRuns(items, handledJobFires).superseded)
|
|
1619
1682
|
supersededJobRuns.add(id);
|
|
1683
|
+
if (items.length > 0)
|
|
1684
|
+
resetInboxBackoff();
|
|
1620
1685
|
for (const item of items)
|
|
1621
1686
|
enqueue(item);
|
|
1622
1687
|
pump();
|
|
1688
|
+
if (stopping || state !== "running")
|
|
1689
|
+
break;
|
|
1623
1690
|
}
|
|
1624
1691
|
catch (err) {
|
|
1625
1692
|
inboxFails += 1;
|
|
@@ -1633,7 +1700,17 @@ export async function runBot(deps) {
|
|
|
1633
1700
|
}
|
|
1634
1701
|
if (stopping || state !== "running")
|
|
1635
1702
|
break;
|
|
1636
|
-
|
|
1703
|
+
if (!useInboxBackoff) {
|
|
1704
|
+
inboxBackoffIndex = 0;
|
|
1705
|
+
await waitForInbox(inboxIntervalMs, false);
|
|
1706
|
+
continue;
|
|
1707
|
+
}
|
|
1708
|
+
const delayMs = items.length > 0
|
|
1709
|
+
? inboxIntervalMs
|
|
1710
|
+
: inboxBackoffSteps[Math.min(inboxBackoffIndex, inboxBackoffSteps.length - 1)];
|
|
1711
|
+
if (items.length === 0)
|
|
1712
|
+
inboxBackoffIndex = Math.min(inboxBackoffIndex + 1, inboxBackoffSteps.length - 1);
|
|
1713
|
+
await waitForInbox(delayMs, true);
|
|
1637
1714
|
}
|
|
1638
1715
|
// Stop, drain and promotion wait for the turns already running. Queued
|
|
1639
1716
|
// messages were never started or acked, so the inbox delivers them again.
|
|
@@ -1655,6 +1732,7 @@ export async function runBot(deps) {
|
|
|
1655
1732
|
return;
|
|
1656
1733
|
}
|
|
1657
1734
|
stopping = true;
|
|
1735
|
+
activeInboxWait?.abort();
|
|
1658
1736
|
// End the turn in progress; its message gets "did not finish", not silence.
|
|
1659
1737
|
// Promotion instead drains that turn, including its reply and ACK, before
|
|
1660
1738
|
// the coordinator snapshots memory and starts the cloud consumer.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export declare const CLIENT_HEALTH_FLAG_TOKEN_TIMEOUT_MS = 1000;
|
|
2
|
+
export declare class MissingFlagTokenError extends Error {
|
|
3
|
+
constructor();
|
|
4
|
+
}
|
|
5
|
+
export declare class ClientHealthFlagTokenTimeoutError extends Error {
|
|
6
|
+
constructor(timeoutMs: number);
|
|
7
|
+
}
|
|
8
|
+
export type ClientHealthFlagTokenProvider = (signal: AbortSignal) => string | Promise<string>;
|
|
9
|
+
/**
|
|
10
|
+
* Obtain a non-interactive ID token, separately bounded from the flag read.
|
|
11
|
+
* ensureCognitoIdToken refreshes expired person sessions and selects machine
|
|
12
|
+
* identity credentials when the current installation is a machine.
|
|
13
|
+
*/
|
|
14
|
+
export declare function getClientHealthFlagToken(provider?: ClientHealthFlagTokenProvider, timeoutMs?: number, callerSignal?: AbortSignal): Promise<string>;
|
|
15
|
+
//# sourceMappingURL=client-health-flag-token.d.ts.map
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { ensureCognitoIdToken } from "../../utils/cognito-session.js";
|
|
2
|
+
export const CLIENT_HEALTH_FLAG_TOKEN_TIMEOUT_MS = 1_000;
|
|
3
|
+
export class MissingFlagTokenError extends Error {
|
|
4
|
+
constructor() {
|
|
5
|
+
super("No cached HQ identity token is available");
|
|
6
|
+
this.name = "MissingFlagTokenError";
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
export class ClientHealthFlagTokenTimeoutError extends Error {
|
|
10
|
+
constructor(timeoutMs) {
|
|
11
|
+
super(`Client-health flag token acquisition exceeded its ${timeoutMs} ms bound`);
|
|
12
|
+
this.name = "ClientHealthFlagTokenTimeoutError";
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Obtain a non-interactive ID token, separately bounded from the flag read.
|
|
17
|
+
* ensureCognitoIdToken refreshes expired person sessions and selects machine
|
|
18
|
+
* identity credentials when the current installation is a machine.
|
|
19
|
+
*/
|
|
20
|
+
export async function getClientHealthFlagToken(provider, timeoutMs = CLIENT_HEALTH_FLAG_TOKEN_TIMEOUT_MS, callerSignal) {
|
|
21
|
+
if (callerSignal?.aborted) {
|
|
22
|
+
throw callerSignal.reason ?? new Error("Token acquisition was aborted");
|
|
23
|
+
}
|
|
24
|
+
const controller = new AbortController();
|
|
25
|
+
const signal = callerSignal ? AbortSignal.any([callerSignal, controller.signal]) : controller.signal;
|
|
26
|
+
const timeout = new ClientHealthFlagTokenTimeoutError(timeoutMs);
|
|
27
|
+
const timer = setTimeout(() => controller.abort(timeout), timeoutMs);
|
|
28
|
+
let onAbort;
|
|
29
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
30
|
+
onAbort = () => reject(signal.reason ?? new Error("Token acquisition was aborted"));
|
|
31
|
+
if (signal.aborted)
|
|
32
|
+
onAbort();
|
|
33
|
+
else
|
|
34
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
35
|
+
});
|
|
36
|
+
try {
|
|
37
|
+
const acquire = provider ?? ((requestSignal) => ensureCognitoIdToken({ interactive: false, signal: requestSignal }));
|
|
38
|
+
const acquisition = Promise.resolve().then(() => {
|
|
39
|
+
if (signal.aborted) {
|
|
40
|
+
throw signal.reason ?? new Error("Token acquisition was aborted");
|
|
41
|
+
}
|
|
42
|
+
return acquire(signal);
|
|
43
|
+
});
|
|
44
|
+
const token = await Promise.race([acquisition, aborted]);
|
|
45
|
+
if (signal.aborted) {
|
|
46
|
+
throw signal.reason ?? new Error("Token acquisition was aborted");
|
|
47
|
+
}
|
|
48
|
+
if (typeof token !== "string" || token.trim().length === 0) {
|
|
49
|
+
throw new MissingFlagTokenError();
|
|
50
|
+
}
|
|
51
|
+
return token;
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
if (error instanceof Error &&
|
|
55
|
+
error.message.startsWith("No valid HQ session and interactive login is disabled")) {
|
|
56
|
+
throw new MissingFlagTokenError();
|
|
57
|
+
}
|
|
58
|
+
throw error;
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
clearTimeout(timer);
|
|
62
|
+
if (onAbort)
|
|
63
|
+
signal.removeEventListener("abort", onAbort);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=client-health-flag-token.js.map
|
|
@@ -7,6 +7,7 @@ export interface ClientHealthReasonCodesFlagDependencies {
|
|
|
7
7
|
env?: NodeJS.ProcessEnv;
|
|
8
8
|
createClient?: (options: FlagClientOptions) => ClientHealthFlagClient;
|
|
9
9
|
getToken?: FlagClientOptions["getToken"];
|
|
10
|
+
tokenTimeoutMs?: number;
|
|
10
11
|
fetch?: typeof fetch;
|
|
11
12
|
reportFailure?: (error: unknown) => void;
|
|
12
13
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { loadCachedTokens } from "@indigoai-us/hq-cloud";
|
|
2
1
|
import { createFlagClient, } from "@indigoai-us/hq-flags-client";
|
|
3
2
|
import { resolveFlagRegistryEndpoint } from "../flag-registry-endpoint.js";
|
|
3
|
+
import { getClientHealthFlagToken } from "./client-health-flag-token.js";
|
|
4
4
|
export const CLIENT_HEALTH_REASON_CODES_FLAG = "client-health-sync-reason-codes";
|
|
5
5
|
export const CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK = false;
|
|
6
6
|
export const CLIENT_HEALTH_REASON_CODES_FLAG_TIMEOUT_MS = 150;
|
|
@@ -39,9 +39,8 @@ export async function clientHealthReasonCodesEnabledForCompany(company, dependen
|
|
|
39
39
|
return CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK;
|
|
40
40
|
}
|
|
41
41
|
const companySlug = company.companySlug?.trim() || "";
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
const doFetch = dependencies.fetch ?? globalThis.fetch;
|
|
42
|
+
if (callerSignal?.aborted)
|
|
43
|
+
return CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK;
|
|
45
44
|
let client;
|
|
46
45
|
let failed = false;
|
|
47
46
|
let failureReported = false;
|
|
@@ -53,6 +52,18 @@ export async function clientHealthReasonCodesEnabledForCompany(company, dependen
|
|
|
53
52
|
failureReported = true;
|
|
54
53
|
(dependencies.reportFailure ?? reportFlagFailure)(error);
|
|
55
54
|
};
|
|
55
|
+
let token;
|
|
56
|
+
try {
|
|
57
|
+
token = await getClientHealthFlagToken(dependencies.getToken, dependencies.tokenTimeoutMs, callerSignal);
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
if (!callerSignal?.aborted)
|
|
61
|
+
reportFailureOnce(error);
|
|
62
|
+
return CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK;
|
|
63
|
+
}
|
|
64
|
+
const deadline = AbortSignal.timeout(CLIENT_HEALTH_REASON_CODES_FLAG_TIMEOUT_MS);
|
|
65
|
+
const signal = callerSignal ? AbortSignal.any([callerSignal, deadline]) : deadline;
|
|
66
|
+
const doFetch = dependencies.fetch ?? globalThis.fetch;
|
|
56
67
|
try {
|
|
57
68
|
client = (dependencies.createClient ?? createFlagClient)({
|
|
58
69
|
endpoint,
|
|
@@ -61,7 +72,7 @@ export async function clientHealthReasonCodesEnabledForCompany(company, dependen
|
|
|
61
72
|
? { companyIdentifiers: [companySlug] }
|
|
62
73
|
: {}),
|
|
63
74
|
env: {},
|
|
64
|
-
getToken:
|
|
75
|
+
getToken: () => token,
|
|
65
76
|
fetch: (input, init) => doFetch(input, {
|
|
66
77
|
...init,
|
|
67
78
|
signal: combineSignals(init?.signal ?? undefined, signal),
|
|
@@ -7,6 +7,7 @@ export interface ClientHealthSentryFlagDependencies {
|
|
|
7
7
|
env?: NodeJS.ProcessEnv;
|
|
8
8
|
createClient?: (options: FlagClientOptions) => ClientHealthFlagClient;
|
|
9
9
|
getToken?: FlagClientOptions["getToken"];
|
|
10
|
+
tokenTimeoutMs?: number;
|
|
10
11
|
fetch?: typeof fetch;
|
|
11
12
|
reportFailure?: (error: unknown) => void;
|
|
12
13
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { loadCachedTokens } from "@indigoai-us/hq-cloud";
|
|
2
1
|
import { createFlagClient, } from "@indigoai-us/hq-flags-client";
|
|
3
2
|
import { resolveFlagRegistryEndpoint } from "../flag-registry-endpoint.js";
|
|
3
|
+
import { getClientHealthFlagToken } from "./client-health-flag-token.js";
|
|
4
4
|
export const CLIENT_HEALTH_SENTRY_DIRECT_FLAG = "client-health.sentry-direct";
|
|
5
5
|
export const CLIENT_HEALTH_SENTRY_DIRECT_FALLBACK = false;
|
|
6
6
|
export const CLIENT_HEALTH_SENTRY_DIRECT_TIMEOUT_MS = 150;
|
|
@@ -31,10 +31,8 @@ export async function clientHealthSentryDirectEnabledForCompany(company, depende
|
|
|
31
31
|
if (!/^cmp_[A-Za-z0-9]{3,128}$/.test(companyUid)) {
|
|
32
32
|
return CLIENT_HEALTH_SENTRY_DIRECT_FALLBACK;
|
|
33
33
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
const signal = callerSignal ? AbortSignal.any([callerSignal, deadline]) : deadline;
|
|
37
|
-
const doFetch = dependencies.fetch ?? globalThis.fetch;
|
|
34
|
+
if (callerSignal?.aborted)
|
|
35
|
+
return CLIENT_HEALTH_SENTRY_DIRECT_FALLBACK;
|
|
38
36
|
let client;
|
|
39
37
|
let failed = false;
|
|
40
38
|
let failureReported = false;
|
|
@@ -46,6 +44,19 @@ export async function clientHealthSentryDirectEnabledForCompany(company, depende
|
|
|
46
44
|
failureReported = true;
|
|
47
45
|
(dependencies.reportFailure ?? reportFlagFailure)(error);
|
|
48
46
|
};
|
|
47
|
+
let token;
|
|
48
|
+
try {
|
|
49
|
+
token = await getClientHealthFlagToken(dependencies.getToken, dependencies.tokenTimeoutMs, callerSignal);
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
if (!callerSignal?.aborted)
|
|
53
|
+
reportFailureOnce(error);
|
|
54
|
+
return CLIENT_HEALTH_SENTRY_DIRECT_FALLBACK;
|
|
55
|
+
}
|
|
56
|
+
const companySlug = company.companySlug?.trim() || "";
|
|
57
|
+
const deadline = AbortSignal.timeout(CLIENT_HEALTH_SENTRY_DIRECT_TIMEOUT_MS);
|
|
58
|
+
const signal = callerSignal ? AbortSignal.any([callerSignal, deadline]) : deadline;
|
|
59
|
+
const doFetch = dependencies.fetch ?? globalThis.fetch;
|
|
49
60
|
try {
|
|
50
61
|
client = (dependencies.createClient ?? createFlagClient)({
|
|
51
62
|
endpoint: resolveFlagRegistryEndpoint(undefined, env),
|
|
@@ -54,7 +65,7 @@ export async function clientHealthSentryDirectEnabledForCompany(company, depende
|
|
|
54
65
|
? { companyIdentifiers: [companySlug] }
|
|
55
66
|
: {}),
|
|
56
67
|
env: {},
|
|
57
|
-
getToken:
|
|
68
|
+
getToken: () => token,
|
|
58
69
|
fetch: (input, init) => doFetch(input, {
|
|
59
70
|
...init,
|
|
60
71
|
signal: combineSignals(init?.signal ?? undefined, signal),
|
|
@@ -50,6 +50,8 @@ export interface DrainDeps {
|
|
|
50
50
|
afterRead?: () => void;
|
|
51
51
|
/** Prefix marks archives that belong to one provider turn. */
|
|
52
52
|
archivePrefix?: string;
|
|
53
|
+
/** Terminal messages with no permitted delivery path are kept separately. */
|
|
54
|
+
archiveDirectory?: "delivered" | "undeliverable";
|
|
53
55
|
}
|
|
54
56
|
export interface LaneDrainHookInput {
|
|
55
57
|
transcript_path?: string | null;
|
|
@@ -68,6 +70,13 @@ export declare function mintClaimId(): string;
|
|
|
68
70
|
export declare function assertSafeLaneId(laneId: string): void;
|
|
69
71
|
export declare function dropboxDir(hqRoot: string, laneId: string): string;
|
|
70
72
|
export declare function dropboxInboxPath(hqRoot: string, laneId: string): string;
|
|
73
|
+
/** A recovery drain can claim the inbox, draining files, or dead inflight files. */
|
|
74
|
+
export declare function dropboxInboxHasMessages(hqRoot: string, laneId: string): boolean;
|
|
75
|
+
/**
|
|
76
|
+
* Return a failed recovery claim to the append-only inbox without replacing
|
|
77
|
+
* messages appended while resume validation or spawn was in progress.
|
|
78
|
+
*/
|
|
79
|
+
export declare function requeueDropboxMessages(hqRoot: string, laneId: string, messages: readonly DropboxMessage[]): void;
|
|
71
80
|
/**
|
|
72
81
|
* Worker tool-call liveness (watcher.ts reads it). A Claude worker run with
|
|
73
82
|
* `--output-format json` writes nothing to worker.out until it exits, so the
|
|
@@ -65,6 +65,75 @@ export function dropboxDir(hqRoot, laneId) {
|
|
|
65
65
|
export function dropboxInboxPath(hqRoot, laneId) {
|
|
66
66
|
return path.join(dropboxDir(hqRoot, laneId), INBOX_NAME);
|
|
67
67
|
}
|
|
68
|
+
function directoryHasRecoverableFiles(dir, nestedInboxOnly = false) {
|
|
69
|
+
let names;
|
|
70
|
+
try {
|
|
71
|
+
names = fs.readdirSync(dir).filter((name) => !name.startsWith("."));
|
|
72
|
+
}
|
|
73
|
+
catch (err) {
|
|
74
|
+
if (err.code === "ENOENT")
|
|
75
|
+
return false;
|
|
76
|
+
throw err;
|
|
77
|
+
}
|
|
78
|
+
for (const name of names) {
|
|
79
|
+
const candidate = path.join(dir, name);
|
|
80
|
+
let stat;
|
|
81
|
+
try {
|
|
82
|
+
stat = fs.statSync(candidate);
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
if (err.code === "ENOENT")
|
|
86
|
+
continue;
|
|
87
|
+
throw err;
|
|
88
|
+
}
|
|
89
|
+
if (nestedInboxOnly) {
|
|
90
|
+
if (!stat.isDirectory())
|
|
91
|
+
continue;
|
|
92
|
+
try {
|
|
93
|
+
const inbox = fs.statSync(path.join(candidate, INBOX_NAME));
|
|
94
|
+
if (inbox.isFile() && inbox.size > 0)
|
|
95
|
+
return true;
|
|
96
|
+
}
|
|
97
|
+
catch (err) {
|
|
98
|
+
if (err.code !== "ENOENT")
|
|
99
|
+
throw err;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
else if (stat.isFile() && stat.size > 0) {
|
|
103
|
+
return true;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
function fileHasBytes(filePath) {
|
|
109
|
+
try {
|
|
110
|
+
const stat = fs.statSync(filePath);
|
|
111
|
+
return stat.isFile() && stat.size > 0;
|
|
112
|
+
}
|
|
113
|
+
catch (err) {
|
|
114
|
+
if (err.code === "ENOENT")
|
|
115
|
+
return false;
|
|
116
|
+
throw err;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
/** A recovery drain can claim the inbox, draining files, or dead inflight files. */
|
|
120
|
+
export function dropboxInboxHasMessages(hqRoot, laneId) {
|
|
121
|
+
const box = dropboxDir(hqRoot, laneId);
|
|
122
|
+
return fileHasBytes(path.join(box, INBOX_NAME)) ||
|
|
123
|
+
directoryHasRecoverableFiles(path.join(box, DRAINING_DIR)) ||
|
|
124
|
+
directoryHasRecoverableFiles(path.join(box, INFLIGHT_DIR), true);
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Return a failed recovery claim to the append-only inbox without replacing
|
|
128
|
+
* messages appended while resume validation or spawn was in progress.
|
|
129
|
+
*/
|
|
130
|
+
export function requeueDropboxMessages(hqRoot, laneId, messages) {
|
|
131
|
+
if (messages.length === 0)
|
|
132
|
+
return;
|
|
133
|
+
const inbox = dropboxInboxPath(hqRoot, laneId);
|
|
134
|
+
fs.mkdirSync(path.dirname(inbox), { recursive: true });
|
|
135
|
+
fs.appendFileSync(inbox, messages.map((message) => `${JSON.stringify(message)}\n`).join(""), "utf8");
|
|
136
|
+
}
|
|
68
137
|
/**
|
|
69
138
|
* Worker tool-call liveness (watcher.ts reads it). A Claude worker run with
|
|
70
139
|
* `--output-format json` writes nothing to worker.out until it exits, so the
|
|
@@ -220,7 +289,7 @@ export function drainDropboxAtPath(box, depsInput = {}) {
|
|
|
220
289
|
return [];
|
|
221
290
|
const inbox = path.join(box, INBOX_NAME);
|
|
222
291
|
const draining = path.join(box, DRAINING_DIR);
|
|
223
|
-
const delivered = path.join(box, DELIVERED_DIR);
|
|
292
|
+
const delivered = path.join(box, deps.archiveDirectory ?? DELIVERED_DIR);
|
|
224
293
|
const inflightRoot = path.join(box, INFLIGHT_DIR);
|
|
225
294
|
try {
|
|
226
295
|
deps.mkdirSync(box, { recursive: true });
|
|
@@ -81,6 +81,8 @@ export interface ReconcileDeps {
|
|
|
81
81
|
readRegistryFlag?: typeof readLanesRegistryFlag;
|
|
82
82
|
/** Active-company fallback for questions raised from an unscoped lane. */
|
|
83
83
|
resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
|
|
84
|
+
/** Resume queued messages after this pass wins a local dead-worker CAS. */
|
|
85
|
+
recoverQueuedMessages?: (lane: LaneRecord) => Promise<void>;
|
|
84
86
|
/** Detached mesh seam for dead-watcher recovery. */
|
|
85
87
|
mesh?: LaneMesh | null;
|
|
86
88
|
/** Bound on one terminal mesh emit during dead-host recovery. Default 10 s. */
|
|
@@ -1172,6 +1172,18 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1172
1172
|
}
|
|
1173
1173
|
}
|
|
1174
1174
|
}
|
|
1175
|
+
if (!deps.dryRun && deps.recoverQueuedMessages) {
|
|
1176
|
+
try {
|
|
1177
|
+
await deps.recoverQueuedMessages(result.record);
|
|
1178
|
+
}
|
|
1179
|
+
catch (err) {
|
|
1180
|
+
markDirty(summary, {
|
|
1181
|
+
kind: "lane",
|
|
1182
|
+
id: lane.lane_id,
|
|
1183
|
+
why: `queued message recovery failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1184
|
+
});
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1175
1187
|
}
|
|
1176
1188
|
}
|
|
1177
1189
|
catch (err) {
|