@indigoai-us/hq-cli 5.345.187 → 5.345.189
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -0
- package/dist/command-catalog.generated.d.ts +34 -0
- package/dist/command-catalog.generated.js +42 -0
- package/dist/commands/agent-inbox.js +43 -11
- package/dist/commands/agent-mcp.js +5 -2
- package/dist/commands/bot.js +15 -3
- package/dist/commands/core-company-index-scope-flag.d.ts +2 -1
- package/dist/commands/core-company-index-scope-flag.js +2 -1
- package/dist/commands/core.js +0 -5
- package/dist/commands/dm-attach.d.ts +73 -0
- package/dist/commands/dm-attach.js +104 -0
- package/dist/commands/dm.d.ts +10 -0
- package/dist/commands/dm.js +114 -2
- package/dist/commands/sources.js +15 -0
- package/dist/hq-cloud-sync-runner/bin/sync-runner.js +57330 -63803
- package/dist/hq-cloud-sync-runner/package.json +1 -1
- package/dist/lib/agent-kit/inbox-fetch.d.ts +92 -0
- package/dist/lib/agent-kit/inbox-fetch.js +135 -0
- package/dist/lib/agent-kit/mcp/tools.d.ts +11 -2
- package/dist/lib/agent-kit/mcp/tools.js +17 -3
- package/dist/lib/bot/fleet-local.d.ts +3 -1
- package/dist/lib/bot/fleet-local.js +9 -3
- package/dist/lib/bot/fleet-python.d.ts +6 -0
- package/dist/lib/bot/fleet-python.js +24 -5
- package/dist/lib/bot/probe.d.ts +7 -1
- package/dist/lib/bot/probe.js +11 -3
- package/dist/lib/bot/run.d.ts +9 -0
- package/dist/lib/bot/run.js +114 -8
- package/dist/lib/bot/runtime-failure.d.ts +3 -0
- package/dist/lib/bot/runtime-failure.js +20 -2
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/dist/lib/index-render/session-scope.d.ts +3 -3
- package/dist/lib/index-render/session-scope.js +11 -9
- package/dist/lib/index-render/shared.d.ts +6 -4
- package/dist/lib/index-render/shared.js +33 -4
- package/dist/utils/cognito-session.d.ts +5 -3
- package/dist/utils/cognito-session.js +43 -46
- package/dist/utils/hq-root-resolver.d.ts +27 -1
- package/dist/utils/hq-root-resolver.js +65 -33
- package/package.json +15 -15
|
@@ -1 +1 @@
|
|
|
1
|
-
{"name":"hq-cli-daemon-sync-runner","type":"module","version":"6.18.
|
|
1
|
+
{"name":"hq-cli-daemon-sync-runner","type":"module","version":"6.18.107","bundleSha256":"4ffaf3a579e9222aa2e9a1850e355d4a152d6d3323cf0583154cf28e1d45459a","formatVersion":1}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bounded, once-retried read of the agent's server inbox.
|
|
3
|
+
*
|
|
4
|
+
* `hq agent inbox` and the `hq_inbox_read` MCP tool used to do one token mint
|
|
5
|
+
* and one GET with no deadline and no retry, so three transient conditions
|
|
6
|
+
* each showed up as a local-mirror fallback that cleared minutes later
|
|
7
|
+
* (feedback 3e376c49, DEV-4065):
|
|
8
|
+
*
|
|
9
|
+
* - the machine-token cache is shared with the kit poller and heartbeat,
|
|
10
|
+
* which re-mint in the same minute each hour; hq-cloud serialises mints
|
|
11
|
+
* behind a 20 s lock and throws "... held by another process ..." when it
|
|
12
|
+
* gives up waiting;
|
|
13
|
+
* - the server bounds its inbox pull at 18 s and answers 503 with
|
|
14
|
+
* `Retry-After: 2` when the listing stalls;
|
|
15
|
+
* - a half-open socket can hang the GET until the OS drops it.
|
|
16
|
+
*
|
|
17
|
+
* Each attempt (mint plus GET) runs under one deadline. A first attempt that
|
|
18
|
+
* fails for one of those reasons is retried exactly once after a short delay
|
|
19
|
+
* (the server's `Retry-After`, capped, when it sent one). The second attempt
|
|
20
|
+
* mints a fresh token, because the first failure may have been the mint. Auth
|
|
21
|
+
* and not-found answers, and a 200 whose body is invalid, are never retried.
|
|
22
|
+
* Pure apart from the injected `sleep`/`random`, so tests drive it with fake
|
|
23
|
+
* timers.
|
|
24
|
+
*/
|
|
25
|
+
import type { MergedInbox } from "./inbox-state.js";
|
|
26
|
+
/** Env override for the per-attempt deadline; values under the floor are raised to it. */
|
|
27
|
+
export declare const AGENT_INBOX_TIMEOUT_ENV = "HQ_AGENT_INBOX_TIMEOUT_MS";
|
|
28
|
+
/** Delay before the one retry when the server sent no usable `Retry-After`. */
|
|
29
|
+
export declare const AGENT_INBOX_RETRY_BASE_MS = 1500;
|
|
30
|
+
/** Jitter added to the base delay, so clients that fail together do not retry together. */
|
|
31
|
+
export declare const AGENT_INBOX_RETRY_JITTER_MS = 500;
|
|
32
|
+
/** Longest wait honoured from a `Retry-After` header. */
|
|
33
|
+
export declare const AGENT_INBOX_RETRY_AFTER_CAP_MS = 5000;
|
|
34
|
+
/** Total attempts: the first read plus one retry. */
|
|
35
|
+
export declare const AGENT_INBOX_MAX_ATTEMPTS = 2;
|
|
36
|
+
export interface InboxFetchResponse {
|
|
37
|
+
status: number;
|
|
38
|
+
/** Decoded JSON body, or `undefined` when it did not decode. Never a raw string of response bytes. */
|
|
39
|
+
body: unknown;
|
|
40
|
+
/** Parsed numeric `Retry-After` header in seconds, when the server sent one. */
|
|
41
|
+
retryAfterSeconds?: number;
|
|
42
|
+
}
|
|
43
|
+
export interface FetchServerInboxDeps {
|
|
44
|
+
/** Mint (or fetch from cache) the machine token. Called again for the retry. */
|
|
45
|
+
mint: (signal: AbortSignal) => Promise<string>;
|
|
46
|
+
/** GET the inbox with `token`. Must not throw on a non-2xx status. */
|
|
47
|
+
fetch: (token: string, signal: AbortSignal) => Promise<InboxFetchResponse>;
|
|
48
|
+
sleep?: (ms: number) => Promise<void>;
|
|
49
|
+
/** Uniform [0, 1) source for the jitter. */
|
|
50
|
+
random?: () => number;
|
|
51
|
+
/** Per-attempt deadline covering mint plus GET. */
|
|
52
|
+
timeoutMs?: number;
|
|
53
|
+
}
|
|
54
|
+
export type FetchServerInboxOutcome = {
|
|
55
|
+
ok: true;
|
|
56
|
+
response: {
|
|
57
|
+
status: number;
|
|
58
|
+
body: unknown;
|
|
59
|
+
};
|
|
60
|
+
attempts: number;
|
|
61
|
+
} | {
|
|
62
|
+
ok: false;
|
|
63
|
+
error: unknown;
|
|
64
|
+
attempts: number;
|
|
65
|
+
};
|
|
66
|
+
/** Per-attempt deadline from the env, defaulting to the kit's request timeout. */
|
|
67
|
+
export declare function resolveAgentInboxTimeoutMs(env?: NodeJS.ProcessEnv): number;
|
|
68
|
+
/** Whether a thrown first attempt is worth one more try. */
|
|
69
|
+
export declare function isRetryableInboxError(err: unknown): boolean;
|
|
70
|
+
export declare function isRetryableInboxStatus(status: number): boolean;
|
|
71
|
+
/** `Retry-After` as seconds, or undefined for a missing, non-numeric or negative value. */
|
|
72
|
+
export declare function parseRetryAfterSeconds(header: string | null | undefined): number | undefined;
|
|
73
|
+
/**
|
|
74
|
+
* Read the server inbox with at most AGENT_INBOX_MAX_ATTEMPTS attempts, each
|
|
75
|
+
* bounded by `timeoutMs`. Never throws: the outcome carries either the last
|
|
76
|
+
* response or the last error, plus how many attempts were made.
|
|
77
|
+
*/
|
|
78
|
+
export declare function fetchServerInboxWithRetry(deps: FetchServerInboxDeps): Promise<FetchServerInboxOutcome>;
|
|
79
|
+
/** Suffix added to the degradation note when the retry also failed. */
|
|
80
|
+
export declare const AGENT_INBOX_AFTER_ATTEMPTS_SUFFIX = " (after 2 attempts)";
|
|
81
|
+
export interface AnnotatedInbox extends MergedInbox {
|
|
82
|
+
/** True whenever the answer is not a complete server read. */
|
|
83
|
+
stale: boolean;
|
|
84
|
+
/** Present when more than one attempt was made, so a hard failure is distinct from a single blip. */
|
|
85
|
+
attempts?: number;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Add `stale` and, when a retry happened, `attempts` to a merged inbox. A
|
|
89
|
+
* degraded note after a retry also says so in words.
|
|
90
|
+
*/
|
|
91
|
+
export declare function annotateInboxAttempts(merged: MergedInbox, attempts: number): AnnotatedInbox;
|
|
92
|
+
//# sourceMappingURL=inbox-fetch.d.ts.map
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bounded, once-retried read of the agent's server inbox.
|
|
3
|
+
*
|
|
4
|
+
* `hq agent inbox` and the `hq_inbox_read` MCP tool used to do one token mint
|
|
5
|
+
* and one GET with no deadline and no retry, so three transient conditions
|
|
6
|
+
* each showed up as a local-mirror fallback that cleared minutes later
|
|
7
|
+
* (feedback 3e376c49, DEV-4065):
|
|
8
|
+
*
|
|
9
|
+
* - the machine-token cache is shared with the kit poller and heartbeat,
|
|
10
|
+
* which re-mint in the same minute each hour; hq-cloud serialises mints
|
|
11
|
+
* behind a 20 s lock and throws "... held by another process ..." when it
|
|
12
|
+
* gives up waiting;
|
|
13
|
+
* - the server bounds its inbox pull at 18 s and answers 503 with
|
|
14
|
+
* `Retry-After: 2` when the listing stalls;
|
|
15
|
+
* - a half-open socket can hang the GET until the OS drops it.
|
|
16
|
+
*
|
|
17
|
+
* Each attempt (mint plus GET) runs under one deadline. A first attempt that
|
|
18
|
+
* fails for one of those reasons is retried exactly once after a short delay
|
|
19
|
+
* (the server's `Retry-After`, capped, when it sent one). The second attempt
|
|
20
|
+
* mints a fresh token, because the first failure may have been the mint. Auth
|
|
21
|
+
* and not-found answers, and a 200 whose body is invalid, are never retried.
|
|
22
|
+
* Pure apart from the injected `sleep`/`random`, so tests drive it with fake
|
|
23
|
+
* timers.
|
|
24
|
+
*/
|
|
25
|
+
import { networkTransportErrorCode } from "../../utils/network-transport-error.js";
|
|
26
|
+
import { KIT_REQUEST_TIMEOUT_MIN_MS, KIT_REQUEST_TIMEOUT_MS, KitRequestTimeoutError, withRequestTimeout } from "./run/request-timeout.js";
|
|
27
|
+
/** Env override for the per-attempt deadline; values under the floor are raised to it. */
|
|
28
|
+
export const AGENT_INBOX_TIMEOUT_ENV = "HQ_AGENT_INBOX_TIMEOUT_MS";
|
|
29
|
+
/** Delay before the one retry when the server sent no usable `Retry-After`. */
|
|
30
|
+
export const AGENT_INBOX_RETRY_BASE_MS = 1_500;
|
|
31
|
+
/** Jitter added to the base delay, so clients that fail together do not retry together. */
|
|
32
|
+
export const AGENT_INBOX_RETRY_JITTER_MS = 500;
|
|
33
|
+
/** Longest wait honoured from a `Retry-After` header. */
|
|
34
|
+
export const AGENT_INBOX_RETRY_AFTER_CAP_MS = 5_000;
|
|
35
|
+
/** Total attempts: the first read plus one retry. */
|
|
36
|
+
export const AGENT_INBOX_MAX_ATTEMPTS = 2;
|
|
37
|
+
// 504 is absent on purpose: vaultApiFetch already retries a 504 once
|
|
38
|
+
// (Lambda invoke errors), and a second outer retry would make three GETs.
|
|
39
|
+
const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503]);
|
|
40
|
+
const LOCK_HELD_RE = /held by another process/i;
|
|
41
|
+
const TRANSPORT_MESSAGE_RE = /\b(fetch failed|ECONNRESET|ETIMEDOUT|EAI_AGAIN)\b/;
|
|
42
|
+
/** Per-attempt deadline from the env, defaulting to the kit's request timeout. */
|
|
43
|
+
export function resolveAgentInboxTimeoutMs(env = process.env) {
|
|
44
|
+
const raw = env[AGENT_INBOX_TIMEOUT_ENV]?.trim();
|
|
45
|
+
if (!raw)
|
|
46
|
+
return KIT_REQUEST_TIMEOUT_MS;
|
|
47
|
+
const n = Number(raw);
|
|
48
|
+
if (!Number.isFinite(n))
|
|
49
|
+
return KIT_REQUEST_TIMEOUT_MS;
|
|
50
|
+
return Math.max(KIT_REQUEST_TIMEOUT_MIN_MS, Math.floor(n));
|
|
51
|
+
}
|
|
52
|
+
/** Whether a thrown first attempt is worth one more try. */
|
|
53
|
+
export function isRetryableInboxError(err) {
|
|
54
|
+
if (err instanceof KitRequestTimeoutError)
|
|
55
|
+
return true;
|
|
56
|
+
if (!(err instanceof Error))
|
|
57
|
+
return false;
|
|
58
|
+
if (LOCK_HELD_RE.test(err.message))
|
|
59
|
+
return true;
|
|
60
|
+
if (networkTransportErrorCode(err) !== null)
|
|
61
|
+
return true;
|
|
62
|
+
if (err instanceof TypeError && err.message === "fetch failed")
|
|
63
|
+
return true;
|
|
64
|
+
return TRANSPORT_MESSAGE_RE.test(err.message);
|
|
65
|
+
}
|
|
66
|
+
export function isRetryableInboxStatus(status) {
|
|
67
|
+
return RETRYABLE_STATUSES.has(status);
|
|
68
|
+
}
|
|
69
|
+
/** `Retry-After` as seconds, or undefined for a missing, non-numeric or negative value. */
|
|
70
|
+
export function parseRetryAfterSeconds(header) {
|
|
71
|
+
if (header === null || header === undefined)
|
|
72
|
+
return undefined;
|
|
73
|
+
const trimmed = header.trim();
|
|
74
|
+
if (!/^\d+(\.\d+)?$/.test(trimmed))
|
|
75
|
+
return undefined;
|
|
76
|
+
const n = Number(trimmed);
|
|
77
|
+
return Number.isFinite(n) && n >= 0 ? n : undefined;
|
|
78
|
+
}
|
|
79
|
+
function retryDelayMs(retryAfterSeconds, random) {
|
|
80
|
+
if (retryAfterSeconds !== undefined) {
|
|
81
|
+
return Math.min(AGENT_INBOX_RETRY_AFTER_CAP_MS, Math.round(retryAfterSeconds * 1000));
|
|
82
|
+
}
|
|
83
|
+
return AGENT_INBOX_RETRY_BASE_MS + Math.floor(random() * AGENT_INBOX_RETRY_JITTER_MS);
|
|
84
|
+
}
|
|
85
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
86
|
+
/**
|
|
87
|
+
* Read the server inbox with at most AGENT_INBOX_MAX_ATTEMPTS attempts, each
|
|
88
|
+
* bounded by `timeoutMs`. Never throws: the outcome carries either the last
|
|
89
|
+
* response or the last error, plus how many attempts were made.
|
|
90
|
+
*/
|
|
91
|
+
export async function fetchServerInboxWithRetry(deps) {
|
|
92
|
+
const sleep = deps.sleep ?? defaultSleep;
|
|
93
|
+
const random = deps.random ?? Math.random;
|
|
94
|
+
const timeoutMs = deps.timeoutMs ?? KIT_REQUEST_TIMEOUT_MS;
|
|
95
|
+
let attempts = 0;
|
|
96
|
+
for (;;) {
|
|
97
|
+
attempts += 1;
|
|
98
|
+
const last = attempts >= AGENT_INBOX_MAX_ATTEMPTS;
|
|
99
|
+
let retryAfterSeconds;
|
|
100
|
+
try {
|
|
101
|
+
const res = await withRequestTimeout(timeoutMs, undefined, async (signal) => {
|
|
102
|
+
const token = await deps.mint(signal);
|
|
103
|
+
return deps.fetch(token, signal);
|
|
104
|
+
});
|
|
105
|
+
if (last || !isRetryableInboxStatus(res.status)) {
|
|
106
|
+
return { ok: true, response: { status: res.status, body: res.body }, attempts };
|
|
107
|
+
}
|
|
108
|
+
retryAfterSeconds = res.retryAfterSeconds;
|
|
109
|
+
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
if (last || !isRetryableInboxError(err))
|
|
112
|
+
return { ok: false, error: err, attempts };
|
|
113
|
+
}
|
|
114
|
+
await sleep(retryDelayMs(retryAfterSeconds, random));
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/** Suffix added to the degradation note when the retry also failed. */
|
|
118
|
+
export const AGENT_INBOX_AFTER_ATTEMPTS_SUFFIX = ` (after ${AGENT_INBOX_MAX_ATTEMPTS} attempts)`;
|
|
119
|
+
/**
|
|
120
|
+
* Add `stale` and, when a retry happened, `attempts` to a merged inbox. A
|
|
121
|
+
* degraded note after a retry also says so in words.
|
|
122
|
+
*/
|
|
123
|
+
export function annotateInboxAttempts(merged, attempts) {
|
|
124
|
+
const stale = merged.source !== "server";
|
|
125
|
+
const retried = attempts > 1;
|
|
126
|
+
const note = merged.note && retried && stale ? `${merged.note}${AGENT_INBOX_AFTER_ATTEMPTS_SUFFIX}` : merged.note;
|
|
127
|
+
return {
|
|
128
|
+
entries: merged.entries,
|
|
129
|
+
source: merged.source,
|
|
130
|
+
stale,
|
|
131
|
+
...(note ? { note } : {}),
|
|
132
|
+
...(retried ? { attempts } : {}),
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
//# sourceMappingURL=inbox-fetch.js.map
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
* tool result.
|
|
17
17
|
*/
|
|
18
18
|
import type { ExternalMachineCreds } from "../creds.js";
|
|
19
|
+
import { type FetchServerInboxDeps } from "../inbox-fetch.js";
|
|
19
20
|
import type { AgentKitPaths } from "../paths.js";
|
|
20
21
|
import { type McpTool } from "./jsonrpc.js";
|
|
21
22
|
export interface HqRunResult {
|
|
@@ -32,7 +33,8 @@ export interface McpToolClients {
|
|
|
32
33
|
paths: Pick<AgentKitPaths, "inboxDir" | "attachmentsDir">;
|
|
33
34
|
/** Run `hq <args…>` as the machine identity; never throws. */
|
|
34
35
|
runHq: (args: string[]) => Promise<HqRunResult>;
|
|
35
|
-
|
|
36
|
+
/** Machine token; `signal` cancels this caller's wait on a shared mint. */
|
|
37
|
+
getToken: (signal?: AbortSignal) => Promise<string>;
|
|
36
38
|
/**
|
|
37
39
|
* Whether this kit mirrors the vault to disk. Off by default, and it decides
|
|
38
40
|
* what `hq_search` can honestly do: the local qmd index only exists when
|
|
@@ -47,15 +49,22 @@ export interface McpToolClients {
|
|
|
47
49
|
* key through the presign API, as the machine identity.
|
|
48
50
|
*/
|
|
49
51
|
downloadVaultObject: (scopeUid: string, key: string) => Promise<Buffer>;
|
|
50
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Authenticated JSON call against the control plane. `retryAfterSeconds` is
|
|
54
|
+
* the server's numeric `Retry-After`, when it sent one.
|
|
55
|
+
*/
|
|
51
56
|
apiJson: (token: string, path: string, init?: {
|
|
52
57
|
method?: string;
|
|
53
58
|
body?: Record<string, unknown>;
|
|
54
59
|
query?: Record<string, string>;
|
|
60
|
+
signal?: AbortSignal;
|
|
55
61
|
}) => Promise<{
|
|
56
62
|
status: number;
|
|
57
63
|
body: unknown;
|
|
64
|
+
retryAfterSeconds?: number;
|
|
58
65
|
}>;
|
|
66
|
+
/** Test seam for hq_inbox_read's bounded retry (sleep, jitter source, per-attempt deadline). */
|
|
67
|
+
inboxRetry?: Pick<FetchServerInboxDeps, "sleep" | "random" | "timeoutMs">;
|
|
59
68
|
}
|
|
60
69
|
export declare function clampOutput(text: string, max?: number): string;
|
|
61
70
|
/** Vault paths are company-anchored and must not escape. */
|
|
@@ -19,6 +19,7 @@ import { CLI_VERSION } from "../../../cli-version.js";
|
|
|
19
19
|
import { peekIdToken } from "../../../utils/id-token.js";
|
|
20
20
|
import { PERSON_AGENT_AUDIENCE_MESSAGE, callerMayUseAudience, defaultAudienceFor, isAgentUid, isRecipientUid } from "../../../lib/notify-recipient.js";
|
|
21
21
|
import { tokenIsMachinePrincipal } from "../../../lib/caller-principal.js";
|
|
22
|
+
import { annotateInboxAttempts, fetchServerInboxWithRetry, resolveAgentInboxTimeoutMs } from "../inbox-fetch.js";
|
|
22
23
|
import { redactAgentInboxItem } from "../inbox-redact.js";
|
|
23
24
|
import { markInboxDone, mergeAgentInbox, readDoneIds, summarizeInboxEntry } from "../inbox-state.js";
|
|
24
25
|
import { AttachmentCompanyUnresolvedError, fetchChatAttachment } from "./attachments.js";
|
|
@@ -286,10 +287,21 @@ export function buildAgentMcpTools(clients) {
|
|
|
286
287
|
const limit = int(args, "limit", { min: 1, max: 100, fallback: 20 });
|
|
287
288
|
const includeDone = args.include_done === true;
|
|
288
289
|
const done = readDoneIds(clients.paths);
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
290
|
+
// Same bounded, once-retried server read as `hq agent inbox`.
|
|
291
|
+
let attempts = 1;
|
|
292
|
+
const merged = await mergeAgentInbox(clients.paths, async () => {
|
|
293
|
+
const outcome = await fetchServerInboxWithRetry({
|
|
294
|
+
timeoutMs: resolveAgentInboxTimeoutMs(process.env),
|
|
295
|
+
...clients.inboxRetry,
|
|
296
|
+
mint: (signal) => clients.getToken(signal),
|
|
297
|
+
fetch: (token, signal) => clients.apiJson(token, `/v1/agents/${encodeURIComponent(creds.entityUid)}/inbox`, { signal }),
|
|
298
|
+
});
|
|
299
|
+
attempts = outcome.attempts;
|
|
300
|
+
if (!outcome.ok)
|
|
301
|
+
throw outcome.error;
|
|
302
|
+
return outcome.response;
|
|
292
303
|
});
|
|
304
|
+
const { entries, note: serverNote, source, stale, attempts: attemptsMade } = annotateInboxAttempts(merged, attempts);
|
|
293
305
|
const byId = new Map(entries.map((e) => [e.id, e]));
|
|
294
306
|
const rows = [...byId.values()]
|
|
295
307
|
.filter((e) => includeDone || !done.has(e.id))
|
|
@@ -299,6 +311,8 @@ export function buildAgentMcpTools(clients) {
|
|
|
299
311
|
return textResult(clampOutput(JSON.stringify({
|
|
300
312
|
count: rows.length,
|
|
301
313
|
source,
|
|
314
|
+
stale,
|
|
315
|
+
...(attemptsMade !== undefined ? { attempts: attemptsMade } : {}),
|
|
302
316
|
messages: rows,
|
|
303
317
|
...(serverNote ? { note: serverNote } : {}),
|
|
304
318
|
...(hasFiles ? { attachmentsHint: ATTACHMENTS_HINT } : {}),
|
|
@@ -139,7 +139,9 @@ export declare const HQ_FLEET_LOCAL_RELEASE: FleetLocalReleasePin;
|
|
|
139
139
|
export declare const HQ_FLEET_LOCAL_PUBLIC_KEY = "-----BEGIN PUBLIC KEY-----\nMCowBQYDK2VwAyEASJoapvo0WMTg+Wu3Jvt0nzlXLOBEr+G1aJRi0JkEb4A=\n-----END PUBLIC KEY-----\n";
|
|
140
140
|
export declare function fleetHome(home?: string, env?: NodeJS.ProcessEnv): string;
|
|
141
141
|
export declare function verifyFleetLocalRelease(files: FleetLocalReleaseFiles, release?: FleetLocalReleasePin, publicKey?: string): void;
|
|
142
|
-
|
|
142
|
+
/** `gh auth token` fallback for the release download token. */
|
|
143
|
+
export declare const FLEET_RELEASE_TOKEN_TIMEOUT_MS = 15000;
|
|
144
|
+
export declare function resolveFleetReleaseToken(env?: NodeJS.ProcessEnv, run?: FleetLocalExec): string;
|
|
143
145
|
export declare function fetchHqFleetReleaseLinks(release: FleetLocalReleasePin): Promise<Map<string, string> | null>;
|
|
144
146
|
export declare const FLEET_LOCAL_METADATA_DOWNLOAD_TIMEOUT_MS = 30000;
|
|
145
147
|
export declare const FLEET_LOCAL_ARCHIVE_DOWNLOAD_TIMEOUT_MS: number;
|
|
@@ -105,18 +105,24 @@ export function verifyFleetLocalRelease(files, release = HQ_FLEET_LOCAL_RELEASE,
|
|
|
105
105
|
throw expectedError(`HQ Fleet Local ${release.version} manifest signature is invalid.`);
|
|
106
106
|
}
|
|
107
107
|
}
|
|
108
|
-
|
|
108
|
+
/** `gh auth token` fallback for the release download token. */
|
|
109
|
+
export const FLEET_RELEASE_TOKEN_TIMEOUT_MS = 15_000;
|
|
110
|
+
export function resolveFleetReleaseToken(env = process.env, run = execFileSync) {
|
|
109
111
|
for (const name of ["HQ_FLEET_RELEASE_GITHUB_TOKEN", "GH_TOKEN", "GITHUB_TOKEN"]) {
|
|
110
112
|
const value = env[name]?.trim();
|
|
111
113
|
if (value)
|
|
112
114
|
return value;
|
|
113
115
|
}
|
|
114
116
|
try {
|
|
115
|
-
|
|
117
|
+
// gh can block on a keychain prompt; a create must fail, not hang.
|
|
118
|
+
const output = run("gh", ["auth", "token"], {
|
|
116
119
|
encoding: "utf8",
|
|
117
120
|
stdio: ["ignore", "pipe", "ignore"],
|
|
118
121
|
windowsHide: true,
|
|
119
|
-
|
|
122
|
+
timeout: FLEET_RELEASE_TOKEN_TIMEOUT_MS,
|
|
123
|
+
killSignal: "SIGKILL",
|
|
124
|
+
});
|
|
125
|
+
const value = (Buffer.isBuffer(output) ? output.toString("utf8") : output).trim();
|
|
120
126
|
if (value)
|
|
121
127
|
return value;
|
|
122
128
|
}
|
|
@@ -26,6 +26,12 @@ export interface FleetPython {
|
|
|
26
26
|
export declare const HQ_FLEET_PYTHON_VERSION = "3.13.17";
|
|
27
27
|
export declare const HQ_FLEET_PYTHON_TOOL: FleetPythonToolPin;
|
|
28
28
|
export declare const FLEET_PYTHON_TOOL_DOWNLOAD_TIMEOUT_MS: number;
|
|
29
|
+
/** Unpacking the pinned installer archive. */
|
|
30
|
+
export declare const FLEET_PYTHON_EXTRACT_TIMEOUT_MS: number;
|
|
31
|
+
/** `uv python install`: downloads and unpacks the private Python. */
|
|
32
|
+
export declare const FLEET_PYTHON_INSTALL_TIMEOUT_MS: number;
|
|
33
|
+
/** `uv python find`: local lookup only. */
|
|
34
|
+
export declare const FLEET_PYTHON_FIND_TIMEOUT_MS = 60000;
|
|
29
35
|
/** One lock for the whole machine: the installer and the private Python are shared by every bot. */
|
|
30
36
|
export declare const FLEET_PYTHON_LOCK_NAME = "hq-fleet-python";
|
|
31
37
|
/**
|
|
@@ -18,6 +18,12 @@ export const HQ_FLEET_PYTHON_TOOL = {
|
|
|
18
18
|
},
|
|
19
19
|
};
|
|
20
20
|
export const FLEET_PYTHON_TOOL_DOWNLOAD_TIMEOUT_MS = 5 * 60_000;
|
|
21
|
+
/** Unpacking the pinned installer archive. */
|
|
22
|
+
export const FLEET_PYTHON_EXTRACT_TIMEOUT_MS = 2 * 60_000;
|
|
23
|
+
/** `uv python install`: downloads and unpacks the private Python. */
|
|
24
|
+
export const FLEET_PYTHON_INSTALL_TIMEOUT_MS = 15 * 60_000;
|
|
25
|
+
/** `uv python find`: local lookup only. */
|
|
26
|
+
export const FLEET_PYTHON_FIND_TIMEOUT_MS = 60_000;
|
|
21
27
|
/** One lock for the whole machine: the installer and the private Python are shared by every bot. */
|
|
22
28
|
export const FLEET_PYTHON_LOCK_NAME = "hq-fleet-python";
|
|
23
29
|
function expectedError(message) {
|
|
@@ -27,9 +33,13 @@ function toolBinaryName(platform) {
|
|
|
27
33
|
return platform === "win32" ? "uv.exe" : "uv";
|
|
28
34
|
}
|
|
29
35
|
/** The last lines a failed child process wrote, or its exit code, so the user sees why. */
|
|
30
|
-
function causeOf(error) {
|
|
36
|
+
function causeOf(error, timeoutMs) {
|
|
31
37
|
if (error && typeof error === "object") {
|
|
32
38
|
const failure = error;
|
|
39
|
+
// A child killed by execFileSync's timeout: say so instead of a bare signal.
|
|
40
|
+
if (failure.code === "ETIMEDOUT") {
|
|
41
|
+
return timeoutMs !== undefined ? `timed out after ${Math.round(timeoutMs / 1000)}s` : "timed out";
|
|
42
|
+
}
|
|
33
43
|
const stderr = failure.stderr === undefined ? "" : (Buffer.isBuffer(failure.stderr) ? failure.stderr.toString("utf8") : String(failure.stderr));
|
|
34
44
|
const tail = stderr.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).slice(-3).join(" ");
|
|
35
45
|
if (tail)
|
|
@@ -132,10 +142,15 @@ async function installTool(deps) {
|
|
|
132
142
|
const extracted = path.join(staging, "extracted");
|
|
133
143
|
fs.mkdirSync(extracted, { mode: 0o700 });
|
|
134
144
|
try {
|
|
135
|
-
run(tarExecutable(platform, deps.env ?? process.env), ["-xf", archive, "-C", extracted], {
|
|
145
|
+
run(tarExecutable(platform, deps.env ?? process.env), ["-xf", archive, "-C", extracted], {
|
|
146
|
+
stdio: ["ignore", "ignore", "pipe"],
|
|
147
|
+
windowsHide: true,
|
|
148
|
+
timeout: FLEET_PYTHON_EXTRACT_TIMEOUT_MS,
|
|
149
|
+
killSignal: "SIGKILL",
|
|
150
|
+
});
|
|
136
151
|
}
|
|
137
152
|
catch (error) {
|
|
138
|
-
throw expectedError(`HQ Fleet Local could not unpack its runtime installer (${causeOf(error)}).`);
|
|
153
|
+
throw expectedError(`HQ Fleet Local could not unpack its runtime installer (${causeOf(error, FLEET_PYTHON_EXTRACT_TIMEOUT_MS)}).`);
|
|
139
154
|
}
|
|
140
155
|
const unpacked = findFile(extracted, toolBinaryName(platform));
|
|
141
156
|
if (!unpacked)
|
|
@@ -179,10 +194,12 @@ async function ensureFleetPythonLocked(root, deps) {
|
|
|
179
194
|
stdio: ["ignore", "ignore", "pipe"],
|
|
180
195
|
env,
|
|
181
196
|
windowsHide: true,
|
|
197
|
+
timeout: FLEET_PYTHON_INSTALL_TIMEOUT_MS,
|
|
198
|
+
killSignal: "SIGKILL",
|
|
182
199
|
});
|
|
183
200
|
}
|
|
184
201
|
catch (error) {
|
|
185
|
-
throw expectedError(`HQ Fleet Local could not install its private Python ${version} (${causeOf(error)}). If this mentions a download, check your internet connection and retry.`);
|
|
202
|
+
throw expectedError(`HQ Fleet Local could not install its private Python ${version} (${causeOf(error, FLEET_PYTHON_INSTALL_TIMEOUT_MS)}). If this mentions a download, check your internet connection and retry.`);
|
|
186
203
|
}
|
|
187
204
|
let found;
|
|
188
205
|
try {
|
|
@@ -191,11 +208,13 @@ async function ensureFleetPythonLocked(root, deps) {
|
|
|
191
208
|
env,
|
|
192
209
|
encoding: "utf8",
|
|
193
210
|
windowsHide: true,
|
|
211
|
+
timeout: FLEET_PYTHON_FIND_TIMEOUT_MS,
|
|
212
|
+
killSignal: "SIGKILL",
|
|
194
213
|
});
|
|
195
214
|
found = (Buffer.isBuffer(output) ? output.toString("utf8") : output).trim();
|
|
196
215
|
}
|
|
197
216
|
catch (error) {
|
|
198
|
-
throw expectedError(`HQ Fleet Local installed its private Python but could not locate it (${causeOf(error)}).`);
|
|
217
|
+
throw expectedError(`HQ Fleet Local installed its private Python but could not locate it (${causeOf(error, FLEET_PYTHON_FIND_TIMEOUT_MS)}).`);
|
|
199
218
|
}
|
|
200
219
|
const python = path.resolve(found);
|
|
201
220
|
if (!found || !fs.existsSync(python) || !isInside(pythonDir, python, platform)) {
|
package/dist/lib/bot/probe.d.ts
CHANGED
|
@@ -20,7 +20,9 @@ export declare const PROBE_SYSTEM_PROMPT = "You are an HQ bot being checked befo
|
|
|
20
20
|
/** A probe that has not answered by then is reported as `transient`. */
|
|
21
21
|
export declare const PROBE_DEFAULT_TIMEOUT_MS = 120000;
|
|
22
22
|
/** The plain sentence for a failed probe: what is wrong and the one fix. */
|
|
23
|
-
export declare function probeFailureText(cls: RuntimeFailureClass, runtime: BotRuntimeId | string
|
|
23
|
+
export declare function probeFailureText(cls: RuntimeFailureClass, runtime: BotRuntimeId | string, usageLimit?: {
|
|
24
|
+
resetsAt?: string;
|
|
25
|
+
}): string;
|
|
24
26
|
export interface RuntimeProbeResult {
|
|
25
27
|
ok: boolean;
|
|
26
28
|
/** null when ok; otherwise the same classes a bot's failure DM uses. */
|
|
@@ -30,6 +32,10 @@ export interface RuntimeProbeResult {
|
|
|
30
32
|
runtime: BotRuntimeId;
|
|
31
33
|
/** The model asked for; null = the runtime's own default. */
|
|
32
34
|
model: string | null;
|
|
35
|
+
/** The tool's plan ran out of usage (class `transient`). */
|
|
36
|
+
usageLimit?: boolean;
|
|
37
|
+
/** When that limit resets, as the tool wrote it ("9pm (Asia/Dhaka)"). */
|
|
38
|
+
resetsAt?: string;
|
|
33
39
|
/** The CLI was too old for `from`, and the probe answered on `to`. */
|
|
34
40
|
modelFallback?: {
|
|
35
41
|
from: string;
|
package/dist/lib/bot/probe.js
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import * as fs from "node:fs";
|
|
16
16
|
import * as os from "node:os";
|
|
17
17
|
import * as path from "node:path";
|
|
18
|
-
import { classifyRuntimeError } from "./runtime-failure.js";
|
|
18
|
+
import { classifyRuntimeError, isRuntimeUsageLimit, usageLimitResetsAt } from "./runtime-failure.js";
|
|
19
19
|
import { runtimeDisplayName } from "./runtime-sign-in.js";
|
|
20
20
|
import { RuntimeError, runRuntimeTurn, runtimeFor } from "./runtime/index.js";
|
|
21
21
|
import { turnEnv } from "./run.js";
|
|
@@ -24,8 +24,12 @@ export const PROBE_SYSTEM_PROMPT = "You are an HQ bot being checked before it is
|
|
|
24
24
|
/** A probe that has not answered by then is reported as `transient`. */
|
|
25
25
|
export const PROBE_DEFAULT_TIMEOUT_MS = 120_000;
|
|
26
26
|
/** The plain sentence for a failed probe: what is wrong and the one fix. */
|
|
27
|
-
export function probeFailureText(cls, runtime) {
|
|
27
|
+
export function probeFailureText(cls, runtime, usageLimit) {
|
|
28
28
|
const tool = runtimeDisplayName(runtime);
|
|
29
|
+
if (cls === "transient" && usageLimit) {
|
|
30
|
+
const when = usageLimit.resetsAt ? ` It resets ${usageLimit.resetsAt}.` : "";
|
|
31
|
+
return `${tool} has hit its plan's usage limit.${when} Check again after it resets, or pick another tool.`;
|
|
32
|
+
}
|
|
29
33
|
switch (cls) {
|
|
30
34
|
case "signed-out":
|
|
31
35
|
return `${tool} is not signed in on this computer. Sign in to ${tool} in HQ, then check again.`;
|
|
@@ -84,11 +88,15 @@ export async function probeBotRuntime(opts) {
|
|
|
84
88
|
const raw = err instanceof RuntimeError ? `${err.message}\n${err.stderrExcerpt}` : err instanceof Error ? err.message : String(err);
|
|
85
89
|
// Same rule as a bot turn: the runtime's reported error decides, never a tail of its output.
|
|
86
90
|
const cls = classifyRuntimeError(err);
|
|
91
|
+
const limited = cls === "transient" && isRuntimeUsageLimit(raw);
|
|
92
|
+
const resetsAt = limited ? usageLimitResetsAt(raw) : undefined;
|
|
87
93
|
return {
|
|
88
94
|
ok: false,
|
|
89
95
|
class: cls,
|
|
90
|
-
detail: probeFailureText(cls, opts.runtime),
|
|
96
|
+
detail: probeFailureText(cls, opts.runtime, limited ? { resetsAt } : undefined),
|
|
91
97
|
...base,
|
|
98
|
+
...(limited ? { usageLimit: true } : {}),
|
|
99
|
+
...(resetsAt ? { resetsAt } : {}),
|
|
92
100
|
diagnostic: raw.slice(0, 400),
|
|
93
101
|
durationMs: now() - started,
|
|
94
102
|
};
|
package/dist/lib/bot/run.d.ts
CHANGED
|
@@ -71,6 +71,15 @@ export declare const API_BACKOFF_CAP_MS: number;
|
|
|
71
71
|
/** Honor Retry-After up to this; a bogus 30-day header must not park the bot. */
|
|
72
72
|
export declare const API_RETRY_AFTER_CAP_MS: number;
|
|
73
73
|
export declare const API_RETRY_AFTER_FLOOR_MS = 1000;
|
|
74
|
+
/**
|
|
75
|
+
* Delays between startup identity-check tries when HQ cannot be reached
|
|
76
|
+
* (five tries over about three minutes). After the last one the bot stops in
|
|
77
|
+
* a visible failed state the person can retry.
|
|
78
|
+
*/
|
|
79
|
+
export declare const STARTUP_VERIFY_RETRY_DELAYS_MS: readonly number[];
|
|
80
|
+
export declare function isTransientApiError(err: unknown): boolean;
|
|
81
|
+
/** Plain failed-state reason after the startup retries run out. */
|
|
82
|
+
export declare function startupUnreachableReason(name: string): string;
|
|
74
83
|
/**
|
|
75
84
|
* How long to wait after a failed inbox poll or heartbeat.
|
|
76
85
|
* 429/5xx (and transport errors) back off exponentially, honouring Retry-After
|