@indigoai-us/hq-cli 5.225.0 → 5.226.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -1
- package/dist/command-catalog.generated.d.ts +94 -0
- package/dist/command-catalog.generated.js +120 -0
- package/dist/command-registration-plan.d.ts +18 -0
- package/dist/command-registration-plan.js +6 -0
- package/dist/commands/agent-enroll.d.ts +2 -0
- package/dist/commands/agent-enroll.js +17 -2
- package/dist/commands/cloud.d.ts +10 -0
- package/dist/commands/cloud.js +52 -47
- package/dist/commands/lanes.d.ts +5 -0
- package/dist/commands/lanes.js +498 -312
- package/dist/commands/staff/reset-test-account.d.ts +110 -0
- package/dist/commands/staff/reset-test-account.js +268 -0
- package/dist/commands/staff/test-account.d.ts +51 -0
- package/dist/commands/staff/test-account.js +196 -0
- package/dist/lib/doctor/client-health-flag-token.d.ts +15 -0
- package/dist/lib/doctor/client-health-flag-token.js +66 -0
- package/dist/lib/doctor/client-health-reason-codes-flag.d.ts +1 -0
- package/dist/lib/doctor/client-health-reason-codes-flag.js +16 -5
- package/dist/lib/doctor/client-health-sentry-flag.d.ts +1 -0
- package/dist/lib/doctor/client-health-sentry-flag.js +17 -6
- package/dist/lib/lanes/dropbox.d.ts +9 -0
- package/dist/lib/lanes/dropbox.js +70 -1
- package/dist/lib/lanes/reconcile.d.ts +2 -0
- package/dist/lib/lanes/reconcile.js +12 -0
- package/dist/lib/lanes/watcher-main.js +16 -0
- package/dist/lib/lanes/watcher.d.ts +4 -1
- package/dist/lib/lanes/watcher.js +77 -3
- package/dist/utils/cognito-session.d.ts +8 -0
- package/dist/utils/cognito-session.js +198 -3
- package/dist/utils/upload-author.d.ts +15 -0
- package/dist/utils/upload-author.js +39 -0
- package/package.json +1 -1
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { ensureCognitoIdToken } from "../../utils/cognito-session.js";
|
|
2
|
+
export const CLIENT_HEALTH_FLAG_TOKEN_TIMEOUT_MS = 1_000;
|
|
3
|
+
export class MissingFlagTokenError extends Error {
|
|
4
|
+
constructor() {
|
|
5
|
+
super("No cached HQ identity token is available");
|
|
6
|
+
this.name = "MissingFlagTokenError";
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
export class ClientHealthFlagTokenTimeoutError extends Error {
|
|
10
|
+
constructor(timeoutMs) {
|
|
11
|
+
super(`Client-health flag token acquisition exceeded its ${timeoutMs} ms bound`);
|
|
12
|
+
this.name = "ClientHealthFlagTokenTimeoutError";
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Obtain a non-interactive ID token, separately bounded from the flag read.
|
|
17
|
+
* ensureCognitoIdToken refreshes expired person sessions and selects machine
|
|
18
|
+
* identity credentials when the current installation is a machine.
|
|
19
|
+
*/
|
|
20
|
+
export async function getClientHealthFlagToken(provider, timeoutMs = CLIENT_HEALTH_FLAG_TOKEN_TIMEOUT_MS, callerSignal) {
|
|
21
|
+
if (callerSignal?.aborted) {
|
|
22
|
+
throw callerSignal.reason ?? new Error("Token acquisition was aborted");
|
|
23
|
+
}
|
|
24
|
+
const controller = new AbortController();
|
|
25
|
+
const signal = callerSignal ? AbortSignal.any([callerSignal, controller.signal]) : controller.signal;
|
|
26
|
+
const timeout = new ClientHealthFlagTokenTimeoutError(timeoutMs);
|
|
27
|
+
const timer = setTimeout(() => controller.abort(timeout), timeoutMs);
|
|
28
|
+
let onAbort;
|
|
29
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
30
|
+
onAbort = () => reject(signal.reason ?? new Error("Token acquisition was aborted"));
|
|
31
|
+
if (signal.aborted)
|
|
32
|
+
onAbort();
|
|
33
|
+
else
|
|
34
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
35
|
+
});
|
|
36
|
+
try {
|
|
37
|
+
const acquire = provider ?? ((requestSignal) => ensureCognitoIdToken({ interactive: false, signal: requestSignal }));
|
|
38
|
+
const acquisition = Promise.resolve().then(() => {
|
|
39
|
+
if (signal.aborted) {
|
|
40
|
+
throw signal.reason ?? new Error("Token acquisition was aborted");
|
|
41
|
+
}
|
|
42
|
+
return acquire(signal);
|
|
43
|
+
});
|
|
44
|
+
const token = await Promise.race([acquisition, aborted]);
|
|
45
|
+
if (signal.aborted) {
|
|
46
|
+
throw signal.reason ?? new Error("Token acquisition was aborted");
|
|
47
|
+
}
|
|
48
|
+
if (typeof token !== "string" || token.trim().length === 0) {
|
|
49
|
+
throw new MissingFlagTokenError();
|
|
50
|
+
}
|
|
51
|
+
return token;
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
if (error instanceof Error &&
|
|
55
|
+
error.message.startsWith("No valid HQ session and interactive login is disabled")) {
|
|
56
|
+
throw new MissingFlagTokenError();
|
|
57
|
+
}
|
|
58
|
+
throw error;
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
clearTimeout(timer);
|
|
62
|
+
if (onAbort)
|
|
63
|
+
signal.removeEventListener("abort", onAbort);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=client-health-flag-token.js.map
|
|
@@ -7,6 +7,7 @@ export interface ClientHealthReasonCodesFlagDependencies {
|
|
|
7
7
|
env?: NodeJS.ProcessEnv;
|
|
8
8
|
createClient?: (options: FlagClientOptions) => ClientHealthFlagClient;
|
|
9
9
|
getToken?: FlagClientOptions["getToken"];
|
|
10
|
+
tokenTimeoutMs?: number;
|
|
10
11
|
fetch?: typeof fetch;
|
|
11
12
|
reportFailure?: (error: unknown) => void;
|
|
12
13
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { loadCachedTokens } from "@indigoai-us/hq-cloud";
|
|
2
1
|
import { createFlagClient, } from "@indigoai-us/hq-flags-client";
|
|
3
2
|
import { resolveFlagRegistryEndpoint } from "../flag-registry-endpoint.js";
|
|
3
|
+
import { getClientHealthFlagToken } from "./client-health-flag-token.js";
|
|
4
4
|
export const CLIENT_HEALTH_REASON_CODES_FLAG = "client-health-sync-reason-codes";
|
|
5
5
|
export const CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK = false;
|
|
6
6
|
export const CLIENT_HEALTH_REASON_CODES_FLAG_TIMEOUT_MS = 150;
|
|
@@ -39,9 +39,8 @@ export async function clientHealthReasonCodesEnabledForCompany(company, dependen
|
|
|
39
39
|
return CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK;
|
|
40
40
|
}
|
|
41
41
|
const companySlug = company.companySlug?.trim() || "";
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
const doFetch = dependencies.fetch ?? globalThis.fetch;
|
|
42
|
+
if (callerSignal?.aborted)
|
|
43
|
+
return CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK;
|
|
45
44
|
let client;
|
|
46
45
|
let failed = false;
|
|
47
46
|
let failureReported = false;
|
|
@@ -53,6 +52,18 @@ export async function clientHealthReasonCodesEnabledForCompany(company, dependen
|
|
|
53
52
|
failureReported = true;
|
|
54
53
|
(dependencies.reportFailure ?? reportFlagFailure)(error);
|
|
55
54
|
};
|
|
55
|
+
let token;
|
|
56
|
+
try {
|
|
57
|
+
token = await getClientHealthFlagToken(dependencies.getToken, dependencies.tokenTimeoutMs, callerSignal);
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
if (!callerSignal?.aborted)
|
|
61
|
+
reportFailureOnce(error);
|
|
62
|
+
return CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK;
|
|
63
|
+
}
|
|
64
|
+
const deadline = AbortSignal.timeout(CLIENT_HEALTH_REASON_CODES_FLAG_TIMEOUT_MS);
|
|
65
|
+
const signal = callerSignal ? AbortSignal.any([callerSignal, deadline]) : deadline;
|
|
66
|
+
const doFetch = dependencies.fetch ?? globalThis.fetch;
|
|
56
67
|
try {
|
|
57
68
|
client = (dependencies.createClient ?? createFlagClient)({
|
|
58
69
|
endpoint,
|
|
@@ -61,7 +72,7 @@ export async function clientHealthReasonCodesEnabledForCompany(company, dependen
|
|
|
61
72
|
? { companyIdentifiers: [companySlug] }
|
|
62
73
|
: {}),
|
|
63
74
|
env: {},
|
|
64
|
-
getToken:
|
|
75
|
+
getToken: () => token,
|
|
65
76
|
fetch: (input, init) => doFetch(input, {
|
|
66
77
|
...init,
|
|
67
78
|
signal: combineSignals(init?.signal ?? undefined, signal),
|
|
@@ -7,6 +7,7 @@ export interface ClientHealthSentryFlagDependencies {
|
|
|
7
7
|
env?: NodeJS.ProcessEnv;
|
|
8
8
|
createClient?: (options: FlagClientOptions) => ClientHealthFlagClient;
|
|
9
9
|
getToken?: FlagClientOptions["getToken"];
|
|
10
|
+
tokenTimeoutMs?: number;
|
|
10
11
|
fetch?: typeof fetch;
|
|
11
12
|
reportFailure?: (error: unknown) => void;
|
|
12
13
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { loadCachedTokens } from "@indigoai-us/hq-cloud";
|
|
2
1
|
import { createFlagClient, } from "@indigoai-us/hq-flags-client";
|
|
3
2
|
import { resolveFlagRegistryEndpoint } from "../flag-registry-endpoint.js";
|
|
3
|
+
import { getClientHealthFlagToken } from "./client-health-flag-token.js";
|
|
4
4
|
export const CLIENT_HEALTH_SENTRY_DIRECT_FLAG = "client-health.sentry-direct";
|
|
5
5
|
export const CLIENT_HEALTH_SENTRY_DIRECT_FALLBACK = false;
|
|
6
6
|
export const CLIENT_HEALTH_SENTRY_DIRECT_TIMEOUT_MS = 150;
|
|
@@ -31,10 +31,8 @@ export async function clientHealthSentryDirectEnabledForCompany(company, depende
|
|
|
31
31
|
if (!/^cmp_[A-Za-z0-9]{3,128}$/.test(companyUid)) {
|
|
32
32
|
return CLIENT_HEALTH_SENTRY_DIRECT_FALLBACK;
|
|
33
33
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
const signal = callerSignal ? AbortSignal.any([callerSignal, deadline]) : deadline;
|
|
37
|
-
const doFetch = dependencies.fetch ?? globalThis.fetch;
|
|
34
|
+
if (callerSignal?.aborted)
|
|
35
|
+
return CLIENT_HEALTH_SENTRY_DIRECT_FALLBACK;
|
|
38
36
|
let client;
|
|
39
37
|
let failed = false;
|
|
40
38
|
let failureReported = false;
|
|
@@ -46,6 +44,19 @@ export async function clientHealthSentryDirectEnabledForCompany(company, depende
|
|
|
46
44
|
failureReported = true;
|
|
47
45
|
(dependencies.reportFailure ?? reportFlagFailure)(error);
|
|
48
46
|
};
|
|
47
|
+
let token;
|
|
48
|
+
try {
|
|
49
|
+
token = await getClientHealthFlagToken(dependencies.getToken, dependencies.tokenTimeoutMs, callerSignal);
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
if (!callerSignal?.aborted)
|
|
53
|
+
reportFailureOnce(error);
|
|
54
|
+
return CLIENT_HEALTH_SENTRY_DIRECT_FALLBACK;
|
|
55
|
+
}
|
|
56
|
+
const companySlug = company.companySlug?.trim() || "";
|
|
57
|
+
const deadline = AbortSignal.timeout(CLIENT_HEALTH_SENTRY_DIRECT_TIMEOUT_MS);
|
|
58
|
+
const signal = callerSignal ? AbortSignal.any([callerSignal, deadline]) : deadline;
|
|
59
|
+
const doFetch = dependencies.fetch ?? globalThis.fetch;
|
|
49
60
|
try {
|
|
50
61
|
client = (dependencies.createClient ?? createFlagClient)({
|
|
51
62
|
endpoint: resolveFlagRegistryEndpoint(undefined, env),
|
|
@@ -54,7 +65,7 @@ export async function clientHealthSentryDirectEnabledForCompany(company, depende
|
|
|
54
65
|
? { companyIdentifiers: [companySlug] }
|
|
55
66
|
: {}),
|
|
56
67
|
env: {},
|
|
57
|
-
getToken:
|
|
68
|
+
getToken: () => token,
|
|
58
69
|
fetch: (input, init) => doFetch(input, {
|
|
59
70
|
...init,
|
|
60
71
|
signal: combineSignals(init?.signal ?? undefined, signal),
|
|
@@ -50,6 +50,8 @@ export interface DrainDeps {
|
|
|
50
50
|
afterRead?: () => void;
|
|
51
51
|
/** Prefix marks archives that belong to one provider turn. */
|
|
52
52
|
archivePrefix?: string;
|
|
53
|
+
/** Terminal messages with no permitted delivery path are kept separately. */
|
|
54
|
+
archiveDirectory?: "delivered" | "undeliverable";
|
|
53
55
|
}
|
|
54
56
|
export interface LaneDrainHookInput {
|
|
55
57
|
transcript_path?: string | null;
|
|
@@ -68,6 +70,13 @@ export declare function mintClaimId(): string;
|
|
|
68
70
|
export declare function assertSafeLaneId(laneId: string): void;
|
|
69
71
|
export declare function dropboxDir(hqRoot: string, laneId: string): string;
|
|
70
72
|
export declare function dropboxInboxPath(hqRoot: string, laneId: string): string;
|
|
73
|
+
/** A recovery drain can claim the inbox, draining files, or dead inflight files. */
|
|
74
|
+
export declare function dropboxInboxHasMessages(hqRoot: string, laneId: string): boolean;
|
|
75
|
+
/**
|
|
76
|
+
* Return a failed recovery claim to the append-only inbox without replacing
|
|
77
|
+
* messages appended while resume validation or spawn was in progress.
|
|
78
|
+
*/
|
|
79
|
+
export declare function requeueDropboxMessages(hqRoot: string, laneId: string, messages: readonly DropboxMessage[]): void;
|
|
71
80
|
/**
|
|
72
81
|
* Worker tool-call liveness (watcher.ts reads it). A Claude worker run with
|
|
73
82
|
* `--output-format json` writes nothing to worker.out until it exits, so the
|
|
@@ -65,6 +65,75 @@ export function dropboxDir(hqRoot, laneId) {
|
|
|
65
65
|
export function dropboxInboxPath(hqRoot, laneId) {
|
|
66
66
|
return path.join(dropboxDir(hqRoot, laneId), INBOX_NAME);
|
|
67
67
|
}
|
|
68
|
+
function directoryHasRecoverableFiles(dir, nestedInboxOnly = false) {
|
|
69
|
+
let names;
|
|
70
|
+
try {
|
|
71
|
+
names = fs.readdirSync(dir).filter((name) => !name.startsWith("."));
|
|
72
|
+
}
|
|
73
|
+
catch (err) {
|
|
74
|
+
if (err.code === "ENOENT")
|
|
75
|
+
return false;
|
|
76
|
+
throw err;
|
|
77
|
+
}
|
|
78
|
+
for (const name of names) {
|
|
79
|
+
const candidate = path.join(dir, name);
|
|
80
|
+
let stat;
|
|
81
|
+
try {
|
|
82
|
+
stat = fs.statSync(candidate);
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
if (err.code === "ENOENT")
|
|
86
|
+
continue;
|
|
87
|
+
throw err;
|
|
88
|
+
}
|
|
89
|
+
if (nestedInboxOnly) {
|
|
90
|
+
if (!stat.isDirectory())
|
|
91
|
+
continue;
|
|
92
|
+
try {
|
|
93
|
+
const inbox = fs.statSync(path.join(candidate, INBOX_NAME));
|
|
94
|
+
if (inbox.isFile() && inbox.size > 0)
|
|
95
|
+
return true;
|
|
96
|
+
}
|
|
97
|
+
catch (err) {
|
|
98
|
+
if (err.code !== "ENOENT")
|
|
99
|
+
throw err;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
else if (stat.isFile() && stat.size > 0) {
|
|
103
|
+
return true;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
function fileHasBytes(filePath) {
|
|
109
|
+
try {
|
|
110
|
+
const stat = fs.statSync(filePath);
|
|
111
|
+
return stat.isFile() && stat.size > 0;
|
|
112
|
+
}
|
|
113
|
+
catch (err) {
|
|
114
|
+
if (err.code === "ENOENT")
|
|
115
|
+
return false;
|
|
116
|
+
throw err;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
/** A recovery drain can claim the inbox, draining files, or dead inflight files. */
|
|
120
|
+
export function dropboxInboxHasMessages(hqRoot, laneId) {
|
|
121
|
+
const box = dropboxDir(hqRoot, laneId);
|
|
122
|
+
return fileHasBytes(path.join(box, INBOX_NAME)) ||
|
|
123
|
+
directoryHasRecoverableFiles(path.join(box, DRAINING_DIR)) ||
|
|
124
|
+
directoryHasRecoverableFiles(path.join(box, INFLIGHT_DIR), true);
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Return a failed recovery claim to the append-only inbox without replacing
|
|
128
|
+
* messages appended while resume validation or spawn was in progress.
|
|
129
|
+
*/
|
|
130
|
+
export function requeueDropboxMessages(hqRoot, laneId, messages) {
|
|
131
|
+
if (messages.length === 0)
|
|
132
|
+
return;
|
|
133
|
+
const inbox = dropboxInboxPath(hqRoot, laneId);
|
|
134
|
+
fs.mkdirSync(path.dirname(inbox), { recursive: true });
|
|
135
|
+
fs.appendFileSync(inbox, messages.map((message) => `${JSON.stringify(message)}\n`).join(""), "utf8");
|
|
136
|
+
}
|
|
68
137
|
/**
|
|
69
138
|
* Worker tool-call liveness (watcher.ts reads it). A Claude worker run with
|
|
70
139
|
* `--output-format json` writes nothing to worker.out until it exits, so the
|
|
@@ -220,7 +289,7 @@ export function drainDropboxAtPath(box, depsInput = {}) {
|
|
|
220
289
|
return [];
|
|
221
290
|
const inbox = path.join(box, INBOX_NAME);
|
|
222
291
|
const draining = path.join(box, DRAINING_DIR);
|
|
223
|
-
const delivered = path.join(box, DELIVERED_DIR);
|
|
292
|
+
const delivered = path.join(box, deps.archiveDirectory ?? DELIVERED_DIR);
|
|
224
293
|
const inflightRoot = path.join(box, INFLIGHT_DIR);
|
|
225
294
|
try {
|
|
226
295
|
deps.mkdirSync(box, { recursive: true });
|
|
@@ -81,6 +81,8 @@ export interface ReconcileDeps {
|
|
|
81
81
|
readRegistryFlag?: typeof readLanesRegistryFlag;
|
|
82
82
|
/** Active-company fallback for questions raised from an unscoped lane. */
|
|
83
83
|
resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
|
|
84
|
+
/** Resume queued messages after this pass wins a local dead-worker CAS. */
|
|
85
|
+
recoverQueuedMessages?: (lane: LaneRecord) => Promise<void>;
|
|
84
86
|
/** Detached mesh seam for dead-watcher recovery. */
|
|
85
87
|
mesh?: LaneMesh | null;
|
|
86
88
|
/** Bound on one terminal mesh emit during dead-host recovery. Default 10 s. */
|
|
@@ -1172,6 +1172,18 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1172
1172
|
}
|
|
1173
1173
|
}
|
|
1174
1174
|
}
|
|
1175
|
+
if (!deps.dryRun && deps.recoverQueuedMessages) {
|
|
1176
|
+
try {
|
|
1177
|
+
await deps.recoverQueuedMessages(result.record);
|
|
1178
|
+
}
|
|
1179
|
+
catch (err) {
|
|
1180
|
+
markDirty(summary, {
|
|
1181
|
+
kind: "lane",
|
|
1182
|
+
id: lane.lane_id,
|
|
1183
|
+
why: `queued message recovery failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1184
|
+
});
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1175
1187
|
}
|
|
1176
1188
|
}
|
|
1177
1189
|
catch (err) {
|
|
@@ -5,6 +5,9 @@
|
|
|
5
5
|
* Usage: node watcher-main.js <lane-id>
|
|
6
6
|
* Requires HQ_ROOT in the environment.
|
|
7
7
|
*/
|
|
8
|
+
import { spawnSync } from "node:child_process";
|
|
9
|
+
import * as path from "node:path";
|
|
10
|
+
import { fileURLToPath } from "node:url";
|
|
8
11
|
import { runWatcherLoop } from "./watcher.js";
|
|
9
12
|
import { LanesError } from "./errors.js";
|
|
10
13
|
import { openMeshTransport } from "../mesh/client.js";
|
|
@@ -27,8 +30,21 @@ function main() {
|
|
|
27
30
|
process.stderr.write("hq lanes watcher: requires lane id argv and HQ_ROOT in the environment\n");
|
|
28
31
|
process.exit(1);
|
|
29
32
|
}
|
|
33
|
+
const cliEntry = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../index.js");
|
|
30
34
|
void runWatcherLoop(hqRoot, laneId, {
|
|
31
35
|
resolveQuestionActiveCompany,
|
|
36
|
+
recoverQueuedMessages: async () => {
|
|
37
|
+
const result = spawnSync(process.execPath, [cliEntry, "lanes", "_recover-queued-messages", laneId], {
|
|
38
|
+
env: process.env,
|
|
39
|
+
encoding: "utf8",
|
|
40
|
+
timeout: 120_000,
|
|
41
|
+
});
|
|
42
|
+
if (result.error)
|
|
43
|
+
throw result.error;
|
|
44
|
+
if (result.status !== 0) {
|
|
45
|
+
throw new Error(`recovery command exited ${result.status ?? result.signal ?? "without status"}: ${(result.stderr ?? "").trim()}`);
|
|
46
|
+
}
|
|
47
|
+
},
|
|
32
48
|
}).catch((err) => {
|
|
33
49
|
process.stderr.write(`hq lanes watcher: ${err instanceof Error ? err.message : String(err)}${err instanceof Error && err.stack ? `\n${err.stack}` : ""}\n`);
|
|
34
50
|
process.exit(1);
|
|
@@ -48,7 +48,7 @@ export declare const WATCHER_HEARTBEAT_MS: number;
|
|
|
48
48
|
* once the file would exceed this, the oldest complete lines are dropped.
|
|
49
49
|
*/
|
|
50
50
|
export declare const WATCHER_OUT_MAX_BYTES: number;
|
|
51
|
-
export declare const WATCHER_DECISION_EVENTS: readonly ["liveness_transition", "silence_deadline", "envelope_observed", "envelope_invalid", "envelope_repair", "classification", "terminal_write", "mesh_event", "mesh_event_failed"];
|
|
51
|
+
export declare const WATCHER_DECISION_EVENTS: readonly ["liveness_transition", "silence_deadline", "envelope_observed", "envelope_invalid", "envelope_repair", "classification", "terminal_write", "message_recovery", "message_recovery_failed", "message_undeliverable", "mesh_event", "mesh_event_failed"];
|
|
52
52
|
/** Terminal reason when the envelope failed to parse after the repair turn (spec §4). */
|
|
53
53
|
export { ENVELOPE_INVALID_REASON };
|
|
54
54
|
export type WatcherDecisionEvent = (typeof WATCHER_DECISION_EVENTS)[number];
|
|
@@ -222,6 +222,8 @@ export interface WatcherLoopDeps {
|
|
|
222
222
|
retrySpawnDeps?: Omit<SpawnLaneDeps, "admissionHost">;
|
|
223
223
|
/** Active-company fallback when an unscoped lane asks from its envelope. */
|
|
224
224
|
resolveQuestionActiveCompany?: () => Promise<LaneCompanyRef>;
|
|
225
|
+
/** Resume an atomically queued inbox after this watcher wins terminal CAS. */
|
|
226
|
+
recoverQueuedMessages?: (lane: LaneRecord) => Promise<void>;
|
|
225
227
|
}
|
|
226
228
|
/**
|
|
227
229
|
* Wrap one mesh call. A failed emit is logged as `mesh_event_failed` with
|
|
@@ -231,6 +233,7 @@ export interface WatcherLoopDeps {
|
|
|
231
233
|
export declare function emitMeshSoft(event: string, logDecision: (record: Record<string, unknown>) => void, at: string, fn: (signal: AbortSignal) => Promise<void>, timeoutMs?: number): Promise<boolean>;
|
|
232
234
|
/** Question ids an ask envelope raised, as `raiseEnvelopeQuestions` mints them. */
|
|
233
235
|
export declare function envelopeQuestionIds(laneId: string, envelopeRef: string, envelope: LaneEnvelope, questionIndexes?: readonly number[]): string[];
|
|
236
|
+
/** Detached and synchronous classifiers share one process-identity claim. */
|
|
234
237
|
export declare function runWatcherLoop(hqRoot: string, laneId: string, deps?: WatcherLoopDeps): Promise<void>;
|
|
235
238
|
export type EnvelopeRepairDecision = {
|
|
236
239
|
action: "spawned";
|
|
@@ -37,16 +37,16 @@ import { LanesError } from "./errors.js";
|
|
|
37
37
|
import { questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
38
38
|
import { isTerminalLaneState, ENVELOPE_INVALID_REASON, parseLaneEnvelope, STORY_STATUSES, } from "./types.js";
|
|
39
39
|
import { laneStories, storyStatusForEnvelope } from "./stories.js";
|
|
40
|
-
import { readLane, retryUpdateLane } from "./store.js";
|
|
40
|
+
import { readLane, retryUpdateLane, withLaneHireLocks } from "./store.js";
|
|
41
41
|
import { readWorkerRunResult } from "./delivery.js";
|
|
42
42
|
import { applyDefaultQuotaHold, quotaSignalFromFields, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
43
43
|
import { PROVIDER_REJECTED_REASON, QUOTA_STATUS, isProviderLoginFailure, providerRejectionDetails, readProviderRejection, } from "./provider-rejection.js";
|
|
44
|
-
import { readToolActivity, toolActivityPath } from "./dropbox.js";
|
|
44
|
+
import { dropboxInboxHasMessages, readToolActivity, toolActivityPath } from "./dropbox.js";
|
|
45
45
|
import { loadLanesConfig, providerProgressEventMinIntervalMs, providerQuotaDefaultHoldMs } from "./config.js";
|
|
46
46
|
import { resolveAdmissionHost as resolveProcessAdmissionHost } from "./admission-host.js";
|
|
47
47
|
import { createLaneMesh, emitLaneDecisionEvents, emitLaneProgressEvent, emitLaneTerminalStateEvent, laneStoryStatusSuppressionKey, progressEventDue, } from "./mesh-events.js";
|
|
48
48
|
import { envelopeQuestionId } from "./questions.js";
|
|
49
|
-
import { laneRunDir, missingPidProbe, probePidRecord, readPidRecord, readProcStarttime, readSpawnManifest, envelopeRepairMode, spawnEnvelopeRepair, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
49
|
+
import { laneRunDir, missingPidProbe, probePidRecord, readPidRecord, readProcStarttime, readSpawnManifest, envelopeRepairMode, spawnEnvelopeRepair, writePidRecord, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
50
50
|
/** Spec §5.7: no signal for 20 minutes while the process is still alive. */
|
|
51
51
|
export const SILENCE_MS = 20 * 60 * 1000;
|
|
52
52
|
export const WATCHER_POLL_MS = 5_000;
|
|
@@ -66,6 +66,9 @@ export const WATCHER_DECISION_EVENTS = [
|
|
|
66
66
|
"envelope_repair",
|
|
67
67
|
"classification",
|
|
68
68
|
"terminal_write",
|
|
69
|
+
"message_recovery",
|
|
70
|
+
"message_recovery_failed",
|
|
71
|
+
"message_undeliverable",
|
|
69
72
|
"mesh_event",
|
|
70
73
|
"mesh_event_failed",
|
|
71
74
|
];
|
|
@@ -670,7 +673,56 @@ function defaultSleep(ms) {
|
|
|
670
673
|
setTimeout(resolve, ms);
|
|
671
674
|
});
|
|
672
675
|
}
|
|
676
|
+
const WATCHER_LOOP_OWNER_FILE = "watcher-loop.pid";
|
|
677
|
+
function claimWatcherLoop(hqRoot, laneId, deps) {
|
|
678
|
+
const runDir = laneRunDir(hqRoot, laneId);
|
|
679
|
+
const ownerPath = path.join(runDir, WATCHER_LOOP_OWNER_FILE);
|
|
680
|
+
const isAlive = deps.isAlive ?? isPidAlive;
|
|
681
|
+
const readStarttime = deps.readStarttime ?? readProcStarttime;
|
|
682
|
+
const owner = withLaneHireLocks(hqRoot, [laneId], () => {
|
|
683
|
+
fs.mkdirSync(runDir, { recursive: true });
|
|
684
|
+
const existing = readPidRecord(ownerPath);
|
|
685
|
+
if (existing !== null && probePidRecord(existing, isAlive, readStarttime).alive) {
|
|
686
|
+
return null;
|
|
687
|
+
}
|
|
688
|
+
if (existing !== null)
|
|
689
|
+
fs.unlinkSync(ownerPath);
|
|
690
|
+
writePidRecord(ownerPath, {
|
|
691
|
+
pid: process.pid,
|
|
692
|
+
starttime: readStarttime(process.pid) ?? readProcStarttime(process.pid) ?? undefined,
|
|
693
|
+
});
|
|
694
|
+
const stat = fs.statSync(ownerPath);
|
|
695
|
+
return { dev: stat.dev, ino: stat.ino };
|
|
696
|
+
});
|
|
697
|
+
if (owner === null)
|
|
698
|
+
return null;
|
|
699
|
+
return () => {
|
|
700
|
+
withLaneHireLocks(hqRoot, [laneId], () => {
|
|
701
|
+
try {
|
|
702
|
+
const current = fs.statSync(ownerPath);
|
|
703
|
+
if (current.dev === owner.dev && current.ino === owner.ino)
|
|
704
|
+
fs.unlinkSync(ownerPath);
|
|
705
|
+
}
|
|
706
|
+
catch (err) {
|
|
707
|
+
if (err.code !== "ENOENT")
|
|
708
|
+
throw err;
|
|
709
|
+
}
|
|
710
|
+
});
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
/** Detached and synchronous classifiers share one process-identity claim. */
|
|
673
714
|
export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
|
|
715
|
+
const release = claimWatcherLoop(hqRoot, laneId, deps);
|
|
716
|
+
if (release === null)
|
|
717
|
+
return;
|
|
718
|
+
try {
|
|
719
|
+
await runWatcherLoopOwned(hqRoot, laneId, deps);
|
|
720
|
+
}
|
|
721
|
+
finally {
|
|
722
|
+
release();
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
async function runWatcherLoopOwned(hqRoot, laneId, deps) {
|
|
674
726
|
const sleep = deps.sleep ?? defaultSleep;
|
|
675
727
|
const now = deps.now ?? (() => Date.now());
|
|
676
728
|
const isAlive = deps.isAlive ?? isPidAlive;
|
|
@@ -1118,6 +1170,28 @@ export async function runWatcherLoop(hqRoot, laneId, deps = {}) {
|
|
|
1118
1170
|
await emitMeshSoft(`lane_${updated.state}`, logDecision, tick.emit.at, (signal) => emitLaneTerminalStateEvent(mesh, updated, updated.state, updated.terminal_reason ?? classification.reason, signal), meshRequestTimeoutMs);
|
|
1119
1171
|
}
|
|
1120
1172
|
}
|
|
1173
|
+
if (terminalWrite.changed && deps.recoverQueuedMessages) {
|
|
1174
|
+
try {
|
|
1175
|
+
if (dropboxInboxHasMessages(hqRoot, laneId)) {
|
|
1176
|
+
await deps.recoverQueuedMessages(updated);
|
|
1177
|
+
logDecision({
|
|
1178
|
+
event: "message_recovery",
|
|
1179
|
+
at: tick.emit.at,
|
|
1180
|
+
state: updated.state,
|
|
1181
|
+
});
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
catch (err) {
|
|
1185
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1186
|
+
process.stderr.write(`hq lanes watcher: queued message recovery failed for ${laneId}: ${message}\n`);
|
|
1187
|
+
logDecision({
|
|
1188
|
+
event: "message_recovery_failed",
|
|
1189
|
+
at: tick.emit.at,
|
|
1190
|
+
state: updated.state,
|
|
1191
|
+
error: message,
|
|
1192
|
+
});
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1121
1195
|
}
|
|
1122
1196
|
}
|
|
1123
1197
|
lastSignalMs = tick.nextLastSignalMs;
|
|
@@ -114,6 +114,8 @@ export type EnsureCognitoTokenOptions = {
|
|
|
114
114
|
* cached. Explicit `"person"` never uses machine creds.
|
|
115
115
|
*/
|
|
116
116
|
tokenSource?: CognitoTokenSource;
|
|
117
|
+
/** Cancels bounded, non-interactive person-session refreshes. */
|
|
118
|
+
signal?: AbortSignal;
|
|
117
119
|
};
|
|
118
120
|
/**
|
|
119
121
|
* On-disk person Cognito cache path for `env` (`HQ_STATE_DIR` or `~/.hq`).
|
|
@@ -196,6 +198,12 @@ export declare function readPersonTokenFile(home?: string, env?: NodeJS.ProcessE
|
|
|
196
198
|
* the response subject differs, put the bytes back and leave them there.
|
|
197
199
|
*/
|
|
198
200
|
export declare function refreshActivePersonTokens(cached: CognitoTokens, env?: NodeJS.ProcessEnv): Promise<CognitoTokens>;
|
|
201
|
+
/**
|
|
202
|
+
* Client-health's short-lived flag read cannot use hq-cloud's unbounded refresh
|
|
203
|
+
* helper. Reuse its lockfile protocol, attach the caller signal to the refresh
|
|
204
|
+
* request, and persist only after the request has completed before the deadline.
|
|
205
|
+
*/
|
|
206
|
+
export declare function refreshActivePersonTokensForClientHealth(cached: CognitoTokens, signal: AbortSignal): Promise<CognitoTokens>;
|
|
199
207
|
/** Derive actor kind from an ID token's custom:entity* claims (no verify). */
|
|
200
208
|
export declare function actorKindFromIdToken(idToken: string | undefined): CognitoActorKind;
|
|
201
209
|
/**
|