@indigoai-us/hq-cli 5.345.48 → 5.345.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +2 -1
- package/dist/command-catalog.generated.d.ts +64 -11
- package/dist/command-catalog.generated.js +77 -11
- package/dist/commands/anywhere-runtime-flag.js +5 -3
- package/dist/commands/db-destroy.js +13 -11
- package/dist/commands/db-dump.js +3 -3
- package/dist/commands/db-remote-deps.d.ts +3 -0
- package/dist/commands/db-remote-deps.js +13 -0
- package/dist/commands/db-sql.js +4 -3
- package/dist/commands/db-usage.js +4 -3
- package/dist/commands/dm-forward-flag.d.ts +4 -0
- package/dist/commands/dm-forward-flag.js +7 -0
- package/dist/commands/dm.d.ts +8 -0
- package/dist/commands/dm.js +219 -27
- package/dist/commands/feedback.d.ts +10 -0
- package/dist/commands/feedback.js +45 -0
- package/dist/commands/lanes-message.js +2 -2
- package/dist/commands/lanes.js +2 -2
- package/dist/commands/mcp-serve.js +17 -4
- package/dist/lib/anywhere-person-setting.d.ts +1 -0
- package/dist/lib/anywhere-person-setting.js +12 -0
- package/dist/lib/anywhere-runtime-fast-flag.js +5 -3
- package/dist/lib/anywhere-runtime-flag-key.d.ts +3 -1
- package/dist/lib/anywhere-runtime-flag-key.js +6 -1
- package/dist/lib/daemon/flag.d.ts +1 -1
- package/dist/lib/daemon/flag.js +20 -9
- package/dist/lib/daemon/hq-cloud.d.ts +2 -2
- package/dist/lib/daemon/hq-cloud.js +38 -9
- package/dist/lib/daemon/run.js +45 -19
- package/dist/lib/daemon/server.d.ts +1 -1
- package/dist/lib/db/app-ref.d.ts +21 -0
- package/dist/lib/db/app-ref.js +45 -0
- package/dist/lib/lanes/keep-alive-audit.d.ts +5 -1
- package/dist/lib/lanes/keep-alive-audit.js +85 -3
- package/dist/lib/lanes/reconcile.js +6 -4
- package/dist/lib/lanes/registry-flag.d.ts +11 -1
- package/dist/lib/lanes/registry-flag.js +68 -29
- package/dist/lib/lanes/watcher.js +63 -2
- package/dist/utils/resident-package-preload.js +4 -1
- package/dist/utils/self-update.js +6 -5
- package/dist/utils/version-gate.d.ts +11 -4
- package/dist/utils/version-gate.js +132 -41
- package/docs/anywhere-runtime.md +5 -3
- package/package.json +1 -1
package/dist/lib/daemon/flag.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Feature flags read by the daemon. The daemon runs for weeks, so each read
|
|
3
3
|
* asks the registry again instead of holding a per-process answer, and it
|
|
4
|
-
* reads as
|
|
4
|
+
* reads as each flag's fallback unless the registry or a local override decides it.
|
|
5
5
|
*/
|
|
6
6
|
import { createFlagClient, readLocalOverride, } from "@indigoai-us/hq-flags-client";
|
|
7
7
|
import { ensureCognitoIdToken } from "../../utils/cognito-session.js";
|
|
8
8
|
import { resolveFlagRegistryEndpoint } from "../flag-registry-endpoint.js";
|
|
9
9
|
import { resolveProcessFlagCompanyContext } from "../flag-registry.js";
|
|
10
|
+
import { HQ_ANYWHERE_RUNTIME_FLAG, hqAnywhereRuntimeLocallyDisabled } from "../anywhere-runtime-flag-key.js";
|
|
10
11
|
/** Outposts hand running services to the next daemon on a CLI update instead of restarting them all. */
|
|
11
12
|
export const SUPERVISOR_HANDOFF_FLAG = "daemon.supervisor-handoff";
|
|
12
13
|
/** Self-updates build the new CLI beside the running one and swap it in, so `hq` never disappears mid-update. */
|
|
@@ -14,6 +15,9 @@ export const STAGED_SELF_UPDATE_FLAG = "cli.staged-self-update";
|
|
|
14
15
|
const READ_TIMEOUT_MS = 5_000;
|
|
15
16
|
export async function readDaemonFlag(flagKey, options = {}) {
|
|
16
17
|
const env = options.env ?? process.env;
|
|
18
|
+
if (flagKey === HQ_ANYWHERE_RUNTIME_FLAG && hqAnywhereRuntimeLocallyDisabled(env))
|
|
19
|
+
return false;
|
|
20
|
+
const fallback = flagKey === HQ_ANYWHERE_RUNTIME_FLAG;
|
|
17
21
|
const override = readLocalOverride(flagKey, env);
|
|
18
22
|
const localOverrideMode = options.localOverrideMode ?? "respect";
|
|
19
23
|
if (override !== null && (localOverrideMode === "respect" || (localOverrideMode === "off-only" && !override)))
|
|
@@ -28,10 +32,10 @@ export async function readDaemonFlag(flagKey, options = {}) {
|
|
|
28
32
|
let client;
|
|
29
33
|
try {
|
|
30
34
|
const company = resolveProcessFlagCompanyContext(env);
|
|
31
|
-
// Signed out reads as
|
|
35
|
+
// Signed out reads as the flag fallback without reaching the registry.
|
|
32
36
|
const token = await (options.getToken ?? (() => ensureCognitoIdToken({ interactive: false })))();
|
|
33
37
|
if (typeof token !== "string" || token.trim() === "")
|
|
34
|
-
return
|
|
38
|
+
return fallback;
|
|
35
39
|
client = (options.createClient ?? createFlagClient)({
|
|
36
40
|
endpoint: resolveFlagRegistryEndpoint(undefined, env),
|
|
37
41
|
getToken: () => token,
|
|
@@ -42,20 +46,27 @@ export async function readDaemonFlag(flagKey, options = {}) {
|
|
|
42
46
|
...(company.companySlug ? { companyIdentifiers: [company.companySlug] } : {}),
|
|
43
47
|
onError,
|
|
44
48
|
});
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
49
|
+
try {
|
|
50
|
+
await client.ready();
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
onError(error);
|
|
54
|
+
}
|
|
55
|
+
const resolution = client.explain(flagKey, { fallback });
|
|
56
|
+
if (failed && options.throwOnError && !fallback) {
|
|
48
57
|
throw new Error("hq-flags registry read failed", { cause: reportedError });
|
|
49
58
|
}
|
|
50
|
-
|
|
59
|
+
if (resolution.source === "snapshot")
|
|
60
|
+
return resolution.value === true;
|
|
61
|
+
return fallback && resolution.value === true;
|
|
51
62
|
}
|
|
52
63
|
catch (error) {
|
|
53
64
|
if (!failed)
|
|
54
65
|
onError(error);
|
|
55
|
-
if (options.throwOnError && failed) {
|
|
66
|
+
if (options.throwOnError && failed && !fallback) {
|
|
56
67
|
throw new Error("hq-flags registry read failed", { cause: reportedError ?? error });
|
|
57
68
|
}
|
|
58
|
-
return
|
|
69
|
+
return fallback;
|
|
59
70
|
}
|
|
60
71
|
finally {
|
|
61
72
|
try {
|
|
@@ -14,8 +14,6 @@ export interface HqCloudCopy {
|
|
|
14
14
|
}
|
|
15
15
|
export interface PickedHqCloud extends HqCloudCopy {
|
|
16
16
|
source: "bundled" | "managed";
|
|
17
|
-
/** Set when a version is pinned but that version is not the installed managed copy. */
|
|
18
|
-
pinnedVersionMissing?: string;
|
|
19
17
|
}
|
|
20
18
|
export type ManagedHqCloud = PickedHqCloud & {
|
|
21
19
|
source: "managed";
|
|
@@ -85,6 +83,8 @@ export interface UpdateHqCloudDeps {
|
|
|
85
83
|
run?: (file: string, args: string[], shell: boolean, env: NodeJS.ProcessEnv) => NpmRunResult;
|
|
86
84
|
bundled?: HqCloudCopy | null;
|
|
87
85
|
platform?: NodeJS.Platform;
|
|
86
|
+
/** Test seam for bounded install retry delays. */
|
|
87
|
+
sleep?: (ms: number) => void;
|
|
88
88
|
}
|
|
89
89
|
/** Install or update the updater's copy, as this (the login) user. */
|
|
90
90
|
export declare function updateManagedHqCloud(daemonRoot: string, env: NodeJS.ProcessEnv, deps?: UpdateHqCloudDeps): HqCloudUpdateOutcome;
|
|
@@ -54,10 +54,11 @@ export function pickHqCloud(bundled, managed, usable = () => true, pinnedVersion
|
|
|
54
54
|
if (managed && managed.version === pinnedVersion && b && m && m[0] === b[0] && usable(managed)) {
|
|
55
55
|
return { ...managed, source: "managed" };
|
|
56
56
|
}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
//
|
|
60
|
-
|
|
57
|
+
if (bundled.version === pinnedVersion && b && usable(bundled))
|
|
58
|
+
return { ...bundled, source: "bundled" };
|
|
59
|
+
// A pin is an exact version contract. Do not silently run another installed
|
|
60
|
+
// package while npm has not yet delivered the pinned version.
|
|
61
|
+
return null;
|
|
61
62
|
}
|
|
62
63
|
return pickNewest(bundled, managed, b, m, usable);
|
|
63
64
|
}
|
|
@@ -160,6 +161,17 @@ export function currentHqCloudAfterLeaseRelease(daemonRoot, expectedVersion) {
|
|
|
160
161
|
return null;
|
|
161
162
|
return { ...managed, source: "managed" };
|
|
162
163
|
}
|
|
164
|
+
function isTransientNpmFailure(result) {
|
|
165
|
+
// spawnSync's ETIMEDOUT means this attempt hit our full 10-minute process
|
|
166
|
+
// timeout. Retrying that result would multiply one stall across every attempt.
|
|
167
|
+
if (result.error?.code === "ETIMEDOUT")
|
|
168
|
+
return false;
|
|
169
|
+
const text = `${result.stderr ?? ""}\n${result.stdout ?? ""}\n${result.error?.message ?? ""}`;
|
|
170
|
+
return /\b(?:ETARGET|EAI_AGAIN|ECONNRESET|ECONNREFUSED|ENETUNREACH|ENOTFOUND|ETIMEDOUT|EHOSTUNREACH|ENETWORK|ECONNABORTED)\b|\bnotarget\b|network timeout|fetch failed|socket hang up/i.test(text);
|
|
171
|
+
}
|
|
172
|
+
function sleepSync(ms) {
|
|
173
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
174
|
+
}
|
|
163
175
|
function runNpm(file, args, shell, env) {
|
|
164
176
|
return spawnSync(file, args, {
|
|
165
177
|
encoding: "utf8",
|
|
@@ -180,7 +192,7 @@ export function updateManagedHqCloud(daemonRoot, env, deps = {}) {
|
|
|
180
192
|
const platform = deps.platform ?? process.platform;
|
|
181
193
|
const pinnedVersion = readDaemonConfig(path.join(daemonRoot, "config.json")).hqCloudVersion;
|
|
182
194
|
const bundled = deps.bundled !== undefined ? deps.bundled : bundledHqCloud();
|
|
183
|
-
const from =
|
|
195
|
+
const from = managedHqCloud(daemonRoot)?.version ?? bundled?.version ?? null;
|
|
184
196
|
if (!bundled)
|
|
185
197
|
return { ok: false, from, to: null, detail: "this CLI install has no bundled hq-cloud to match" };
|
|
186
198
|
const prefix = managedHqCloudDir(daemonRoot);
|
|
@@ -203,9 +215,26 @@ export function updateManagedHqCloud(daemonRoot, env, deps = {}) {
|
|
|
203
215
|
version = latest;
|
|
204
216
|
}
|
|
205
217
|
const cmd = hqCloudInstallCommand(prefix, version, platform);
|
|
206
|
-
const
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
218
|
+
const retryDelays = [500, 1_000, 2_000];
|
|
219
|
+
const sleep = deps.sleep ?? sleepSync;
|
|
220
|
+
let r;
|
|
221
|
+
for (let attempt = 0;; attempt++) {
|
|
222
|
+
r = run(cmd.file, cmd.args, cmd.shell, env);
|
|
223
|
+
if (r.status === 0)
|
|
224
|
+
break;
|
|
225
|
+
const retryable = pinnedVersion !== undefined && pinnedVersion !== null && isTransientNpmFailure(r);
|
|
226
|
+
const delay = retryDelays[attempt];
|
|
227
|
+
if (!retryable || delay === undefined) {
|
|
228
|
+
const retries = retryable ? attempt : 0;
|
|
229
|
+
return {
|
|
230
|
+
ok: false,
|
|
231
|
+
from,
|
|
232
|
+
to: null,
|
|
233
|
+
detail: retries > 0 ? `pinned hq-cloud install failed after ${retries + 1} attempts: ${npmFailure(r)}` : npmFailure(r),
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
sleep(delay);
|
|
237
|
+
}
|
|
238
|
+
return { ok: true, from, to: managedHqCloud(daemonRoot)?.version ?? null };
|
|
210
239
|
}
|
|
211
240
|
//# sourceMappingURL=hq-cloud.js.map
|
package/dist/lib/daemon/run.js
CHANGED
|
@@ -105,17 +105,26 @@ const HANDOFF_FLAG_TIMEOUT_MS = 10_000;
|
|
|
105
105
|
const HQD_PERSON_SETTING_GRACE_MS = 30 * 60_000;
|
|
106
106
|
export function syncRunnerSourceForPlan(stagedRunnerPath, hqCloud, legacyBundledRunnerPath = null, staged = { version: null }, currentSyncRunnerSource = null) {
|
|
107
107
|
// A managed copy lives in the daemon directory, so it survives a CLI
|
|
108
|
-
// replacement.
|
|
109
|
-
//
|
|
110
|
-
// eligible package, so it must not displace a newer staged bundle unless it
|
|
111
|
-
// is the pinned version. The staged bundle is built from the bundled copy and
|
|
112
|
-
// otherwise only stands in for that copy (hq-cli#1547).
|
|
108
|
+
// replacement. A configured pin is an exact version contract: only that
|
|
109
|
+
// installed package or an exactly matching staged bundle may start.
|
|
113
110
|
const pin = staged.pinnedVersion;
|
|
114
|
-
|
|
115
|
-
(
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
111
|
+
if (pin) {
|
|
112
|
+
if (hqCloud?.version === pin) {
|
|
113
|
+
if (hqCloud.source === "managed") {
|
|
114
|
+
return { kind: "managed-full-package", runnerPath: path.join(hqCloud.root, "dist", "bin", "sync-runner.js") };
|
|
115
|
+
}
|
|
116
|
+
if (stagedRunnerPath && staged.version === pin)
|
|
117
|
+
return { kind: "staged-sync", runnerPath: stagedRunnerPath };
|
|
118
|
+
return { kind: "bundled-full-package", runnerPath: path.join(hqCloud.root, "dist", "bin", "sync-runner.js") };
|
|
119
|
+
}
|
|
120
|
+
if (stagedRunnerPath && staged.version === pin)
|
|
121
|
+
return { kind: "staged-sync", runnerPath: stagedRunnerPath };
|
|
122
|
+
// A transient install gap must not stop a runner that is already serving
|
|
123
|
+
// sync. On cold start there is no current source, so mismatched bundles
|
|
124
|
+
// remain unavailable and cannot be started.
|
|
125
|
+
if (currentSyncRunnerSource?.runnerPath)
|
|
126
|
+
return currentSyncRunnerSource;
|
|
127
|
+
return { kind: "unavailable", runnerPath: null };
|
|
119
128
|
}
|
|
120
129
|
const managedYieldsToStaged = stagedRunnerPath !== null &&
|
|
121
130
|
hqCloud?.source === "managed" &&
|
|
@@ -408,6 +417,7 @@ export async function runDaemon(opts) {
|
|
|
408
417
|
let legacySyncRunnerPathChecked = false;
|
|
409
418
|
let legacySyncRunnerPath = null;
|
|
410
419
|
let lastPlannedSyncRunnerSource = null;
|
|
420
|
+
let lastPlannedSyncRunnerVersion = null;
|
|
411
421
|
let packageUseLeaseReleased = false;
|
|
412
422
|
let hqCloudVersionBeforeRelease = null;
|
|
413
423
|
let startupPlanOpen = true;
|
|
@@ -580,6 +590,7 @@ export async function runDaemon(opts) {
|
|
|
580
590
|
let warnedMissingPin = null;
|
|
581
591
|
let warnedMissingSyncRunner = false;
|
|
582
592
|
const daemonSyncRunnerPids = new Set();
|
|
593
|
+
let supervisor = null;
|
|
583
594
|
const plan = () => {
|
|
584
595
|
const config = readDaemonConfig(paths.configPath);
|
|
585
596
|
const legacy = legacyUnits();
|
|
@@ -594,19 +605,34 @@ export async function runDaemon(opts) {
|
|
|
594
605
|
: currentHqCloud(paths.root);
|
|
595
606
|
if (!packageUseLeaseReleased)
|
|
596
607
|
hqCloudVersionBeforeRelease = hqCloud?.version ?? staged?.metadata.version ?? null;
|
|
597
|
-
const missingPin = hqCloud?.pinnedVersionMissing ?? null;
|
|
598
|
-
if (missingPin !== warnedMissingPin) {
|
|
599
|
-
warnedMissingPin = missingPin;
|
|
600
|
-
if (missingPin && hqCloud) {
|
|
601
|
-
log("warn", `hq-cloud is pinned to ${missingPin}, which is not installed; running ${hqCloud.version} (${hqCloud.source}) until the updater installs it`);
|
|
602
|
-
}
|
|
603
|
-
}
|
|
604
608
|
if (!packageUseLeaseReleased && !legacySyncRunnerPathChecked) {
|
|
605
609
|
legacySyncRunnerPath = resolveSyncRunnerPath();
|
|
606
610
|
legacySyncRunnerPathChecked = true;
|
|
607
611
|
}
|
|
608
|
-
const syncSource = syncRunnerSourceForPlan(staged?.runnerPath ?? null, hqCloud, packageUseLeaseReleased ? null : legacySyncRunnerPath, { version: staged?.metadata.version ?? null, pinnedVersion: config.hqCloudVersion ?? null },
|
|
612
|
+
const syncSource = syncRunnerSourceForPlan(staged?.runnerPath ?? null, hqCloud, packageUseLeaseReleased ? null : legacySyncRunnerPath, { version: staged?.metadata.version ?? null, pinnedVersion: config.hqCloudVersion ?? null }, supervisor?.state().units.some((unit) => unit.id === "sync" && unit.status === "running" && typeof unit.pid === "number")
|
|
613
|
+
? lastPlannedSyncRunnerSource
|
|
614
|
+
: null);
|
|
615
|
+
const retainedLiveSource = syncSource === lastPlannedSyncRunnerSource;
|
|
616
|
+
const syncRunnerVersion = retainedLiveSource
|
|
617
|
+
? lastPlannedSyncRunnerVersion
|
|
618
|
+
: syncSource.kind === "staged-sync"
|
|
619
|
+
? staged?.metadata.version ?? null
|
|
620
|
+
: hqCloud?.version ?? null;
|
|
621
|
+
const missingPin = config.hqCloudVersion && syncRunnerVersion !== config.hqCloudVersion ? config.hqCloudVersion : null;
|
|
622
|
+
const pinWarningKey = missingPin
|
|
623
|
+
? `${missingPin}:${retainedLiveSource ? syncRunnerVersion ?? "unknown" : "unavailable"}`
|
|
624
|
+
: null;
|
|
625
|
+
if (pinWarningKey !== warnedMissingPin) {
|
|
626
|
+
warnedMissingPin = pinWarningKey;
|
|
627
|
+
if (missingPin && retainedLiveSource) {
|
|
628
|
+
log("warn", `hq-cloud is pinned to ${missingPin}, but the running sync runner is ${syncRunnerVersion ?? "unknown"}; keeping it running while the updater retries`);
|
|
629
|
+
}
|
|
630
|
+
else if (missingPin && syncSource.kind === "unavailable") {
|
|
631
|
+
log("warn", `hq-cloud is pinned to ${missingPin}, which is not installed; sync is unavailable until the updater installs it`);
|
|
632
|
+
}
|
|
633
|
+
}
|
|
609
634
|
lastPlannedSyncRunnerSource = syncSource.runnerPath ? syncSource : null;
|
|
635
|
+
lastPlannedSyncRunnerVersion = syncSource.runnerPath ? syncRunnerVersion : null;
|
|
610
636
|
if (!syncSource.runnerPath && !warnedMissingSyncRunner) {
|
|
611
637
|
warnedMissingSyncRunner = true;
|
|
612
638
|
log("warn", "the hq-cloud sync runner was not found in this CLI install; sync is unavailable");
|
|
@@ -797,7 +823,7 @@ export async function runDaemon(opts) {
|
|
|
797
823
|
// Set once the shutdown path exists below; a restart request before then is dropped.
|
|
798
824
|
let restartDaemon = () => false;
|
|
799
825
|
let extendSyncShutdownCap;
|
|
800
|
-
|
|
826
|
+
supervisor = createSupervisor({
|
|
801
827
|
machineType,
|
|
802
828
|
host: opts.host,
|
|
803
829
|
pid: process.pid,
|
|
@@ -39,7 +39,7 @@ export interface HqdServerOptions {
|
|
|
39
39
|
env?: NodeJS.ProcessEnv;
|
|
40
40
|
/** Work-mesh root override for mesh.presence (tests). */
|
|
41
41
|
workMeshRoot?: string;
|
|
42
|
-
/** Test seam for the hq-anywhere-runtime
|
|
42
|
+
/** Test seam for the hq-anywhere-runtime fail-safe gate. */
|
|
43
43
|
mcpEnabled?: () => boolean | Promise<boolean>;
|
|
44
44
|
/** Test seam for the live hq-flags read used by the daemon's bounded runtime monitor. */
|
|
45
45
|
runtimeFlagReader?: (options: ReadDaemonFlagOptions) => Promise<boolean>;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve the `--app` value of the `hq db` commands to an app id. The value can
|
|
3
|
+
* be the app id, its name or its subdomain. Only apps in the given company are
|
|
4
|
+
* searched, so a name never resolves to another company's app.
|
|
5
|
+
*/
|
|
6
|
+
import { deployApiRequest } from "../deploy-api.js";
|
|
7
|
+
export interface AppRefRow {
|
|
8
|
+
id?: unknown;
|
|
9
|
+
name?: unknown;
|
|
10
|
+
subdomain?: unknown;
|
|
11
|
+
}
|
|
12
|
+
export declare class AppRefError extends Error {
|
|
13
|
+
constructor(message: string);
|
|
14
|
+
}
|
|
15
|
+
export declare function matchAppRef(apps: AppRefRow[], ref: string): string;
|
|
16
|
+
export declare function resolveAppRef(input: {
|
|
17
|
+
token: string;
|
|
18
|
+
companySlug: string;
|
|
19
|
+
ref: string;
|
|
20
|
+
}, request?: typeof deployApiRequest): Promise<string>;
|
|
21
|
+
//# sourceMappingURL=app-ref.d.ts.map
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolve the `--app` value of the `hq db` commands to an app id. The value can
|
|
3
|
+
* be the app id, its name or its subdomain. Only apps in the given company are
|
|
4
|
+
* searched, so a name never resolves to another company's app.
|
|
5
|
+
*/
|
|
6
|
+
import { deployApiRequest } from "../deploy-api.js";
|
|
7
|
+
import { listDeployApps } from "../deploy-apps.js";
|
|
8
|
+
export class AppRefError extends Error {
|
|
9
|
+
constructor(message) {
|
|
10
|
+
super(message);
|
|
11
|
+
this.name = "AppRefError";
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
15
|
+
export function matchAppRef(apps, ref) {
|
|
16
|
+
const ids = (rows) => [
|
|
17
|
+
...new Set(rows.map((row) => row.id).filter((id) => typeof id === "string")),
|
|
18
|
+
];
|
|
19
|
+
const byId = ids(apps.filter((row) => row.id === ref));
|
|
20
|
+
if (byId.length === 1)
|
|
21
|
+
return byId[0];
|
|
22
|
+
const byName = ids(apps.filter((row) => row.name === ref || row.subdomain === ref));
|
|
23
|
+
if (byName.length === 1)
|
|
24
|
+
return byName[0];
|
|
25
|
+
if (byName.length > 1) {
|
|
26
|
+
throw new AppRefError(`More than one app in this company is called "${ref}" (${byName.join(", ")}). Use the app id.`);
|
|
27
|
+
}
|
|
28
|
+
throw new AppRefError(`No app called "${ref}" was found in this company.`);
|
|
29
|
+
}
|
|
30
|
+
export async function resolveAppRef(input, request = deployApiRequest) {
|
|
31
|
+
const ref = input.ref.trim();
|
|
32
|
+
if (UUID_RE.test(ref))
|
|
33
|
+
return ref;
|
|
34
|
+
const headers = { "X-Org-Slug": input.companySlug };
|
|
35
|
+
const apps = await listDeployApps(async (cursor) => {
|
|
36
|
+
const route = cursor ? `/api/apps?cursor=${encodeURIComponent(cursor)}` : "/api/apps";
|
|
37
|
+
const res = await request(input.token, route, "GET", undefined, headers);
|
|
38
|
+
if (!res.ok) {
|
|
39
|
+
throw new AppRefError(`Could not look up the app "${ref}" (hq-deploy returned ${res.status}). Use the app id.`);
|
|
40
|
+
}
|
|
41
|
+
return (await res.json());
|
|
42
|
+
});
|
|
43
|
+
return matchAppRef(apps, ref);
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=app-ref.js.map
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
type KeepAliveAuditVerdict = "on_time" | "late" | "early_wake" | "message_wake" | "work_triggered" | "self_wake" | "early_by_keep_alive" | "restarted_untimed" | "not_restarted" | "stranded" | "keep_alive_off" | "waiting";
|
|
1
|
+
type KeepAliveAuditVerdict = "on_time" | "late" | "early_wake" | "message_wake" | "work_triggered" | "self_wake" | "early_by_keep_alive" | "restarted_untimed" | "not_restarted" | "stranded" | "keep_alive_off" | "finished_story" | "idle_no_queued_work" | "waiting";
|
|
2
2
|
interface KeepAliveAuditRestart {
|
|
3
3
|
at: string | null;
|
|
4
4
|
trigger: string;
|
|
@@ -17,6 +17,7 @@ interface KeepAliveAuditRow {
|
|
|
17
17
|
blocked_outcome?: "woken" | "skipped";
|
|
18
18
|
blocked_outcome_reason?: string;
|
|
19
19
|
verdict: KeepAliveAuditVerdict;
|
|
20
|
+
queued_work_at_audit_time?: boolean;
|
|
20
21
|
remediation?: string;
|
|
21
22
|
terminal_reason?: string;
|
|
22
23
|
}
|
|
@@ -25,6 +26,8 @@ interface KeepAliveAuditSummary {
|
|
|
25
26
|
not_restarted: number;
|
|
26
27
|
stranded: number;
|
|
27
28
|
restarted_untimed: number;
|
|
29
|
+
finished_story: number;
|
|
30
|
+
idle_no_queued_work: number;
|
|
28
31
|
late_restarts_after_window: number;
|
|
29
32
|
late_over_10m: number;
|
|
30
33
|
early_wake_by_others: number;
|
|
@@ -53,6 +56,7 @@ export interface StopEnvelope {
|
|
|
53
56
|
name: string;
|
|
54
57
|
mtimeMs: number;
|
|
55
58
|
decision: "done" | "blocked";
|
|
59
|
+
storyComplete: boolean;
|
|
56
60
|
nextCheckAt?: string;
|
|
57
61
|
}
|
|
58
62
|
export declare function readStopEnvelopes(hqRoot: string, laneId: string): {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
|
-
import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS } from "./keep-alive.js";
|
|
3
|
+
import { LEAD_LANE_KEEP_ALIVE_BASE_BACKOFF_MS, LEAD_LANE_KEEP_ALIVE_NEXT_CHECK_MAX_DELAY_MS, doneKeepAliveVerdict, hasPendingKeepAliveWork, undeliveredAnsweredQuestions } from "./keep-alive.js";
|
|
4
|
+
import { dropboxDir, dropboxInboxPath } from "./dropbox.js";
|
|
4
5
|
import { LanesError } from "./errors.js";
|
|
5
6
|
import { laneRunDir } from "./spawn.js";
|
|
6
7
|
import { compareEnvelopeNames } from "./envelope-sequence.js";
|
|
@@ -63,6 +64,8 @@ const VERDICTS = [
|
|
|
63
64
|
"keep_alive_off",
|
|
64
65
|
"not_restarted",
|
|
65
66
|
"stranded",
|
|
67
|
+
"finished_story",
|
|
68
|
+
"idle_no_queued_work",
|
|
66
69
|
"waiting",
|
|
67
70
|
];
|
|
68
71
|
function parseTime(value) {
|
|
@@ -174,7 +177,7 @@ export function readStopEnvelopes(hqRoot, laneId) {
|
|
|
174
177
|
if (!isRecord(raw) || raw.schema_version !== "hq-lane-envelope.v1") {
|
|
175
178
|
throw new LanesError("keep_alive_audit_envelope_invalid", `Lane envelope is not hq-lane-envelope.v1: ${file}`, { lane_id: laneId, path: file });
|
|
176
179
|
}
|
|
177
|
-
const doneStop = raw.decision === "done"
|
|
180
|
+
const doneStop = raw.decision === "done";
|
|
178
181
|
const blockedStop = raw.decision === "blocked" && typeof raw.next_check_at === "string" &&
|
|
179
182
|
isIsoDateTime(raw.next_check_at) && Date.parse(raw.next_check_at) > stat.mtimeMs;
|
|
180
183
|
if (!doneStop && !blockedStop)
|
|
@@ -183,6 +186,7 @@ export function readStopEnvelopes(hqRoot, laneId) {
|
|
|
183
186
|
name,
|
|
184
187
|
mtimeMs: stat.mtimeMs,
|
|
185
188
|
decision: blockedStop ? "blocked" : "done",
|
|
189
|
+
storyComplete: raw.story_complete === true,
|
|
186
190
|
...(typeof raw.next_check_at === "string" ? { nextCheckAt: raw.next_check_at } : {}),
|
|
187
191
|
});
|
|
188
192
|
}
|
|
@@ -428,6 +432,48 @@ function firstRestartAfter(lane, stop, nextEnvelopeMs, workerResumeEvents) {
|
|
|
428
432
|
// untimed instead of using the lane's mutable current state as a proxy.
|
|
429
433
|
return nextEnvelopeMs === undefined ? null : { at: null, trigger: "unknown", caller: null };
|
|
430
434
|
}
|
|
435
|
+
/**
|
|
436
|
+
* When the work a finished standing lane is now waiting on first appeared.
|
|
437
|
+
* The earliest signal wins: if work was queued at 10:05 and more arrived at
|
|
438
|
+
* 11:59, keep-alive has owed a launch since 10:05. Returns undefined when no
|
|
439
|
+
* signal carries a time, which leaves the caller unable to open a window.
|
|
440
|
+
*/
|
|
441
|
+
function keepAliveQueuedWorkAt(hqRoot, lane) {
|
|
442
|
+
const candidates = [];
|
|
443
|
+
const pendingAt = parseTime(lane.pending_resume?.at);
|
|
444
|
+
if (pendingAt !== undefined)
|
|
445
|
+
candidates.push(pendingAt);
|
|
446
|
+
for (const candidate of [dropboxInboxPath(hqRoot, lane.lane_id), dropboxDir(hqRoot, lane.lane_id)]) {
|
|
447
|
+
try {
|
|
448
|
+
candidates.push(fs.statSync(candidate).mtimeMs);
|
|
449
|
+
break;
|
|
450
|
+
}
|
|
451
|
+
catch (err) {
|
|
452
|
+
if (err.code !== "ENOENT")
|
|
453
|
+
throw err;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
const questions = [];
|
|
457
|
+
for (const questionId of listQuestionIds(hqRoot)) {
|
|
458
|
+
try {
|
|
459
|
+
const question = tryReadQuestion(hqRoot, questionId);
|
|
460
|
+
if (question)
|
|
461
|
+
questions.push(question);
|
|
462
|
+
}
|
|
463
|
+
catch {
|
|
464
|
+
// One unreadable question file must not fail the whole audit.
|
|
465
|
+
continue;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
for (const question of undeliveredAnsweredQuestions(lane, questions)) {
|
|
469
|
+
const answeredAt = parseTime(question.answer?.answered_at);
|
|
470
|
+
if (answeredAt !== undefined)
|
|
471
|
+
candidates.push(answeredAt);
|
|
472
|
+
}
|
|
473
|
+
if (candidates.length === 0)
|
|
474
|
+
return undefined;
|
|
475
|
+
return Math.min(...candidates);
|
|
476
|
+
}
|
|
431
477
|
function classifyVerdict(input) {
|
|
432
478
|
const { dueAt, restart, lane, nowMs } = input;
|
|
433
479
|
if (restart?.trigger === "unknown" && laneWasKeepAliveOffAtStop(lane, input.stoppedAt))
|
|
@@ -478,6 +524,34 @@ function classifyVerdict(input) {
|
|
|
478
524
|
}
|
|
479
525
|
if (laneWasKeepAliveOffAtStop(lane, input.stoppedAt))
|
|
480
526
|
return "keep_alive_off";
|
|
527
|
+
// Split the unrestarted stops by whether keep-alive owes this lane a wake at
|
|
528
|
+
// all. The discriminator is the shared done-state policy, not whether the
|
|
529
|
+
// envelope declared a next_check_at: a stop with story_complete false is owed
|
|
530
|
+
// a wake with or without one. Only done stops with no later envelope reach
|
|
531
|
+
// this branch, because a later envelope means the lane carried on.
|
|
532
|
+
if (input.decision === "done" && input.nextEnvelopeAt === undefined) {
|
|
533
|
+
const eligibility = doneKeepAliveVerdict(lane, {
|
|
534
|
+
decision: "done",
|
|
535
|
+
story_complete: input.storyComplete,
|
|
536
|
+
...(input.nextCheckAt === undefined ? {} : { next_check_at: input.nextCheckAt }),
|
|
537
|
+
}, input.hasQueuedWork, { nowMs });
|
|
538
|
+
// Keep-alive is specified not to wake either case, so neither is a
|
|
539
|
+
// keep-alive liveness failure and neither counts against the KPI.
|
|
540
|
+
if (eligibility === "story_complete")
|
|
541
|
+
return "finished_story";
|
|
542
|
+
if (eligibility === "standing_idle")
|
|
543
|
+
return "idle_no_queued_work";
|
|
544
|
+
// A finished story becomes eligible again only once work is queued, which
|
|
545
|
+
// can be long after the stop. Open the failure window when that work
|
|
546
|
+
// appeared, not at the stop's own due time, or every normal exit from
|
|
547
|
+
// idle_no_queued_work reads as a liveness failure the moment it happens.
|
|
548
|
+
if (input.storyComplete) {
|
|
549
|
+
if (input.queuedWorkAt === undefined)
|
|
550
|
+
return "waiting";
|
|
551
|
+
const anchor = Math.max(dueAt, input.queuedWorkAt);
|
|
552
|
+
return nowMs > anchor + KEEP_ALIVE_NOT_RESTARTED_GRACE_MS ? "not_restarted" : "waiting";
|
|
553
|
+
}
|
|
554
|
+
}
|
|
481
555
|
if (nowMs > dueAt + KEEP_ALIVE_NOT_RESTARTED_GRACE_MS)
|
|
482
556
|
return "not_restarted";
|
|
483
557
|
return "waiting";
|
|
@@ -546,6 +620,10 @@ export function buildKeepAliveAudit(input) {
|
|
|
546
620
|
try {
|
|
547
621
|
const { allMtimes, stops } = readStopEnvelopes(input.hqRoot, lane.lane_id);
|
|
548
622
|
const keepAliveResumeEvents = readKeepAliveResumeEvents(input.hqRoot, lane.lane_id);
|
|
623
|
+
// Read at audit time, not stop time: for a stop with no restart the
|
|
624
|
+
// question is whether anything is going to resume the lane now.
|
|
625
|
+
const hasQueuedWork = lane.keep_alive_standing === true && hasPendingKeepAliveWork(input.hqRoot, lane);
|
|
626
|
+
const queuedWorkAt = hasQueuedWork ? keepAliveQueuedWorkAt(input.hqRoot, lane) : undefined;
|
|
549
627
|
const laneWorkerResumeEvents = readLaneWorkerResumeEvents(input.hqRoot, lane.lane_id);
|
|
550
628
|
const terminalClassifications = readTerminalClassifications(input.hqRoot, lane.lane_id);
|
|
551
629
|
for (const stop of stops) {
|
|
@@ -572,7 +650,8 @@ export function buildKeepAliveAudit(input) {
|
|
|
572
650
|
blocked_outcome: blockedOutcome.status,
|
|
573
651
|
...(blockedOutcome.reason === undefined ? {} : { blocked_outcome_reason: blockedOutcome.reason }),
|
|
574
652
|
} : {}),
|
|
575
|
-
verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, lane, nowMs, resumeEvents: keepAliveResumeEvents, ...(nextEnvelopeMs === undefined ? {} : { nextEnvelopeAt: nextEnvelopeMs }), hqRoot: input.hqRoot }),
|
|
653
|
+
verdict: classifyVerdict({ dueAt: due.at, stoppedAt: stop.mtimeMs, nextCheckAt: stop.nextCheckAt, restart, decision: stop.decision, storyComplete: stop.storyComplete, hasQueuedWork, ...(queuedWorkAt === undefined ? {} : { queuedWorkAt }), lane, nowMs, resumeEvents: keepAliveResumeEvents, ...(nextEnvelopeMs === undefined ? {} : { nextEnvelopeAt: nextEnvelopeMs }), hqRoot: input.hqRoot }),
|
|
654
|
+
...(stop.decision === "done" ? { queued_work_at_audit_time: hasQueuedWork } : {}),
|
|
576
655
|
});
|
|
577
656
|
}
|
|
578
657
|
}
|
|
@@ -611,6 +690,8 @@ export function buildKeepAliveAudit(input) {
|
|
|
611
690
|
not_restarted: byVerdict.not_restarted,
|
|
612
691
|
stranded: byVerdict.stranded,
|
|
613
692
|
restarted_untimed: byVerdict.restarted_untimed,
|
|
693
|
+
finished_story: byVerdict.finished_story,
|
|
694
|
+
idle_no_queued_work: byVerdict.idle_no_queued_work,
|
|
614
695
|
late_restarts_after_window: lateRestartsAfterWindow,
|
|
615
696
|
late_over_10m: lateOverTenMinutes,
|
|
616
697
|
early_wake_by_others: byVerdict.early_wake,
|
|
@@ -642,6 +723,7 @@ export function formatKeepAliveAuditText(result) {
|
|
|
642
723
|
"",
|
|
643
724
|
`Since ${result.since}; ${counts}`,
|
|
644
725
|
`not_restarted=${result.summary.not_restarted}, stranded=${result.summary.stranded}, late_restarts_after_window=${result.summary.late_restarts_after_window}, late_over_10m=${result.summary.late_over_10m}, early_wake_by_others=${result.summary.early_wake_by_others}, self_wake=${result.summary.self_wake}`,
|
|
726
|
+
`finished_story=${result.summary.finished_story}, idle_no_queued_work=${result.summary.idle_no_queued_work} (neither counts against the keep-alive KPI)`,
|
|
645
727
|
`Stopped and not restarted on their own: ${result.summary.stopped_and_not_restarted_on_their_own}`,
|
|
646
728
|
...result.rows.filter((row) => row.remediation).map((row) => `Stranded ${row.lane}: ${row.remediation}`),
|
|
647
729
|
...result.companyless_lead_lanes.map((lane) => `Company-less lead ${lane.lane} (${lane.project}): ${lane.reason}; repair with hq lanes repair-company ${lane.lane} --company <slug>`),
|
|
@@ -22,7 +22,7 @@ import { classifySessionLiveness, defaultLivenessDeps, loadLivenessIndex, readDa
|
|
|
22
22
|
import { ANSWERED_QUESTIONS_FILE, armLaneWatcher, envelopeRepairMode, isPidRecordAlive, laneRunDir, nextEnvelopePath, readPidRecord, readProcStarttime, readSpawnManifest, markSpawnFailure, stopLaneProcesses, spawnEnvelopeRepair, spawnLaneWorker, retryLaneOnAlternateAccount, } from "./spawn.js";
|
|
23
23
|
import { appendCappedHistory, lanesStoreRoot, parseLaneRecord, parseQuestionRecord, readLane, readQuestion, listQuestionIds, updateLane, updateQuestion, } from "./store.js";
|
|
24
24
|
import { applyThrottleDecision, decideAdmission, defaultThrottleDeps, emptyThrottleState, evaluateThrottle, isCpuCheckEnabled, readAvailableMemMb, readThrottleState, resolveCpuBusyFraction, sampleCpuBusyObservation, recordCpuBusyObservation, throttleStatePath, writeThrottleState, } from "./throttle.js";
|
|
25
|
-
import { readLanesRegistryFlag, readLanesRegistryFlagFailOff } from "./registry-flag.js";
|
|
25
|
+
import { readLanesRegistryFlag, readLanesRegistryFlagFailOff, readLanesRegistryFlagFailOpen } from "./registry-flag.js";
|
|
26
26
|
import { LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, LANE_LEAD_ADMISSION_PRIORITY_FLAG, LANE_QUOTA_ACCOUNT_FAILOVER_FLAG, } from "./flag-keys.js";
|
|
27
27
|
import { checkLaneDiskSpace, configuredCodexRolloutHomes, laneDiskAccount, retainCodexRollouts } from "./disk-guard.js";
|
|
28
28
|
import { laneCompanyUidSource, resolveLaneCompany } from "./lane-company.js";
|
|
@@ -1216,6 +1216,7 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
1216
1216
|
}
|
|
1217
1217
|
}
|
|
1218
1218
|
const readRegistryFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
1219
|
+
const readRegistryFlagFailOpen = deps.readRegistryFlag ?? readLanesRegistryFlagFailOpen;
|
|
1219
1220
|
const storyCloseCompanies = new Map();
|
|
1220
1221
|
for (const lane of lanes) {
|
|
1221
1222
|
if (lane.depth < 1 || lane.state !== "interrupted" || lane.keep_alive === true ||
|
|
@@ -1303,10 +1304,10 @@ export async function runPassBody(hqRoot, deps) {
|
|
|
1303
1304
|
readRegistryFlag(LEAD_LANE_KEEP_ALIVE_FLAG, options),
|
|
1304
1305
|
readRegistryFlag(LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, options),
|
|
1305
1306
|
hasBlockedKeepAliveLane
|
|
1306
|
-
?
|
|
1307
|
+
? readRegistryFlagFailOpen(LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, options)
|
|
1307
1308
|
: Promise.resolve(false),
|
|
1308
1309
|
readRegistryFlag(LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, options),
|
|
1309
|
-
|
|
1310
|
+
readRegistryFlagFailOpen(LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, options),
|
|
1310
1311
|
]);
|
|
1311
1312
|
keepAliveFlagByCompany.set(key, keepAliveEnabled);
|
|
1312
1313
|
blockedResumeFlagByCompany.set(key, blockedResumeEnabled);
|
|
@@ -3585,6 +3586,7 @@ async function runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, n
|
|
|
3585
3586
|
if (company === null)
|
|
3586
3587
|
return summary;
|
|
3587
3588
|
const readFlag = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
3589
|
+
const readFlagFailOpen = deps.readRegistryFlag ?? readLanesRegistryFlagFailOpen;
|
|
3588
3590
|
const options = { company, env: deps.env ?? process.env };
|
|
3589
3591
|
let keepAliveEnabled;
|
|
3590
3592
|
let lineageAware;
|
|
@@ -3595,7 +3597,7 @@ async function runTerminalKeepAliveCandidatePass(hqRoot, lanes, summary, deps, n
|
|
|
3595
3597
|
readFlag(LEAD_LANE_KEEP_ALIVE_FLAG, options),
|
|
3596
3598
|
readFlag(LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, options),
|
|
3597
3599
|
readFlag(LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, options),
|
|
3598
|
-
|
|
3600
|
+
readFlagFailOpen(LANE_KEEP_ALIVE_DUE_WAKE_PRECISE_FLAG, options),
|
|
3599
3601
|
]);
|
|
3600
3602
|
}
|
|
3601
3603
|
catch (err) {
|
|
@@ -19,12 +19,22 @@ export interface ReadLanesFlagOptions {
|
|
|
19
19
|
reportFailure?: (error: unknown) => void;
|
|
20
20
|
}
|
|
21
21
|
export declare function readLanesFlagLocalOverride(flagKey: string, env?: NodeJS.ProcessEnv): boolean | null;
|
|
22
|
+
export type LanesRegistryFlagResult = {
|
|
23
|
+
state: "resolved";
|
|
24
|
+
value: boolean;
|
|
25
|
+
} | {
|
|
26
|
+
state: "unknown";
|
|
27
|
+
};
|
|
22
28
|
/**
|
|
23
29
|
* Read a lanes flag once for each company identity in this CLI process. The
|
|
24
|
-
* cache contains only an in-flight or settled
|
|
30
|
+
* cache contains only an in-flight or settled decision result; it is never
|
|
25
31
|
* written to disk. A local override is checked on every call and wins first.
|
|
26
32
|
*/
|
|
27
33
|
export declare function readLanesRegistryFlag(flagKey: string, options?: ReadLanesFlagOptions): Promise<boolean>;
|
|
34
|
+
/** Read a lanes flag while preserving registry failure as an explicit unknown state. */
|
|
35
|
+
export declare function readLanesRegistryFlagResult(flagKey: string, options?: ReadLanesFlagOptions): Promise<LanesRegistryFlagResult>;
|
|
36
|
+
/** Delivery and wake call sites opt in to availability when the registry is unreadable. */
|
|
37
|
+
export declare function readLanesRegistryFlagFailOpen(flagKey: string, options?: ReadLanesFlagOptions): Promise<boolean>;
|
|
28
38
|
/**
|
|
29
39
|
* Flag readers normally report their own failures and return the safe default.
|
|
30
40
|
* Keep enforcement callers safe if an injected or unexpected reader rejects.
|