@indigoai-us/hq-cli 5.345.20 → 5.345.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/dist/commands/cloud.js +1 -1
- package/dist/commands/doctor.d.ts +3 -0
- package/dist/commands/doctor.js +78 -4
- package/dist/commands/integrations-connect.js +7 -3
- package/dist/commands/lanes.js +41 -26
- package/dist/commands/personal-targeted-push-flag.d.ts +1 -1
- package/dist/commands/personal-targeted-push-flag.js +6 -2
- package/dist/lib/bot/continuity-download.js +4 -1
- package/dist/lib/core-utils/sentry-report.js +69 -13
- package/dist/lib/doctor/client-health-reason-codes-flag.d.ts +6 -0
- package/dist/lib/doctor/client-health-reason-codes-flag.js +28 -7
- package/dist/lib/doctor/json-output.d.ts +4 -0
- package/dist/lib/doctor/json-output.js +5 -2
- package/dist/lib/lanes/autopilot.js +5 -1
- package/dist/lib/lanes/grants.d.ts +3 -0
- package/dist/lib/lanes/grants.js +10 -3
- package/dist/lib/lanes/host-capacity.d.ts +5 -0
- package/dist/lib/lanes/host-capacity.js +18 -0
- package/dist/lib/lanes/index.d.ts +1 -1
- package/dist/lib/lanes/index.js +1 -1
- package/dist/lib/lanes/provider-rejection.d.ts +2 -0
- package/dist/lib/lanes/provider-rejection.js +16 -1
- package/dist/lib/lanes/quota.d.ts +5 -5
- package/dist/lib/lanes/quota.js +45 -9
- package/dist/lib/lanes/spawn.d.ts +1 -1
- package/dist/lib/lanes/spawn.js +1 -0
- package/dist/lib/lanes/watcher.d.ts +0 -12
- package/dist/lib/lanes/watcher.js +8 -2
- package/dist/utils/safe-caught-error.d.ts +3 -0
- package/dist/utils/safe-caught-error.js +16 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [5.345.22] — 2026-10-06
|
|
6
|
+
- Fix Claude quota holds so time-only reset messages keep accounts unavailable until the provider's stated UTC reset, and expose the hold state in host capacity output.
|
|
7
|
+
|
|
8
|
+
## [5.345.21] — 2026-10-06
|
|
9
|
+
- `hq sync push --personal <path>` now waits up to 500 ms for the startup flag registry snapshot before evaluating `cli.personal-targeted-push-paths`, while preserving the legacy refusal when the registry is unavailable.
|
|
10
|
+
|
|
11
|
+
- Hook timeout reports now show elapsed time against the configured budget and include available phase and hook attribution.
|
|
12
|
+
- Reconcile hire-grant carve accounting across resume, reparent and grant updates; decreases no longer fail because sibling grants exceed the senior allocation.
|
|
13
|
+
- Route eligible Linux pull-request checks and local release guards through the registered ARM64 self-hosted runner. Keep the shared npm/OIDC publish job on its existing runner.
|
|
14
|
+
- Gate `hq doctor --json --client-health-report` status reporting for sync FAIL/WARN results behind the separate, default-off `client-health-sync-check-status` hq-flag; reason-code eligibility remains independent.
|
|
15
|
+
|
|
5
16
|
## [5.345.20] — 2026-10-06
|
|
6
17
|
- Keep bundled scaffold scripts and `hq core register-project` aligned with the shared source, including company UID lookup from `companies/manifest.yaml`.
|
|
7
18
|
|
package/dist/commands/cloud.js
CHANGED
|
@@ -840,7 +840,7 @@ export function registerCloudCommands(program) {
|
|
|
840
840
|
"to have already resolved entity + credentials. Pick one.");
|
|
841
841
|
}
|
|
842
842
|
// Closes hq-cli#25 — see `assertNoPersonalPositionalPaths` doc-block.
|
|
843
|
-
const targetedPersonalPathsEnabled = Boolean(options.personal && paths && paths.length > 0 && personalTargetedPushPathsEnabled());
|
|
843
|
+
const targetedPersonalPathsEnabled = Boolean(options.personal && paths && paths.length > 0 && await personalTargetedPushPathsEnabled());
|
|
844
844
|
assertNoPersonalPositionalPaths(options, paths, targetedPersonalPathsEnabled);
|
|
845
845
|
if (targetedPersonalPathsEnabled) {
|
|
846
846
|
explicitPersonalPaths = resolveExplicitPersonalPushPaths(options.hqRoot, paths);
|
|
@@ -57,6 +57,8 @@ export interface RunDoctorOptions {
|
|
|
57
57
|
json?: boolean;
|
|
58
58
|
/** Resolve hq-flags independently for each company represented in sync results. */
|
|
59
59
|
resolveClientHealthReasonCodesForCompany?: (companySlug: string | null) => Promise<boolean>;
|
|
60
|
+
/** Resolve the separate status-report flag per company in sync results. */
|
|
61
|
+
resolveClientHealthCheckStatusForCompany?: (companySlug: string | null) => Promise<boolean>;
|
|
60
62
|
/** In text mode, also print every PASS result. Default: false. */
|
|
61
63
|
verbose?: boolean;
|
|
62
64
|
/** In text mode, emit ANSI colour. Default: false (no escape sequences). */
|
|
@@ -117,6 +119,7 @@ export interface RunDoctorResult {
|
|
|
117
119
|
* read; all other default tiers remain local unless explicitly opted in.
|
|
118
120
|
*/
|
|
119
121
|
export declare function runDoctor(options?: RunDoctorOptions): Promise<RunDoctorResult>;
|
|
122
|
+
export declare function createClientHealthCheckStatusCompanyResolver(hqRoot: string): (companySlug: string | null) => Promise<boolean>;
|
|
120
123
|
/** Register the top-level `hq doctor` command so it appears in `hq --help`. */
|
|
121
124
|
export declare function registerDoctorCommand(program: Command): void;
|
|
122
125
|
//# sourceMappingURL=doctor.d.ts.map
|
package/dist/commands/doctor.js
CHANGED
|
@@ -32,7 +32,7 @@ import { DEEP_FAMILY_ID, DEEP_FAMILY_TITLE, runDeepGuardTests, } from "../lib/do
|
|
|
32
32
|
import { PARITY_FAMILY_ID, PARITY_FAMILY_TITLE, runParityReplay, } from "../lib/doctor/deep/parity.js";
|
|
33
33
|
import { applyFixes } from "../lib/doctor/fix/apply.js";
|
|
34
34
|
import { renderDoctorFixJson } from "../lib/doctor/fix/json-output.js";
|
|
35
|
-
import { clientHealthReasonCodesEnabled as isActiveCompanyClientHealthReasonCodesEnabled, clientHealthReasonCodesEnabledForCompany, } from "../lib/doctor/client-health-reason-codes-flag.js";
|
|
35
|
+
import { clientHealthCheckStatusEnabled as isActiveCompanyClientHealthCheckStatusEnabled, clientHealthCheckStatusEnabledForCompany, clientHealthReasonCodesEnabled as isActiveCompanyClientHealthReasonCodesEnabled, clientHealthReasonCodesEnabledForCompany, } from "../lib/doctor/client-health-reason-codes-flag.js";
|
|
36
36
|
import { isCompanyJournalSlug } from "../lib/doctor/checks/sync-health.js";
|
|
37
37
|
import { readCompaniesManifestMap } from "../lib/work-context/repo-remote.js";
|
|
38
38
|
import * as readline from "node:readline";
|
|
@@ -166,15 +166,21 @@ export async function runDoctor(options = {}) {
|
|
|
166
166
|
});
|
|
167
167
|
}
|
|
168
168
|
if (options.json) {
|
|
169
|
-
const clientHealthReasonCodesEnabledCheckIds =
|
|
170
|
-
|
|
171
|
-
|
|
169
|
+
const [clientHealthReasonCodesEnabledCheckIds, clientHealthCheckStatusEnabledCheckIds] = await Promise.all([
|
|
170
|
+
options.resolveClientHealthReasonCodesForCompany
|
|
171
|
+
? resolveClientHealthReasonCodesCheckIds(families, options.resolveClientHealthReasonCodesForCompany)
|
|
172
|
+
: undefined,
|
|
173
|
+
options.resolveClientHealthCheckStatusForCompany
|
|
174
|
+
? resolveClientHealthCheckStatusCheckIds(families, options.resolveClientHealthCheckStatusForCompany)
|
|
175
|
+
: undefined,
|
|
176
|
+
]);
|
|
172
177
|
write(renderJson(buildDoctorJson({
|
|
173
178
|
hqRoot: hqRoot ?? "",
|
|
174
179
|
families,
|
|
175
180
|
platform,
|
|
176
181
|
scope,
|
|
177
182
|
clientHealthReasonCodesEnabledCheckIds,
|
|
183
|
+
clientHealthCheckStatusEnabledCheckIds,
|
|
178
184
|
})));
|
|
179
185
|
}
|
|
180
186
|
else {
|
|
@@ -220,6 +226,34 @@ async function resolveClientHealthReasonCodesCheckIds(families, resolveForCompan
|
|
|
220
226
|
}
|
|
221
227
|
return enabledChecks;
|
|
222
228
|
}
|
|
229
|
+
async function resolveClientHealthCheckStatusCheckIds(families, resolveForCompany) {
|
|
230
|
+
const checksByCompany = new Map();
|
|
231
|
+
for (const run of families) {
|
|
232
|
+
if (run.family.id !== "sync")
|
|
233
|
+
continue;
|
|
234
|
+
for (const result of run.results) {
|
|
235
|
+
if (result.status !== "FAIL" && result.status !== "WARN")
|
|
236
|
+
continue;
|
|
237
|
+
const scoped = /^sync\.(?:journal|manifest)\.(.+)$/.exec(result.checkId);
|
|
238
|
+
const companySlug = scoped && isCompanyJournalSlug(scoped[1]) ? scoped[1] : null;
|
|
239
|
+
const checkIds = checksByCompany.get(companySlug) ?? [];
|
|
240
|
+
checkIds.push(result.checkId);
|
|
241
|
+
checksByCompany.set(companySlug, checkIds);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
const enabledChecks = new Set();
|
|
245
|
+
const decisions = await Promise.all([...checksByCompany.entries()].map(async ([companySlug, checkIds]) => ({
|
|
246
|
+
checkIds,
|
|
247
|
+
enabled: await resolveForCompany(companySlug),
|
|
248
|
+
})));
|
|
249
|
+
for (const decision of decisions) {
|
|
250
|
+
if (decision.enabled) {
|
|
251
|
+
for (const checkId of decision.checkIds)
|
|
252
|
+
enabledChecks.add(checkId);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
return enabledChecks;
|
|
256
|
+
}
|
|
223
257
|
function createClientHealthReasonCodeCompanyResolver(hqRoot) {
|
|
224
258
|
const companies = readCompaniesManifestMap(hqRoot);
|
|
225
259
|
return async (companySlug) => {
|
|
@@ -245,6 +279,37 @@ function createClientHealthReasonCodeCompanyResolver(hqRoot) {
|
|
|
245
279
|
});
|
|
246
280
|
};
|
|
247
281
|
}
|
|
282
|
+
export function createClientHealthCheckStatusCompanyResolver(hqRoot) {
|
|
283
|
+
const companies = readCompaniesManifestMap(hqRoot);
|
|
284
|
+
return async (companySlug) => {
|
|
285
|
+
if (companySlug === null) {
|
|
286
|
+
return isActiveCompanyClientHealthCheckStatusEnabled();
|
|
287
|
+
}
|
|
288
|
+
const scopedCompanyUid = /^company:(cmp_[A-Za-z0-9]{3,128})$/.exec(companySlug);
|
|
289
|
+
if (scopedCompanyUid) {
|
|
290
|
+
return clientHealthCheckStatusEnabledForCompany({
|
|
291
|
+
companyUid: scopedCompanyUid[1],
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(companySlug)) {
|
|
295
|
+
return false;
|
|
296
|
+
}
|
|
297
|
+
const manifestUid = companies?.[companySlug]?.cloud_uid;
|
|
298
|
+
const activeSlug = process.env.HQ_COMPANY_SLUG?.trim() || "";
|
|
299
|
+
const activeUid = process.env.HQ_COMPANY_UID?.trim() || "";
|
|
300
|
+
const companyUid = typeof manifestUid === "string" && manifestUid.trim()
|
|
301
|
+
? manifestUid.trim()
|
|
302
|
+
: companySlug === activeSlug
|
|
303
|
+
? activeUid
|
|
304
|
+
: "";
|
|
305
|
+
if (!/^cmp_[A-Za-z0-9]{3,128}$/.test(companyUid))
|
|
306
|
+
return false;
|
|
307
|
+
return clientHealthCheckStatusEnabledForCompany({
|
|
308
|
+
companyUid,
|
|
309
|
+
companySlug,
|
|
310
|
+
});
|
|
311
|
+
};
|
|
312
|
+
}
|
|
248
313
|
/** Register the top-level `hq doctor` command so it appears in `hq --help`. */
|
|
249
314
|
export function registerDoctorCommand(program) {
|
|
250
315
|
program
|
|
@@ -334,6 +399,14 @@ export function registerDoctorCommand(program) {
|
|
|
334
399
|
: undefined;
|
|
335
400
|
})()
|
|
336
401
|
: undefined;
|
|
402
|
+
const clientHealthCheckStatusResolver = json && opts.clientHealthReport === true
|
|
403
|
+
? (() => {
|
|
404
|
+
const hqRoot = resolveHqRoot();
|
|
405
|
+
return hqRoot
|
|
406
|
+
? createClientHealthCheckStatusCompanyResolver(hqRoot)
|
|
407
|
+
: undefined;
|
|
408
|
+
})()
|
|
409
|
+
: undefined;
|
|
337
410
|
const result = await runDoctor({
|
|
338
411
|
json,
|
|
339
412
|
// `src/main.ts` also declares --verbose on the root program. When
|
|
@@ -350,6 +423,7 @@ export function registerDoctorCommand(program) {
|
|
|
350
423
|
company: opts.company,
|
|
351
424
|
only: parseOnly(opts.only),
|
|
352
425
|
resolveClientHealthReasonCodesForCompany: clientHealthReasonCodesResolver,
|
|
426
|
+
resolveClientHealthCheckStatusForCompany: clientHealthCheckStatusResolver,
|
|
353
427
|
});
|
|
354
428
|
// Set the exit code rather than calling process.exit, so the CLI's
|
|
355
429
|
// normal shutdown (telemetry flush) still runs. Non-zero means either an
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
import chalk from "chalk";
|
|
20
20
|
import open from "open";
|
|
21
21
|
import { ensureCognitoIdToken } from "../utils/cognito-session.js";
|
|
22
|
+
import { safeCaughtError } from "../utils/safe-caught-error.js";
|
|
22
23
|
import { runResidentCommandWithPackageLease, } from "../utils/resident-package-preload.js";
|
|
23
24
|
import { consoleOrigin } from "../utils/console-origin.js";
|
|
24
25
|
export { consoleOrigin };
|
|
@@ -29,6 +30,9 @@ import { completeOAuth, discoverDocs, installIntegration, listCatalog, pullBluep
|
|
|
29
30
|
import { OAUTH_LOOPBACK_TIMEOUT_CODE, startLoopbackListener, } from "./integrations-oauth.js";
|
|
30
31
|
/** hq-pro's machine code for "this endpoint needs a browser sign-in". */
|
|
31
32
|
const OAUTH_REQUIRED_CODE = "INTEGRATION_FACTORY_OAUTH_REQUIRED";
|
|
33
|
+
function warnBrowserOpenFailure(error) {
|
|
34
|
+
console.warn("[hq-cli:integrations-connect] browser open failed", safeCaughtError(error));
|
|
35
|
+
}
|
|
32
36
|
/**
|
|
33
37
|
* A server-authoritative "this endpoint is OAuth-protected, run the browser
|
|
34
38
|
* sign-in instead" signal, in EITHER of the two shapes hq-pro emits it:
|
|
@@ -679,7 +683,7 @@ async function consoleHandoff(target, opts) {
|
|
|
679
683
|
console.error(chalk.yellow(`Note: --scopes cannot be carried into a console sign-in, so this connect will use ${target.label}'s default permissions.`));
|
|
680
684
|
}
|
|
681
685
|
if (url && opts.browser !== false)
|
|
682
|
-
await open(url).catch(() =>
|
|
686
|
+
await open(url).catch((error) => warnBrowserOpenFailure(error));
|
|
683
687
|
console.error(chalk.dim("When the browser says it connected, run `hq integrations list` to confirm."));
|
|
684
688
|
return null;
|
|
685
689
|
}
|
|
@@ -712,7 +716,7 @@ export async function bringYourOwnClientHandoff(target, opts, reason, redirectUr
|
|
|
712
716
|
console.error("The console shows the callback URL to register and takes the credentials.");
|
|
713
717
|
printConsoleDestination(target, opts, url);
|
|
714
718
|
if (url && opts.browser !== false)
|
|
715
|
-
await open(url).catch(() =>
|
|
719
|
+
await open(url).catch((error) => warnBrowserOpenFailure(error));
|
|
716
720
|
return null;
|
|
717
721
|
}
|
|
718
722
|
/**
|
|
@@ -837,7 +841,7 @@ async function userClientSignIn(token, companyUid, target, opts, client, request
|
|
|
837
841
|
if (googleHint)
|
|
838
842
|
console.error(chalk.yellow(googleHint));
|
|
839
843
|
if (opts.browser !== false)
|
|
840
|
-
await open(started.authorizationUrl).catch(() =>
|
|
844
|
+
await open(started.authorizationUrl).catch((error) => warnBrowserOpenFailure(error));
|
|
841
845
|
console.error(chalk.dim("The browser finishes on the HQ console. When it says connected, " +
|
|
842
846
|
"run `hq integrations list` to confirm."));
|
|
843
847
|
return null;
|
package/dist/commands/lanes.js
CHANGED
|
@@ -62,7 +62,7 @@ import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, lan
|
|
|
62
62
|
import { sessionIdFromEnv } from "../lib/mesh/live/session-identity.js";
|
|
63
63
|
import { listLaneReviews } from "../lib/lanes/reviews.js";
|
|
64
64
|
import { effectiveAccountHome } from "../lib/lanes/account-home.js";
|
|
65
|
-
import { AccountResolutionError, compareLaneQueueOrder, LANE_PROVIDERS, LANE_STATES, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, isFinishedResumableHireChild, aliveDirectChildren, assignedChildHireGrants, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, laneStoryLedgerEnabled, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listLaneIds, listQuestionIds, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneRecord, parseLaneState, parseQuestionRecord, parseSeniorKind, readAllLanes, readLane, sanitizeHostComponent, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, loadWorkerProfileWithRegistry, registryWorkerRecord, runReconcilePass, DEAD_HOST_MS, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, dropboxDir, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, providerSessionPortable, sessionNotPortableMessage, withLaneAdmissionLock, withAllLaneAdmissionLocks, readWorkerRunResult, resumeCommandRefusal, withLaneHireLocks, raiseLaneQuestion, sha256Text, } from "../lib/lanes/index.js";
|
|
65
|
+
import { AccountResolutionError, compareLaneQueueOrder, LANE_PROVIDERS, LANE_STATES, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, isFinishedResumableHireChild, aliveDirectChildren, childrenHoldingHireGrantCarve, laneHoldsHireGrantCarve, assignedChildHireGrants, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, laneStoryLedgerEnabled, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listLaneIds, listQuestionIds, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneRecord, parseLaneState, parseQuestionRecord, parseSeniorKind, readAllLanes, readLane, sanitizeHostComponent, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, loadWorkerProfileWithRegistry, registryWorkerRecord, runReconcilePass, DEAD_HOST_MS, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, dropboxDir, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, providerSessionPortable, sessionNotPortableMessage, withLaneAdmissionLock, withAllLaneAdmissionLocks, readWorkerRunResult, resumeCommandRefusal, withLaneHireLocks, raiseLaneQuestion, sha256Text, } from "../lib/lanes/index.js";
|
|
66
66
|
import { laneStories, storyWarning, } from "../lib/lanes/stories.js";
|
|
67
67
|
import { resolveLaneKeepAlive } from "../lib/lanes/profile.js";
|
|
68
68
|
import { laneCompanyUidSource, resolveLaneCompany } from "../lib/lanes/lane-company.js";
|
|
@@ -507,7 +507,12 @@ function laneHireFields(lane, lanes, aliveChildrenByParent, finishedChildrenDoNo
|
|
|
507
507
|
};
|
|
508
508
|
}
|
|
509
509
|
function hireGrantFinishedChildrenDoNotCount(ctx, lane) {
|
|
510
|
-
const resolution = resolveLaneCompany(ctx.hqRoot(),
|
|
510
|
+
const resolution = resolveLaneCompany(ctx.hqRoot(), {
|
|
511
|
+
lane_id: lane.lane_id,
|
|
512
|
+
company: lane.company ?? null,
|
|
513
|
+
mesh_company: lane.mesh_company,
|
|
514
|
+
project_id: lane.project_id ?? "",
|
|
515
|
+
});
|
|
511
516
|
const company = resolution.company ?? undefined;
|
|
512
517
|
return readLanesRegistryFlagFailOff(LANE_HIRE_GRANT_FINISHED_LANES_FLAG, {
|
|
513
518
|
company,
|
|
@@ -1036,9 +1041,15 @@ async function withResumeHireAdmission(ctx, lane, fn, grantContext) {
|
|
|
1036
1041
|
!sameHireGrantSenior(lane.senior, grantContext.expected_senior))) {
|
|
1037
1042
|
throw new LanesError("hire_grant_senior_changed", `Lane ${lane.lane_id}'s senior changed after the primary authorized its Outpost dispatch`, { lane_id: lane.lane_id, expected_senior: grantContext.expected_senior, current_senior: lane.senior });
|
|
1038
1043
|
}
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
:
|
|
1044
|
+
let finishedChildrenDoNotCount = false;
|
|
1045
|
+
if (isTerminalLaneState(lane.state) || isFinishedResumableHireChild(lane.state)) {
|
|
1046
|
+
const seniorLaneId = lane.senior.kind === "lane" ? lane.senior.id : undefined;
|
|
1047
|
+
const resumeSenior = seniorLaneId
|
|
1048
|
+
? grantContext?.hierarchy.find((entry) => entry.lane_id === seniorLaneId) ??
|
|
1049
|
+
(listLaneIds(hqRoot).includes(seniorLaneId) ? readLane(hqRoot, seniorLaneId) : lane)
|
|
1050
|
+
: lane;
|
|
1051
|
+
finishedChildrenDoNotCount = await hireGrantFinishedChildrenDoNotCount(ctx, resumeSenior);
|
|
1052
|
+
}
|
|
1042
1053
|
const shouldCheckGrant = isTerminalLaneState(lane.state) ||
|
|
1043
1054
|
(finishedChildrenDoNotCount && isFinishedResumableHireChild(lane.state));
|
|
1044
1055
|
if (!shouldCheckGrant ||
|
|
@@ -5295,6 +5306,7 @@ async function runCapacity(ctx) {
|
|
|
5295
5306
|
env: ctx.env,
|
|
5296
5307
|
hostname: ctx.hostname(),
|
|
5297
5308
|
now: ctx.now(),
|
|
5309
|
+
quotaState: loadQuotaState(hqRoot),
|
|
5298
5310
|
});
|
|
5299
5311
|
printJson({
|
|
5300
5312
|
...snapshot,
|
|
@@ -5736,6 +5748,7 @@ async function applyHireGrant(ctx, laneId, hireGrant, authorization, grantContex
|
|
|
5736
5748
|
initial.senior.kind === "lane" && initial.senior.id
|
|
5737
5749
|
? await readRemoteHireGrantSenior(ctx, hqRoot, initial.senior.id)
|
|
5738
5750
|
: undefined;
|
|
5751
|
+
const finishedChildrenDoNotCount = await hireGrantFinishedChildrenDoNotCount(ctx, initial);
|
|
5739
5752
|
const updated = ctx.withGrantHireLocks(hqRoot, lockedLaneIds, () => {
|
|
5740
5753
|
const current = readLane(hqRoot, laneId);
|
|
5741
5754
|
const currentCallerAuthorization = authorization === "senior"
|
|
@@ -5779,11 +5792,11 @@ async function applyHireGrant(ctx, laneId, hireGrant, authorization, grantContex
|
|
|
5779
5792
|
if (isTerminalLaneState(parent.state)) {
|
|
5780
5793
|
throw new LanesError("hire_grant_senior_inactive", `Senior lane ${parent.lane_id} is ${parent.state} and cannot change grants`, { senior_lane_id: parent.lane_id, state: parent.state });
|
|
5781
5794
|
}
|
|
5782
|
-
const siblings =
|
|
5795
|
+
const siblings = childrenHoldingHireGrantCarve(hierarchy, parent.lane_id, finishedChildrenDoNotCount);
|
|
5783
5796
|
const parentGrant = parent.hire_grant ?? 0;
|
|
5784
5797
|
const assigned = assignedChildHireGrants(siblings, current.lane_id);
|
|
5785
5798
|
const available = Math.max(0, parentGrant - assigned);
|
|
5786
|
-
if (hireGrant > available) {
|
|
5799
|
+
if (hireGrant > (current.hire_grant ?? 0) && hireGrant > available) {
|
|
5787
5800
|
throw new LanesError("hire_grant_carve_exceeded", `Senior lane ${parent.lane_id} has ${available} unassigned hire grant remaining for lane ${laneId}`, {
|
|
5788
5801
|
senior_lane_id: parent.lane_id,
|
|
5789
5802
|
parent_grant: parentGrant,
|
|
@@ -5794,7 +5807,7 @@ async function applyHireGrant(ctx, laneId, hireGrant, authorization, grantContex
|
|
|
5794
5807
|
});
|
|
5795
5808
|
}
|
|
5796
5809
|
}
|
|
5797
|
-
const children =
|
|
5810
|
+
const children = childrenHoldingHireGrantCarve(hierarchy, current.lane_id, finishedChildrenDoNotCount);
|
|
5798
5811
|
const allocated = assignedChildHireGrants(children);
|
|
5799
5812
|
const minimumGrant = Math.max(allocated, children.length);
|
|
5800
5813
|
if (hireGrant < minimumGrant) {
|
|
@@ -7063,6 +7076,7 @@ async function reparentOnOwner(ctx, laneId, opts) {
|
|
|
7063
7076
|
const nextSenior = parseSeniorRef(opts.senior);
|
|
7064
7077
|
assertSeniorExists(hqRoot, nextSenior, ctx);
|
|
7065
7078
|
const initial = readLane(hqRoot, laneId);
|
|
7079
|
+
const finishedChildrenDoNotCount = await hireGrantFinishedChildrenDoNotCount(ctx, initial);
|
|
7066
7080
|
const refreshParents = new Set();
|
|
7067
7081
|
if (initial.senior.kind === "lane" && initial.senior.id)
|
|
7068
7082
|
refreshParents.add(initial.senior.id);
|
|
@@ -7074,7 +7088,7 @@ async function reparentOnOwner(ctx, laneId, opts) {
|
|
|
7074
7088
|
...(initial.senior.kind === "lane" && initial.senior.id ? [initial.senior.id] : []),
|
|
7075
7089
|
...(nextSenior.kind === "lane" && nextSenior.id ? [nextSenior.id] : []),
|
|
7076
7090
|
];
|
|
7077
|
-
return withLaneHireLocks(hqRoot, lockIds, () => runReparentLocked(ctx, laneId, opts));
|
|
7091
|
+
return withLaneHireLocks(hqRoot, lockIds, () => runReparentLocked(ctx, laneId, opts, finishedChildrenDoNotCount));
|
|
7078
7092
|
}
|
|
7079
7093
|
async function runReparentOwnerTarget(ctx, laneId, opts) {
|
|
7080
7094
|
if (!ctx.isSsmOwnerDispatch()) {
|
|
@@ -7094,7 +7108,7 @@ async function runReparentOwnerTarget(ctx, laneId, opts) {
|
|
|
7094
7108
|
changed_descendant_ids: result.changedDescendantIds,
|
|
7095
7109
|
});
|
|
7096
7110
|
}
|
|
7097
|
-
function runReparentLocked(ctx, laneId, opts) {
|
|
7111
|
+
function runReparentLocked(ctx, laneId, opts, finishedChildrenDoNotCount = false) {
|
|
7098
7112
|
const hqRoot = ctx.hqRoot();
|
|
7099
7113
|
if (!opts.senior) {
|
|
7100
7114
|
throw new LanesError("missing_senior", "--senior is required", {});
|
|
@@ -7108,26 +7122,27 @@ function runReparentLocked(ctx, laneId, opts) {
|
|
|
7108
7122
|
}
|
|
7109
7123
|
const seniorCompany = companyOfSenior(hqRoot, nextSenior, ctx);
|
|
7110
7124
|
assertSameTenant(lane.company, nextSenior, seniorCompany, { lane_id: laneId });
|
|
7111
|
-
if (nextSenior.kind === "lane" &&
|
|
7112
|
-
seniorKey(lane.senior) !== seniorKey(nextSenior)) {
|
|
7125
|
+
if (nextSenior.kind === "lane" && seniorKey(lane.senior) !== seniorKey(nextSenior)) {
|
|
7113
7126
|
const parent = readLane(hqRoot, nextSenior.id);
|
|
7114
7127
|
if (isTerminalLaneState(parent.state)) {
|
|
7115
7128
|
throw new LanesError("hire_grant_senior_inactive", `Senior lane ${parent.lane_id} is ${parent.state} and cannot accept lane children`, { senior_lane_id: parent.lane_id, state: parent.state });
|
|
7116
7129
|
}
|
|
7117
|
-
|
|
7118
|
-
|
|
7119
|
-
|
|
7120
|
-
|
|
7121
|
-
|
|
7122
|
-
|
|
7123
|
-
|
|
7124
|
-
|
|
7125
|
-
|
|
7126
|
-
|
|
7127
|
-
|
|
7128
|
-
|
|
7129
|
-
|
|
7130
|
-
|
|
7130
|
+
if (laneHoldsHireGrantCarve(lane, finishedChildrenDoNotCount)) {
|
|
7131
|
+
const children = childrenHoldingHireGrantCarve(loadAllLanes(hqRoot), parent.lane_id, finishedChildrenDoNotCount);
|
|
7132
|
+
if (children.length >= (parent.hire_grant ?? 0)) {
|
|
7133
|
+
throw hireGrantCapacityError(parent, children);
|
|
7134
|
+
}
|
|
7135
|
+
const available = Math.max(0, (parent.hire_grant ?? 0) - assignedChildHireGrants(children));
|
|
7136
|
+
const requested = lane.hire_grant ?? 0;
|
|
7137
|
+
if (requested > available) {
|
|
7138
|
+
throw new LanesError("hire_grant_carve_exceeded", `Senior lane ${parent.lane_id} has ${available} unassigned hire grant remaining for lane ${laneId}`, {
|
|
7139
|
+
senior_lane_id: parent.lane_id,
|
|
7140
|
+
parent_grant: parent.hire_grant ?? 0,
|
|
7141
|
+
assigned_child_grants: assignedChildHireGrants(children),
|
|
7142
|
+
requested_hire_grant: requested,
|
|
7143
|
+
available_hire_grant: available,
|
|
7144
|
+
});
|
|
7145
|
+
}
|
|
7131
7146
|
}
|
|
7132
7147
|
}
|
|
7133
7148
|
if (walkHitsLane(hqRoot, nextSenior, laneId, ctx)) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export declare const CLI_PERSONAL_TARGETED_PUSH_PATHS_FLAG = "cli.personal-targeted-push-paths";
|
|
2
2
|
/** The established full-scope behavior remains in force until an explicit hq-flags enablement. */
|
|
3
|
-
export declare function personalTargetedPushPathsEnabled(): boolean
|
|
3
|
+
export declare function personalTargetedPushPathsEnabled(): Promise<boolean>;
|
|
4
4
|
//# sourceMappingURL=personal-targeted-push-flag.d.ts.map
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
-
import { resolveProcessFlagGate } from "../lib/flag-registry.js";
|
|
1
|
+
import { resolveProcessFlagGate, waitForFlagRegistryRefresh, } from "../lib/flag-registry.js";
|
|
2
2
|
export const CLI_PERSONAL_TARGETED_PUSH_PATHS_FLAG = "cli.personal-targeted-push-paths";
|
|
3
3
|
/** The established full-scope behavior remains in force until an explicit hq-flags enablement. */
|
|
4
|
-
export function personalTargetedPushPathsEnabled() {
|
|
4
|
+
export async function personalTargetedPushPathsEnabled() {
|
|
5
|
+
// This command reads the gate immediately after startup. Give the existing
|
|
6
|
+
// unref'd registry worker one bounded window to publish its company snapshot.
|
|
7
|
+
// An unavailable registry leaves resolveProcessFlagGate on the legacy false.
|
|
8
|
+
await waitForFlagRegistryRefresh(500);
|
|
5
9
|
return resolveProcessFlagGate(CLI_PERSONAL_TARGETED_PUSH_PATHS_FLAG, { fallback: false }, () => false);
|
|
6
10
|
}
|
|
7
11
|
//# sourceMappingURL=personal-targeted-push-flag.js.map
|
|
@@ -4,6 +4,7 @@ import * as path from "node:path";
|
|
|
4
4
|
import { ensureCognitoIdToken } from "../../utils/cognito-session.js";
|
|
5
5
|
import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
|
|
6
6
|
import { vaultApiFetch } from "../../utils/vault-api.js";
|
|
7
|
+
import { safeCaughtError } from "../../utils/safe-caught-error.js";
|
|
7
8
|
async function readOwnVault(input) {
|
|
8
9
|
const client = new S3Client({ region: "us-east-1", credentials: input.credentials });
|
|
9
10
|
try {
|
|
@@ -52,7 +53,9 @@ export async function downloadContinuitySnapshot(input, deps = { credential: ens
|
|
|
52
53
|
}
|
|
53
54
|
}
|
|
54
55
|
finally {
|
|
55
|
-
await reader.cancel().catch(() => {
|
|
56
|
+
await reader.cancel().catch((error) => {
|
|
57
|
+
console.warn("[hq-cli:bot/continuity-download] reader.cancel failed", safeCaughtError(error));
|
|
58
|
+
});
|
|
56
59
|
download.close();
|
|
57
60
|
}
|
|
58
61
|
const bytes = Buffer.concat(chunks);
|
|
@@ -48,6 +48,7 @@ const SENTRY_REPORT_LEVELS = new Set([
|
|
|
48
48
|
const HOOK_TIMEOUT_WARNING_TYPE = "hook_timeout_warning";
|
|
49
49
|
const HOOK_TIMEOUT_EXCEEDED_TYPE = "hook_timeout_exceeded";
|
|
50
50
|
const HOOK_LATE_FINISH_TYPE = "hook_late_finish";
|
|
51
|
+
const HOOK_TIMEOUT_FINGERPRINT = "hook-timeout";
|
|
51
52
|
const HOOK_TIMEOUT_WARNING_METRIC = "hq.hook_timeout_warning";
|
|
52
53
|
const HOOK_TIMEOUT_EXCEEDED_METRIC = "hq.hook_timeout_exceeded";
|
|
53
54
|
const MAX_HOOK_SEQUENCE_ENTRIES = 20;
|
|
@@ -275,6 +276,17 @@ function safeSlowPhase(value) {
|
|
|
275
276
|
return "unknown";
|
|
276
277
|
return SAFE_SLOW_PHASES.has(value) ? value : "other";
|
|
277
278
|
}
|
|
279
|
+
function resolvedSlowPhase(metadata) {
|
|
280
|
+
const reported = safeSlowPhase(metadata.slow_phase);
|
|
281
|
+
if (reported !== "unknown" && reported !== "other")
|
|
282
|
+
return reported;
|
|
283
|
+
// Which measured master-hook phase consumed the most time when the caller's
|
|
284
|
+
// active-phase field was absent or unusable?
|
|
285
|
+
const debugContext = metadata.hook_timeout_debug_context;
|
|
286
|
+
const slowestPhase = debugContext?.phase_timings
|
|
287
|
+
.reduce((slowest, timing) => slowest === undefined || timing.elapsed_ms > slowest.elapsed_ms ? timing : slowest, undefined);
|
|
288
|
+
return slowestPhase ? safeSlowPhase(slowestPhase.phase) : reported;
|
|
289
|
+
}
|
|
278
290
|
function boundedDebugHookEvent(value) {
|
|
279
291
|
return typeof value === "string" && DEBUG_CONTEXT_HOOK_EVENTS.has(value) ? value : "other";
|
|
280
292
|
}
|
|
@@ -534,6 +546,28 @@ function timeoutElapsedMs(metadata) {
|
|
|
534
546
|
function formatNumber(value) {
|
|
535
547
|
return value.toFixed(2).replace(/\.00$|(?<=\.[0-9])0$/, "");
|
|
536
548
|
}
|
|
549
|
+
function formatSeconds(milliseconds) {
|
|
550
|
+
return `${formatNumber(milliseconds / 1_000)}s`;
|
|
551
|
+
}
|
|
552
|
+
// How much of the declared timeout has elapsed, and how much remains?
|
|
553
|
+
function hookTimeoutMessage(event) {
|
|
554
|
+
if (event.type !== HOOK_TIMEOUT_WARNING_TYPE)
|
|
555
|
+
return event.message;
|
|
556
|
+
const elapsedMs = timeoutElapsedMs(event.metadata);
|
|
557
|
+
const declaredTimeoutMs = event.metadata.declared_timeout_ms;
|
|
558
|
+
if (elapsedMs === undefined ||
|
|
559
|
+
typeof declaredTimeoutMs !== "number" ||
|
|
560
|
+
!Number.isFinite(declaredTimeoutMs) ||
|
|
561
|
+
declaredTimeoutMs <= 0) {
|
|
562
|
+
return event.message;
|
|
563
|
+
}
|
|
564
|
+
const elapsed = formatSeconds(elapsedMs);
|
|
565
|
+
const budget = formatSeconds(declaredTimeoutMs);
|
|
566
|
+
if (elapsedMs >= declaredTimeoutMs) {
|
|
567
|
+
return `HQ hook used ${elapsed} of its ${budget} configured timeout (${formatSeconds(elapsedMs - declaredTimeoutMs)} over budget).`;
|
|
568
|
+
}
|
|
569
|
+
return `HQ hook used ${elapsed} of its ${budget} configured timeout (${formatSeconds(declaredTimeoutMs - elapsedMs)} remaining).`;
|
|
570
|
+
}
|
|
537
571
|
function elapsedBucket(elapsedMs) {
|
|
538
572
|
if (elapsedMs === undefined)
|
|
539
573
|
return "unknown";
|
|
@@ -672,9 +706,33 @@ function safeSlowChild(metadata) {
|
|
|
672
706
|
(typeof value !== "string" || value.length === 0 || value === "other")) {
|
|
673
707
|
return "entry-bash-startup";
|
|
674
708
|
}
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
709
|
+
return typeof value === "string" && SAFE_HOOK_SCRIPT.test(value) ? value : "other";
|
|
710
|
+
}
|
|
711
|
+
function resolvedSlowChild(metadata) {
|
|
712
|
+
const directName = safeSlowChild(metadata);
|
|
713
|
+
const directDuration = metadata.slow_child_ms;
|
|
714
|
+
const hasDirectDuration = typeof directDuration === "number" && Number.isFinite(directDuration) && directDuration >= 0;
|
|
715
|
+
if (directName !== "other") {
|
|
716
|
+
return { name: directName, ...(hasDirectDuration ? { durationMs: directDuration } : {}) };
|
|
717
|
+
}
|
|
718
|
+
const currentScript = hookScript(metadata);
|
|
719
|
+
const currentEvent = stringMetadata(metadata, "hook_event", "");
|
|
720
|
+
const sequence = metadata.hook_sequence;
|
|
721
|
+
if (Array.isArray(sequence)) {
|
|
722
|
+
// Which named hook from this event's serialized sequence took longest
|
|
723
|
+
// when dispatch could not directly name its slow child?
|
|
724
|
+
const slowestNamedHook = sequence
|
|
725
|
+
.filter((entry) => entry.event === currentEvent &&
|
|
726
|
+
entry.script !== currentScript &&
|
|
727
|
+
SAFE_HOOK_SCRIPT.test(entry.script) &&
|
|
728
|
+
Number.isFinite(entry.ms) &&
|
|
729
|
+
entry.ms >= 0)
|
|
730
|
+
.reduce((slowest, entry) => (slowest === undefined || entry.ms > slowest.ms ? entry : slowest), undefined);
|
|
731
|
+
if (slowestNamedHook) {
|
|
732
|
+
return { name: slowestNamedHook.script, durationMs: slowestNamedHook.ms };
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
return { name: directName, ...(hasDirectDuration ? { durationMs: directDuration } : {}) };
|
|
678
736
|
}
|
|
679
737
|
function isMachineWideHookSlowness(metadata) {
|
|
680
738
|
const spawnMs = metadata.spawn_ms;
|
|
@@ -717,7 +775,7 @@ function hookTimeoutTags(event) {
|
|
|
717
775
|
const shell = stringMetadata(metadata, "shell");
|
|
718
776
|
const nproc = metadata.nproc;
|
|
719
777
|
const loadAverage = parsedLoadAverage(metadata);
|
|
720
|
-
const
|
|
778
|
+
const slowChild = resolvedSlowChild(metadata);
|
|
721
779
|
return {
|
|
722
780
|
hook_timeout_kind: hookTimeoutKind(event),
|
|
723
781
|
hook_event: hookEvent(metadata),
|
|
@@ -738,11 +796,9 @@ function hookTimeoutTags(event) {
|
|
|
738
796
|
load_bucket: typeof loadAverage === "string" ? loadAverage : loadBucket(loadAverage),
|
|
739
797
|
root_storage_class: rootStorageClassTag(metadata),
|
|
740
798
|
spawn_ms_bucket: spawnMsBucket(metadata),
|
|
741
|
-
slow_child:
|
|
742
|
-
slow_phase:
|
|
743
|
-
slow_child_ms_bucket:
|
|
744
|
-
? elapsedBucket(slowChildMs)
|
|
745
|
-
: "unknown",
|
|
799
|
+
slow_child: slowChild.name,
|
|
800
|
+
slow_phase: resolvedSlowPhase(metadata),
|
|
801
|
+
slow_child_ms_bucket: elapsedBucket(slowChild.durationMs),
|
|
746
802
|
machine_wide: isMachineWideHookSlowness(metadata) ? "true" : "false",
|
|
747
803
|
nproc: typeof nproc === "number" || typeof nproc === "string"
|
|
748
804
|
? safeTagValue(String(nproc), SAFE_NPROC)
|
|
@@ -777,7 +833,7 @@ function sentryEnvelope(event, overrides, additionalTags = {}) {
|
|
|
777
833
|
/** Build the same scrubbed event projection that the live SDK path uses. */
|
|
778
834
|
function scrubbedSentryEnvelope(event, overrides, additionalTags = {}) {
|
|
779
835
|
return beforeSend({
|
|
780
|
-
message: event
|
|
836
|
+
message: hookTimeoutMessage(event),
|
|
781
837
|
...sentryEnvelope(event, overrides, additionalTags),
|
|
782
838
|
}, {});
|
|
783
839
|
}
|
|
@@ -787,7 +843,7 @@ function hookTimeoutExceededOverrides(event) {
|
|
|
787
843
|
return undefined;
|
|
788
844
|
return {
|
|
789
845
|
level: "warning",
|
|
790
|
-
fingerprint: [
|
|
846
|
+
fingerprint: [HOOK_TIMEOUT_FINGERPRINT, hookTimeoutKind(event)],
|
|
791
847
|
};
|
|
792
848
|
}
|
|
793
849
|
/**
|
|
@@ -1090,7 +1146,7 @@ export async function reportSentryEvent(stdin, options = {}, dependencies = defa
|
|
|
1090
1146
|
: {};
|
|
1091
1147
|
const envelope = sentryEnvelope(event, overrides, { ...debugContextTags, ...tags });
|
|
1092
1148
|
try {
|
|
1093
|
-
dependencies.sentry.captureMessage(event
|
|
1149
|
+
dependencies.sentry.captureMessage(hookTimeoutMessage(event), envelope);
|
|
1094
1150
|
rateLimit.commitCapture();
|
|
1095
1151
|
if (await flushWithin(dependencies.sentry, timeoutMs))
|
|
1096
1152
|
return result;
|
|
@@ -1104,7 +1160,7 @@ export async function reportSentryEvent(stdin, options = {}, dependencies = defa
|
|
|
1104
1160
|
}
|
|
1105
1161
|
const envelope = sentryEnvelope(event, overrides, debugContextTags);
|
|
1106
1162
|
try {
|
|
1107
|
-
dependencies.sentry.captureMessage(event
|
|
1163
|
+
dependencies.sentry.captureMessage(hookTimeoutMessage(event), envelope);
|
|
1108
1164
|
if (await flushWithin(dependencies.sentry, timeoutMs))
|
|
1109
1165
|
return result;
|
|
1110
1166
|
}
|
|
@@ -2,6 +2,8 @@ import { type FlagClient, type FlagClientOptions } from "@indigoai-us/hq-flags-c
|
|
|
2
2
|
export declare const CLIENT_HEALTH_REASON_CODES_FLAG = "client-health-sync-reason-codes";
|
|
3
3
|
export declare const CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK = false;
|
|
4
4
|
export declare const CLIENT_HEALTH_REASON_CODES_FLAG_TIMEOUT_MS = 150;
|
|
5
|
+
export declare const CLIENT_HEALTH_CHECK_STATUS_FLAG = "client-health-sync-check-status";
|
|
6
|
+
export declare const CLIENT_HEALTH_CHECK_STATUS_FLAG_FALLBACK = false;
|
|
5
7
|
type ClientHealthFlagClient = Pick<FlagClient, "ready" | "isEnabled" | "close">;
|
|
6
8
|
export interface ClientHealthReasonCodesFlagDependencies {
|
|
7
9
|
env?: NodeJS.ProcessEnv;
|
|
@@ -21,7 +23,11 @@ export interface ClientHealthReasonCodesCompany {
|
|
|
21
23
|
* off so a flag outage cannot delay the doctor's health result.
|
|
22
24
|
*/
|
|
23
25
|
export declare function clientHealthReasonCodesEnabled(dependencies?: ClientHealthReasonCodesFlagDependencies, callerSignal?: AbortSignal): Promise<boolean>;
|
|
26
|
+
/** Resolve the separate status-report gate for the active company. */
|
|
27
|
+
export declare function clientHealthCheckStatusEnabled(dependencies?: ClientHealthReasonCodesFlagDependencies, callerSignal?: AbortSignal): Promise<boolean>;
|
|
24
28
|
/** Resolve the same default-off gate for one explicitly identified company. */
|
|
25
29
|
export declare function clientHealthReasonCodesEnabledForCompany(company: ClientHealthReasonCodesCompany, dependencies?: ClientHealthReasonCodesFlagDependencies, callerSignal?: AbortSignal): Promise<boolean>;
|
|
30
|
+
/** Resolve the separate default-off check-status gate for one company. */
|
|
31
|
+
export declare function clientHealthCheckStatusEnabledForCompany(company: ClientHealthReasonCodesCompany, dependencies?: ClientHealthReasonCodesFlagDependencies, callerSignal?: AbortSignal): Promise<boolean>;
|
|
26
32
|
export {};
|
|
27
33
|
//# sourceMappingURL=client-health-reason-codes-flag.d.ts.map
|
|
@@ -4,12 +4,14 @@ import { getClientHealthFlagToken } from "./client-health-flag-token.js";
|
|
|
4
4
|
export const CLIENT_HEALTH_REASON_CODES_FLAG = "client-health-sync-reason-codes";
|
|
5
5
|
export const CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK = false;
|
|
6
6
|
export const CLIENT_HEALTH_REASON_CODES_FLAG_TIMEOUT_MS = 150;
|
|
7
|
+
export const CLIENT_HEALTH_CHECK_STATUS_FLAG = "client-health-sync-check-status";
|
|
8
|
+
export const CLIENT_HEALTH_CHECK_STATUS_FLAG_FALLBACK = false;
|
|
7
9
|
function safeErrorClass(error) {
|
|
8
10
|
const name = error instanceof Error ? error.name : "UnknownError";
|
|
9
11
|
return /^[A-Za-z][A-Za-z0-9]{0,63}$/.test(name) ? name : "UnknownError";
|
|
10
12
|
}
|
|
11
13
|
function reportFlagFailure(error) {
|
|
12
|
-
process.stderr.write(`HQ client-health
|
|
14
|
+
process.stderr.write(`HQ client-health flag lookup failed (${safeErrorClass(error)}); using default off.\n`);
|
|
13
15
|
}
|
|
14
16
|
function combineSignals(first, second) {
|
|
15
17
|
return first ? AbortSignal.any([first, second]) : second;
|
|
@@ -30,13 +32,32 @@ export async function clientHealthReasonCodesEnabled(dependencies = {}, callerSi
|
|
|
30
32
|
companySlug: env.HQ_COMPANY_SLUG?.trim() || undefined,
|
|
31
33
|
}, dependencies, callerSignal);
|
|
32
34
|
}
|
|
35
|
+
/** Resolve the separate status-report gate for the active company. */
|
|
36
|
+
export async function clientHealthCheckStatusEnabled(dependencies = {}, callerSignal) {
|
|
37
|
+
const env = dependencies.env ?? process.env;
|
|
38
|
+
const companyUid = env.HQ_COMPANY_UID?.trim() || "";
|
|
39
|
+
if (!/^cmp_[A-Za-z0-9]{3,128}$/.test(companyUid)) {
|
|
40
|
+
return CLIENT_HEALTH_CHECK_STATUS_FLAG_FALLBACK;
|
|
41
|
+
}
|
|
42
|
+
return clientHealthCheckStatusEnabledForCompany({
|
|
43
|
+
companyUid,
|
|
44
|
+
companySlug: env.HQ_COMPANY_SLUG?.trim() || undefined,
|
|
45
|
+
}, dependencies, callerSignal);
|
|
46
|
+
}
|
|
33
47
|
/** Resolve the same default-off gate for one explicitly identified company. */
|
|
34
48
|
export async function clientHealthReasonCodesEnabledForCompany(company, dependencies = {}, callerSignal) {
|
|
49
|
+
return clientHealthFlagEnabledForCompany(CLIENT_HEALTH_REASON_CODES_FLAG, CLIENT_HEALTH_REASON_CODES_FLAG_FALLBACK, company, dependencies, callerSignal);
|
|
50
|
+
}
|
|
51
|
+
/** Resolve the separate default-off check-status gate for one company. */
|
|
52
|
+
export async function clientHealthCheckStatusEnabledForCompany(company, dependencies = {}, callerSignal) {
|
|
53
|
+
return clientHealthFlagEnabledForCompany(CLIENT_HEALTH_CHECK_STATUS_FLAG, CLIENT_HEALTH_CHECK_STATUS_FLAG_FALLBACK, company, dependencies, callerSignal);
|
|
54
|
+
}
|
|
55
|
+
async function clientHealthFlagEnabledForCompany(flagKey, fallback, company, dependencies, callerSignal) {
|
|
35
56
|
const env = dependencies.env ?? process.env;
|
|
36
57
|
const endpoint = resolveFlagRegistryEndpoint(undefined, env);
|
|
37
58
|
const companyUid = company.companyUid.trim();
|
|
38
|
-
if (!/^cmp_[A-Za-z0-9]{3,128}$/.test(companyUid)) {
|
|
39
|
-
return
|
|
59
|
+
if (!endpoint || !/^cmp_[A-Za-z0-9]{3,128}$/.test(companyUid)) {
|
|
60
|
+
return fallback;
|
|
40
61
|
}
|
|
41
62
|
const companySlug = company.companySlug?.trim() || "";
|
|
42
63
|
if (callerSignal?.aborted)
|
|
@@ -44,7 +65,7 @@ export async function clientHealthReasonCodesEnabledForCompany(company, dependen
|
|
|
44
65
|
let client;
|
|
45
66
|
let failed = false;
|
|
46
67
|
let failureReported = false;
|
|
47
|
-
let enabled =
|
|
68
|
+
let enabled = fallback;
|
|
48
69
|
const reportFailureOnce = (error) => {
|
|
49
70
|
failed = true;
|
|
50
71
|
if (failureReported)
|
|
@@ -83,8 +104,8 @@ export async function clientHealthReasonCodesEnabledForCompany(company, dependen
|
|
|
83
104
|
});
|
|
84
105
|
await client.ready();
|
|
85
106
|
if (!failed && !signal.aborted) {
|
|
86
|
-
enabled = client.isEnabled(
|
|
87
|
-
fallback
|
|
107
|
+
enabled = client.isEnabled(flagKey, {
|
|
108
|
+
fallback,
|
|
88
109
|
});
|
|
89
110
|
}
|
|
90
111
|
}
|
|
@@ -100,7 +121,7 @@ export async function clientHealthReasonCodesEnabledForCompany(company, dependen
|
|
|
100
121
|
}
|
|
101
122
|
}
|
|
102
123
|
return failed || signal.aborted
|
|
103
|
-
?
|
|
124
|
+
? fallback
|
|
104
125
|
: enabled;
|
|
105
126
|
}
|
|
106
127
|
//# sourceMappingURL=client-health-reason-codes-flag.js.map
|
|
@@ -54,6 +54,8 @@ export interface DoctorJsonResult {
|
|
|
54
54
|
reasonCode?: SyncReasonCode;
|
|
55
55
|
/** Present only when this result's company opted in through hq-flags. */
|
|
56
56
|
clientHealthReasonCodesEnabled?: true;
|
|
57
|
+
/** Present only when this result's company opted in to status reporting through hq-flags. */
|
|
58
|
+
clientHealthCheckStatusEnabled?: true;
|
|
57
59
|
/** What was inspected. Present only when the result carries one. */
|
|
58
60
|
target?: string;
|
|
59
61
|
/** Human-readable explanation. */
|
|
@@ -100,6 +102,8 @@ export interface BuildDoctorJsonInput {
|
|
|
100
102
|
scope?: readonly string[];
|
|
101
103
|
/** Check ids whose own company enabled diagnostic reason codes in hq-flags. */
|
|
102
104
|
clientHealthReasonCodesEnabledCheckIds?: ReadonlySet<string>;
|
|
105
|
+
/** Check ids whose own company enabled per-check status reporting in hq-flags. */
|
|
106
|
+
clientHealthCheckStatusEnabledCheckIds?: ReadonlySet<string>;
|
|
103
107
|
}
|
|
104
108
|
/** Assemble the machine-readable document from a completed run. */
|
|
105
109
|
export declare function buildDoctorJson(input: BuildDoctorJsonInput): DoctorJsonDocument;
|
|
@@ -34,7 +34,7 @@ export function buildDoctorJson(input) {
|
|
|
34
34
|
const results = [];
|
|
35
35
|
for (const run of input.families) {
|
|
36
36
|
for (const result of run.results) {
|
|
37
|
-
results.push(toJsonResult(run.family.id, result, input.clientHealthReasonCodesEnabledCheckIds));
|
|
37
|
+
results.push(toJsonResult(run.family.id, result, input.clientHealthReasonCodesEnabledCheckIds, input.clientHealthCheckStatusEnabledCheckIds));
|
|
38
38
|
}
|
|
39
39
|
}
|
|
40
40
|
return {
|
|
@@ -48,7 +48,7 @@ export function buildDoctorJson(input) {
|
|
|
48
48
|
};
|
|
49
49
|
}
|
|
50
50
|
/** Convert one {@link CheckResult} into its JSON form, omitting empty optionals. */
|
|
51
|
-
function toJsonResult(familyId, result, clientHealthReasonCodesEnabledCheckIds) {
|
|
51
|
+
function toJsonResult(familyId, result, clientHealthReasonCodesEnabledCheckIds, clientHealthCheckStatusEnabledCheckIds) {
|
|
52
52
|
const remediation = deriveRemediation(result);
|
|
53
53
|
return {
|
|
54
54
|
status: result.status,
|
|
@@ -58,6 +58,9 @@ function toJsonResult(familyId, result, clientHealthReasonCodesEnabledCheckIds)
|
|
|
58
58
|
...(clientHealthReasonCodesEnabledCheckIds?.has(result.checkId)
|
|
59
59
|
? { clientHealthReasonCodesEnabled: true }
|
|
60
60
|
: {}),
|
|
61
|
+
...(clientHealthCheckStatusEnabledCheckIds?.has(result.checkId)
|
|
62
|
+
? { clientHealthCheckStatusEnabled: true }
|
|
63
|
+
: {}),
|
|
61
64
|
...(result.target !== undefined ? { target: result.target } : {}),
|
|
62
65
|
message: result.message,
|
|
63
66
|
...(result.details !== undefined ? { details: result.details } : {}),
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import * as fs from "node:fs";
|
|
3
3
|
import * as path from "node:path";
|
|
4
|
+
import { safeCaughtError } from "../../utils/safe-caught-error.js";
|
|
4
5
|
import { laneRunsOnHost } from "./admission.js";
|
|
5
6
|
import { isControllerAliasOutpostId } from "./admission-host.js";
|
|
6
7
|
import { LEAD_REGISTRY_REL, validateOutpostLeadRegistry } from "./autopilot-registry.js";
|
|
@@ -783,7 +784,10 @@ async function runLeadAutopilotPassUnlocked(input) {
|
|
|
783
784
|
input.outpostId === registered.host &&
|
|
784
785
|
lead.host.kind === "local" &&
|
|
785
786
|
lead.host.id === input.hostname &&
|
|
786
|
-
await (input.readFlag ?? readLanesRegistryFlag)(LEAD_AUTOPILOT_OUTPOST_NATIVE_LEADS_FLAG, { company }).catch(() =>
|
|
787
|
+
await (input.readFlag ?? readLanesRegistryFlag)(LEAD_AUTOPILOT_OUTPOST_NATIVE_LEADS_FLAG, { company }).catch((error) => {
|
|
788
|
+
console.warn("[hq-cli:lanes/autopilot] outpost-native lead flag read failed", safeCaughtError(error));
|
|
789
|
+
return false;
|
|
790
|
+
});
|
|
787
791
|
const hostMatchesRegistry = registeredLocalHost
|
|
788
792
|
? lead.host.kind === "local"
|
|
789
793
|
: outpostNativeLead ||
|
|
@@ -2,7 +2,10 @@ import { type LaneRecord } from "./types.js";
|
|
|
2
2
|
import { LanesError } from "./errors.js";
|
|
3
3
|
export type LaneHireGrantRow = Pick<LaneRecord, "lane_id" | "state" | "senior" | "hire_grant">;
|
|
4
4
|
export declare function isFinishedResumableHireChild(state: LaneRecord["state"]): boolean;
|
|
5
|
+
export declare function laneHoldsHireGrantCarve(lane: Pick<LaneHireGrantRow, "state">, finishedChildrenDoNotCount?: boolean): boolean;
|
|
5
6
|
export declare function aliveDirectChildren<T extends LaneHireGrantRow>(lanes: readonly T[], parentLaneId: string, finishedChildrenDoNotCount?: boolean): T[];
|
|
7
|
+
/** Children whose existing hire grants occupy their senior's carve. */
|
|
8
|
+
export declare function childrenHoldingHireGrantCarve<T extends LaneHireGrantRow>(lanes: readonly T[], parentLaneId: string, finishedChildrenDoNotCount?: boolean): T[];
|
|
6
9
|
export declare function assignedChildHireGrants(children: readonly LaneHireGrantRow[], exceptLaneId?: string): number;
|
|
7
10
|
export interface HireGrantAdmissionSnapshot {
|
|
8
11
|
parent: LaneHireGrantRow;
|
package/dist/lib/lanes/grants.js
CHANGED
|
@@ -4,11 +4,18 @@ import { LanesError } from "./errors.js";
|
|
|
4
4
|
export function isFinishedResumableHireChild(state) {
|
|
5
5
|
return state === "awaiting_input" || state === "blocked" || state === "done";
|
|
6
6
|
}
|
|
7
|
+
export function laneHoldsHireGrantCarve(lane, finishedChildrenDoNotCount = false) {
|
|
8
|
+
return !isTerminalLaneState(lane.state) &&
|
|
9
|
+
!(finishedChildrenDoNotCount && isFinishedResumableHireChild(lane.state));
|
|
10
|
+
}
|
|
7
11
|
export function aliveDirectChildren(lanes, parentLaneId, finishedChildrenDoNotCount = false) {
|
|
12
|
+
return childrenHoldingHireGrantCarve(lanes, parentLaneId, finishedChildrenDoNotCount);
|
|
13
|
+
}
|
|
14
|
+
/** Children whose existing hire grants occupy their senior's carve. */
|
|
15
|
+
export function childrenHoldingHireGrantCarve(lanes, parentLaneId, finishedChildrenDoNotCount = false) {
|
|
8
16
|
return lanes.filter((lane) => lane.senior.kind === "lane" &&
|
|
9
17
|
lane.senior.id === parentLaneId &&
|
|
10
|
-
|
|
11
|
-
!(finishedChildrenDoNotCount && isFinishedResumableHireChild(lane.state)));
|
|
18
|
+
laneHoldsHireGrantCarve(lane, finishedChildrenDoNotCount));
|
|
12
19
|
}
|
|
13
20
|
export function assignedChildHireGrants(children, exceptLaneId) {
|
|
14
21
|
return children.reduce((sum, child) => child.lane_id === exceptLaneId ? sum : sum + (child.hire_grant ?? 0), 0);
|
|
@@ -101,7 +108,7 @@ export function withHireGrantAdmission(hqRoot, parentLaneId, fn, suppliedHierarc
|
|
|
101
108
|
}
|
|
102
109
|
parent = suppliedParent;
|
|
103
110
|
}
|
|
104
|
-
const children =
|
|
111
|
+
const children = childrenHoldingHireGrantCarve(allLanes, parentLaneId, finishedChildrenDoNotCount);
|
|
105
112
|
const parentGrant = parent.hire_grant ?? 0;
|
|
106
113
|
const availableChildGrant = Math.max(0, parentGrant - assignedChildHireGrants(children));
|
|
107
114
|
return fn({ parent, children, parentGrant, availableChildGrant });
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type AuthState } from "./accounts.js";
|
|
2
|
+
import { type QuotaState } from "./quota.js";
|
|
2
3
|
import type { LaneHost, LaneProvider, LaneRecord, LanesConfig } from "./types.js";
|
|
3
4
|
export interface HostCapacityAccount {
|
|
4
5
|
enabled: boolean;
|
|
@@ -8,6 +9,9 @@ export interface HostCapacityAccount {
|
|
|
8
9
|
max_running: number;
|
|
9
10
|
running: number;
|
|
10
11
|
queued: number;
|
|
12
|
+
/** Account-wide quota status when this snapshot is produced on its host. */
|
|
13
|
+
quota_held?: boolean;
|
|
14
|
+
quota_reset_at?: string;
|
|
11
15
|
/** Exact lane states used for Claude account balancing; absent on older Outposts. */
|
|
12
16
|
lane_counts?: {
|
|
13
17
|
running: number;
|
|
@@ -54,6 +58,7 @@ export declare function buildHostCapacitySnapshot(input: {
|
|
|
54
58
|
env?: NodeJS.ProcessEnv;
|
|
55
59
|
hostname?: string;
|
|
56
60
|
now: Date;
|
|
61
|
+
quotaState?: QuotaState;
|
|
57
62
|
}): HostCapacitySnapshot;
|
|
58
63
|
/** Reject malformed remote snapshots before they can influence host selection. */
|
|
59
64
|
export declare function parseHostCapacitySnapshot(raw: unknown): HostCapacitySnapshot;
|
|
@@ -2,6 +2,7 @@ import { accountStatus } from "./accounts.js";
|
|
|
2
2
|
import { accountMaxRunning } from "./config.js";
|
|
3
3
|
import { laneOccupiesRunningSlot, laneRunsOnHost, } from "./admission.js";
|
|
4
4
|
import { LanesError } from "./errors.js";
|
|
5
|
+
import { quotaBlockFor } from "./quota.js";
|
|
5
6
|
function admissionHost(host) {
|
|
6
7
|
return host.kind === "outpost"
|
|
7
8
|
? { hostname: "", outpostId: host.id }
|
|
@@ -33,6 +34,10 @@ export function buildHostCapacitySnapshot(input) {
|
|
|
33
34
|
});
|
|
34
35
|
const accountLanes = providerLanes.filter((lane) => lane.account === id);
|
|
35
36
|
const liveLanes = accountLanes.filter((lane) => lane.state === "running" || lane.state === "queued" || lane.state === "awaiting_input");
|
|
37
|
+
const quotaBlock = input.quotaState
|
|
38
|
+
? quotaBlockFor(input.quotaState, provider, id, undefined, input.now)
|
|
39
|
+
: undefined;
|
|
40
|
+
const quotaResetAt = input.quotaState?.providers[provider]?.[id]?.all_models_quota_reset_at;
|
|
36
41
|
accounts[id] = {
|
|
37
42
|
enabled: account.enabled,
|
|
38
43
|
logged_in: status.logged_in,
|
|
@@ -40,6 +45,12 @@ export function buildHostCapacitySnapshot(input) {
|
|
|
40
45
|
max_running: accountMaxRunning(providerConfig, id),
|
|
41
46
|
running: running.filter((lane) => lane.account === id).length,
|
|
42
47
|
queued: queued.filter((lane) => lane.account === id).length,
|
|
48
|
+
...(quotaBlock !== undefined
|
|
49
|
+
? {
|
|
50
|
+
quota_held: quotaBlock.blocked,
|
|
51
|
+
...(quotaResetAt ? { quota_reset_at: quotaResetAt } : {}),
|
|
52
|
+
}
|
|
53
|
+
: {}),
|
|
43
54
|
lane_counts: {
|
|
44
55
|
running: liveLanes.filter((lane) => lane.state === "running").length,
|
|
45
56
|
queued: liveLanes.filter((lane) => lane.state === "queued").length,
|
|
@@ -108,6 +119,9 @@ export function parseHostCapacitySnapshot(raw) {
|
|
|
108
119
|
if (id.trim() === "" || account === undefined ||
|
|
109
120
|
typeof account.enabled !== "boolean" ||
|
|
110
121
|
!(account.logged_in === null || typeof account.logged_in === "boolean") ||
|
|
122
|
+
(account.quota_held !== undefined && typeof account.quota_held !== "boolean") ||
|
|
123
|
+
(account.quota_reset_at !== undefined &&
|
|
124
|
+
(typeof account.quota_reset_at !== "string" || !Number.isFinite(Date.parse(account.quota_reset_at)))) ||
|
|
111
125
|
(account.auth_state !== undefined &&
|
|
112
126
|
!["file", "status", "absent", "unknown", "home_missing"].includes(account.auth_state))) {
|
|
113
127
|
throw new LanesError("host_capacity_invalid", `Malformed ${provider} account capacity`, {
|
|
@@ -124,6 +138,10 @@ export function parseHostCapacitySnapshot(raw) {
|
|
|
124
138
|
max_running: requireCount(account.max_running, `${provider}.${id}.max_running`, true),
|
|
125
139
|
running: requireCount(account.running, `${provider}.${id}.running`),
|
|
126
140
|
queued: requireCount(account.queued, `${provider}.${id}.queued`),
|
|
141
|
+
...(account.quota_held !== undefined ? { quota_held: account.quota_held } : {}),
|
|
142
|
+
...(account.quota_reset_at !== undefined
|
|
143
|
+
? { quota_reset_at: account.quota_reset_at }
|
|
144
|
+
: {}),
|
|
127
145
|
...(account.lane_counts !== undefined
|
|
128
146
|
? { lane_counts: parseLaneCounts(account.lane_counts, `${provider}.${id}.lane_counts`) }
|
|
129
147
|
: {}),
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
|
|
5
5
|
export { MAX_LANE_DEPTH } from "./constants.js";
|
|
6
|
-
export { assignedChildHireGrants, aliveDirectChildren, isFinishedResumableHireChild, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
|
|
6
|
+
export { assignedChildHireGrants, aliveDirectChildren, childrenHoldingHireGrantCarve, isFinishedResumableHireChild, laneHoldsHireGrantCarve, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
|
|
7
7
|
export type { HireGrantBackfillRow } from "./grants.js";
|
|
8
8
|
export { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, hasEarlierEligibleQueuedLane, laneOccupiesRunningSlot, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, withLaneAdmissionLock, } from "./admission.js";
|
|
9
9
|
export type { AdmissionCapReason, AdmissionHost } from "./admission.js";
|
package/dist/lib/lanes/index.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export { AccountResolutionError, LaneNotFoundError, LaneSchemaError, LaneVersionConflict, LanesConfigError, LanesError, WorkerResolutionError, isLaneVersionConflict, isLanesError, } from "./errors.js";
|
|
5
5
|
export { MAX_LANE_DEPTH } from "./constants.js";
|
|
6
|
-
export { assignedChildHireGrants, aliveDirectChildren, isFinishedResumableHireChild, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
|
|
6
|
+
export { assignedChildHireGrants, aliveDirectChildren, childrenHoldingHireGrantCarve, isFinishedResumableHireChild, laneHoldsHireGrantCarve, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, } from "./grants.js";
|
|
7
7
|
export { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingResume, hasEarlierEligibleQueuedLane, laneOccupiesRunningSlot, laneAdmissionCapReason, laneRunsOnHost, withAllLaneAdmissionLocks, withLaneAdmissionLock, } from "./admission.js";
|
|
8
8
|
export { buildHostCapacitySnapshot, chooseHostForLane, parseHostCapacitySnapshot, } from "./host-capacity.js";
|
|
9
9
|
export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STORY_STATUSES, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
|
|
@@ -36,6 +36,8 @@ export interface ProviderRejection {
|
|
|
36
36
|
retryable: boolean;
|
|
37
37
|
/** Which event on the stream carried the failure. */
|
|
38
38
|
event: "codex.turn.failed" | "claude.result.is_error" | "claude.result.login_failure" | "grok.error";
|
|
39
|
+
/** Filesystem time when the provider output was appended, when read from worker.out. */
|
|
40
|
+
output_at?: string;
|
|
39
41
|
}
|
|
40
42
|
export declare function isProviderLoginFailure(failure: ProviderRejection | null): boolean;
|
|
41
43
|
/**
|
|
@@ -271,9 +271,21 @@ export function parseProviderRejection(provider, text) {
|
|
|
271
271
|
export function readProviderRejection(runDir, provider, workerOutOffset = 0) {
|
|
272
272
|
const outPath = path.join(runDir, "logs", "worker.out");
|
|
273
273
|
let text;
|
|
274
|
+
let outputAt;
|
|
274
275
|
try {
|
|
275
276
|
const bytes = fs.readFileSync(outPath);
|
|
276
277
|
text = bytes.subarray(Math.min(workerOutOffset, bytes.length)).toString("utf8");
|
|
278
|
+
const outputMtime = fs.statSync(outPath).mtime;
|
|
279
|
+
let spawnMtime;
|
|
280
|
+
try {
|
|
281
|
+
spawnMtime = fs.statSync(path.join(runDir, "spawn.json")).mtime;
|
|
282
|
+
}
|
|
283
|
+
catch (err) {
|
|
284
|
+
if (err.code !== "ENOENT")
|
|
285
|
+
throw err;
|
|
286
|
+
}
|
|
287
|
+
if (!spawnMtime || outputMtime >= spawnMtime)
|
|
288
|
+
outputAt = outputMtime.toISOString();
|
|
277
289
|
}
|
|
278
290
|
catch (err) {
|
|
279
291
|
const code = err.code;
|
|
@@ -281,6 +293,9 @@ export function readProviderRejection(runDir, provider, workerOutOffset = 0) {
|
|
|
281
293
|
return null;
|
|
282
294
|
throw err;
|
|
283
295
|
}
|
|
284
|
-
|
|
296
|
+
const failure = parseProviderRejection(provider, text);
|
|
297
|
+
return failure
|
|
298
|
+
? { ...failure, ...(outputAt ? { output_at: outputAt } : {}) }
|
|
299
|
+
: null;
|
|
285
300
|
}
|
|
286
301
|
//# sourceMappingURL=provider-rejection.js.map
|
|
@@ -26,13 +26,13 @@ export interface ModelQuota {
|
|
|
26
26
|
/** Operator-facing marker, e.g. "needs credits for claude-fable-5-1". */
|
|
27
27
|
marker?: string;
|
|
28
28
|
}
|
|
29
|
-
export type QuotaResetSource = "
|
|
29
|
+
export type QuotaResetSource = "parsed" | "derived";
|
|
30
30
|
export interface AccountQuota {
|
|
31
31
|
/** Set when the 429 is an account-wide usage limit with a reset time. */
|
|
32
32
|
all_models_quota_reset_at?: string;
|
|
33
33
|
/**
|
|
34
|
-
*
|
|
35
|
-
*
|
|
34
|
+
* Whether `all_models_quota_reset_at` came from a parsed provider reset or
|
|
35
|
+
* the configured default hold applied to a 429 with no parseable reset.
|
|
36
36
|
*/
|
|
37
37
|
reset_source?: QuotaResetSource;
|
|
38
38
|
models: Record<string, ModelQuota>;
|
|
@@ -57,8 +57,8 @@ export interface QuotaSignal {
|
|
|
57
57
|
* otherwise record no hold at all, and the next lane on the account would
|
|
58
58
|
* hit the same 429. Give such a signal the provider's configured default
|
|
59
59
|
* hold (`quota_default_hold_ms`, fifteen minutes unless set) as its reset,
|
|
60
|
-
* marked `
|
|
61
|
-
*
|
|
60
|
+
* marked `derived`; a signal that already carries a reset keeps it, marked
|
|
61
|
+
* `parsed`. A per-model credits refusal is untouched.
|
|
62
62
|
*/
|
|
63
63
|
export declare function applyDefaultQuotaHold(signal: QuotaSignal, defaultHoldMs: number, now: Date): QuotaSignal;
|
|
64
64
|
export declare function quotaStatePath(hqRoot: string): string;
|
package/dist/lib/lanes/quota.js
CHANGED
|
@@ -27,19 +27,19 @@ export const QUOTA_SCHEMA_VERSION = "hq-lanes-quota.v1";
|
|
|
27
27
|
* otherwise record no hold at all, and the next lane on the account would
|
|
28
28
|
* hit the same 429. Give such a signal the provider's configured default
|
|
29
29
|
* hold (`quota_default_hold_ms`, fifteen minutes unless set) as its reset,
|
|
30
|
-
* marked `
|
|
31
|
-
*
|
|
30
|
+
* marked `derived`; a signal that already carries a reset keeps it, marked
|
|
31
|
+
* `parsed`. A per-model credits refusal is untouched.
|
|
32
32
|
*/
|
|
33
33
|
export function applyDefaultQuotaHold(signal, defaultHoldMs, now) {
|
|
34
34
|
if (signal.reason !== "account_quota_exhausted" || signal.needs_credits)
|
|
35
35
|
return signal;
|
|
36
36
|
if (signal.quota_reset_at)
|
|
37
|
-
return { ...signal, reset_source: "
|
|
37
|
+
return { ...signal, reset_source: "parsed" };
|
|
38
38
|
return {
|
|
39
39
|
...signal,
|
|
40
40
|
quota_reset_at: new Date(now.getTime() + defaultHoldMs).toISOString(),
|
|
41
41
|
account_wide: true,
|
|
42
|
-
reset_source: "
|
|
42
|
+
reset_source: "derived",
|
|
43
43
|
};
|
|
44
44
|
}
|
|
45
45
|
const MONTHS = {
|
|
@@ -119,8 +119,13 @@ export function loadQuotaState(hqRoot) {
|
|
|
119
119
|
if (typeof raw.all_models_quota_reset_at === "string" &&
|
|
120
120
|
raw.all_models_quota_reset_at.length > 0) {
|
|
121
121
|
account.all_models_quota_reset_at = raw.all_models_quota_reset_at;
|
|
122
|
-
|
|
123
|
-
|
|
122
|
+
// Normalize values written by older CLI versions while preserving the
|
|
123
|
+
// distinction between provider-parsed and configured fallback times.
|
|
124
|
+
if (raw.reset_source === "provider" || raw.reset_source === "parsed") {
|
|
125
|
+
account.reset_source = "parsed";
|
|
126
|
+
}
|
|
127
|
+
else if (raw.reset_source === "default_hold" || raw.reset_source === "derived") {
|
|
128
|
+
account.reset_source = "derived";
|
|
124
129
|
}
|
|
125
130
|
}
|
|
126
131
|
accounts[id] = account;
|
|
@@ -145,6 +150,22 @@ function writeQuotaState(hqRoot, state) {
|
|
|
145
150
|
* An explicit provider year is kept as stated.
|
|
146
151
|
*/
|
|
147
152
|
export function parseProviderResetTime(text, now) {
|
|
153
|
+
const timeOnlyMatch = /\bresets\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)\s*\(UTC\)/i.exec(text);
|
|
154
|
+
if (timeOnlyMatch) {
|
|
155
|
+
let hour = Number(timeOnlyMatch[1]);
|
|
156
|
+
const minute = Number(timeOnlyMatch[2] ?? 0);
|
|
157
|
+
const meridiem = (timeOnlyMatch[3] ?? "").toLowerCase();
|
|
158
|
+
if (!Number.isInteger(hour) || hour < 1 || hour > 12 || minute < 0 || minute > 59)
|
|
159
|
+
return null;
|
|
160
|
+
if (meridiem === "pm" && hour < 12)
|
|
161
|
+
hour += 12;
|
|
162
|
+
if (meridiem === "am" && hour === 12)
|
|
163
|
+
hour = 0;
|
|
164
|
+
const candidate = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), hour, minute, 0, 0));
|
|
165
|
+
if (candidate.getTime() <= now.getTime())
|
|
166
|
+
candidate.setUTCDate(candidate.getUTCDate() + 1);
|
|
167
|
+
return candidate.toISOString();
|
|
168
|
+
}
|
|
148
169
|
const resetMatch = /\bresets\s+([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?(?:,\s*(?:(\d{4})\s+)?(\d{1,2})(?::(\d{2}))?\s*(am|pm))?(?:\s*\((UTC)\))?/i.exec(text);
|
|
149
170
|
const codexMatch = /\btry\s+again\s+at\s+([A-Za-z]+)\s+(\d{1,2})(?:st|nd|rd|th)?,\s*(\d{4})\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)(?:\s*\((UTC)\))?/i.exec(text);
|
|
150
171
|
const match = resetMatch ?? codexMatch;
|
|
@@ -263,9 +284,23 @@ export function quotaSignalFromFields(fields, model, now) {
|
|
|
263
284
|
export function readQuotaSignal(runDir, model, now, workerOutOffset = 0) {
|
|
264
285
|
const outPath = path.join(runDir, "logs", "worker.out");
|
|
265
286
|
let text;
|
|
287
|
+
let outputAt;
|
|
266
288
|
try {
|
|
267
289
|
const bytes = fs.readFileSync(outPath);
|
|
268
290
|
text = bytes.subarray(Math.min(workerOutOffset, bytes.length)).toString("utf8");
|
|
291
|
+
// worker.out is appended as provider output arrives. Its mtime anchors a
|
|
292
|
+
// time-only reset to the output event, not to a later reconcile pass.
|
|
293
|
+
outputAt = fs.statSync(outPath).mtime;
|
|
294
|
+
let spawnAt;
|
|
295
|
+
try {
|
|
296
|
+
spawnAt = fs.statSync(path.join(runDir, "spawn.json")).mtime;
|
|
297
|
+
}
|
|
298
|
+
catch (err) {
|
|
299
|
+
if (err.code !== "ENOENT")
|
|
300
|
+
throw err;
|
|
301
|
+
}
|
|
302
|
+
if (outputAt > now || (spawnAt && outputAt < spawnAt))
|
|
303
|
+
outputAt = now;
|
|
269
304
|
}
|
|
270
305
|
catch (err) {
|
|
271
306
|
const code = err.code;
|
|
@@ -276,7 +311,8 @@ export function readQuotaSignal(runDir, model, now, workerOutOffset = 0) {
|
|
|
276
311
|
const fields = quotaFieldsFromOutput(text);
|
|
277
312
|
if (!fields)
|
|
278
313
|
return null;
|
|
279
|
-
|
|
314
|
+
const observedAt = Number.isFinite(outputAt.getTime()) ? outputAt : now;
|
|
315
|
+
return quotaSignalFromFields(fields, model, observedAt);
|
|
280
316
|
}
|
|
281
317
|
export function recordAccountQuota(hqRoot, provider, accountId, signal) {
|
|
282
318
|
return withLaneRecordFileLock(quotaStatePath(hqRoot), () => {
|
|
@@ -303,11 +339,11 @@ export function recordAccountQuota(hqRoot, provider, accountId, signal) {
|
|
|
303
339
|
if (signal.account_wide && signal.quota_reset_at) {
|
|
304
340
|
if (shouldReplaceReset(current.all_models_quota_reset_at, signal.quota_reset_at)) {
|
|
305
341
|
account.all_models_quota_reset_at = signal.quota_reset_at;
|
|
306
|
-
account.reset_source = signal.reset_source ?? "
|
|
342
|
+
account.reset_source = signal.reset_source ?? "parsed";
|
|
307
343
|
}
|
|
308
344
|
else if (current.all_models_quota_reset_at) {
|
|
309
345
|
account.all_models_quota_reset_at = current.all_models_quota_reset_at;
|
|
310
|
-
account.reset_source = current.reset_source ?? "
|
|
346
|
+
account.reset_source = current.reset_source ?? "parsed";
|
|
311
347
|
}
|
|
312
348
|
}
|
|
313
349
|
accounts[accountId] = account;
|
|
@@ -19,7 +19,7 @@ import { type LaneCompanyRef, type LaneProvider, type LaneRecord, type AccountHi
|
|
|
19
19
|
export declare const LANES_RUNS_REL: string;
|
|
20
20
|
/** Written by reconcile before a resume spawn; the worker prompt names this path. */
|
|
21
21
|
export declare const ANSWERED_QUESTIONS_FILE = "answered-questions.json";
|
|
22
|
-
export declare const SPAWN_TERMINAL_REASONS: readonly ["account_not_logged_in", "unknown_account", "account_disabled", "no_enabled_account", "spawn_failed", "not_detached", "watcher_not_armed", "model_unsupported"];
|
|
22
|
+
export declare const SPAWN_TERMINAL_REASONS: readonly ["account_not_logged_in", "unknown_account", "account_disabled", "no_enabled_account", "spawn_failed", "not_detached", "watcher_not_armed", "model_unsupported", "account_quota_exhausted"];
|
|
23
23
|
export type SpawnTerminalReason = (typeof SPAWN_TERMINAL_REASONS)[number];
|
|
24
24
|
declare const PROVIDER_COMMAND: Record<LaneProvider, string>;
|
|
25
25
|
export declare function providerCommandOnPath(command: string, pathEnv: string | undefined): boolean;
|
package/dist/lib/lanes/spawn.js
CHANGED
|
@@ -108,18 +108,6 @@ export declare function classifyLaneExit(input: {
|
|
|
108
108
|
*/
|
|
109
109
|
providerRejection?: ProviderRejection | null;
|
|
110
110
|
}): TerminalClassification;
|
|
111
|
-
/**
|
|
112
|
-
* One decision for an exited worker's output. An explicit provider status
|
|
113
|
-
* wins over the quota text heuristics: a 429 from any provider (including
|
|
114
|
-
* codex `turn.failed` and grok error objects, which the claude-shaped quota
|
|
115
|
-
* reader never sees) becomes a quota signal; any other explicit status is a
|
|
116
|
-
* rejection even when its text sounds like a quota message (a 402 "requires
|
|
117
|
-
* usage credits" is a rejection, not a hold). A failure with no status at
|
|
118
|
-
* all is matched against the quota text heuristics first: a status-less
|
|
119
|
-
* "usage limit" / "rate limit" turn.failed still creates the account hold;
|
|
120
|
-
* any other status-less message is a rejection. Only when the stream
|
|
121
|
-
* carries no failure object does the quota reader's own read apply.
|
|
122
|
-
*/
|
|
123
111
|
export declare function routeProviderFailure(failure: ProviderRejection | null, readQuota: () => QuotaSignal | null, model: string | undefined, now: Date): {
|
|
124
112
|
quota: QuotaSignal | null;
|
|
125
113
|
rejection: ProviderRejection | null;
|
|
@@ -139,15 +139,21 @@ export function classifyLaneExit(input) {
|
|
|
139
139
|
* any other status-less message is a rejection. Only when the stream
|
|
140
140
|
* carries no failure object does the quota reader's own read apply.
|
|
141
141
|
*/
|
|
142
|
+
function validFailureOutputTime(value, fallback) {
|
|
143
|
+
const parsed = new Date(value);
|
|
144
|
+
return Number.isFinite(parsed.getTime()) && parsed.getTime() <= fallback.getTime()
|
|
145
|
+
? parsed
|
|
146
|
+
: fallback;
|
|
147
|
+
}
|
|
142
148
|
export function routeProviderFailure(failure, readQuota, model, now) {
|
|
143
149
|
if (failure && failure.status === QUOTA_STATUS) {
|
|
144
150
|
return {
|
|
145
|
-
quota: quotaSignalFromFields({ api_error_status: QUOTA_STATUS, is_error: true, result: failure.message }, model, now),
|
|
151
|
+
quota: quotaSignalFromFields({ api_error_status: QUOTA_STATUS, is_error: true, result: failure.message }, model, failure.output_at ? validFailureOutputTime(failure.output_at, now) : now),
|
|
146
152
|
rejection: null,
|
|
147
153
|
};
|
|
148
154
|
}
|
|
149
155
|
if (failure && failure.status === null) {
|
|
150
|
-
const quota = quotaSignalFromFields({ api_error_status: undefined, is_error: true, result: failure.message }, model, now);
|
|
156
|
+
const quota = quotaSignalFromFields({ api_error_status: undefined, is_error: true, result: failure.message }, model, failure.output_at ? validFailureOutputTime(failure.output_at, now) : now);
|
|
151
157
|
if (quota)
|
|
152
158
|
return { quota, rejection: null };
|
|
153
159
|
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { redactErrorText } from "./redact-error-text.js";
|
|
2
|
+
/** Render an unknown rejection safely for a concise diagnostic log. */
|
|
3
|
+
export function safeCaughtError(error) {
|
|
4
|
+
const rawName = error instanceof Error ? error.name : "UnknownError";
|
|
5
|
+
// A writable `name` can carry arbitrary text, so only class-shaped names are logged.
|
|
6
|
+
const name = /^[A-Za-z]{1,64}(?:Error|Exception)$/.test(rawName) ? rawName : "Error";
|
|
7
|
+
const rawMessage = error instanceof Error ? error.message : "non-Error rejection";
|
|
8
|
+
const message = redactErrorText(rawMessage)
|
|
9
|
+
.replace(/\b[a-z][a-z0-9+.-]*:\/\/[^\s"<>]+/gi, "[URL]")
|
|
10
|
+
.replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[EMAIL]")
|
|
11
|
+
.replace(/\b(?:agt|cmp|prs|prj|session|sess|usr|user|acct|fin|st|e)_[A-Za-z0-9_-]+\b/gi, "[ID]")
|
|
12
|
+
.replace(/\b[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b/gi, "[ID]")
|
|
13
|
+
.slice(0, 160);
|
|
14
|
+
return `${name}: ${message || "[empty]"}`;
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=safe-caught-error.js.map
|