@indigoai-us/hq-cli 5.345.126 → 5.345.128
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/assets/scaffold/core/scripts/handoff-knowledge-commit.sh +123 -0
- package/assets/scaffold/core/scripts/hq-detach.sh +169 -0
- package/dist/command-catalog.generated.d.ts +40 -2
- package/dist/command-catalog.generated.js +48 -2
- package/dist/commands/auth.js +19 -2
- package/dist/commands/core-bucket-c-handoff-knowledge-fast.d.ts +2 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-fast.js +10 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-native.d.ts +46 -0
- package/dist/commands/core-bucket-c-handoff-knowledge-native.js +284 -0
- package/dist/commands/core-bucket-c-hq-detach-fast.d.ts +2 -0
- package/dist/commands/core-bucket-c-hq-detach-fast.js +10 -0
- package/dist/commands/core-bucket-c-hq-detach-native.d.ts +43 -0
- package/dist/commands/core-bucket-c-hq-detach-native.js +241 -0
- package/dist/commands/core.js +12 -0
- package/dist/commands/lanes-accounts.js +3 -1
- package/dist/commands/lanes-shared.d.ts +7 -0
- package/dist/commands/lanes.js +113 -3
- package/dist/commands/skill.js +9 -4
- package/dist/fast-scaffold.js +28 -0
- package/dist/lib/capacity/lane-login.js +1 -0
- package/dist/lib/lanes/accounts.js +2 -2
- package/dist/lib/lanes/host-capacity.d.ts +4 -0
- package/dist/lib/lanes/host-capacity.js +10 -1
- package/dist/lib/lanes/index.d.ts +2 -2
- package/dist/lib/lanes/index.js +1 -1
- package/dist/lib/lanes/loop.d.ts +8 -0
- package/dist/lib/lanes/loop.js +35 -0
- package/dist/lib/lanes/quota-notice.d.ts +11 -11
- package/dist/lib/lanes/quota-notice.js +20 -29
- package/dist/lib/lanes/quota.d.ts +69 -40
- package/dist/lib/lanes/quota.js +146 -107
- package/dist/lib/lanes/reconcile.d.ts +14 -0
- package/dist/lib/lanes/reconcile.js +77 -23
- package/dist/lib/lanes/remote-control-lane.js +10 -3
- package/dist/lib/lanes/watcher.d.ts +8 -0
- package/dist/lib/lanes/watcher.js +62 -20
- package/dist/utils/resident-package-preload.js +12 -4
- package/package.json +1 -1
package/dist/commands/lanes.js
CHANGED
|
@@ -45,7 +45,7 @@ import { appendProjectFocusOverride, readProjectStoryFocusData, removeProjectFoc
|
|
|
45
45
|
import { appendLaneFocusOverride, assertFocusGuardAllows, makeFocusOverride } from "../lib/lanes/focus-guard.js";
|
|
46
46
|
import { LANE_FOCUS_GUARD_FLAG, LANE_FOCUS_DASHBOARD_FLAG, LANE_HIRE_GRANT_FINISHED_LANES_FLAG, LANE_LIST_SKIP_REMOTE_REFRESH_FLAG, LANE_MESSAGE_ROUTE_OWNER_HOST_FLAG, LANE_ORPHANED_BARE_ID_DROPBOX_REPAIR_FLAG, LANE_PROFILE_PIN_EXHAUSTED_FALLBACK_FLAG, LANE_STATUS_ROW_FIELDS_FLAG, LANE_REAP_STOP_FLAG, LANE_REUSE_REFRESHES_PROFILE_FLAG, LANE_WORKER_CONTINUITY_FLAG, LANE_CONDUCT_LOOP_FLAG, LANE_ENVELOPE_REPAIR_GUIDANCE_FLAG, } from "../lib/lanes/flag-keys.js";
|
|
47
47
|
import { findWorkerContinuityLane, sameLaneHost, workerContinuityPrompt } from "../lib/lanes/worker-continuity.js";
|
|
48
|
-
import { enqueueLoopEnvelope, interruptLoopQueue, interruptLoopQueueForStory, loopControllerIsAlive, loopStatus, validateLoopEnvelope } from "../lib/lanes/loop.js";
|
|
48
|
+
import { commitIdleLoopStop, enqueueLoopEnvelope, interruptLoopQueue, interruptLoopQueueForStory, loopControllerIsAlive, loopStatus, validateLoopEnvelope } from "../lib/lanes/loop.js";
|
|
49
49
|
import { runLaneLoopController, startLaneLoopController } from "./lanes-loop.js";
|
|
50
50
|
import { configuredCodexRolloutHomes, retainCodexRollouts } from "../lib/lanes/disk-guard.js";
|
|
51
51
|
import { isControllerAliasOutpostId } from "../lib/lanes/admission-host.js";
|
|
@@ -76,6 +76,7 @@ import { listLaneReviews } from "../lib/lanes/reviews.js";
|
|
|
76
76
|
import { effectiveAccountHome } from "../lib/lanes/account-home.js";
|
|
77
77
|
import { remoteControlSelectionConfig, remoteControlSessionHome } from "../lib/lanes/remote-control-lane.js";
|
|
78
78
|
import { AccountResolutionError, compareLaneQueueOrder, LANE_PROVIDERS, LANE_STATES, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, isFinishedResumableHireChild, childrenHoldingHireGrantCarve, laneHoldsHireGrantCarve, assignedChildHireGrants, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, laneStoryLedgerEnabled, syncLaneToProjectStoryLedger, removeLaneFromProjectStoryLedger, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listLaneIds, listQuestionIds, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, isLaneProvider, parseLaneRecord, parseLaneState, parseQuestionRecord, parseSeniorKind, readAllLanes, readLane, sanitizeHostComponent, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, loadWorkerProfileWithRegistry, registryWorkerRecord, runReconcilePass, DEAD_HOST_MS, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, dropboxDir, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, providerSessionPortable, isFailedQuotaKeepAliveLane, sessionNotPortableMessage, withLaneAdmissionLock, withAllLaneAdmissionLocks, readWorkerRunResult, resumeCommandRefusal, withLaneHireLocks, raiseLaneQuestion, sha256Text, } from "../lib/lanes/index.js";
|
|
79
|
+
import { admitQueuedNow } from "../lib/lanes/reconcile.js";
|
|
79
80
|
import { laneStories, storyWarning, } from "../lib/lanes/stories.js";
|
|
80
81
|
import { resolveLaneKeepAlive } from "../lib/lanes/profile.js";
|
|
81
82
|
import { laneCompanyUidSource, resolveLaneCompany } from "../lib/lanes/lane-company.js";
|
|
@@ -490,6 +491,82 @@ function livenessDepsFromCtx(ctx) {
|
|
|
490
491
|
hostname: ctx.hostname,
|
|
491
492
|
};
|
|
492
493
|
}
|
|
494
|
+
/**
|
|
495
|
+
* Admit queued lanes in-process after a command frees or adds capacity, so
|
|
496
|
+
* the queue drains without `hq daemon`. Best effort: an admission failure is
|
|
497
|
+
* reported on stderr and never fails the command that freed the slot.
|
|
498
|
+
*/
|
|
499
|
+
async function admitQueuedFromCtx(ctx) {
|
|
500
|
+
try {
|
|
501
|
+
const result = await admitQueuedNow(ctx.hqRoot(), {
|
|
502
|
+
now: ctx.now,
|
|
503
|
+
host: reconcileHostFromCtx(ctx),
|
|
504
|
+
dryRun: false,
|
|
505
|
+
spawnLaneDeps: {
|
|
506
|
+
env: ctx.env,
|
|
507
|
+
existsSync: ctx.existsSync,
|
|
508
|
+
spawnProcess: ctx.spawnProcess,
|
|
509
|
+
proveDetached: ctx.proveDetached,
|
|
510
|
+
killProcess: ctx.killProcess,
|
|
511
|
+
now: ctx.now,
|
|
512
|
+
resolveMachineIdentity: ctx.resolveMachineIdentity,
|
|
513
|
+
runCompaction: ctx.runCompaction,
|
|
514
|
+
locateTranscript: ctx.locateTranscript,
|
|
515
|
+
},
|
|
516
|
+
throttleDeps: ctx.throttleDeps,
|
|
517
|
+
livenessDeps: livenessDepsFromCtx(ctx),
|
|
518
|
+
presenceConnected: ctx.presenceConnected,
|
|
519
|
+
mesh: ctx.mesh,
|
|
520
|
+
env: ctx.env,
|
|
521
|
+
});
|
|
522
|
+
for (const error of result.errors) {
|
|
523
|
+
process.stderr.write(`hq lanes: queue admission: ${error.message}\n`);
|
|
524
|
+
}
|
|
525
|
+
return result.admitted;
|
|
526
|
+
}
|
|
527
|
+
catch (err) {
|
|
528
|
+
process.stderr.write(`hq lanes: queue admission failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
529
|
+
return [];
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
/**
|
|
533
|
+
* End a queued loop lane whose queue holds only a stop envelope and whose
|
|
534
|
+
* controller is not running. Without this the lane waits for admission just
|
|
535
|
+
* to read the stop, and counts as busy for its worker until then.
|
|
536
|
+
*/
|
|
537
|
+
async function finalizeIdleLoopStop(ctx, lane) {
|
|
538
|
+
if (lane.loop_mode !== true || lane.state !== "queued")
|
|
539
|
+
return undefined;
|
|
540
|
+
const hqRoot = ctx.hqRoot();
|
|
541
|
+
const runDirPath = runDir(hqRoot, lane.lane_id);
|
|
542
|
+
if (loopControllerIsAlive(runDirPath))
|
|
543
|
+
return undefined;
|
|
544
|
+
if (!commitIdleLoopStop(runDirPath, lane.lane_id, ctx.now))
|
|
545
|
+
return undefined;
|
|
546
|
+
let stopped;
|
|
547
|
+
try {
|
|
548
|
+
stopped = casUpdateLane(hqRoot, lane.lane_id, (current) => {
|
|
549
|
+
if (current.state !== "queued") {
|
|
550
|
+
throw new LanesError("lane_state_changed", `Lane ${lane.lane_id} left the queue while its stop was committed`, { lane_id: lane.lane_id, state: current.state });
|
|
551
|
+
}
|
|
552
|
+
return {
|
|
553
|
+
...current,
|
|
554
|
+
state: "done",
|
|
555
|
+
queued_reason: undefined,
|
|
556
|
+
queued_details: undefined,
|
|
557
|
+
admission_reserved_at: undefined,
|
|
558
|
+
terminal_reason: "loop_stopped",
|
|
559
|
+
};
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
catch (err) {
|
|
563
|
+
if (err instanceof LanesError && err.error === "lane_state_changed")
|
|
564
|
+
return undefined;
|
|
565
|
+
throw err;
|
|
566
|
+
}
|
|
567
|
+
await emitLaneTerminalStateSoft(ctx, stopped);
|
|
568
|
+
return stopped;
|
|
569
|
+
}
|
|
493
570
|
function reconcileHostFromCtx(ctx) {
|
|
494
571
|
return resolveAdmissionHost({
|
|
495
572
|
hostname: ctx.hostname,
|
|
@@ -3522,6 +3599,17 @@ async function runCreate(ctx, opts) {
|
|
|
3522
3599
|
throw new LanesError("lane_title_requires_remote_control", "--title is only supported with --remote-control", {});
|
|
3523
3600
|
}
|
|
3524
3601
|
const continuityEnabled = opts.remoteControl !== true && opts.worker !== undefined && await readLanesRegistryFlagFailOpen(LANE_WORKER_CONTINUITY_FLAG, { company: company ?? meshCompany, env: ctx.env, defaultValue: true });
|
|
3602
|
+
// A lane that waits only on the admitter (a queued loop lane holding just a
|
|
3603
|
+
// stop, or a queued lane with free capacity) is not busy: finish it here.
|
|
3604
|
+
const preCreateAdmitted = [];
|
|
3605
|
+
if (continuityEnabled) {
|
|
3606
|
+
const workerId = resolvedWorker?.id ?? opts.worker;
|
|
3607
|
+
const waiting = loadAllLanes(hqRoot).filter((lane) => lane.worker === workerId && lane.provider === provider && lane.state === "queued");
|
|
3608
|
+
for (const lane of waiting)
|
|
3609
|
+
await finalizeIdleLoopStop(ctx, lane);
|
|
3610
|
+
if (waiting.length > 0)
|
|
3611
|
+
preCreateAdmitted.push(...await admitQueuedFromCtx(ctx));
|
|
3612
|
+
}
|
|
3525
3613
|
let workerContinuity = continuityEnabled
|
|
3526
3614
|
? findWorkerContinuityLane(loadAllLanes(hqRoot), {
|
|
3527
3615
|
senior,
|
|
@@ -4337,10 +4425,19 @@ async function runCreate(ctx, opts) {
|
|
|
4337
4425
|
: undefined,
|
|
4338
4426
|
});
|
|
4339
4427
|
}
|
|
4428
|
+
// Admit queued lanes (older ones first) now that this create has settled.
|
|
4429
|
+
// Outpost lanes are admitted on their Outpost.
|
|
4430
|
+
const admitted = [...preCreateAdmitted];
|
|
4431
|
+
if (!outpostId) {
|
|
4432
|
+
admitted.push(...await admitQueuedFromCtx(ctx));
|
|
4433
|
+
if (admitted.some((entry) => entry.id === started.lane_id))
|
|
4434
|
+
started = readLane(hqRoot, started.lane_id);
|
|
4435
|
+
}
|
|
4340
4436
|
if (!opts.quiet)
|
|
4341
4437
|
printJson({
|
|
4342
4438
|
ok: true,
|
|
4343
4439
|
action: "create",
|
|
4440
|
+
admitted,
|
|
4344
4441
|
lane_id: started.lane_id,
|
|
4345
4442
|
state: started.state,
|
|
4346
4443
|
queued_reason: started.queued_reason ?? null,
|
|
@@ -7410,7 +7507,7 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
7410
7507
|
});
|
|
7411
7508
|
const block = quotaBlockFor(loadQuotaState(hqRoot), lane.provider, target, model, now);
|
|
7412
7509
|
if (block.blocked) {
|
|
7413
|
-
throw new LanesError("account_quota_held", `Account ${target} for ${lane.provider}
|
|
7510
|
+
throw new LanesError("account_quota_held", `Account ${target} for ${lane.provider} cannot run model ${model ?? "(default)"}: the account's model needs credits${block.marker ? ` (${block.marker})` : ""}. Once credits are added, clear the marker with: hq lanes accounts clear-quota ${lane.provider} ${target}`, { provider: lane.provider, account: target, quota_reset_at: block.quota_reset_at ?? null, marker: block.marker ?? null, model: model ?? null, clear_command: `hq lanes accounts clear-quota ${lane.provider} ${target}` });
|
|
7414
7511
|
}
|
|
7415
7512
|
if (model !== undefined) {
|
|
7416
7513
|
validateLaneModel(lane.provider, model, targetHome, {
|
|
@@ -8705,8 +8802,18 @@ async function runStop(ctx, laneId, reason, quiet = false) {
|
|
|
8705
8802
|
const queued = enqueueLoopEnvelope({ runDir: runDirPath, laneId: lane.lane_id, envelope: { kind: "stop" }, now: ctx.now });
|
|
8706
8803
|
if (!controllerAlive && lane.state !== "queued")
|
|
8707
8804
|
startLaneLoopController(ctx, lane.lane_id);
|
|
8805
|
+
const finalized = !controllerAlive ? await finalizeIdleLoopStop(ctx, readLane(hqRoot, lane.lane_id)) : undefined;
|
|
8806
|
+
const admitted = await admitQueuedFromCtx(ctx);
|
|
8708
8807
|
if (!quiet)
|
|
8709
|
-
printJson({
|
|
8808
|
+
printJson({
|
|
8809
|
+
ok: true,
|
|
8810
|
+
action: "stop",
|
|
8811
|
+
lane_id: lane.lane_id,
|
|
8812
|
+
terminal: finalized !== undefined,
|
|
8813
|
+
queued_stop: queued,
|
|
8814
|
+
...(finalized ? { lane: finalized } : {}),
|
|
8815
|
+
admitted,
|
|
8816
|
+
});
|
|
8710
8817
|
return;
|
|
8711
8818
|
}
|
|
8712
8819
|
if (isTerminalLaneState(lane.state)) {
|
|
@@ -8744,6 +8851,7 @@ async function runStop(ctx, laneId, reason, quiet = false) {
|
|
|
8744
8851
|
}
|
|
8745
8852
|
await emitLaneTerminalStateSoft(ctx, updated);
|
|
8746
8853
|
const cancelled = cancelPendingQuestionsForLane(hqRoot, laneId, at);
|
|
8854
|
+
const admitted = await admitQueuedFromCtx(ctx);
|
|
8747
8855
|
if (!quiet)
|
|
8748
8856
|
printJson({
|
|
8749
8857
|
ok: true,
|
|
@@ -8752,6 +8860,7 @@ async function runStop(ctx, laneId, reason, quiet = false) {
|
|
|
8752
8860
|
terminal: true,
|
|
8753
8861
|
resumable: false,
|
|
8754
8862
|
cancelled_questions: cancelled.map((q) => q.question_id),
|
|
8863
|
+
admitted,
|
|
8755
8864
|
});
|
|
8756
8865
|
}
|
|
8757
8866
|
/** Stderr notice that `stop` ends a lane for good, naming the resumable path. */
|
|
@@ -9540,6 +9649,7 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
9540
9649
|
readClaudeAccountBalanceSignalsEnabled,
|
|
9541
9650
|
reconcileHostFromCtx,
|
|
9542
9651
|
resolveCallerPath,
|
|
9652
|
+
admitQueuedFromCtx,
|
|
9543
9653
|
});
|
|
9544
9654
|
const questionHandlers = createLaneQuestionsHandlers({
|
|
9545
9655
|
CAPACITY_PROBE_TIMEOUT_MS,
|
package/dist/commands/skill.js
CHANGED
|
@@ -190,16 +190,21 @@ function displayNameFromSlug(slug) {
|
|
|
190
190
|
.join(" ");
|
|
191
191
|
}
|
|
192
192
|
export function makeSkillTemplate(input) {
|
|
193
|
-
|
|
193
|
+
// Frontmatter `name` is the slash-command name in Claude Code and Codex, so it
|
|
194
|
+
// must be the slug. A spaced display title there breaks command entry in the
|
|
195
|
+
// Claude desktop app. The display title goes in frontmatter `title` (which
|
|
196
|
+
// HQ Pro shows in the console) and in the heading.
|
|
197
|
+
const title = input.name?.trim() || displayNameFromSlug(input.slug);
|
|
194
198
|
const description = input.description?.trim() ?? "";
|
|
195
199
|
return [
|
|
196
200
|
"---",
|
|
197
|
-
`name: ${JSON.stringify(
|
|
201
|
+
`name: ${JSON.stringify(input.slug)}`,
|
|
202
|
+
`title: ${JSON.stringify(title)}`,
|
|
198
203
|
`description: ${JSON.stringify(description)}`,
|
|
199
204
|
"tags: []",
|
|
200
205
|
"---",
|
|
201
206
|
"",
|
|
202
|
-
`# ${
|
|
207
|
+
`# ${title}`,
|
|
203
208
|
"",
|
|
204
209
|
"Describe when and how to use this skill.",
|
|
205
210
|
"",
|
|
@@ -466,7 +471,7 @@ export function registerSkillCommand(program, deps = {}) {
|
|
|
466
471
|
skill
|
|
467
472
|
.command("create <slug>")
|
|
468
473
|
.description("Register, stamp, surface, and sync a company skill")
|
|
469
|
-
.option("--name <name>", "Display
|
|
474
|
+
.option("--name <name>", "Display title for a new template, written as frontmatter title (frontmatter name is always the slug)")
|
|
470
475
|
.option("--description <text>", "Description when creating a new template")
|
|
471
476
|
.option("--no-sync", "Register locally without uploading the stamped file")
|
|
472
477
|
.option("--surface-only", "Refresh local skill discovery without registration or sync")
|
package/dist/fast-scaffold.js
CHANGED
|
@@ -160,6 +160,30 @@ export function isDefaultOnFastFamily(flagKey) {
|
|
|
160
160
|
}
|
|
161
161
|
/** The relocated scripts that qualify for the entrypoint fast path. */
|
|
162
162
|
export const FAST_SCAFFOLD_SCRIPTS = [
|
|
163
|
+
{
|
|
164
|
+
name: "hq-detach",
|
|
165
|
+
kind: "native",
|
|
166
|
+
modulePath: "./commands/core-bucket-c-hq-detach-fast.js",
|
|
167
|
+
exportName: "runHqDetachFast",
|
|
168
|
+
flagKey: "hq-core-native-fast-route",
|
|
169
|
+
familyFlagKey: "core-native-bucket-c",
|
|
170
|
+
asset: "core/scripts/hq-detach.sh",
|
|
171
|
+
root: "cwd",
|
|
172
|
+
shellCwd: "caller",
|
|
173
|
+
shellHqRoot: "none",
|
|
174
|
+
},
|
|
175
|
+
{
|
|
176
|
+
name: "handoff-knowledge-commit",
|
|
177
|
+
kind: "native",
|
|
178
|
+
modulePath: "./commands/core-bucket-c-handoff-knowledge-fast.js",
|
|
179
|
+
exportName: "runHandoffKnowledgeCommitFast",
|
|
180
|
+
flagKey: "hq-core-native-fast-route",
|
|
181
|
+
familyFlagKey: "core-native-bucket-c",
|
|
182
|
+
asset: "core/scripts/handoff-knowledge-commit.sh",
|
|
183
|
+
root: "cwd",
|
|
184
|
+
shellCwd: "caller",
|
|
185
|
+
shellHqRoot: "none",
|
|
186
|
+
},
|
|
163
187
|
{
|
|
164
188
|
name: "work-mesh-project-registration-offer",
|
|
165
189
|
kind: "native",
|
|
@@ -897,6 +921,10 @@ export function isFastCoreRequest(argv) {
|
|
|
897
921
|
}
|
|
898
922
|
/** Metadata-only help can render without importing the selected command graph. */
|
|
899
923
|
export function isCommandHelpRequest(argv) {
|
|
924
|
+
// `core` owns the argument parser even when global options appear before
|
|
925
|
+
// its subcommand; its child commands pass --help through to their handlers.
|
|
926
|
+
if (argv[2] === "core" || argv[2] === "run")
|
|
927
|
+
return false;
|
|
900
928
|
const args = argv.slice(2);
|
|
901
929
|
const delimiter = args.indexOf("--");
|
|
902
930
|
const beforeDelimiter = delimiter === -1 ? args : args.slice(0, delimiter);
|
|
@@ -82,6 +82,7 @@ function canonicalHome(home, homedir) {
|
|
|
82
82
|
}
|
|
83
83
|
function handlerDeps(printJson) {
|
|
84
84
|
return {
|
|
85
|
+
admitQueuedFromCtx: async () => [],
|
|
85
86
|
automaticBalanceMoveHistoryReason: () => unused("automaticBalanceMoveHistoryReason"),
|
|
86
87
|
ensureDir: (dir) => {
|
|
87
88
|
fs.mkdirSync(dir, { recursive: true });
|
|
@@ -280,7 +280,7 @@ function throwIfQuotaBlocked(block, provider, accountId, identity) {
|
|
|
280
280
|
const label = accountLabel(accountId, identity);
|
|
281
281
|
const identityFields = identity ? { identity } : {};
|
|
282
282
|
if (block.reason === "account_model_unavailable") {
|
|
283
|
-
throw new AccountResolutionError("account_model_unavailable", `${label} cannot run model ${model}: ${block.marker ?? "needs credits"}`, {
|
|
283
|
+
throw new AccountResolutionError("account_model_unavailable", `${label} cannot run model ${model}: ${block.marker ?? "needs credits"}. The account's model needs credits; once they are added, clear the marker with: ${clearQuotaCommand(provider, accountId)}`, {
|
|
284
284
|
provider,
|
|
285
285
|
account: accountId,
|
|
286
286
|
model: block.model ?? null,
|
|
@@ -289,7 +289,7 @@ function throwIfQuotaBlocked(block, provider, accountId, identity) {
|
|
|
289
289
|
});
|
|
290
290
|
}
|
|
291
291
|
const clear = clearQuotaCommand(provider, accountId);
|
|
292
|
-
throw new AccountResolutionError("account_quota_exhausted", `${label} is
|
|
292
|
+
throw new AccountResolutionError("account_quota_exhausted", `${label} is marked unavailable for model ${model} (the account's model needs credits). Clear the marker with: ${clear}`, {
|
|
293
293
|
provider,
|
|
294
294
|
account: accountId,
|
|
295
295
|
model: block.model ?? null,
|
|
@@ -12,6 +12,10 @@ export interface HostCapacityAccount {
|
|
|
12
12
|
/** Account-wide quota status when this snapshot is produced on its host. */
|
|
13
13
|
quota_held?: boolean;
|
|
14
14
|
quota_reset_at?: string;
|
|
15
|
+
/** Last recorded provider usage limit on this account. Informational only. */
|
|
16
|
+
last_quota_hit_at?: string;
|
|
17
|
+
/** The provider's reset time from that limit, when it named one. Informational only. */
|
|
18
|
+
provider_reset_at?: string;
|
|
15
19
|
/** Exact lane states used for Claude account balancing; absent on older Outposts. */
|
|
16
20
|
lane_counts?: {
|
|
17
21
|
running: number;
|
|
@@ -2,7 +2,7 @@ import { accountStatus } from "./accounts.js";
|
|
|
2
2
|
import { accountMaxRunning } from "./config.js";
|
|
3
3
|
import { laneOccupiesRunningSlot, laneRunsOnHost, } from "./admission.js";
|
|
4
4
|
import { LanesError } from "./errors.js";
|
|
5
|
-
import { quotaBlockFor } from "./quota.js";
|
|
5
|
+
import { quotaBlockFor, quotaHitInfo } from "./quota.js";
|
|
6
6
|
function admissionHost(host) {
|
|
7
7
|
return host.kind === "outpost"
|
|
8
8
|
? { hostname: "", outpostId: host.id }
|
|
@@ -38,6 +38,7 @@ export function buildHostCapacitySnapshot(input) {
|
|
|
38
38
|
? quotaBlockFor(input.quotaState, provider, id, undefined, input.now)
|
|
39
39
|
: undefined;
|
|
40
40
|
const quotaResetAt = input.quotaState?.providers[provider]?.[id]?.all_models_quota_reset_at;
|
|
41
|
+
const hit = input.quotaState ? quotaHitInfo(input.quotaState, provider, id, input.now) : undefined;
|
|
41
42
|
accounts[id] = {
|
|
42
43
|
enabled: account.enabled,
|
|
43
44
|
logged_in: status.logged_in,
|
|
@@ -51,6 +52,8 @@ export function buildHostCapacitySnapshot(input) {
|
|
|
51
52
|
...(quotaResetAt ? { quota_reset_at: quotaResetAt } : {}),
|
|
52
53
|
}
|
|
53
54
|
: {}),
|
|
55
|
+
...(hit?.last_quota_hit_at ? { last_quota_hit_at: hit.last_quota_hit_at } : {}),
|
|
56
|
+
...(hit?.provider_reset_at ? { provider_reset_at: hit.provider_reset_at } : {}),
|
|
54
57
|
lane_counts: {
|
|
55
58
|
running: liveLanes.filter((lane) => lane.state === "running").length,
|
|
56
59
|
queued: liveLanes.filter((lane) => lane.state === "queued").length,
|
|
@@ -142,6 +145,12 @@ export function parseHostCapacitySnapshot(raw) {
|
|
|
142
145
|
...(account.quota_reset_at !== undefined
|
|
143
146
|
? { quota_reset_at: account.quota_reset_at }
|
|
144
147
|
: {}),
|
|
148
|
+
...(typeof account.last_quota_hit_at === "string" && Number.isFinite(Date.parse(account.last_quota_hit_at))
|
|
149
|
+
? { last_quota_hit_at: account.last_quota_hit_at }
|
|
150
|
+
: {}),
|
|
151
|
+
...(typeof account.provider_reset_at === "string" && Number.isFinite(Date.parse(account.provider_reset_at))
|
|
152
|
+
? { provider_reset_at: account.provider_reset_at }
|
|
153
|
+
: {}),
|
|
145
154
|
...(account.lane_counts !== undefined
|
|
146
155
|
? { lane_counts: parseLaneCounts(account.lane_counts, `${provider}.${id}.lane_counts`) }
|
|
147
156
|
: {}),
|
|
@@ -44,8 +44,8 @@ export { AUTH_JSON_NAME, CLAUDE_CREDENTIALS_NAME, PROVIDER_CREDENTIAL_FILES, PRO
|
|
|
44
44
|
export type { AccountResolveContext, AccountStatus, AuthState, ClaudeIdentity, ResolvedAccount, } from "./accounts.js";
|
|
45
45
|
export { LIVE_WITHIN_MS, LIVENESS_EVENT_KINDS, SILENT_LOST_MS, TURN_START_LOST_MS, buildLivenessIndexFromSpool, classifySessionLiveness, defaultLivenessDeps, defaultSpoolPath, ingestSpoolLine, listSessionLiveness, livenessIndexPath, loadLivenessIndex, readDaemonPresenceConnected, readLivenessIndex, rebuildLivenessIndex, sessionLiveness, } from "./liveness.js";
|
|
46
46
|
export type { LivenessDeps, LivenessEventKind, LivenessIndex, SessionIndexEntry, } from "./liveness.js";
|
|
47
|
-
export { DEAD_HOST_MS, MAX_AGED_ADVANCES, QUESTION_AGE_MS, RECONCILE_TIMER_NAME, RECONCILE_UNIT_NAME, wakeAnsweredLaneNow, agedAdvanceCount, currentHopStartedAtMs, isDeadlinePassed, laneRunsHere, readTimeoutDeadline, reconcileHeartbeatPath, resolveQuestionAudience, runReconcilePass, shouldAdvanceAudienceEnded, shouldAgeQuestion, timeoutPath, } from "./reconcile.js";
|
|
48
|
-
export type { AnsweredLaneWakeResult, ReconcileChange, ReconcileDeps, ReconcileHost, ReconcileSkip, ReconcileSummary, } from "./reconcile.js";
|
|
47
|
+
export { DEAD_HOST_MS, MAX_AGED_ADVANCES, QUESTION_AGE_MS, RECONCILE_TIMER_NAME, RECONCILE_UNIT_NAME, wakeAnsweredLaneNow, admitQueuedNow, agedAdvanceCount, currentHopStartedAtMs, isDeadlinePassed, laneRunsHere, readTimeoutDeadline, reconcileHeartbeatPath, resolveQuestionAudience, runReconcilePass, shouldAdvanceAudienceEnded, shouldAgeQuestion, timeoutPath, } from "./reconcile.js";
|
|
48
|
+
export type { AnsweredLaneWakeResult, AdmitQueuedNowResult, ReconcileChange, ReconcileDeps, ReconcileHost, ReconcileSkip, ReconcileSummary, } from "./reconcile.js";
|
|
49
49
|
export { RECONCILE_TIMER_INTERVAL, defaultSystemdUserDir, installReconcileTimer, providerPathEnv, renderReconcileService, renderReconcileTimer, uninstallReconcileTimer, } from "./reconcile-install.js";
|
|
50
50
|
export type { ProviderPathEnv, ReconcileInstallOpts, ReconcileInstallResult, RunCommand, RunCommandResult, } from "./reconcile-install.js";
|
|
51
51
|
export { unitPathEnv } from "./reconcile-install.js";
|
package/dist/lib/lanes/index.js
CHANGED
|
@@ -24,7 +24,7 @@ export { CPU_SAMPLE_MS, LANES_CPU_BUSY_OVERRIDE, LANES_CPU_CHECK, PROC_MEMINFO,
|
|
|
24
24
|
export { effectiveAccountHome, } from "./account-home.js";
|
|
25
25
|
export { AUTH_JSON_NAME, CLAUDE_CREDENTIALS_NAME, PROVIDER_CREDENTIAL_FILES, PROVIDER_HOME_ENV, accountEnv, accountStatus, authJsonPath, authStateFor, claudeIdentityDiffers, claudeIdentityFromProbe, clearQuotaCommand, credentialPaths, describeClaudeIdentity, hasCredentialFile, isCredentialProven, isDefaultProviderHome, isLoggedOut, probeInteractiveClaudeIdentity, withProviderHomeEnv, isOccupyingLaneState, lastStartedAt, listAccountStatuses, occupyingLaneCount, resolveAccount, runningLaneCount, selectBalancedAccount, } from "./accounts.js";
|
|
26
26
|
export { LIVE_WITHIN_MS, LIVENESS_EVENT_KINDS, SILENT_LOST_MS, TURN_START_LOST_MS, buildLivenessIndexFromSpool, classifySessionLiveness, defaultLivenessDeps, defaultSpoolPath, ingestSpoolLine, listSessionLiveness, livenessIndexPath, loadLivenessIndex, readDaemonPresenceConnected, readLivenessIndex, rebuildLivenessIndex, sessionLiveness, } from "./liveness.js";
|
|
27
|
-
export { DEAD_HOST_MS, MAX_AGED_ADVANCES, QUESTION_AGE_MS, RECONCILE_TIMER_NAME, RECONCILE_UNIT_NAME, wakeAnsweredLaneNow, agedAdvanceCount, currentHopStartedAtMs, isDeadlinePassed, laneRunsHere, readTimeoutDeadline, reconcileHeartbeatPath, resolveQuestionAudience, runReconcilePass, shouldAdvanceAudienceEnded, shouldAgeQuestion, timeoutPath, } from "./reconcile.js";
|
|
27
|
+
export { DEAD_HOST_MS, MAX_AGED_ADVANCES, QUESTION_AGE_MS, RECONCILE_TIMER_NAME, RECONCILE_UNIT_NAME, wakeAnsweredLaneNow, admitQueuedNow, agedAdvanceCount, currentHopStartedAtMs, isDeadlinePassed, laneRunsHere, readTimeoutDeadline, reconcileHeartbeatPath, resolveQuestionAudience, runReconcilePass, shouldAdvanceAudienceEnded, shouldAgeQuestion, timeoutPath, } from "./reconcile.js";
|
|
28
28
|
export { RECONCILE_TIMER_INTERVAL, defaultSystemdUserDir, installReconcileTimer, providerPathEnv, renderReconcileService, renderReconcileTimer, uninstallReconcileTimer, } from "./reconcile-install.js";
|
|
29
29
|
export { unitPathEnv } from "./reconcile-install.js";
|
|
30
30
|
export { PROVIDER_REJECTED_REASON, PROVIDER_REJECTION_MESSAGE_CAP, parseClaudeResultRejection, parseCodexTurnFailed, parseGrokErrorOutput, parseProviderRejection, readProviderRejection, retryableFromStatus, } from "./provider-rejection.js";
|
package/dist/lib/lanes/loop.d.ts
CHANGED
|
@@ -178,6 +178,14 @@ export declare function applyLoopStallAnswer(input: {
|
|
|
178
178
|
action: "requeued" | "parked" | "already_applied";
|
|
179
179
|
} | undefined;
|
|
180
180
|
export declare function loopStatus(runDir: string, laneId: string): LoopStatus;
|
|
181
|
+
/**
|
|
182
|
+
* Commit a stop envelope that is the only thing left in an idle loop lane's
|
|
183
|
+
* queue: no phase is active and the stop is the FIFO head. The loop
|
|
184
|
+
* controller would do exactly this on its next claim; doing it in-process
|
|
185
|
+
* lets a queued loop lane end without waiting for admission. Returns false
|
|
186
|
+
* and changes nothing when the queue holds anything ahead of the stop.
|
|
187
|
+
*/
|
|
188
|
+
export declare function commitIdleLoopStop(runDir: string, laneId: string, now?: () => Date): boolean;
|
|
181
189
|
export declare function setLoopControllerPid(runDir: string, laneId: string, pid: number | undefined): void;
|
|
182
190
|
export declare function clearLoopControllerPid(runDir: string, laneId: string, pid: number): void;
|
|
183
191
|
export declare function loopControllerIsAlive(runDir: string): boolean;
|
package/dist/lib/lanes/loop.js
CHANGED
|
@@ -596,6 +596,41 @@ export function loopStatus(runDir, laneId) {
|
|
|
596
596
|
updated_at: typeof loop.updated_at === "string" ? loop.updated_at : new Date(0).toISOString(),
|
|
597
597
|
};
|
|
598
598
|
}
|
|
599
|
+
/**
|
|
600
|
+
* Commit a stop envelope that is the only thing left in an idle loop lane's
|
|
601
|
+
* queue: no phase is active and the stop is the FIFO head. The loop
|
|
602
|
+
* controller would do exactly this on its next claim; doing it in-process
|
|
603
|
+
* lets a queued loop lane end without waiting for admission. Returns false
|
|
604
|
+
* and changes nothing when the queue holds anything ahead of the stop.
|
|
605
|
+
*/
|
|
606
|
+
export function commitIdleLoopStop(runDir, laneId, now = () => new Date()) {
|
|
607
|
+
if (envelopeFiles(activeDir(runDir)).length > 0)
|
|
608
|
+
return false;
|
|
609
|
+
const head = envelopeFiles(pendingDir(runDir))[0];
|
|
610
|
+
if (!head)
|
|
611
|
+
return false;
|
|
612
|
+
let headRecord;
|
|
613
|
+
try {
|
|
614
|
+
headRecord = fileEnvelope(head);
|
|
615
|
+
}
|
|
616
|
+
catch {
|
|
617
|
+
return false;
|
|
618
|
+
}
|
|
619
|
+
if (headRecord.envelope.kind !== "stop")
|
|
620
|
+
return false;
|
|
621
|
+
const claimed = claimNextLoopEnvelope(runDir);
|
|
622
|
+
if (!claimed)
|
|
623
|
+
return false;
|
|
624
|
+
if (claimed.record.envelope.kind !== "stop") {
|
|
625
|
+
// Another claimer moved the queue between the check and the claim; put it back.
|
|
626
|
+
fs.renameSync(claimed.file, path.join(pendingDir(runDir), path.basename(claimed.file)));
|
|
627
|
+
return false;
|
|
628
|
+
}
|
|
629
|
+
const stopped = { schema_version: LOOP_SCHEMA, envelope_id: claimed.record.envelope_id, status: "stopped", started_at: claimed.record.queued_at, finished_at: now().toISOString() };
|
|
630
|
+
writeResultAt(runDir, claimed.file, claimed.record, stopped);
|
|
631
|
+
persistStatus(runDir, laneId, { state: "stopped", queue_depth: listLoopQueue(runDir).length, active_envelope_id: null, active_phase: null });
|
|
632
|
+
return true;
|
|
633
|
+
}
|
|
599
634
|
export function setLoopControllerPid(runDir, laneId, pid) {
|
|
600
635
|
persistStatus(runDir, laneId, { pid });
|
|
601
636
|
}
|
|
@@ -1,24 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Tell a lane's senior
|
|
2
|
+
* Tell a lane's senior that a provider usage limit ended that lane.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* A 429 never holds the account (owner decision 2026-10-14): the lane that
|
|
5
|
+
* hit it ends resumable and the senior decides. One message per lane exit,
|
|
6
|
+
* keyed by lane id and record version, so a re-read of the same exit or a
|
|
7
|
+
* retried append does not send it twice.
|
|
8
8
|
*/
|
|
9
|
-
import type {
|
|
10
|
-
import { type
|
|
11
|
-
export interface
|
|
9
|
+
import type { QuotaSignal } from "./quota.js";
|
|
10
|
+
import { type LaneRecord } from "./types.js";
|
|
11
|
+
export interface QuotaNotice {
|
|
12
12
|
message_id: string;
|
|
13
13
|
text: string;
|
|
14
14
|
}
|
|
15
|
-
/**
|
|
16
|
-
export declare function
|
|
15
|
+
/** The notice for a lane that ended on a usage limit or a needs-credits refusal. */
|
|
16
|
+
export declare function quotaLaneNotice(lane: Pick<LaneRecord, "lane_id" | "version" | "provider" | "account" | "model" | "state" | "terminal_reason" | "quota_reset_at">, quota: QuotaSignal | null): QuotaNotice;
|
|
17
17
|
/**
|
|
18
18
|
* Deliver the notice to the lane's senior inbox. A session senior gets it in
|
|
19
19
|
* its monitor-session inbox, a lane senior in that lane's dropbox. The notice
|
|
20
20
|
* is always written to stderr as well, so a senior with no inbox still has it
|
|
21
21
|
* in the watcher log. Returns where it was delivered.
|
|
22
22
|
*/
|
|
23
|
-
export declare function
|
|
23
|
+
export declare function deliverQuotaNotice(hqRoot: string, lane: Pick<LaneRecord, "lane_id" | "senior">, notice: QuotaNotice, at: string): "session_inbox" | "lane_inbox" | "stderr_only";
|
|
24
24
|
//# sourceMappingURL=quota-notice.d.ts.map
|
|
@@ -1,39 +1,30 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Tell a lane's senior
|
|
2
|
+
* Tell a lane's senior that a provider usage limit ended that lane.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* A 429 never holds the account (owner decision 2026-10-14): the lane that
|
|
5
|
+
* hit it ends resumable and the senior decides. One message per lane exit,
|
|
6
|
+
* keyed by lane id and record version, so a re-read of the same exit or a
|
|
7
|
+
* retried append does not send it twice.
|
|
8
8
|
*/
|
|
9
9
|
import * as fs from "node:fs";
|
|
10
10
|
import * as path from "node:path";
|
|
11
11
|
import { monitorSessionDir } from "../monitor.js";
|
|
12
12
|
import { appendDropboxMessageOnce, appendDropboxMessageOnceAtPath } from "./dropbox.js";
|
|
13
13
|
import { LANE_PROVIDERS } from "./types.js";
|
|
14
|
-
/**
|
|
15
|
-
export function
|
|
16
|
-
const
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const said = account.provider_text ? `Provider said: "${account.provider_text}".` : "Provider gave no message text.";
|
|
21
|
-
const clear = `If the account is not actually exhausted (check /usage), clear the hold with: hq lanes accounts clear-quota ${provider} ${accountId}`;
|
|
14
|
+
/** The notice for a lane that ended on a usage limit or a needs-credits refusal. */
|
|
15
|
+
export function quotaLaneNotice(lane, quota) {
|
|
16
|
+
const providerReset = quota?.provider_reset_at ?? quota?.quota_reset_at ?? lane.quota_reset_at;
|
|
17
|
+
const what = lane.terminal_reason === "account_model_unavailable"
|
|
18
|
+
? `cannot run model ${lane.model ?? "(unknown)"}${quota?.marker ? ` (${quota.marker})` : ""}`
|
|
19
|
+
: "hit the provider usage limit";
|
|
22
20
|
const lines = [
|
|
23
|
-
`
|
|
24
|
-
|
|
21
|
+
`Lane ${lane.lane_id} ${what} on ${lane.provider} account ${lane.account ?? "(none)"} and stopped as ${lane.state}.`,
|
|
22
|
+
providerReset ? `The provider says the limit resets at ${providerReset}.` : "The provider did not name a reset time.",
|
|
25
23
|
];
|
|
26
|
-
if (
|
|
27
|
-
lines.push(`
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
lines.push("The provider named no reset time, so this is the default hold.");
|
|
31
|
-
}
|
|
32
|
-
lines.push(clear);
|
|
33
|
-
return {
|
|
34
|
-
message_id: `quota-hold:${provider}:${accountId}:${holdUntil}`,
|
|
35
|
-
text: lines.join("\n"),
|
|
36
|
-
};
|
|
24
|
+
if (quota?.provider_text)
|
|
25
|
+
lines.push(`Provider said: "${quota.provider_text}".`);
|
|
26
|
+
lines.push("No account hold was set; other lanes keep using this account.", `To continue this lane: hq lanes resume ${lane.lane_id} (pass --account <id> to use another account).`);
|
|
27
|
+
return { message_id: `quota-limit:${lane.lane_id}:${lane.version}`, text: lines.join(" ") };
|
|
37
28
|
}
|
|
38
29
|
/**
|
|
39
30
|
* Deliver the notice to the lane's senior inbox. A session senior gets it in
|
|
@@ -41,9 +32,9 @@ export function quotaHoldNotice(state, provider, accountId, laneId) {
|
|
|
41
32
|
* is always written to stderr as well, so a senior with no inbox still has it
|
|
42
33
|
* in the watcher log. Returns where it was delivered.
|
|
43
34
|
*/
|
|
44
|
-
export function
|
|
45
|
-
process.stderr.write(`hq lanes: ${notice.text
|
|
46
|
-
const message = { at, message_id: notice.message_id, kind: "
|
|
35
|
+
export function deliverQuotaNotice(hqRoot, lane, notice, at) {
|
|
36
|
+
process.stderr.write(`hq lanes: ${notice.text}\n`);
|
|
37
|
+
const message = { at, message_id: notice.message_id, kind: "quota_limit", text: notice.text };
|
|
47
38
|
if (lane.senior.kind === "lane" && lane.senior.id) {
|
|
48
39
|
appendDropboxMessageOnce(hqRoot, lane.senior.id, message);
|
|
49
40
|
return "lane_inbox";
|