@indigoai-us/hq-cli 5.345.129 → 5.345.131
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/assets/scaffold/core/scripts/lib/codex-output-schema.mjs +304 -0
- package/assets/scaffold/core/scripts/workflow-runner.mjs +2890 -0
- package/dist/command-catalog.generated.d.ts +29 -0
- package/dist/command-catalog.generated.js +34 -0
- package/dist/commands/capacity.d.ts +1 -2
- package/dist/commands/capacity.js +7 -9
- package/dist/commands/core-pipeline-workflow-runner-native.d.ts +175 -0
- package/dist/commands/core-pipeline-workflow-runner-native.js +3237 -0
- package/dist/commands/core-workflow-run-command.d.ts +4 -0
- package/dist/commands/core-workflow-run-command.js +41 -0
- package/dist/commands/core.js +4 -0
- package/dist/commands/lanes-accounts.js +0 -1
- package/dist/commands/lanes-loop.d.ts +14 -0
- package/dist/commands/lanes-loop.js +43 -2
- package/dist/commands/lanes.js +8 -4
- package/dist/lib/capacity/freshness.d.ts +17 -0
- package/dist/lib/capacity/freshness.js +16 -0
- package/dist/lib/capacity/route.d.ts +7 -0
- package/dist/lib/capacity/route.js +46 -1
- package/dist/lib/core-utils/workflow-runner-schema.d.ts +7 -0
- package/dist/lib/core-utils/workflow-runner-schema.js +259 -0
- package/dist/lib/lanes/config.d.ts +8 -4
- package/dist/lib/lanes/config.js +30 -36
- package/dist/lib/lanes/index.d.ts +1 -1
- package/dist/lib/lanes/index.js +1 -1
- package/dist/lib/lanes/loop.d.ts +5 -0
- package/dist/lib/lanes/loop.js +8 -0
- package/dist/lib/lanes/reconcile.js +21 -27
- package/dist/lib/lanes/throttle.js +5 -9
- package/dist/lib/lanes/types.d.ts +4 -5
- package/dist/lib/lanes/types.js +1 -0
- package/dist/lib/lanes/worker-continuity.d.ts +13 -1
- package/dist/lib/lanes/worker-continuity.js +14 -0
- package/package.json +1 -1
package/dist/lib/lanes/config.js
CHANGED
|
@@ -13,10 +13,14 @@ import { parseConfigEffort } from "./effort.js";
|
|
|
13
13
|
import { LanesConfigError } from "./errors.js";
|
|
14
14
|
import { CONFIG_SCHEMA_VERSION, LANE_PROVIDERS, isLaneProvider, parseMajorSchema, } from "./types.js";
|
|
15
15
|
export const LANES_CONFIG_REL = path.join("personal", "lanes", "config.json");
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
16
|
+
/**
|
|
17
|
+
* Owner directive 2026-10-14: no default lane caps or CPU/memory admission
|
|
18
|
+
* gates. A provider has no running or queued limit unless the operator sets
|
|
19
|
+
* `max_running` / `max_queued` explicitly; this value stands for "no limit"
|
|
20
|
+
* and stays a finite integer so it survives JSON output.
|
|
21
|
+
*/
|
|
22
|
+
export const UNLIMITED_LANES = Number.MAX_SAFE_INTEGER;
|
|
23
|
+
export const DEFAULT_MAX_QUEUED = UNLIMITED_LANES;
|
|
20
24
|
/**
|
|
21
25
|
* Owner directive 2026-09-22: "if it's been an hour since last msg, always
|
|
22
26
|
* run compact". Override per provider with `compact_after_ms`.
|
|
@@ -69,9 +73,9 @@ export const DEFAULT_LIVENESS_WINDOWS = {
|
|
|
69
73
|
silent_lost_ms: DEFAULT_SILENT_LOST_MS,
|
|
70
74
|
};
|
|
71
75
|
export const DEFAULT_PROVIDER_MAX_RUNNING = {
|
|
72
|
-
claude:
|
|
73
|
-
codex:
|
|
74
|
-
grok:
|
|
76
|
+
claude: UNLIMITED_LANES,
|
|
77
|
+
codex: UNLIMITED_LANES,
|
|
78
|
+
grok: UNLIMITED_LANES,
|
|
75
79
|
};
|
|
76
80
|
export const DEFAULT_ACCOUNT_HOMES = {
|
|
77
81
|
claude: "~/.claude",
|
|
@@ -152,14 +156,6 @@ function positiveInt(value, field, fallback) {
|
|
|
152
156
|
}
|
|
153
157
|
return value;
|
|
154
158
|
}
|
|
155
|
-
function nonNegativeInt(value, field, fallback) {
|
|
156
|
-
if (value === undefined)
|
|
157
|
-
return fallback;
|
|
158
|
-
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) {
|
|
159
|
-
throw new LanesConfigError("invalid_config", `${field} must be a non-negative integer`, { field, value });
|
|
160
|
-
}
|
|
161
|
-
return value;
|
|
162
|
-
}
|
|
163
159
|
function positiveNumber(value, field, fallback) {
|
|
164
160
|
if (value === undefined)
|
|
165
161
|
return fallback;
|
|
@@ -168,9 +164,9 @@ function positiveNumber(value, field, fallback) {
|
|
|
168
164
|
}
|
|
169
165
|
return value;
|
|
170
166
|
}
|
|
171
|
-
function
|
|
167
|
+
function optionalUnitInterval(value, field) {
|
|
172
168
|
if (value === undefined)
|
|
173
|
-
return
|
|
169
|
+
return undefined;
|
|
174
170
|
if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) {
|
|
175
171
|
throw new LanesConfigError("invalid_config", `${field} must be a number in [0, 1]`, { field, value });
|
|
176
172
|
}
|
|
@@ -377,30 +373,35 @@ function parseProvider(provider, raw, deps) {
|
|
|
377
373
|
}
|
|
378
374
|
return parsed;
|
|
379
375
|
}
|
|
376
|
+
/**
|
|
377
|
+
* CPU and memory admission gates are opt-in. With no `throttle` keys set,
|
|
378
|
+
* admission never waits on CPU load or free memory.
|
|
379
|
+
*/
|
|
380
380
|
function parseThrottle(raw) {
|
|
381
381
|
const obj = raw === undefined ? {} : raw;
|
|
382
382
|
if (!isPlainObject(obj)) {
|
|
383
383
|
throw new LanesConfigError("invalid_config", "throttle must be an object");
|
|
384
384
|
}
|
|
385
|
-
const
|
|
386
|
-
const
|
|
387
|
-
|
|
388
|
-
|
|
385
|
+
const result = {};
|
|
386
|
+
const pause = optionalUnitInterval(obj.cpu_busy_pause, "throttle.cpu_busy_pause");
|
|
387
|
+
const resumeRaw = optionalUnitInterval(obj.cpu_busy_resume, "throttle.cpu_busy_resume");
|
|
388
|
+
if (pause !== undefined) {
|
|
389
|
+
const resume = resumeRaw ?? pause;
|
|
390
|
+
if (resume > pause) {
|
|
391
|
+
throw new LanesConfigError("invalid_config", "throttle.cpu_busy_resume must be <= throttle.cpu_busy_pause", { cpu_busy_pause: pause, cpu_busy_resume: resume });
|
|
392
|
+
}
|
|
393
|
+
result.cpu_busy_pause = pause;
|
|
394
|
+
result.cpu_busy_resume = resume;
|
|
389
395
|
}
|
|
390
|
-
let minMem = DEFAULT_MIN_AVAILABLE_MEM_MB;
|
|
391
396
|
if (obj.min_available_mem_mb !== undefined) {
|
|
392
397
|
if (typeof obj.min_available_mem_mb !== "number" ||
|
|
393
398
|
!Number.isFinite(obj.min_available_mem_mb) ||
|
|
394
399
|
obj.min_available_mem_mb < 0) {
|
|
395
400
|
throw new LanesConfigError("invalid_config", "throttle.min_available_mem_mb must be a non-negative number", { value: obj.min_available_mem_mb });
|
|
396
401
|
}
|
|
397
|
-
|
|
402
|
+
result.min_available_mem_mb = obj.min_available_mem_mb;
|
|
398
403
|
}
|
|
399
|
-
return
|
|
400
|
-
cpu_busy_pause: pause,
|
|
401
|
-
cpu_busy_resume: resume,
|
|
402
|
-
min_available_mem_mb: minMem,
|
|
403
|
-
};
|
|
404
|
+
return result;
|
|
404
405
|
}
|
|
405
406
|
function parseLiveness(raw) {
|
|
406
407
|
const obj = raw === undefined ? {} : raw;
|
|
@@ -458,16 +459,11 @@ export function documentedDefaultConfig(deps = defaultDeps()) {
|
|
|
458
459
|
return {
|
|
459
460
|
schema_version: CONFIG_SCHEMA_VERSION,
|
|
460
461
|
providers,
|
|
461
|
-
throttle: {
|
|
462
|
-
cpu_busy_pause: DEFAULT_CPU_BUSY_PAUSE,
|
|
463
|
-
cpu_busy_resume: DEFAULT_CPU_BUSY_RESUME,
|
|
464
|
-
min_available_mem_mb: DEFAULT_MIN_AVAILABLE_MEM_MB,
|
|
465
|
-
},
|
|
462
|
+
throttle: {},
|
|
466
463
|
liveness: { ...DEFAULT_LIVENESS_WINDOWS },
|
|
467
464
|
tracker: { checks: { ...DEFAULT_TRACKER_CHECKS } },
|
|
468
465
|
runaway_search_guard: { ...DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG },
|
|
469
466
|
low_disk_min_free_bytes: DEFAULT_LOW_DISK_MIN_FREE_BYTES,
|
|
470
|
-
lead_reserve_per_host: 1,
|
|
471
467
|
codex_rollout_compress_after_hours: DEFAULT_CODEX_ROLLOUT_COMPRESS_AFTER_HOURS,
|
|
472
468
|
codex_rollout_delete_after_days: DEFAULT_CODEX_ROLLOUT_DELETE_AFTER_DAYS,
|
|
473
469
|
session_question_timeout_secs: DEFAULT_SESSION_QUESTION_TIMEOUT_SECS,
|
|
@@ -508,7 +504,6 @@ export function parseLanesConfig(raw, deps = defaultDeps()) {
|
|
|
508
504
|
}
|
|
509
505
|
const compressAfterHours = positiveInt(raw.codex_rollout_compress_after_hours, "codex_rollout_compress_after_hours", DEFAULT_CODEX_ROLLOUT_COMPRESS_AFTER_HOURS);
|
|
510
506
|
const deleteAfterDays = positiveInt(raw.codex_rollout_delete_after_days, "codex_rollout_delete_after_days", DEFAULT_CODEX_ROLLOUT_DELETE_AFTER_DAYS);
|
|
511
|
-
const leadReservePerHost = nonNegativeInt(raw.lead_reserve_per_host, "lead_reserve_per_host", 1);
|
|
512
507
|
if (deleteAfterDays * 24 < compressAfterHours) {
|
|
513
508
|
throw new LanesConfigError("invalid_config", "codex_rollout_delete_after_days must be at least codex_rollout_compress_after_hours / 24", { codex_rollout_compress_after_hours: compressAfterHours, codex_rollout_delete_after_days: deleteAfterDays });
|
|
514
509
|
}
|
|
@@ -522,7 +517,6 @@ export function parseLanesConfig(raw, deps = defaultDeps()) {
|
|
|
522
517
|
tracker: { checks: parseTrackerChecks(raw.tracker) },
|
|
523
518
|
runaway_search_guard: parseRunawaySearchGuard(raw.runaway_search_guard),
|
|
524
519
|
low_disk_min_free_bytes: positiveInt(raw.low_disk_min_free_bytes, "low_disk_min_free_bytes", DEFAULT_LOW_DISK_MIN_FREE_BYTES),
|
|
525
|
-
lead_reserve_per_host: leadReservePerHost,
|
|
526
520
|
codex_rollout_compress_after_hours: compressAfterHours,
|
|
527
521
|
codex_rollout_delete_after_days: deleteAfterDays,
|
|
528
522
|
session_question_timeout_secs: positiveInt(raw.session_question_timeout_secs, "session_question_timeout_secs", DEFAULT_SESSION_QUESTION_TIMEOUT_SECS),
|
|
@@ -12,7 +12,7 @@ export type { HostCapacityAccount, HostCapacityProvider, HostCapacitySnapshot, H
|
|
|
12
12
|
export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STORY_STATUSES, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
|
|
13
13
|
export type { AccountConfig, AttributionHistoryEntry, EnvelopeDecision, EnvelopeQuestion, HostKind, LaneCompanyRef, LaneAccountSelection, LaneAccountSelectionConsidered, LaneAccountSelectionHostCount, LaneEnvelope, LaneHost, LaneProvider, LanePlacementPolicy, LaneRecord, LaneStory, LaneStoryStatus, PendingResume, AccountHistoryEntry, LaneState, LanesConfig, LastEnvelopeRef, LivenessState, LivenessWindows, ProviderConfig, QuestionAnswer, QuestionBodyRef, QuestionPriority, QuestionRecord, QuestionStatus, QueuedReason, RaisedVia, RoutingHistoryEntry, RoutingReason, SeniorHistoryEntry, SeniorKind, SeniorRef, WorkerProfileRepinHistoryEntry, LaneSettingsHistoryEntry, StoryStatus, TerminalLaneState, ThrottleConfig, } from "./types.js";
|
|
14
14
|
export { ACTIVE_LANE_STORY_STATUSES, activeLaneStories, laneStories, laneStory, storyKey, storyStatusForEnvelope, storyWarning, withCurrentStory, withLaneStories, } from "./stories.js";
|
|
15
|
-
export { DEFAULT_ACCOUNT_HOMES, DEFAULT_AUTO_COMPACT_TOKENS, DEFAULT_COMPACT_AFTER_MS,
|
|
15
|
+
export { DEFAULT_ACCOUNT_HOMES, DEFAULT_AUTO_COMPACT_TOKENS, DEFAULT_COMPACT_AFTER_MS, DEFAULT_LIVE_WITHIN_MS, DEFAULT_LIVENESS_WINDOWS, DEFAULT_MAX_QUEUED, DEFAULT_PROVIDER_MAX_RUNNING, DEFAULT_SILENT_LOST_MS, DEFAULT_TURN_START_LOST_MS, LANES_CONFIG_REL, accountMaxRunning, canonicalizeHome, documentedDefaultConfig, expandHome, lanesConfigPath, loadLanesConfig, parseLanesConfig, providerCompactAfterMs, providerAutoCompactTokens, providerQuotaDefaultHoldMs, DEFAULT_QUOTA_HOLD_MS, writeLanesConfig, } from "./config.js";
|
|
16
16
|
export type { ConfigLoadDeps, ConfigWriteDeps } from "./config.js";
|
|
17
17
|
export { resolveLanesHqRoot } from "./hq-root.js";
|
|
18
18
|
export type { LanesHqRootOpts } from "./hq-root.js";
|
package/dist/lib/lanes/index.js
CHANGED
|
@@ -8,7 +8,7 @@ export { compareLaneQueueOrder, hasActiveAdmissionReservation, hasActivePendingR
|
|
|
8
8
|
export { buildHostCapacitySnapshot, chooseHostForLane, parseHostCapacitySnapshot, } from "./host-capacity.js";
|
|
9
9
|
export { CONFIG_SCHEMA_VERSION, ENVELOPE_DECISIONS, ENVELOPE_SCHEMA_VERSION, HISTORY_CAP, HOST_KINDS, LANE_PROVIDERS, LANE_SCHEMA_VERSION, LANE_STORY_STATUSES, LANE_STATES, LIVENESS_STATES, QUESTION_BODY_STORES, QUESTION_SCHEMA_VERSION, QUESTION_STATUSES, QUEUED_REASONS, RAISED_VIA, ROUTING_REASONS, SENIOR_KINDS, STORY_STATUSES, TERMINAL_LANE_STATES, isEnvelopeDecision, isHostKind, isLaneProvider, isLaneState, isLivenessState, isQuestionStatus, isSeniorKind, isTerminalLaneState, parseLaneEnvelope, parseLaneProvider, parseLaneState, parseLivenessState, parseMajorSchema, parseQuestionStatus, parseSeniorKind, } from "./types.js";
|
|
10
10
|
export { ACTIVE_LANE_STORY_STATUSES, activeLaneStories, laneStories, laneStory, storyKey, storyStatusForEnvelope, storyWarning, withCurrentStory, withLaneStories, } from "./stories.js";
|
|
11
|
-
export { DEFAULT_ACCOUNT_HOMES, DEFAULT_AUTO_COMPACT_TOKENS, DEFAULT_COMPACT_AFTER_MS,
|
|
11
|
+
export { DEFAULT_ACCOUNT_HOMES, DEFAULT_AUTO_COMPACT_TOKENS, DEFAULT_COMPACT_AFTER_MS, DEFAULT_LIVE_WITHIN_MS, DEFAULT_LIVENESS_WINDOWS, DEFAULT_MAX_QUEUED, DEFAULT_PROVIDER_MAX_RUNNING, DEFAULT_SILENT_LOST_MS, DEFAULT_TURN_START_LOST_MS, LANES_CONFIG_REL, accountMaxRunning, canonicalizeHome, documentedDefaultConfig, expandHome, lanesConfigPath, loadLanesConfig, parseLanesConfig, providerCompactAfterMs, providerAutoCompactTokens, providerQuotaDefaultHoldMs, DEFAULT_QUOTA_HOLD_MS, writeLanesConfig, } from "./config.js";
|
|
12
12
|
export { resolveLanesHqRoot } from "./hq-root.js";
|
|
13
13
|
export { LANE_STORY_LEDGER_FLAG, buildLaneStatusSnapshot, formatLaneStatusHuman, laneStoryLedgerEnabled, resolveProjectStoryLedger, syncLaneToProjectStoryLedger, removeLaneFromProjectStoryLedger, validateProjectStories, } from "./story-ledger.js";
|
|
14
14
|
export { buildOutpostPickupCommand, parseOutpostPickupPayload, } from "./outpost-dispatch.js";
|
package/dist/lib/lanes/loop.d.ts
CHANGED
|
@@ -188,6 +188,11 @@ export declare function loopStatus(runDir: string, laneId: string): LoopStatus;
|
|
|
188
188
|
export declare function commitIdleLoopStop(runDir: string, laneId: string, now?: () => Date): boolean;
|
|
189
189
|
export declare function setLoopControllerPid(runDir: string, laneId: string, pid: number | undefined): void;
|
|
190
190
|
export declare function clearLoopControllerPid(runDir: string, laneId: string, pid: number): void;
|
|
191
|
+
/**
|
|
192
|
+
* True when a loop lane's controller is alive with no phase active: the lane
|
|
193
|
+
* finished its last phase and is waiting for the next envelope.
|
|
194
|
+
*/
|
|
195
|
+
export declare function loopLaneIsIdle(runDir: string, laneId: string): boolean;
|
|
191
196
|
export declare function loopControllerIsAlive(runDir: string): boolean;
|
|
192
197
|
/** Run queued phases serially. Sessions are isolated by story and provider. */
|
|
193
198
|
export declare function runLoopLane(options: LoopRunnerOptions): Promise<LoopRunSummary>;
|
package/dist/lib/lanes/loop.js
CHANGED
|
@@ -648,6 +648,14 @@ export function clearLoopControllerPid(runDir, laneId, pid) {
|
|
|
648
648
|
if (currentPid === pid)
|
|
649
649
|
persistStatus(runDir, laneId, { pid: undefined });
|
|
650
650
|
}
|
|
651
|
+
/**
|
|
652
|
+
* True when a loop lane's controller is alive with no phase active: the lane
|
|
653
|
+
* finished its last phase and is waiting for the next envelope.
|
|
654
|
+
*/
|
|
655
|
+
export function loopLaneIsIdle(runDir, laneId) {
|
|
656
|
+
const status = loopStatus(runDir, laneId);
|
|
657
|
+
return status.state === "waiting" && status.active_envelope_id === null && loopControllerIsAlive(runDir);
|
|
658
|
+
}
|
|
651
659
|
export function loopControllerIsAlive(runDir) {
|
|
652
660
|
let pid;
|
|
653
661
|
try {
|
|
@@ -8,11 +8,11 @@
|
|
|
8
8
|
import * as fs from "node:fs";
|
|
9
9
|
import * as path from "node:path";
|
|
10
10
|
import { parseUserYaml } from "../../utils/user-yaml-error.js";
|
|
11
|
-
import { compareLaneQueueOrder, compareLaneQueueOrderWithLeads,
|
|
11
|
+
import { compareLaneQueueOrder, compareLaneQueueOrderWithLeads, leadLaneIds, hasActiveAdmissionReservation, hasActivePendingResume, laneAdmissionCapReason, laneOccupiesRunningSlot, laneRunsOnHost, withAllLaneAdmissionLocks, } from "./admission.js";
|
|
12
12
|
import { accountStatus, isCredentialProven, resolveAccount } from "./accounts.js";
|
|
13
13
|
import { effectiveAccountHome } from "./account-home.js";
|
|
14
14
|
import { enforceRegistryApprovalGate } from "./approval-gate.js";
|
|
15
|
-
import { loopStatus } from "./loop.js";
|
|
15
|
+
import { loopLaneIsIdle, loopStatus } from "./loop.js";
|
|
16
16
|
import { loadLanesConfig, providerQuotaDefaultHoldMs } from "./config.js";
|
|
17
17
|
import { MAX_LANE_DEPTH } from "./constants.js";
|
|
18
18
|
import { laneDepthCap } from "./depth-cap.js";
|
|
@@ -33,7 +33,7 @@ import { DEFAULT_RUNAWAY_SEARCH_GUARD_CONFIG, listHostProcesses, runRunawayToolG
|
|
|
33
33
|
import { LEAD_LANE_KEEP_ALIVE_FLAG, LEAD_LANE_KEEP_ALIVE_RESUME_BLOCKED_FLAG, LEAD_LANE_KEEP_ALIVE_IMMEDIATE_FIRST_RESUME_FLAG, LEAD_LANE_KEEP_ALIVE_FIRST_RESUME_SETTLE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_GRACE_MS, LEAD_LANE_KEEP_ALIVE_BLOCKED_RESUME_TEXT, queueLaneTerminalResume, LEAD_LANE_KEEP_ALIVE_INTERRUPTED_RESUME_TEXT, LEAD_LANE_KEEP_ALIVE_MAX_RESUMES, LEAD_LANE_KEEP_ALIVE_RESUME_TEXT, LANE_SUPERSESSION_LINEAGE_AWARE_FLAG, doneKeepAliveVerdict, dueKeepAliveNextCheckMs, hasPendingKeepAliveWork, keepAliveNextCheckAtMs, keepAliveExclusion, loadRetiredLaneIds, leadLaneKeepAliveBackoffMs, undeliveredAnsweredQuestions, } from "./keep-alive.js";
|
|
34
34
|
import { isFailedQuotaKeepAliveLane, providerSupportsSessionResume } from "./delivery.js";
|
|
35
35
|
import { LANE_PROVIDERS, isIsoDateTime, isTerminalLaneState } from "./types.js";
|
|
36
|
-
import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort,
|
|
36
|
+
import { applyDefaultQuotaHold, clearAccountQuotaAfterSuccessfulTurnBestEffort, readQuotaSignal, recordAccountQuota, } from "./quota.js";
|
|
37
37
|
import { deliverQuotaNotice, quotaLaneNotice } from "./quota-notice.js";
|
|
38
38
|
import { answeredQuestionEntries, questionRecordRefusalCode, raiseEnvelopeQuestions, resolveQuestionCompanyWithFallback, } from "./questions.js";
|
|
39
39
|
import { SILENCE_MS, appendWatcherEvent, classifyLaneExit, isPidAlive, missingExpectedEnvelopeFile, readLatestEnvelopeFile, readLatestEnvelopeRecord, watcherEventsPath, watcherOutLogPath, writeWatcherDecision, } from "./watcher.js";
|
|
@@ -1641,6 +1641,23 @@ async function heartbeatAndDeadHosts(hqRoot, lanes, summary, deps, now) {
|
|
|
1641
1641
|
// Watcher owns terminal classification of a process that has just exited.
|
|
1642
1642
|
continue;
|
|
1643
1643
|
}
|
|
1644
|
+
if (lane.loop_mode === true && loopLaneIsIdle(runDir, lane.lane_id)) {
|
|
1645
|
+
// An idle loop lane has no worker between phases. Its controller is
|
|
1646
|
+
// alive and waiting for the next envelope, so the lane is not finished.
|
|
1647
|
+
try {
|
|
1648
|
+
const result = casLaneIf(hqRoot, lane.lane_id, (current) => current.state === "running" && current.last_host_report_at !== nowIso, (current) => ({ ...current, last_host_report_at: nowIso }), deps.dryRun);
|
|
1649
|
+
if (result.changed)
|
|
1650
|
+
summary.heartbeats.push(lane.lane_id);
|
|
1651
|
+
}
|
|
1652
|
+
catch (err) {
|
|
1653
|
+
markDirty(summary, {
|
|
1654
|
+
kind: "lane",
|
|
1655
|
+
id: lane.lane_id,
|
|
1656
|
+
why: `heartbeat CAS failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1657
|
+
});
|
|
1658
|
+
}
|
|
1659
|
+
continue;
|
|
1660
|
+
}
|
|
1644
1661
|
const hadPids = workerPid !== null || watcherPid !== null;
|
|
1645
1662
|
if (!hadPids && nowMs - lastHostReportMs(lane) < DEAD_HOST_MS) {
|
|
1646
1663
|
try {
|
|
@@ -2045,7 +2062,6 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
2045
2062
|
if (localQueued.length === 0 || deps.skipAdmissionWithoutCpuSample === true)
|
|
2046
2063
|
return;
|
|
2047
2064
|
const config = loadLanesConfig(hqRoot);
|
|
2048
|
-
const admissionQuotaState = loadQuotaState(hqRoot);
|
|
2049
2065
|
const candidateLeadIds = leadLaneIds(lanes);
|
|
2050
2066
|
const flagReader = deps.readRegistryFlag ?? readLanesRegistryFlag;
|
|
2051
2067
|
const leadAdmissionEnabledById = new Map();
|
|
@@ -2140,25 +2156,6 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
2140
2156
|
}
|
|
2141
2157
|
}
|
|
2142
2158
|
const projected = [...running, ...spawned];
|
|
2143
|
-
const laneResolution = lane.provider === "claude" ? resolveLaneCompany(hqRoot, lane) : undefined;
|
|
2144
|
-
const laneCompany = laneResolution?.company ?? undefined;
|
|
2145
|
-
const laneCompanyKey = laneCompany?.uid ?? laneCompany?.slug ?? lane.company?.uid ?? lane.company?.slug;
|
|
2146
|
-
let reserveEnabled = false;
|
|
2147
|
-
if (lane.provider === "claude" && laneCompanyKey) {
|
|
2148
|
-
let enabled = leadFlagByCompany.get(laneCompanyKey);
|
|
2149
|
-
if (!enabled) {
|
|
2150
|
-
enabled = readLanesRegistryFlagFailOff(LANE_LEAD_ADMISSION_PRIORITY_FLAG, {
|
|
2151
|
-
company: laneCompany,
|
|
2152
|
-
...(laneCompany && laneResolution ? { companyUidSource: laneCompanyUidSource(laneResolution.source) } : {}),
|
|
2153
|
-
env: deps.env,
|
|
2154
|
-
}, flagReader);
|
|
2155
|
-
leadFlagByCompany.set(laneCompanyKey, enabled);
|
|
2156
|
-
}
|
|
2157
|
-
reserveEnabled = await enabled;
|
|
2158
|
-
}
|
|
2159
|
-
const reserveBlocked = reserveEnabled && lane.provider === "claude" &&
|
|
2160
|
-
!priorityLeads.has(lane.lane_id) &&
|
|
2161
|
-
freeClaudeAccountSlots(projected, config, deps.host, now.getTime(), admissionQuotaState) <= config.lead_reserve_per_host;
|
|
2162
2159
|
const alternateAccount = availableBalancedAccount(hqRoot, lane, projected, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
2163
2160
|
const admittedLane = alternateAccount !== undefined
|
|
2164
2161
|
? writeAdmissionAccount(hqRoot, lane, alternateAccount, deps.dryRun)
|
|
@@ -2166,10 +2163,7 @@ async function admitQueued(hqRoot, lanes, summary, deps, now, onlyLaneIds, exclu
|
|
|
2166
2163
|
const baseCap = laneAdmissionCapReason(running, admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
2167
2164
|
const currentCap = laneAdmissionCapReason([...running, ...spawned], admittedLane, config, deps.host, profileMaxConcurrent, now.getTime());
|
|
2168
2165
|
let blockedBy;
|
|
2169
|
-
if (
|
|
2170
|
-
blockedBy = "lead_reserve";
|
|
2171
|
-
}
|
|
2172
|
-
else if (currentCap !== undefined) {
|
|
2166
|
+
if (currentCap !== undefined) {
|
|
2173
2167
|
blockedBy = baseCap ?? "queue_position";
|
|
2174
2168
|
}
|
|
2175
2169
|
if (!blockedBy && !decision.admit)
|
|
@@ -13,7 +13,6 @@ import { execFileSync as nodeExecFileSync } from "node:child_process";
|
|
|
13
13
|
import * as fs from "node:fs";
|
|
14
14
|
import * as os from "node:os";
|
|
15
15
|
import * as path from "node:path";
|
|
16
|
-
import { DEFAULT_CPU_BUSY_PAUSE, DEFAULT_CPU_BUSY_RESUME, DEFAULT_MIN_AVAILABLE_MEM_MB, } from "./config.js";
|
|
17
16
|
// Reconcile normally measures the interval between cumulative CPU snapshots.
|
|
18
17
|
// A five-second window establishes a baseline or recovers from stale counters.
|
|
19
18
|
export const CPU_SAMPLE_MS = 5_000;
|
|
@@ -294,17 +293,18 @@ export function evaluateThrottle(input) {
|
|
|
294
293
|
const pause = input.throttle.cpu_busy_pause;
|
|
295
294
|
const resume = input.throttle.cpu_busy_resume;
|
|
296
295
|
const minMem = input.throttle.min_available_mem_mb;
|
|
296
|
+
// Each gate is opt-in: an unset threshold never blocks admission.
|
|
297
297
|
let cpuBlocks = false;
|
|
298
|
-
if (input.cpuCheckEnabled && input.busy !== null) {
|
|
298
|
+
if (pause !== undefined && input.cpuCheckEnabled && input.busy !== null) {
|
|
299
299
|
if (input.state.cpuPaused) {
|
|
300
|
-
cpuBlocks = input.busy > resume;
|
|
300
|
+
cpuBlocks = input.busy > (resume ?? pause);
|
|
301
301
|
}
|
|
302
302
|
else {
|
|
303
303
|
cpuBlocks = input.busy >= pause;
|
|
304
304
|
}
|
|
305
305
|
}
|
|
306
306
|
let memBlocks = false;
|
|
307
|
-
if (input.availableMemMb !== null) {
|
|
307
|
+
if (minMem !== undefined && input.availableMemMb !== null) {
|
|
308
308
|
memBlocks = input.availableMemMb < minMem;
|
|
309
309
|
}
|
|
310
310
|
if (!cpuBlocks && !memBlocks) {
|
|
@@ -448,11 +448,7 @@ export function writeThrottleState(filePath, state, deps = defaultThrottleDeps()
|
|
|
448
448
|
throw err;
|
|
449
449
|
}
|
|
450
450
|
}
|
|
451
|
-
export async function decideAdmission(hqRoot, throttle = {
|
|
452
|
-
cpu_busy_pause: DEFAULT_CPU_BUSY_PAUSE,
|
|
453
|
-
cpu_busy_resume: DEFAULT_CPU_BUSY_RESUME,
|
|
454
|
-
min_available_mem_mb: DEFAULT_MIN_AVAILABLE_MEM_MB,
|
|
455
|
-
}, deps = defaultThrottleDeps(), cpuBusyFraction, cpuObservation) {
|
|
451
|
+
export async function decideAdmission(hqRoot, throttle = {}, deps = defaultThrottleDeps(), cpuBusyFraction, cpuObservation) {
|
|
456
452
|
const filePath = throttleStatePath(hqRoot);
|
|
457
453
|
const loaded = readThrottleState(filePath, deps);
|
|
458
454
|
if (loaded === "unreadable") {
|
|
@@ -1055,10 +1055,11 @@ export interface ProviderConfig {
|
|
|
1055
1055
|
quota_default_hold_ms?: number;
|
|
1056
1056
|
accounts: Record<string, AccountConfig>;
|
|
1057
1057
|
}
|
|
1058
|
+
/** Opt-in admission gates. An unset key means that gate is off. */
|
|
1058
1059
|
export interface ThrottleConfig {
|
|
1059
|
-
cpu_busy_pause
|
|
1060
|
-
cpu_busy_resume
|
|
1061
|
-
min_available_mem_mb
|
|
1060
|
+
cpu_busy_pause?: number;
|
|
1061
|
+
cpu_busy_resume?: number;
|
|
1062
|
+
min_available_mem_mb?: number;
|
|
1062
1063
|
}
|
|
1063
1064
|
/** Session liveness windows from spec §9. Missing keys keep these documented defaults. */
|
|
1064
1065
|
export interface LivenessWindows {
|
|
@@ -1096,8 +1097,6 @@ export interface LanesConfig {
|
|
|
1096
1097
|
runaway_search_guard?: RunawaySearchGuardConfig;
|
|
1097
1098
|
/** Optional free-space threshold override for the low-disk admission gate. */
|
|
1098
1099
|
low_disk_min_free_bytes?: number;
|
|
1099
|
-
/** Claude account slots kept available per host for senior lead lanes. */
|
|
1100
|
-
lead_reserve_per_host: number;
|
|
1101
1100
|
/** Retention ages for Codex session rollout maintenance. */
|
|
1102
1101
|
codex_rollout_compress_after_hours?: number;
|
|
1103
1102
|
codex_rollout_delete_after_days?: number;
|
package/dist/lib/lanes/types.js
CHANGED
|
@@ -9,10 +9,22 @@ export interface WorkerContinuityLookup {
|
|
|
9
9
|
export interface WorkerContinuityResult {
|
|
10
10
|
previous?: LaneRecord;
|
|
11
11
|
resumable: boolean;
|
|
12
|
-
fallbackReason?: "provider_unsupported" | "session_id_missing" | "state_unsafe" | "host_mismatch";
|
|
12
|
+
fallbackReason?: "provider_unsupported" | "session_id_missing" | "state_unsafe" | "host_mismatch" | "settings_mismatch";
|
|
13
13
|
}
|
|
14
14
|
export declare function sameLaneHost(left: LaneRecord["host"], right: LaneRecord["host"]): boolean;
|
|
15
15
|
/** Select the prior worker lane within exactly one senior/company/provider scope. */
|
|
16
16
|
export declare function findWorkerContinuityLane(lanes: readonly LaneRecord[], lookup: WorkerContinuityLookup): WorkerContinuityResult;
|
|
17
|
+
export interface WorkerContinuitySettings {
|
|
18
|
+
project: LaneRecord["project_id"];
|
|
19
|
+
/** Resolved model for the new lane; undefined when the provider chooses. */
|
|
20
|
+
model?: string;
|
|
21
|
+
/** Resolved effort for the new lane; undefined when none is passed. */
|
|
22
|
+
effort?: string;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Refuse reuse when the earlier session ran under a different project, model,
|
|
26
|
+
* or effort. A prior lane that recorded no model does not match a pinned one.
|
|
27
|
+
*/
|
|
28
|
+
export declare function applyContinuitySettings(result: WorkerContinuityResult, settings: WorkerContinuitySettings): WorkerContinuityResult;
|
|
17
29
|
export declare function workerContinuityPrompt(previous: LaneRecord, resultSummary?: string, resultRef?: string): string;
|
|
18
30
|
//# sourceMappingURL=worker-continuity.d.ts.map
|
|
@@ -65,6 +65,20 @@ export function findWorkerContinuityLane(lanes, lookup) {
|
|
|
65
65
|
}
|
|
66
66
|
return { previous, resumable: true };
|
|
67
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* Refuse reuse when the earlier session ran under a different project, model,
|
|
70
|
+
* or effort. A prior lane that recorded no model does not match a pinned one.
|
|
71
|
+
*/
|
|
72
|
+
export function applyContinuitySettings(result, settings) {
|
|
73
|
+
const previous = result.previous;
|
|
74
|
+
if (!result.resumable || !previous)
|
|
75
|
+
return result;
|
|
76
|
+
if (previous.project_id === settings.project && previous.model === settings.model &&
|
|
77
|
+
previous.effort === settings.effort) {
|
|
78
|
+
return result;
|
|
79
|
+
}
|
|
80
|
+
return { previous, resumable: false, fallbackReason: "settings_mismatch" };
|
|
81
|
+
}
|
|
68
82
|
export function workerContinuityPrompt(previous, resultSummary, resultRef) {
|
|
69
83
|
const summary = resultSummary?.trim();
|
|
70
84
|
const resultPath = resultRef?.trim()
|