@indigoai-us/hq-cli 5.281.0 → 5.283.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/assets/scaffold/.claude/hooks/validate-policy-frontmatter.sh +485 -0
- package/assets/scaffold/core/scripts/eval-trigger.sh +193 -0
- package/assets/scaffold/core/scripts/policy-age-report.sh +156 -0
- package/assets/scaffold/core/scripts/policy-retire.sh +199 -0
- package/assets/scaffold/core/scripts/read-policy-frontmatter.sh +32 -0
- package/dist/command-catalog.generated.d.ts +67 -0
- package/dist/command-catalog.generated.js +83 -0
- package/dist/commands/core-knowledge-prefs.d.ts +10 -0
- package/dist/commands/core-knowledge-prefs.js +125 -0
- package/dist/commands/core-native-policy-flag.d.ts +9 -0
- package/dist/commands/core-native-policy-flag.js +12 -0
- package/dist/commands/core-ontology-native.d.ts +0 -1
- package/dist/commands/core-ontology-native.js +2 -30
- package/dist/commands/core-ontology.d.ts +4 -0
- package/dist/commands/core-ontology.js +8 -3
- package/dist/commands/core-policy-frontmatter.d.ts +8 -0
- package/dist/commands/core-policy-frontmatter.js +52 -0
- package/dist/commands/core-policy.d.ts +15 -0
- package/dist/commands/core-policy.js +982 -0
- package/dist/commands/core.js +14 -1
- package/dist/commands/lanes-message.js +1 -0
- package/dist/commands/lanes-move.js +2 -0
- package/dist/commands/lanes-resume.js +1 -0
- package/dist/commands/lanes-shared.d.ts +10 -1
- package/dist/commands/lanes-shared.js +2 -1
- package/dist/commands/lanes.d.ts +8 -0
- package/dist/commands/lanes.js +324 -93
- package/dist/fast-lanes-list-route.d.ts +8 -0
- package/dist/fast-lanes-list-route.js +30 -0
- package/dist/fast-lanes-list.d.ts +3 -0
- package/dist/fast-lanes-list.js +252 -0
- package/dist/fast-scaffold.d.ts +5 -0
- package/dist/fast-scaffold.js +10 -0
- package/dist/index.js +10 -1
- package/dist/lib/checkpoint-ontology/merge-copies.d.ts +24 -1
- package/dist/lib/checkpoint-ontology/merge-copies.js +70 -1
- package/dist/lib/lanes/flag-keys.d.ts +3 -0
- package/dist/lib/lanes/flag-keys.js +3 -0
- package/dist/lib/lanes/index.d.ts +1 -1
- package/dist/lib/lanes/index.js +1 -1
- package/dist/lib/lanes/keep-alive.js +1 -0
- package/dist/lib/lanes/outpost-dispatch.d.ts +1 -1
- package/dist/lib/lanes/outpost-dispatch.js +4 -2
- package/dist/lib/lanes/outpost-hire-cache.d.ts +3 -0
- package/dist/lib/lanes/outpost-hire-cache.js +11 -3
- package/dist/lib/lanes/outpost-snapshot-cache.d.ts +26 -0
- package/dist/lib/lanes/outpost-snapshot-cache.js +215 -0
- package/dist/lib/lanes/reconcile.d.ts +7 -1
- package/dist/lib/lanes/reconcile.js +88 -16
- package/dist/lib/lanes/spawn.js +5 -0
- package/dist/lib/lanes/store.d.ts +4 -0
- package/dist/lib/lanes/store.js +66 -0
- package/dist/lib/lanes/throttle.d.ts +29 -2
- package/dist/lib/lanes/throttle.js +145 -5
- package/dist/lib/lanes/types.d.ts +8 -1
- package/dist/lib/lanes/types.js +2 -1
- package/dist/lib/lanes/watcher.js +1 -0
- package/dist/lib/policy-validator.d.ts +2 -0
- package/dist/lib/policy-validator.js +136 -0
- package/dist/native-fast-route.d.ts +2 -0
- package/dist/native-fast-route.js +6 -2
- package/dist/utils/contribution-table.d.ts +1 -1
- package/package.json +1 -1
package/dist/commands/lanes.js
CHANGED
|
@@ -31,13 +31,14 @@ import { withCliFeatureUsed } from "../utils/cli-telemetry.js";
|
|
|
31
31
|
import { resolveLanesHqRoot } from "../lib/lanes/hq-root.js";
|
|
32
32
|
import { isSsmRunCommandProcess } from "../lib/lanes/ssm-command-context.js";
|
|
33
33
|
import { readLanesRegistryFlag } from "../lib/lanes/registry-flag.js";
|
|
34
|
+
import { getOrFetchOutpostSnapshot, OUTPOST_SNAPSHOT_CACHE_TTL_MS, outpostLaneListSnapshotKey, OutpostSnapshotInvocationMemo, } from "../lib/lanes/outpost-snapshot-cache.js";
|
|
34
35
|
import { createOutpostHireGrantContext, parseOutpostHireGrantContext, selectOutpostHireGrantHierarchy, sameHireGrantSenior, } from "../lib/lanes/hire-grant-dispatch.js";
|
|
35
36
|
import { createOutpostProfileRepinContext, parseOutpostProfileRepinContext, } from "../lib/lanes/outpost-profile-repin.js";
|
|
36
37
|
import { createOutpostRegistryWorkerApprovalContext, parseOutpostRegistryWorkerApprovalContext, } from "../lib/lanes/outpost-registry-worker-approval.js";
|
|
37
38
|
import { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity, resolveAdmissionHost, } from "../lib/lanes/admission-host.js";
|
|
38
39
|
export { OUTPOST_DIR, OUTPOST_ID_PATH, readOutpostIdentity } from "../lib/lanes/admission-host.js";
|
|
39
40
|
import { buildOutpostCapacityCommand, buildOutpostIdentityBackfillCommand, buildOutpostLaneCommand, buildOutpostQuestionListCommand, buildOutpostPickupCommand, createOutpostsClient, dispatchErrorMessage, dispatchOutpostPickup, buildOutpostLaneListCommand, outpostRejectedEffortOption, parseOutpostPickupPayload, PICKUP_WAIT_MS, stageOutpostPayload, } from "../lib/lanes/outpost-dispatch.js";
|
|
40
|
-
import { outpostHireSummaryCacheKey, readOutpostHireSummaryCache, writeOutpostHireSummaryCache, } from "../lib/lanes/outpost-hire-cache.js";
|
|
41
|
+
import { outpostHireSummaryCacheKey, LANE_OUTPOST_SNAPSHOT_CACHE_FLAG, OUTPOST_HIRE_SUMMARY_CACHE_TTL_MS, readOutpostHireSummaryCache, readOutpostHireSummaryCacheIncludingStale, writeOutpostHireSummaryCache, } from "../lib/lanes/outpost-hire-cache.js";
|
|
41
42
|
import { buildHostCapacitySnapshot, chooseHostForLane, parseHostCapacitySnapshot, } from "../lib/lanes/host-capacity.js";
|
|
42
43
|
import { BALANCE_MOVABLE_STATES, selectClaudeAccount, } from "../lib/lanes/claude-balance.js";
|
|
43
44
|
import { parseUserYaml } from "../utils/user-yaml-error.js";
|
|
@@ -51,7 +52,7 @@ import { createLaneMesh, emitLaneDecisionEvents, emitLaneTerminalStateEvent, lan
|
|
|
51
52
|
import { sessionIdFromEnv } from "../lib/mesh/live/session-identity.js";
|
|
52
53
|
import { listLaneReviews } from "../lib/lanes/reviews.js";
|
|
53
54
|
import { effectiveAccountHome } from "../lib/lanes/account-home.js";
|
|
54
|
-
import { AccountResolutionError, compareLaneQueueOrder, LANE_PROVIDERS, LANE_STATES, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, aliveDirectChildren, assignedChildHireGrants, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, laneStoryLedgerEnabled, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listLaneIds, listQuestionIds, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneRecord, parseLaneState, parseQuestionRecord, parseSeniorKind, readLane, sanitizeHostComponent, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, loadWorkerProfileWithRegistry, registryWorkerRecord, runReconcilePass, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, dropboxDir, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, providerSessionPortable, sessionNotPortableMessage, withLaneAdmissionLock, withAllLaneAdmissionLocks, readWorkerRunResult, resumeCommandRefusal, withLaneHireLocks, raiseLaneQuestion, sha256Text, } from "../lib/lanes/index.js";
|
|
55
|
+
import { AccountResolutionError, compareLaneQueueOrder, LANE_PROVIDERS, LANE_STATES, QUESTION_STATUSES, SENIOR_KINDS, accountStatus, appendCappedHistory, canonicalizeHome, createLane, decideAdmission, defaultLivenessDeps, defaultSystemdUserDir, defaultThrottleDeps, installReconcileTimer, providerPathEnv, probeDrainHookRegistration, isLanesError, isLaneVersionConflict, isTerminalLaneState, aliveDirectChildren, assignedChildHireGrants, hireGrantCapacityError, planHireGrantBackfill, terminalChildResumeHireGrantError, withHireGrantAdmission, laneStoryLedgerEnabled, validateProjectStories, buildLaneStatusSnapshot, formatLaneStatusHuman, listLaneIds, listQuestionIds, loadLanesConfig, laneRunsHere, MAX_LANE_DEPTH, MONITOR_ERROR_EXIT_CODE, readLiveMonitorEvidence, resolveMachineIdentity as lookupMachineIdentity, loadLivenessIndex, parseLaneProvider, parseLaneRecord, parseLaneState, parseQuestionRecord, parseSeniorKind, readAllLanes, readLane, sanitizeHostComponent, retryUpdateLane, syncLaneRecordFromOwner, readQuestion, resolveAccount, resolveLaneWorker, ignoredWorkerProfileFields, loadWorkerProfile, loadWorkerProfileWithRegistry, registryWorkerRecord, runReconcilePass, sessionLiveness, tryReadLane, uninstallReconcileTimer, updateLane, updateQuestion, dropboxDir, laneDeliveryMechanism, hasActiveAdmissionReservation, hasActivePendingResume, hasRecordedSessionId, hasEarlierEligibleQueuedLane, laneAdmissionCapReason, providerSupportsSessionResume, providerSessionPortable, sessionNotPortableMessage, withLaneAdmissionLock, withAllLaneAdmissionLocks, readWorkerRunResult, resumeCommandRefusal, withLaneHireLocks, raiseLaneQuestion, sha256Text, } from "../lib/lanes/index.js";
|
|
55
56
|
import { activeLaneStories, laneStories, laneStory, storyWarning, } from "../lib/lanes/stories.js";
|
|
56
57
|
import { resolveLaneKeepAlive } from "../lib/lanes/profile.js";
|
|
57
58
|
import { resolveLaneCompany } from "../lib/lanes/lane-company.js";
|
|
@@ -79,6 +80,7 @@ export const LANE_REMOTE_MIRROR_SYNC_FLAG = "lanes.remote-done-mirror";
|
|
|
79
80
|
const REMOTE_MIRROR_REFRESH_BATCH_SIZE = 100;
|
|
80
81
|
/** The lane → mesh seam. Defined in lib/lanes/mesh-events.ts; the watcher shares it. */
|
|
81
82
|
export { emitLaneDecisionEvents };
|
|
83
|
+
export { LANE_OUTPOST_SNAPSHOT_CACHE_FLAG };
|
|
82
84
|
function withoutProviderHome(value) {
|
|
83
85
|
if (Array.isArray(value))
|
|
84
86
|
return value.map(withoutProviderHome);
|
|
@@ -294,6 +296,8 @@ function resolveDeps(deps = {}) {
|
|
|
294
296
|
})),
|
|
295
297
|
fetchText: deps.fetchText ?? defaultFetchText,
|
|
296
298
|
lanePlacementEnabled: deps.lanePlacementEnabled ?? defaultLanePlacementEnabled,
|
|
299
|
+
outpostSnapshotCacheEnabled: deps.outpostSnapshotCacheEnabled ?? (() => readLanesRegistryFlag(LANE_OUTPOST_SNAPSHOT_CACHE_FLAG, { env })),
|
|
300
|
+
startOutpostLaneSnapshotRefresh: deps.startOutpostLaneSnapshotRefresh ?? ((root) => startOutpostLaneSnapshotRefresh(root, env)),
|
|
297
301
|
storyLedgerEnabled: deps.storyLedgerEnabled ?? laneStoryLedgerEnabled,
|
|
298
302
|
isSsmOwnerDispatch: deps.isSsmOwnerDispatch ?? (() => isSsmRunCommandProcess()),
|
|
299
303
|
withGrantHireLocks: deps.withGrantHireLocks ?? withLaneHireLocks,
|
|
@@ -460,7 +464,7 @@ function readQuestionBody(hqRoot, question) {
|
|
|
460
464
|
return stored.body;
|
|
461
465
|
}
|
|
462
466
|
function loadAllLanes(hqRoot) {
|
|
463
|
-
return
|
|
467
|
+
return readAllLanes(hqRoot);
|
|
464
468
|
}
|
|
465
469
|
function laneHireFields(lane, lanes, aliveChildrenByParent) {
|
|
466
470
|
const aliveChildCount = aliveChildrenByParent
|
|
@@ -697,7 +701,7 @@ function laneListQueryFlags(query) {
|
|
|
697
701
|
return [
|
|
698
702
|
...(query.laneIds?.length ? ["--lane"] : []),
|
|
699
703
|
...(query.active ? ["--active"] : []),
|
|
700
|
-
...(query.hierarchy
|
|
704
|
+
...(query.hierarchy?.length ? ["--hierarchy"] : []),
|
|
701
705
|
...(query.snapshot ? ["--snapshot"] : []),
|
|
702
706
|
];
|
|
703
707
|
}
|
|
@@ -726,7 +730,7 @@ async function dispatchOutpostLaneList(ctx, outpostId, query, timeoutSeconds, wa
|
|
|
726
730
|
waitMs,
|
|
727
731
|
});
|
|
728
732
|
}
|
|
729
|
-
async function fetchOutpostListHireSummaries(ctx, outpostId, lanes) {
|
|
733
|
+
async function fetchOutpostListHireSummaries(ctx, outpostId, lanes, localLanes) {
|
|
730
734
|
const fallback = (error) => new Map(lanes.map((lane) => [lane.lane_id, {
|
|
731
735
|
error: new LanesError(error.error, error.message, {
|
|
732
736
|
...error.details,
|
|
@@ -734,65 +738,91 @@ async function fetchOutpostListHireSummaries(ctx, outpostId, lanes) {
|
|
|
734
738
|
lane_id: lane.lane_id,
|
|
735
739
|
}),
|
|
736
740
|
}]));
|
|
737
|
-
|
|
741
|
+
const summaryReadError = (err) => {
|
|
742
|
+
if (isLanesError(err)) {
|
|
743
|
+
if (err.error === "outpost_lane_list_invalid") {
|
|
744
|
+
return new LanesError("outpost_lane_hire_summary_invalid", err.message, err.details);
|
|
745
|
+
}
|
|
746
|
+
if (err.error === "outpost_lane_list_unavailable") {
|
|
747
|
+
return new LanesError("outpost_lane_hire_summary_unavailable", err.message, err.details);
|
|
748
|
+
}
|
|
749
|
+
return err;
|
|
750
|
+
}
|
|
751
|
+
return new LanesError("outpost_lane_hire_summary_unavailable", `Could not read hire summaries from Outpost ${outpostId}: ${dispatchErrorMessage(err)}`, { outpost_id: outpostId });
|
|
752
|
+
};
|
|
753
|
+
const dispatchOptions = {
|
|
754
|
+
timeoutSeconds: OUTPOST_HIRE_SUMMARY_TIMEOUT_SECONDS,
|
|
755
|
+
waitMs: OUTPOST_HIRE_SUMMARY_WAIT_MS,
|
|
756
|
+
};
|
|
757
|
+
let ownerLanes;
|
|
738
758
|
try {
|
|
739
|
-
|
|
759
|
+
ownerLanes = await readOutpostLaneRecords(ctx, outpostId, {
|
|
760
|
+
hierarchy: lanes.map((lane) => lane.lane_id),
|
|
761
|
+
snapshot: true,
|
|
762
|
+
}, dispatchOptions);
|
|
740
763
|
}
|
|
741
764
|
catch (err) {
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
const
|
|
750
|
-
if (
|
|
751
|
-
|
|
765
|
+
return fallback(summaryReadError(err));
|
|
766
|
+
}
|
|
767
|
+
const requestedOwnerIds = new Set(lanes.map((lane) => lane.lane_id));
|
|
768
|
+
const returnedOwnerIds = new Set(ownerLanes
|
|
769
|
+
.filter((lane) => requestedOwnerIds.has(lane.lane_id))
|
|
770
|
+
.map((lane) => lane.lane_id));
|
|
771
|
+
const missingOwnerLanes = lanes.filter((lane) => !returnedOwnerIds.has(lane.lane_id));
|
|
772
|
+
const retryErrors = new Map();
|
|
773
|
+
if (returnedOwnerIds.size > 0 && missingOwnerLanes.length > 0) {
|
|
774
|
+
const retries = await mapWithConcurrency(missingOwnerLanes, OUTPOST_HIRE_SUMMARY_CONCURRENCY, async (lane) => {
|
|
775
|
+
try {
|
|
776
|
+
const rows = await readOutpostLaneRecords(ctx, outpostId, {
|
|
777
|
+
hierarchy: [lane.lane_id],
|
|
778
|
+
snapshot: true,
|
|
779
|
+
}, dispatchOptions);
|
|
780
|
+
return { laneId: lane.lane_id, rows };
|
|
781
|
+
}
|
|
782
|
+
catch (err) {
|
|
783
|
+
return { laneId: lane.lane_id, error: summaryReadError(err) };
|
|
784
|
+
}
|
|
785
|
+
});
|
|
786
|
+
for (const retry of retries) {
|
|
787
|
+
if (retry.error)
|
|
788
|
+
retryErrors.set(retry.laneId, retry.error);
|
|
789
|
+
else if (retry.rows)
|
|
790
|
+
ownerLanes.push(...retry.rows);
|
|
791
|
+
}
|
|
752
792
|
}
|
|
793
|
+
const hierarchy = mergeOwnerLaneHierarchy(localLanes, ownerLanes);
|
|
753
794
|
const outcomes = new Map();
|
|
754
795
|
for (const lane of lanes) {
|
|
755
|
-
const
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
outcomes.set(lane.lane_id, { error
|
|
796
|
+
const matchingLanes = ownerLanes.filter((ownerLane) => ownerLane.lane_id === lane.lane_id);
|
|
797
|
+
if (matchingLanes.length === 0) {
|
|
798
|
+
const error = retryErrors.get(lane.lane_id) ?? new LanesError("outpost_lane_hire_summary_missing", `Outpost ${outpostId} did not return lane ${lane.lane_id} in its lane list`, { outpost_id: outpostId, lane_id: lane.lane_id });
|
|
799
|
+
outcomes.set(lane.lane_id, { error });
|
|
759
800
|
continue;
|
|
760
801
|
}
|
|
761
|
-
if (
|
|
762
|
-
outcomes.set(lane.lane_id, { error: new LanesError("outpost_lane_hire_summary_invalid", `Outpost ${outpostId} returned duplicate lane rows for ${lane.lane_id}`, { outpost_id: outpostId, lane_id: lane.lane_id
|
|
802
|
+
if (matchingLanes.length !== 1) {
|
|
803
|
+
outcomes.set(lane.lane_id, { error: new LanesError("outpost_lane_hire_summary_invalid", `Outpost ${outpostId} returned duplicate lane rows for ${lane.lane_id}`, { outpost_id: outpostId, lane_id: lane.lane_id }) });
|
|
763
804
|
continue;
|
|
764
805
|
}
|
|
765
|
-
const
|
|
766
|
-
|
|
767
|
-
outcomes.set(lane.lane_id, {
|
|
768
|
-
|
|
769
|
-
laneId: lane.lane_id,
|
|
770
|
-
rawLane: row,
|
|
771
|
-
hireGrant: row.hire_grant,
|
|
772
|
-
hireAliveChildren: row.hire_alive_children,
|
|
773
|
-
hire: row.hire,
|
|
774
|
-
hireSummary: row.hire_summary,
|
|
775
|
-
commandId: dispatched.commandId,
|
|
776
|
-
}) });
|
|
777
|
-
}
|
|
778
|
-
catch (err) {
|
|
779
|
-
outcomes.set(lane.lane_id, { error: isLanesError(err)
|
|
780
|
-
? err
|
|
781
|
-
: new LanesError("outpost_lane_hire_summary_invalid", `Outpost ${outpostId} returned an invalid hire summary for lane ${lane.lane_id}`, { outpost_id: outpostId, lane_id: lane.lane_id, command_id: dispatched.commandId }) });
|
|
806
|
+
const ownerLane = matchingLanes[0];
|
|
807
|
+
if (ownerLane.host.kind !== "outpost" || ownerLane.host.id !== outpostId) {
|
|
808
|
+
outcomes.set(lane.lane_id, { error: new LanesError("outpost_lane_hire_summary_invalid", `Outpost ${outpostId} returned a lane owned by a different host`, { outpost_id: outpostId, lane_id: lane.lane_id }) });
|
|
809
|
+
continue;
|
|
782
810
|
}
|
|
811
|
+
outcomes.set(lane.lane_id, { fields: laneHireFields(ownerLane, hierarchy) });
|
|
783
812
|
}
|
|
784
813
|
return outcomes;
|
|
785
814
|
}
|
|
786
|
-
async function batchOwnerLaneHireFields(ctx, lanes) {
|
|
815
|
+
async function batchOwnerLaneHireFields(ctx, lanes, localLanes, options = {}) {
|
|
787
816
|
const outcomes = new Map();
|
|
788
817
|
const groups = new Map();
|
|
789
818
|
const nowMs = ctx.now().getTime();
|
|
790
|
-
const cached =
|
|
819
|
+
const cached = readOutpostHireSummaryCacheIncludingStale(ctx.hqRoot(), nowMs);
|
|
791
820
|
for (const lane of lanes) {
|
|
792
821
|
if (lane.host.kind !== "outpost")
|
|
793
822
|
continue;
|
|
794
823
|
const cachedSummary = cached.get(outpostHireSummaryCacheKey(lane.host.id, lane.lane_id));
|
|
795
|
-
if (cachedSummary
|
|
824
|
+
if (cachedSummary && !options.forceRefresh &&
|
|
825
|
+
nowMs - Date.parse(cachedSummary.cached_at) <= OUTPOST_HIRE_SUMMARY_CACHE_TTL_MS) {
|
|
796
826
|
outcomes.set(lane.lane_id, {
|
|
797
827
|
fields: {
|
|
798
828
|
hire_grant: cachedSummary.hire_grant,
|
|
@@ -807,7 +837,7 @@ async function batchOwnerLaneHireFields(ctx, lanes) {
|
|
|
807
837
|
group.push(lane);
|
|
808
838
|
groups.set(lane.host.id, group);
|
|
809
839
|
}
|
|
810
|
-
const fetchedGroups = await mapWithConcurrency([...groups.entries()], OUTPOST_HIRE_SUMMARY_CONCURRENCY, ([outpostId, group]) => fetchOutpostListHireSummaries(ctx, outpostId, group));
|
|
840
|
+
const fetchedGroups = await mapWithConcurrency([...groups.entries()], OUTPOST_HIRE_SUMMARY_CONCURRENCY, ([outpostId, group]) => fetchOutpostListHireSummaries(ctx, outpostId, group, localLanes));
|
|
811
841
|
const cacheUpdates = [];
|
|
812
842
|
for (const groupOutcomes of fetchedGroups) {
|
|
813
843
|
for (const [laneId, outcome] of groupOutcomes) {
|
|
@@ -947,6 +977,9 @@ function laneHostFromCtx(ctx, expectedId) {
|
|
|
947
977
|
return { kind: "local", id: hostname };
|
|
948
978
|
}
|
|
949
979
|
async function readOutpostCapacity(ctx, outpostId) {
|
|
980
|
+
return readOutpostSnapshotWithCache(ctx, JSON.stringify(["capacity", outpostId]), (value) => isOutpostCapacitySnapshot(value, outpostId), () => readOutpostCapacityUncached(ctx, outpostId));
|
|
981
|
+
}
|
|
982
|
+
async function readOutpostCapacityUncached(ctx, outpostId) {
|
|
950
983
|
let result;
|
|
951
984
|
try {
|
|
952
985
|
result = await ctx.dispatchOutpostExec({
|
|
@@ -980,7 +1013,68 @@ async function readOutpostCapacity(ctx, outpostId) {
|
|
|
980
1013
|
}
|
|
981
1014
|
return snapshot;
|
|
982
1015
|
}
|
|
983
|
-
async function
|
|
1016
|
+
async function readOutpostSnapshotWithCache(ctx, key, isValue, fetch) {
|
|
1017
|
+
let enabled = false;
|
|
1018
|
+
try {
|
|
1019
|
+
enabled = await ctx.outpostSnapshotCacheEnabled();
|
|
1020
|
+
}
|
|
1021
|
+
catch (err) {
|
|
1022
|
+
process.stderr.write(`lanes snapshot cache flag read failed; using uncached Outpost read: ${dispatchErrorMessage(err)}\n`);
|
|
1023
|
+
}
|
|
1024
|
+
if (!enabled)
|
|
1025
|
+
return fetch();
|
|
1026
|
+
return getOrFetchOutpostSnapshot({ hqRoot: ctx.hqRoot(), key, isValue, fetch });
|
|
1027
|
+
}
|
|
1028
|
+
function startOutpostLaneSnapshotRefresh(root, env) {
|
|
1029
|
+
let entry;
|
|
1030
|
+
try {
|
|
1031
|
+
const currentEntry = process.argv[1];
|
|
1032
|
+
if (!currentEntry)
|
|
1033
|
+
throw new Error("process.argv[1] is empty");
|
|
1034
|
+
entry = fs.realpathSync(currentEntry);
|
|
1035
|
+
}
|
|
1036
|
+
catch (err) {
|
|
1037
|
+
process.stderr.write(`lanes snapshot refresh could not resolve the running CLI entry: ${dispatchErrorMessage(err)}\n`);
|
|
1038
|
+
return;
|
|
1039
|
+
}
|
|
1040
|
+
try {
|
|
1041
|
+
const child = spawn(process.execPath, [...process.execArgv, entry, "lanes", "list"], {
|
|
1042
|
+
cwd: root,
|
|
1043
|
+
detached: true,
|
|
1044
|
+
stdio: ["ignore", "ignore", "inherit"],
|
|
1045
|
+
env: { ...env, HQ_ROOT: root, HQ_NO_UPDATE_CHECK: "1", HQ_INTERNAL_LANES_REFRESH_REMOTE_SNAPSHOT: "1" },
|
|
1046
|
+
});
|
|
1047
|
+
child.once("error", (err) => {
|
|
1048
|
+
process.stderr.write(`lanes snapshot refresh could not start: ${dispatchErrorMessage(err)}\n`);
|
|
1049
|
+
});
|
|
1050
|
+
child.unref();
|
|
1051
|
+
}
|
|
1052
|
+
catch (err) {
|
|
1053
|
+
process.stderr.write(`lanes snapshot refresh could not start: ${dispatchErrorMessage(err)}\n`);
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
function isLaneRecordArray(value) {
|
|
1057
|
+
if (!Array.isArray(value))
|
|
1058
|
+
return false;
|
|
1059
|
+
try {
|
|
1060
|
+
for (const lane of value)
|
|
1061
|
+
parseLaneRecord(lane);
|
|
1062
|
+
return true;
|
|
1063
|
+
}
|
|
1064
|
+
catch {
|
|
1065
|
+
return false;
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
function isOutpostCapacitySnapshot(value, outpostId) {
|
|
1069
|
+
try {
|
|
1070
|
+
const snapshot = parseHostCapacitySnapshot(value);
|
|
1071
|
+
return snapshot.host.kind === "outpost" && snapshot.host.id === outpostId;
|
|
1072
|
+
}
|
|
1073
|
+
catch {
|
|
1074
|
+
return false;
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
async function loadHostCapacitySnapshots(ctx, hqRoot, config = loadLanesConfig(hqRoot), localLanes = loadAllLanes(hqRoot)) {
|
|
984
1078
|
const localHost = laneHostFromCtx(ctx);
|
|
985
1079
|
let outpostIds;
|
|
986
1080
|
try {
|
|
@@ -1007,7 +1101,7 @@ async function loadHostCapacitySnapshots(ctx, hqRoot, config = loadLanesConfig(h
|
|
|
1007
1101
|
.sort();
|
|
1008
1102
|
const local = buildHostCapacitySnapshot({
|
|
1009
1103
|
host: localHost,
|
|
1010
|
-
lanes:
|
|
1104
|
+
lanes: localLanes,
|
|
1011
1105
|
config,
|
|
1012
1106
|
existsSync: ctx.existsSync,
|
|
1013
1107
|
hostname: ctx.hostname(),
|
|
@@ -1053,25 +1147,30 @@ function normalizedLaneListQuery(query) {
|
|
|
1053
1147
|
const laneIds = query.laneIds === undefined
|
|
1054
1148
|
? undefined
|
|
1055
1149
|
: [...new Set(query.laneIds)].sort((a, b) => a.localeCompare(b));
|
|
1150
|
+
const hierarchy = query.hierarchy === undefined
|
|
1151
|
+
? undefined
|
|
1152
|
+
: [...new Set(query.hierarchy)].sort((a, b) => a.localeCompare(b));
|
|
1056
1153
|
return {
|
|
1057
1154
|
...(laneIds === undefined ? {} : { laneIds }),
|
|
1058
1155
|
...(query.active ? { active: true } : {}),
|
|
1059
|
-
...(
|
|
1156
|
+
...(hierarchy === undefined ? {} : { hierarchy }),
|
|
1060
1157
|
...(query.snapshot ? { snapshot: true } : {}),
|
|
1061
1158
|
...(query.strictSnapshot ? { strictSnapshot: true } : {}),
|
|
1062
1159
|
};
|
|
1063
1160
|
}
|
|
1064
|
-
async function readOutpostLaneRecordsUncached(ctx, outpostId, query) {
|
|
1161
|
+
async function readOutpostLaneRecordsUncached(ctx, outpostId, query, dispatchOptions = {}) {
|
|
1065
1162
|
const targetedLookup = query.hierarchy !== undefined || query.laneIds !== undefined;
|
|
1066
1163
|
const waitMs = targetedLookup ? outpostSingleLaneWaitMs(ctx.env) : CAPACITY_PROBE_TIMEOUT_MS;
|
|
1164
|
+
const dispatchWaitMs = dispatchOptions.waitMs ?? waitMs;
|
|
1165
|
+
const dispatchTimeoutSeconds = dispatchOptions.timeoutSeconds ?? Math.ceil(dispatchWaitMs / 1000);
|
|
1067
1166
|
let result;
|
|
1068
1167
|
try {
|
|
1069
|
-
result = await dispatchOutpostLaneList(ctx, outpostId, query,
|
|
1168
|
+
result = await dispatchOutpostLaneList(ctx, outpostId, query, dispatchTimeoutSeconds, dispatchWaitMs);
|
|
1070
1169
|
}
|
|
1071
1170
|
catch (err) {
|
|
1072
1171
|
const message = dispatchErrorMessage(err);
|
|
1073
1172
|
if (/timed out waiting for outpost pickup/i.test(message)) {
|
|
1074
|
-
throw new LanesError("outpost_lane_list_unavailable", outpostPickupTimeoutMessage(outpostId,
|
|
1173
|
+
throw new LanesError("outpost_lane_list_unavailable", outpostPickupTimeoutMessage(outpostId, dispatchWaitMs, "reading the requested lane"), { outpost_id: outpostId, wait_ms: dispatchWaitMs, wait_env: OUTPOST_SINGLE_LANE_WAIT_ENV });
|
|
1075
1174
|
}
|
|
1076
1175
|
if (isLanesError(err))
|
|
1077
1176
|
throw err;
|
|
@@ -1130,24 +1229,14 @@ async function readOutpostLaneRecordsUncached(ctx, outpostId, query) {
|
|
|
1130
1229
|
}
|
|
1131
1230
|
return lanes;
|
|
1132
1231
|
}
|
|
1133
|
-
async function readOutpostLaneRecords(ctx, outpostId, requestedQuery = {}) {
|
|
1232
|
+
async function readOutpostLaneRecords(ctx, outpostId, requestedQuery = {}, dispatchOptions = {}) {
|
|
1134
1233
|
const query = normalizedLaneListQuery(requestedQuery);
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
outpostLaneListMemo.set(ctx, byOutpost);
|
|
1140
|
-
}
|
|
1141
|
-
const existing = byOutpost.get(outpostId);
|
|
1142
|
-
if (existing) {
|
|
1143
|
-
if (existing.queryKey !== queryKey) {
|
|
1144
|
-
throw new LanesError("outpost_lane_list_query_conflict", `Outpost ${outpostId} was already read with a different lane-list query in this invocation`, { outpost_id: outpostId });
|
|
1145
|
-
}
|
|
1146
|
-
return existing.promise;
|
|
1234
|
+
let invocationMemo = outpostLaneListMemo.get(ctx);
|
|
1235
|
+
if (!invocationMemo) {
|
|
1236
|
+
invocationMemo = new OutpostSnapshotInvocationMemo();
|
|
1237
|
+
outpostLaneListMemo.set(ctx, invocationMemo);
|
|
1147
1238
|
}
|
|
1148
|
-
|
|
1149
|
-
byOutpost.set(outpostId, { queryKey, promise });
|
|
1150
|
-
return promise;
|
|
1239
|
+
return invocationMemo.getOrCreate(outpostId, query, () => readOutpostSnapshotWithCache(ctx, outpostLaneListSnapshotKey(outpostId, query), isLaneRecordArray, () => readOutpostLaneRecordsUncached(ctx, outpostId, query, dispatchOptions)));
|
|
1151
1240
|
}
|
|
1152
1241
|
/** Refresh only primary-created, non-terminal Outpost mirrors in one bounded owner query per Outpost. */
|
|
1153
1242
|
async function refreshRemoteLaneMirrors(ctx, hqRoot, records, parentLaneIds) {
|
|
@@ -1307,6 +1396,16 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
|
|
|
1307
1396
|
if (!(err instanceof LanesError) || err.error !== "lane_not_found")
|
|
1308
1397
|
throw err;
|
|
1309
1398
|
}
|
|
1399
|
+
const requestedUlid = laneUlidPart(laneId);
|
|
1400
|
+
if (requestedUlid) {
|
|
1401
|
+
const localMatches = listLaneIds(ctx.hqRoot()).filter((storedId) => laneUlidPart(storedId) === requestedUlid);
|
|
1402
|
+
if (localMatches.length > 1) {
|
|
1403
|
+
throw new LanesError("lane_ulid_ambiguous", `More than one local lane record has ULID ${requestedUlid}`, { lane_id: laneId, matches: localMatches });
|
|
1404
|
+
}
|
|
1405
|
+
const localMatch = localMatches[0];
|
|
1406
|
+
if (localMatch)
|
|
1407
|
+
return { lane: readLane(ctx.hqRoot(), localMatch), remoteOnly: false };
|
|
1408
|
+
}
|
|
1310
1409
|
const separator = laneId.lastIndexOf("_");
|
|
1311
1410
|
const hostSuffix = separator < 0 ? "" : laneId.slice(separator + 1);
|
|
1312
1411
|
if (!hostSuffix || sanitizeHostComponent(hostSuffix) !== hostSuffix) {
|
|
@@ -1373,7 +1472,7 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
|
|
|
1373
1472
|
let ownerLanes;
|
|
1374
1473
|
try {
|
|
1375
1474
|
ownerLanes = await readOutpostLaneRecords(ctx, outpostId, {
|
|
1376
|
-
hierarchy: laneId,
|
|
1475
|
+
hierarchy: [laneId],
|
|
1377
1476
|
snapshot: true,
|
|
1378
1477
|
});
|
|
1379
1478
|
}
|
|
@@ -1407,6 +1506,13 @@ async function resolveCommandLaneTarget(ctx, laneId, options = {}) {
|
|
|
1407
1506
|
ownerHost: match.lane.host,
|
|
1408
1507
|
};
|
|
1409
1508
|
}
|
|
1509
|
+
function laneUlidPart(laneId) {
|
|
1510
|
+
const separator = laneId.indexOf("_");
|
|
1511
|
+
if (separator !== 26)
|
|
1512
|
+
return undefined;
|
|
1513
|
+
const ulid = laneId.slice(0, separator);
|
|
1514
|
+
return /^[0-9A-HJKMNP-TV-Z]{26}$/i.test(ulid) ? ulid.toUpperCase() : undefined;
|
|
1515
|
+
}
|
|
1410
1516
|
function commandLaneNotFound(laneId, outpostHint, details = {}) {
|
|
1411
1517
|
const hint = outpostHint
|
|
1412
1518
|
? "; pass --outpost <id> to confirm a registered Outpost"
|
|
@@ -1957,11 +2063,16 @@ async function applyThrottle(hqRoot, config, ctx, current, provider, pinnedAccou
|
|
|
1957
2063
|
if (current.action !== "start" &&
|
|
1958
2064
|
!(current.action === "queue" && current.reason === "queue_position"))
|
|
1959
2065
|
return current;
|
|
1960
|
-
const { decision } = await decideAdmission(hqRoot, config.throttle, ctx.throttleDeps);
|
|
2066
|
+
const { decision, state } = await decideAdmission(hqRoot, config.throttle, ctx.throttleDeps);
|
|
1961
2067
|
if (decision.admit)
|
|
1962
2068
|
return current;
|
|
2069
|
+
const throttle = {
|
|
2070
|
+
last_busy: state.last_busy ?? null,
|
|
2071
|
+
busy_source: state.busy_source,
|
|
2072
|
+
sampled_at: state.sampled_at,
|
|
2073
|
+
};
|
|
1963
2074
|
if (current.action === "queue")
|
|
1964
|
-
return { action: "queue", reason: "throttle" };
|
|
2075
|
+
return { action: "queue", reason: "throttle", throttle };
|
|
1965
2076
|
const providerCfg = config.providers[provider];
|
|
1966
2077
|
const host = options.host ?? reconcileHostFromCtx(ctx);
|
|
1967
2078
|
const nowMs = options.nowMs ?? ctx.now().getTime();
|
|
@@ -1993,7 +2104,7 @@ async function applyThrottle(hqRoot, config, ctx, current, provider, pinnedAccou
|
|
|
1993
2104
|
}),
|
|
1994
2105
|
};
|
|
1995
2106
|
}
|
|
1996
|
-
return { action: "queue", reason: "throttle" };
|
|
2107
|
+
return { action: "queue", reason: "throttle", throttle };
|
|
1997
2108
|
}
|
|
1998
2109
|
async function assessLaneAdmission(ctx, lane, options) {
|
|
1999
2110
|
const hqRoot = ctx.hqRoot();
|
|
@@ -2351,6 +2462,7 @@ function markDispatchFailure(hqRoot, laneId, details) {
|
|
|
2351
2462
|
...current,
|
|
2352
2463
|
state: "failed",
|
|
2353
2464
|
queued_reason: undefined,
|
|
2465
|
+
queued_details: undefined,
|
|
2354
2466
|
terminal_reason: DISPATCH_TERMINAL_REASON,
|
|
2355
2467
|
terminal_details: details,
|
|
2356
2468
|
};
|
|
@@ -2374,6 +2486,7 @@ export function pickupFailureUpdater(reasonRaw, details) {
|
|
|
2374
2486
|
...current,
|
|
2375
2487
|
state: "failed",
|
|
2376
2488
|
queued_reason: undefined,
|
|
2489
|
+
queued_details: undefined,
|
|
2377
2490
|
pickup_validation_pending: undefined,
|
|
2378
2491
|
pickup_validation_pending_since: undefined,
|
|
2379
2492
|
terminal_reason: reason,
|
|
@@ -3241,6 +3354,7 @@ async function runCreate(ctx, opts) {
|
|
|
3241
3354
|
...current,
|
|
3242
3355
|
state: "failed",
|
|
3243
3356
|
queued_reason: undefined,
|
|
3357
|
+
queued_details: undefined,
|
|
3244
3358
|
terminal_reason: "mesh_thread_registration_failed",
|
|
3245
3359
|
}));
|
|
3246
3360
|
}
|
|
@@ -3385,6 +3499,7 @@ async function dispatchCreatedOutpostLane(ctx, input) {
|
|
|
3385
3499
|
...current,
|
|
3386
3500
|
state: "cancelled",
|
|
3387
3501
|
queued_reason: undefined,
|
|
3502
|
+
queued_details: undefined,
|
|
3388
3503
|
terminal_reason: "outpost_self_target",
|
|
3389
3504
|
outpost_command_id: dispatched.commandId,
|
|
3390
3505
|
}));
|
|
@@ -3793,7 +3908,7 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
|
|
|
3793
3908
|
let grantHierarchy;
|
|
3794
3909
|
if (action === "grant") {
|
|
3795
3910
|
const hierarchy = mergeOwnerLaneRecords(loadAllLanes(ctx.hqRoot()), await readOutpostLaneRecords(ctx, outpostId, {
|
|
3796
|
-
hierarchy: lane.lane_id,
|
|
3911
|
+
hierarchy: [lane.lane_id],
|
|
3797
3912
|
snapshot: true,
|
|
3798
3913
|
}));
|
|
3799
3914
|
grantHierarchy = hierarchy;
|
|
@@ -3802,7 +3917,7 @@ async function dispatchOutpostLaneCommand(ctx, lane, action, options, behavior =
|
|
|
3802
3917
|
else if ((action === "resume" && lane.senior.kind === "lane") ||
|
|
3803
3918
|
(action === "message" && (lane.senior.kind === "lane" || lane.senior.kind === "session"))) {
|
|
3804
3919
|
const hierarchy = mergeOwnerLaneRecords(loadAllLanes(ctx.hqRoot()), await readOutpostLaneRecords(ctx, outpostId, {
|
|
3805
|
-
hierarchy: lane.lane_id,
|
|
3920
|
+
hierarchy: [lane.lane_id],
|
|
3806
3921
|
snapshot: true,
|
|
3807
3922
|
}));
|
|
3808
3923
|
grantContext = createOutpostHireGrantContext(lane, hierarchy, action === "message" ? options.supersededMessageCallerId : undefined, action === "message" ? options.supersededMessageSessionId : undefined);
|
|
@@ -4297,6 +4412,7 @@ async function runPickup(ctx, opts) {
|
|
|
4297
4412
|
...live,
|
|
4298
4413
|
state: "queued",
|
|
4299
4414
|
queued_reason: admission.reason ?? "throttle",
|
|
4415
|
+
queued_details: undefined,
|
|
4300
4416
|
queued_at: live.queued_at ?? ctx.now().toISOString(),
|
|
4301
4417
|
};
|
|
4302
4418
|
});
|
|
@@ -4349,7 +4465,7 @@ async function runPickup(ctx, opts) {
|
|
|
4349
4465
|
lane: started,
|
|
4350
4466
|
});
|
|
4351
4467
|
}
|
|
4352
|
-
async function runList(ctx, opts) {
|
|
4468
|
+
async function runList(ctx, opts, fastRouteFlags) {
|
|
4353
4469
|
if (opts.json && opts.human) {
|
|
4354
4470
|
throw new LanesError("conflicting_output_format", "Use either --json or --human for hq lanes list", { json: true, human: true });
|
|
4355
4471
|
}
|
|
@@ -4359,8 +4475,11 @@ async function runList(ctx, opts) {
|
|
|
4359
4475
|
new Set(requestedLaneIds).size !== requestedLaneIds.length) {
|
|
4360
4476
|
throw new LanesError("lane_list_filter_invalid", "--lane must be repeated with distinct, non-empty lane ids", { lane_ids: requestedLaneIds });
|
|
4361
4477
|
}
|
|
4362
|
-
if (opts.hierarchy !== undefined &&
|
|
4363
|
-
|
|
4478
|
+
if (opts.hierarchy !== undefined &&
|
|
4479
|
+
(!Array.isArray(opts.hierarchy) || opts.hierarchy.length === 0 ||
|
|
4480
|
+
opts.hierarchy.some((laneId) => typeof laneId !== "string" || !laneId.trim() || laneId.trim() !== laneId) ||
|
|
4481
|
+
new Set(opts.hierarchy).size !== opts.hierarchy.length)) {
|
|
4482
|
+
throw new LanesError("lane_list_hierarchy_invalid", "--hierarchy must be repeated with distinct, non-empty lane ids without surrounding whitespace", {});
|
|
4364
4483
|
}
|
|
4365
4484
|
if (opts.snapshot && (opts.human || opts.company || opts.project || opts.story || opts.state || opts.senior)) {
|
|
4366
4485
|
throw new LanesError("lane_list_snapshot_invalid", "--snapshot only supports owner lane filters and JSON output", {});
|
|
@@ -4385,7 +4504,7 @@ async function runList(ctx, opts) {
|
|
|
4385
4504
|
: loadAllLanes(hqRoot);
|
|
4386
4505
|
const hierarchyIds = opts.hierarchy === undefined
|
|
4387
4506
|
? undefined
|
|
4388
|
-
: new Set(
|
|
4507
|
+
: new Set(opts.hierarchy.flatMap((laneId) => selectOutpostHireGrantHierarchy(laneId, laneSource).map((lane) => lane.lane_id)));
|
|
4389
4508
|
let selectedLanes = laneSource
|
|
4390
4509
|
.filter((lane) => {
|
|
4391
4510
|
if (requestedLaneIds.length > 0 && !requestedIds.has(lane.lane_id))
|
|
@@ -4452,7 +4571,33 @@ async function runList(ctx, opts) {
|
|
|
4452
4571
|
ownerEligibleLaneIds.add(lane.lane_id);
|
|
4453
4572
|
}
|
|
4454
4573
|
const ownerLanes = selectedLanes.filter((lane) => ownerEligibleLaneIds.has(lane.lane_id));
|
|
4574
|
+
const refreshRemote = ctx.env.HQ_INTERNAL_LANES_REFRESH_REMOTE_SNAPSHOT === "1";
|
|
4575
|
+
let snapshotCacheEnabled = fastRouteFlags?.snapshotCacheEnabled ?? false;
|
|
4576
|
+
if (fastRouteFlags === undefined &&
|
|
4577
|
+
(refreshRemote || (!opts.fresh && ownerLanes.length > 0))) {
|
|
4578
|
+
try {
|
|
4579
|
+
snapshotCacheEnabled = await ctx.outpostSnapshotCacheEnabled();
|
|
4580
|
+
}
|
|
4581
|
+
catch (err) {
|
|
4582
|
+
process.stderr.write(`lanes snapshot cache flag read failed; using wait-for-remote list: ${dispatchErrorMessage(err)}\n`);
|
|
4583
|
+
}
|
|
4584
|
+
}
|
|
4585
|
+
if (refreshRemote) {
|
|
4586
|
+
if (snapshotCacheEnabled && ownerLanes.length > 0) {
|
|
4587
|
+
const outcomes = await batchOwnerLaneHireFields(ctx, ownerLanes, allLanes, { forceRefresh: true });
|
|
4588
|
+
const failures = [...outcomes.entries()].filter(([, outcome]) => "error" in outcome);
|
|
4589
|
+
if (failures.length > 0) {
|
|
4590
|
+
const firstOutcome = failures[0][1];
|
|
4591
|
+
const firstError = "error" in firstOutcome ? firstOutcome.error : undefined;
|
|
4592
|
+
process.stderr.write(`lanes snapshot refresh failed for ${failures.length} lane(s): ${firstError?.error ?? "outpost_lane_hire_summary_missing"}: ${firstError?.message ?? "owner summary outcome was missing"}\n`);
|
|
4593
|
+
throw firstError ?? new LanesError("outpost_lane_hire_summary_missing", "Background Outpost lane summary refresh returned a missing outcome", { lane_ids: failures.map(([laneId]) => laneId) });
|
|
4594
|
+
}
|
|
4595
|
+
}
|
|
4596
|
+
return;
|
|
4597
|
+
}
|
|
4455
4598
|
const hireByLane = new Map();
|
|
4599
|
+
const remoteSnapshotMetadata = new Map();
|
|
4600
|
+
let remoteLanesLoading = false;
|
|
4456
4601
|
for (const lane of selectedLanes) {
|
|
4457
4602
|
if (!ownerEligibleLaneIds.has(lane.lane_id)) {
|
|
4458
4603
|
hireByLane.set(lane.lane_id, {
|
|
@@ -4462,8 +4607,48 @@ async function runList(ctx, opts) {
|
|
|
4462
4607
|
continue;
|
|
4463
4608
|
}
|
|
4464
4609
|
}
|
|
4465
|
-
if (ownerLanes.length > 0) {
|
|
4466
|
-
const
|
|
4610
|
+
if (snapshotCacheEnabled && !opts.fresh && ownerLanes.length > 0) {
|
|
4611
|
+
const nowMs = ctx.now().getTime();
|
|
4612
|
+
const cached = readOutpostHireSummaryCacheIncludingStale(hqRoot, nowMs);
|
|
4613
|
+
let refreshNeeded = false;
|
|
4614
|
+
for (const lane of ownerLanes) {
|
|
4615
|
+
if (lane.host.kind !== "outpost")
|
|
4616
|
+
continue;
|
|
4617
|
+
const summary = cached.get(outpostHireSummaryCacheKey(lane.host.id, lane.lane_id));
|
|
4618
|
+
if (!summary) {
|
|
4619
|
+
hireByLane.set(lane.lane_id, {
|
|
4620
|
+
...laneHireFields(lane, allLanes, aliveChildrenByParent),
|
|
4621
|
+
hire_source: "local",
|
|
4622
|
+
});
|
|
4623
|
+
remoteSnapshotMetadata.set(lane.lane_id, { ageSeconds: null, stale: true, loading: true });
|
|
4624
|
+
refreshNeeded = true;
|
|
4625
|
+
remoteLanesLoading = true;
|
|
4626
|
+
continue;
|
|
4627
|
+
}
|
|
4628
|
+
const ageMs = Math.max(0, nowMs - Date.parse(summary.cached_at));
|
|
4629
|
+
const stale = ageMs >= OUTPOST_SNAPSHOT_CACHE_TTL_MS;
|
|
4630
|
+
hireByLane.set(lane.lane_id, {
|
|
4631
|
+
hire_grant: summary.hire_grant,
|
|
4632
|
+
hire_alive_children: summary.hire_alive_children,
|
|
4633
|
+
hire: summary.hire,
|
|
4634
|
+
hire_summary: summary.hire_summary,
|
|
4635
|
+
hire_source: "owner",
|
|
4636
|
+
});
|
|
4637
|
+
remoteSnapshotMetadata.set(lane.lane_id, {
|
|
4638
|
+
ageSeconds: Math.floor(ageMs / 1_000),
|
|
4639
|
+
stale,
|
|
4640
|
+
loading: stale,
|
|
4641
|
+
});
|
|
4642
|
+
if (stale) {
|
|
4643
|
+
refreshNeeded = true;
|
|
4644
|
+
remoteLanesLoading = true;
|
|
4645
|
+
}
|
|
4646
|
+
}
|
|
4647
|
+
if (refreshNeeded)
|
|
4648
|
+
ctx.startOutpostLaneSnapshotRefresh(hqRoot);
|
|
4649
|
+
}
|
|
4650
|
+
else if (ownerLanes.length > 0) {
|
|
4651
|
+
const ownerSummaries = await batchOwnerLaneHireFields(ctx, ownerLanes, allLanes);
|
|
4467
4652
|
for (const lane of ownerLanes) {
|
|
4468
4653
|
const outcome = ownerSummaries.get(lane.lane_id);
|
|
4469
4654
|
if (outcome && "fields" in outcome) {
|
|
@@ -4494,7 +4679,16 @@ async function runList(ctx, opts) {
|
|
|
4494
4679
|
story_ids: laneStories(lane).map((story) => story.story_id),
|
|
4495
4680
|
no_story_reason: lane.no_story_reason ?? null,
|
|
4496
4681
|
...(remoteQueued
|
|
4497
|
-
? { state: "remote", remote_state: lane.state, queued_reason: undefined }
|
|
4682
|
+
? { state: "remote", remote_state: lane.state, queued_reason: undefined, queued_details: undefined }
|
|
4683
|
+
: {}),
|
|
4684
|
+
...(remoteSnapshotMetadata.has(lane.lane_id)
|
|
4685
|
+
? {
|
|
4686
|
+
remote_snapshot_age_s: remoteSnapshotMetadata.get(lane.lane_id).ageSeconds,
|
|
4687
|
+
remote_snapshot_stale: remoteSnapshotMetadata.get(lane.lane_id).stale,
|
|
4688
|
+
...(remoteSnapshotMetadata.get(lane.lane_id).loading
|
|
4689
|
+
? { remote_snapshot_loading: true }
|
|
4690
|
+
: {}),
|
|
4691
|
+
}
|
|
4498
4692
|
: {}),
|
|
4499
4693
|
overdue: Boolean(lane.overdue) || isOverdue(hqRoot, lane.lane_id, now),
|
|
4500
4694
|
stalled: Boolean(lane.stalled),
|
|
@@ -4519,8 +4713,10 @@ async function runList(ctx, opts) {
|
|
|
4519
4713
|
};
|
|
4520
4714
|
});
|
|
4521
4715
|
const placementCompany = company ?? processLanePlacementCompany(ctx);
|
|
4522
|
-
const
|
|
4523
|
-
|
|
4716
|
+
const placementEnabled = fastRouteFlags?.lanePlacementEnabled ??
|
|
4717
|
+
await readLanePlacementEnabled(ctx, placementCompany);
|
|
4718
|
+
const host_capacity = placementEnabled
|
|
4719
|
+
? (await loadHostCapacitySnapshots(ctx, hqRoot, undefined, allLanes)).entries
|
|
4524
4720
|
: undefined;
|
|
4525
4721
|
const result = {
|
|
4526
4722
|
ok: true,
|
|
@@ -4529,24 +4725,44 @@ async function runList(ctx, opts) {
|
|
|
4529
4725
|
...(remoteRefresh.unverifiedRemote.size > 0
|
|
4530
4726
|
? { unverified_remote_lanes: [...remoteRefresh.unverifiedRemote.keys()] }
|
|
4531
4727
|
: {}),
|
|
4728
|
+
...(remoteLanesLoading ? { remote_lanes_loading: true } : {}),
|
|
4532
4729
|
...(host_capacity !== undefined ? { host_capacity } : {}),
|
|
4533
4730
|
};
|
|
4534
4731
|
if (opts.human)
|
|
4535
|
-
console.log(formatLaneListHuman(lanes));
|
|
4732
|
+
console.log(formatLaneListHuman(lanes, remoteLanesLoading));
|
|
4536
4733
|
else
|
|
4537
4734
|
printJson(result);
|
|
4538
4735
|
}
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4736
|
+
/** Keeps the placement-enabled cache-first path on the full lane command graph. */
|
|
4737
|
+
export async function runFastCachedLanesListFromEntrypoint() {
|
|
4738
|
+
try {
|
|
4739
|
+
await runList(resolveDeps(), { json: true }, {
|
|
4740
|
+
snapshotCacheEnabled: true,
|
|
4741
|
+
lanePlacementEnabled: true,
|
|
4742
|
+
});
|
|
4743
|
+
}
|
|
4744
|
+
catch (err) {
|
|
4745
|
+
failJson(err);
|
|
4746
|
+
}
|
|
4747
|
+
}
|
|
4748
|
+
function formatLaneListHuman(lanes, remoteLanesLoading = false) {
|
|
4749
|
+
const rows = lanes.map((lane) => {
|
|
4543
4750
|
const ids = lane.story_ids;
|
|
4544
4751
|
const storyLabel = ids.length > 0
|
|
4545
4752
|
? "stories: " + ids.join(", ")
|
|
4546
4753
|
: "no story: " + (lane.no_story_reason ?? "reason not recorded");
|
|
4754
|
+
const snapshotNote = lane.remote_snapshot_stale && lane.remote_snapshot_age_s !== null &&
|
|
4755
|
+
lane.remote_snapshot_age_s !== undefined
|
|
4756
|
+
? ` remote snapshot stale (${lane.remote_snapshot_age_s}s)`
|
|
4757
|
+
: "";
|
|
4547
4758
|
return lane.lane_id + " " + lane.state + " " + lane.provider + " " + lane.project_id +
|
|
4548
|
-
" " + storyLabel;
|
|
4549
|
-
})
|
|
4759
|
+
" " + storyLabel + snapshotNote;
|
|
4760
|
+
});
|
|
4761
|
+
if (rows.length === 0)
|
|
4762
|
+
rows.push("No lanes.");
|
|
4763
|
+
if (remoteLanesLoading)
|
|
4764
|
+
rows.push("Remote lanes loading.");
|
|
4765
|
+
return rows.join("\n");
|
|
4550
4766
|
}
|
|
4551
4767
|
async function runStatus(ctx, opts) {
|
|
4552
4768
|
if (!ctx.storyLedgerEnabled()) {
|
|
@@ -4732,7 +4948,7 @@ async function runShow(ctx, laneId) {
|
|
|
4732
4948
|
storedLane.host.kind === "outpost" &&
|
|
4733
4949
|
!laneRunsHere(storedLane, reconcileHostFromCtx(ctx));
|
|
4734
4950
|
const lane = remoteQueued
|
|
4735
|
-
? { ...storedLane, state: "remote", remote_state: storedLane.state, queued_reason: undefined }
|
|
4951
|
+
? { ...storedLane, state: "remote", remote_state: storedLane.state, queued_reason: undefined, queued_details: undefined }
|
|
4736
4952
|
: storedLane;
|
|
4737
4953
|
const askDeliveryFailed = readJsonFile(askDeliveryFailedPath(hqRoot, laneId));
|
|
4738
4954
|
const keepAliveState = resolveLaneKeepAlive(hqRoot, storedLane);
|
|
@@ -5228,7 +5444,7 @@ async function runGrant(ctx, laneId, opts) {
|
|
|
5228
5444
|
await dispatchOutpostLaneCommand(ctx, initial, "grant", { hire: String(hireGrant) });
|
|
5229
5445
|
return;
|
|
5230
5446
|
}
|
|
5231
|
-
const updated = await applyHireGrant(ctx,
|
|
5447
|
+
const updated = await applyHireGrant(ctx, initial.lane_id, hireGrant, "senior");
|
|
5232
5448
|
printGrantResult(hqRoot, updated);
|
|
5233
5449
|
}
|
|
5234
5450
|
async function runApprove(ctx, laneId, opts) {
|
|
@@ -5449,6 +5665,7 @@ async function runAdmit(ctx, laneId, opts) {
|
|
|
5449
5665
|
});
|
|
5450
5666
|
}
|
|
5451
5667
|
let blockedBy;
|
|
5668
|
+
let throttleTelemetry;
|
|
5452
5669
|
if (opts.force) {
|
|
5453
5670
|
const callerAuthorization = forceAdmissionAuthorization(ctx, lane);
|
|
5454
5671
|
if (!callerAuthorization.authorized) {
|
|
@@ -5456,9 +5673,14 @@ async function runAdmit(ctx, laneId, opts) {
|
|
|
5456
5673
|
}
|
|
5457
5674
|
const config = loadLanesConfig(hqRoot);
|
|
5458
5675
|
const lanes = loadAllLanes(hqRoot).filter((row) => row.lane_id !== laneId && laneRunsHere(row, host));
|
|
5459
|
-
const { decision } = await decideAdmission(hqRoot, config.throttle, ctx.throttleDeps);
|
|
5676
|
+
const { decision, state } = await decideAdmission(hqRoot, config.throttle, ctx.throttleDeps);
|
|
5460
5677
|
if (!decision.admit) {
|
|
5461
5678
|
blockedBy = "throttle";
|
|
5679
|
+
throttleTelemetry = {
|
|
5680
|
+
last_busy: state.last_busy ?? null,
|
|
5681
|
+
busy_source: state.busy_source,
|
|
5682
|
+
sampled_at: state.sampled_at,
|
|
5683
|
+
};
|
|
5462
5684
|
}
|
|
5463
5685
|
else if (hasEarlierEligibleQueuedLane(lanes, lane, config, host, (row) => row.worker_profile === undefined
|
|
5464
5686
|
? undefined
|
|
@@ -5471,6 +5693,7 @@ async function runAdmit(ctx, laneId, opts) {
|
|
|
5471
5693
|
queueAt: lane.queued_at ?? lane.created_at,
|
|
5472
5694
|
alreadyQueued: true,
|
|
5473
5695
|
});
|
|
5696
|
+
throttleTelemetry = admission.throttle;
|
|
5474
5697
|
if (admission.action === "refuse" && admission.error) {
|
|
5475
5698
|
throw admission.error;
|
|
5476
5699
|
}
|
|
@@ -5487,7 +5710,7 @@ async function runAdmit(ctx, laneId, opts) {
|
|
|
5487
5710
|
state: current.state,
|
|
5488
5711
|
});
|
|
5489
5712
|
}
|
|
5490
|
-
return { ...current, queued_reason: blockedBy };
|
|
5713
|
+
return { ...current, queued_reason: blockedBy, queued_details: undefined };
|
|
5491
5714
|
});
|
|
5492
5715
|
printJson({
|
|
5493
5716
|
ok: true,
|
|
@@ -5495,6 +5718,9 @@ async function runAdmit(ctx, laneId, opts) {
|
|
|
5495
5718
|
lane_id: laneId,
|
|
5496
5719
|
admitted: false,
|
|
5497
5720
|
blocked_by: blockedBy,
|
|
5721
|
+
...(blockedBy === "throttle" && throttleTelemetry !== undefined
|
|
5722
|
+
? throttleTelemetry
|
|
5723
|
+
: {}),
|
|
5498
5724
|
lane: updated,
|
|
5499
5725
|
});
|
|
5500
5726
|
return;
|
|
@@ -5862,6 +6088,7 @@ async function prepareAccountMove(ctx, lane, input) {
|
|
|
5862
6088
|
else if (queued) {
|
|
5863
6089
|
next.state = "queued";
|
|
5864
6090
|
next.queued_reason = admission?.reason ?? "throttle";
|
|
6091
|
+
next.queued_details = undefined;
|
|
5865
6092
|
next.queued_at = at;
|
|
5866
6093
|
}
|
|
5867
6094
|
else {
|
|
@@ -6944,6 +7171,7 @@ async function runStop(ctx, laneId, reason, quiet = false) {
|
|
|
6944
7171
|
...current,
|
|
6945
7172
|
state: stopCommittedState(current.state),
|
|
6946
7173
|
queued_reason: undefined,
|
|
7174
|
+
queued_details: undefined,
|
|
6947
7175
|
...(operatorReason === undefined ? {} : {
|
|
6948
7176
|
stop_history: [
|
|
6949
7177
|
...(current.stop_history ?? []),
|
|
@@ -7018,6 +7246,7 @@ async function runInterrupt(ctx, laneId, reason) {
|
|
|
7018
7246
|
...current,
|
|
7019
7247
|
state: "interrupted",
|
|
7020
7248
|
queued_reason: undefined,
|
|
7249
|
+
queued_details: undefined,
|
|
7021
7250
|
interrupted_at: at,
|
|
7022
7251
|
terminal_reason: "operator_interrupt",
|
|
7023
7252
|
...(operatorReason === undefined ? {} : {
|
|
@@ -7742,8 +7971,10 @@ export function registerLanesCommand(parent, deps = {}) {
|
|
|
7742
7971
|
.option("--senior <ref>", "Filter by senior ref")
|
|
7743
7972
|
.option("--lane <id>", "Filter by lane id (repeatable)", (value, previous = []) => [...previous, value], [])
|
|
7744
7973
|
.option("--active", "Include only lanes that are not terminal")
|
|
7745
|
-
.addOption(new Option("--hierarchy <lane-id>").hideHelp()
|
|
7974
|
+
.addOption(new Option("--hierarchy <lane-id>").hideHelp()
|
|
7975
|
+
.argParser((value, previous = []) => [...previous, value]))
|
|
7746
7976
|
.addOption(new Option("--snapshot").hideHelp())
|
|
7977
|
+
.option("--fresh", "Wait for fresh remote owner summaries")
|
|
7747
7978
|
.option("--json", "Emit JSON on stdout (default)")
|
|
7748
7979
|
.option("--human", "Emit human-readable output")
|
|
7749
7980
|
.action(wrap(async (opts) => {
|