@adhdev/daemon-core 1.0.18-rc.2 → 1.0.18-rc.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-adapters/cli-state-engine.d.ts +77 -0
- package/dist/cli-adapters/provider-cli-shared.d.ts +33 -0
- package/dist/cli-adapters/pty-transport.d.ts +2 -1
- package/dist/commands/cli-manager.d.ts +9 -0
- package/dist/commands/process-lifecycle.d.ts +43 -0
- package/dist/commands/upgrade-helper.d.ts +1 -0
- package/dist/commands/windows-atomic-upgrade.d.ts +17 -1
- package/dist/config/mesh-json-config.d.ts +62 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +6674 -4237
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +6361 -3927
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/coordinator-prompt.d.ts +76 -0
- package/dist/mesh/mesh-active-work.d.ts +1 -1
- package/dist/mesh/mesh-disk-retention.d.ts +105 -0
- package/dist/mesh/mesh-ledger.d.ts +28 -1
- package/dist/mesh/mesh-node-identity.d.ts +23 -0
- package/dist/mesh/mesh-refine-gates.d.ts +89 -0
- package/dist/mesh/mesh-remote-event-pull.d.ts +3 -0
- package/dist/mesh/mesh-runtime-store.d.ts +11 -0
- package/dist/mesh/mesh-work-queue.d.ts +24 -1
- package/dist/providers/auto-approve-modes.d.ts +14 -0
- package/dist/providers/chat-message-normalization.d.ts +22 -0
- package/dist/providers/cli-provider-instance-types.d.ts +4 -0
- package/dist/providers/cli-provider-instance.d.ts +78 -0
- package/dist/providers/contracts.d.ts +17 -0
- package/dist/providers/provider-schema.d.ts +1 -0
- package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +14 -0
- package/dist/providers/types/interactive-prompt.d.ts +18 -0
- package/dist/repo-mesh-types.d.ts +38 -6
- package/dist/shared-types.d.ts +4 -2
- package/package.json +3 -3
- package/src/boot/daemon-lifecycle.ts +10 -0
- package/src/cli-adapters/cli-state-engine.ts +220 -3
- package/src/cli-adapters/provider-cli-adapter.ts +41 -11
- package/src/cli-adapters/provider-cli-shared.ts +48 -0
- package/src/cli-adapters/pty-transport.d.ts +2 -1
- package/src/cli-adapters/pty-transport.ts +1 -1
- package/src/cli-adapters/session-host-transport.ts +8 -3
- package/src/commands/cli-manager.ts +72 -8
- package/src/commands/high-family/mesh-coordinator-launch.ts +17 -2
- package/src/commands/high-family/mesh-events.ts +13 -2
- package/src/commands/med-family/mesh-crud.ts +155 -0
- package/src/commands/med-family/mesh-queue.ts +74 -1
- package/src/commands/process-lifecycle.ts +248 -0
- package/src/commands/router-refine.ts +71 -4
- package/src/commands/router.ts +8 -0
- package/src/commands/upgrade-helper.ts +106 -82
- package/src/commands/windows-atomic-upgrade.ts +281 -35
- package/src/config/mesh-json-config.ts +111 -0
- package/src/index.ts +21 -1
- package/src/mesh/coordinator-prompt.ts +258 -11
- package/src/mesh/mesh-active-work.ts +11 -1
- package/src/mesh/mesh-completion-synthesis.ts +12 -1
- package/src/mesh/mesh-disk-retention.ts +370 -0
- package/src/mesh/mesh-event-classify.ts +19 -0
- package/src/mesh/mesh-event-forwarding.ts +64 -6
- package/src/mesh/mesh-events-utils.ts +42 -0
- package/src/mesh/mesh-ledger.ts +77 -0
- package/src/mesh/mesh-node-identity.ts +49 -0
- package/src/mesh/mesh-queue-assignment.ts +210 -11
- package/src/mesh/mesh-reconcile-loop.ts +306 -3
- package/src/mesh/mesh-refine-gates.ts +300 -0
- package/src/mesh/mesh-remote-event-pull.ts +68 -47
- package/src/mesh/mesh-runtime-store.ts +21 -0
- package/src/mesh/mesh-work-queue.ts +85 -2
- package/src/providers/auto-approve-modes.ts +103 -0
- package/src/providers/chat-message-normalization.ts +53 -0
- package/src/providers/cli-provider-instance-types.ts +53 -0
- package/src/providers/cli-provider-instance.ts +497 -15
- package/src/providers/contracts.ts +25 -1
- package/src/providers/provider-schema.ts +83 -0
- package/src/providers/sdk/v1/builders/cli/detect-status.ts +23 -0
- package/src/providers/sdk/v1/builders/cli/parse-approval.ts +20 -0
- package/src/providers/sdk/v1/schemas/cli/provider.schema.json +44 -0
- package/src/providers/types/interactive-prompt.ts +77 -0
- package/src/repo-mesh-types.ts +112 -12
- package/src/session-host/managed-host.ts +34 -0
- package/src/shared-types.ts +9 -1
- package/src/status/reporter.ts +1 -0
- package/src/status/snapshot.ts +2 -0
|
@@ -907,6 +907,55 @@ export function isMeshNodeHealthLaunchable(node: any): boolean {
|
|
|
907
907
|
return health === 'online' || health === 'unknown';
|
|
908
908
|
}
|
|
909
909
|
|
|
910
|
+
/** Resolve the git telemetry object a node carries, in the same precedence
|
|
911
|
+
* resolveEffectiveMeshNodeHealth uses: a fresh `node.git`, else the cached inline
|
|
912
|
+
* status git (`node.cachedStatus.git`). Returns an empty record when neither is
|
|
913
|
+
* present (no telemetry). */
|
|
914
|
+
function resolveEffectiveNodeGit(node: any): Record<string, any> {
|
|
915
|
+
const git = readObjectRecord(node?.git);
|
|
916
|
+
if (Object.keys(git).length > 0) return git;
|
|
917
|
+
const cachedStatus = readObjectRecord(node?.cachedStatus);
|
|
918
|
+
return readObjectRecord(cachedStatus.git);
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
/**
|
|
922
|
+
* Launch FRESHNESS gate — distinct from the health gate above.
|
|
923
|
+
*
|
|
924
|
+
* `deriveMeshNodeHealthFromGit` reports a clean-tree node with `behind > 0` as
|
|
925
|
+
* 'online', so the health gate (isMeshNodeHealthLaunchable) happily lets a STALE
|
|
926
|
+
* node — one whose branch is N commits behind its upstream — win auto-launch fitness
|
|
927
|
+
* routing and run a fresh worker against out-of-date code. `behind` is deliberately
|
|
928
|
+
* NOT folded into deriveMeshNodeHealthFromGit because "behind" is not universally
|
|
929
|
+
* unhealthy (the MAGI planner and other callers share that resolver); it is only
|
|
930
|
+
* unhealthy for *spawning new work*. This gate encodes exactly that launch-time axis.
|
|
931
|
+
*
|
|
932
|
+
* Returns false (NOT fresh → skip / de-rank) only when git telemetry is PRESENT and
|
|
933
|
+
* proves staleness:
|
|
934
|
+
* - behind count exceeds `maxBehind` (default 0 — any behind blocks), OR
|
|
935
|
+
* - a submodule is out of sync (gitlink points off upstream — cannot be caught up
|
|
936
|
+
* by a simple worktree ff and would launch against a mismatched submodule).
|
|
937
|
+
*
|
|
938
|
+
* When telemetry is absent it returns true (fresh), preserving the online/unknown-pass
|
|
939
|
+
* philosophy: we never block on missing data, only on data that proves the node stale.
|
|
940
|
+
*/
|
|
941
|
+
export function isMeshNodeFreshEnoughToLaunch(node: any, opts?: { maxBehind?: number }): boolean {
|
|
942
|
+
const git = resolveEffectiveNodeGit(node);
|
|
943
|
+
// No git telemetry at all → cannot prove stale → do not block.
|
|
944
|
+
if (Object.keys(git).length === 0) return true;
|
|
945
|
+
// Not a git repo / no branch: leave this to the health gate (it returns 'degraded'
|
|
946
|
+
// and blocks there); freshness has nothing to add.
|
|
947
|
+
if (readBooleanValue(git.isGitRepo) === false) return true;
|
|
948
|
+
const submoduleDrift = getGitSubmoduleDriftState(git);
|
|
949
|
+
if (submoduleDrift.outOfSync) return false;
|
|
950
|
+
const behind = readNumberValue(git.behind);
|
|
951
|
+
// No behind datum reported → treat as fresh (don't infer staleness from absence).
|
|
952
|
+
if (behind === undefined) return true;
|
|
953
|
+
const maxBehind = Number.isFinite(opts?.maxBehind as number) && (opts?.maxBehind as number) >= 0
|
|
954
|
+
? Math.floor(opts!.maxBehind as number)
|
|
955
|
+
: 0;
|
|
956
|
+
return behind <= maxBehind;
|
|
957
|
+
}
|
|
958
|
+
|
|
910
959
|
function readMeshNodeLabel(status: Record<string, unknown>, node: any): string {
|
|
911
960
|
return readStringValue(status.nodeId, normalizeMeshNodeId(node)) ?? 'unknown';
|
|
912
961
|
}
|
|
@@ -6,20 +6,22 @@ import { getMesh } from '../config/mesh-config.js';
|
|
|
6
6
|
import { detectCLI } from '../detection/cli-detector.js';
|
|
7
7
|
import { LOG } from '../logging/logger.js';
|
|
8
8
|
import { appendLedgerEntry } from './mesh-ledger.js';
|
|
9
|
-
import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, getActiveDirectDispatches, isTaskReadonly, taskDependenciesSatisfied, meshTaskNotBeforeReady, meshTaskPriorityRank } from './mesh-work-queue.js';
|
|
9
|
+
import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, getActiveDirectDispatches, isTaskReadonly, taskDependenciesSatisfied, meshTaskNotBeforeReady, meshTaskPriorityRank, requeueTask } from './mesh-work-queue.js';
|
|
10
10
|
import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
|
|
11
11
|
import { fastForwardMeshNode } from './mesh-fast-forward.js';
|
|
12
12
|
import { createSessionDelivery, updateSessionDeliveryStatus } from './mesh-delivery-policy.js';
|
|
13
13
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
14
14
|
import { traceMeshEventDrop } from './mesh-event-trace.js';
|
|
15
15
|
import { awaitWithWarmupDeadline, resolveWarmupDeadlineOpts } from './mesh-warmup-deadline.js';
|
|
16
|
-
import {
|
|
16
|
+
import { delegatedWorkerAutoApproveSettings, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy, resolveMaxParallelTasks, resolveMaxReadonlyParallelTasks, resolveCoordinatorIdlePushPolicy } from '../repo-mesh-types.js';
|
|
17
|
+
import { loadRepoMeshJsonConfig } from '../config/mesh-json-config.js';
|
|
18
|
+
import type { RepoMeshDeclarativeConfig } from '../config/mesh-json-config.js';
|
|
17
19
|
import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
18
20
|
import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, canonicalDaemonId, expandDaemonIdForms, normalizeMeshWorkspaceForCompare, meshWorkspacesEquivalent, sessionIdsEquivalent, normalizeNodeCapabilitySlots, isMeshTaskDifficulty, withStatusProbeMarker, type MeshNodeIdentified, type NodeCapabilitySlot, type MeshTaskDifficulty } from '@adhdev/mesh-shared';
|
|
19
21
|
import { resolveNodeCapabilitySlots } from './mesh-node-slots.js';
|
|
20
22
|
import { findTerminalLedgerEvidenceForTask, hasUnterminalDirectDispatchLedgerEntry } from './mesh-events-stale.js';
|
|
21
23
|
import { readNonEmptyString } from './mesh-events-utils.js';
|
|
22
|
-
import { readMeshNodeDaemonId, isMeshNodeHealthLaunchable } from './mesh-node-identity.js';
|
|
24
|
+
import { readMeshNodeDaemonId, isMeshNodeHealthLaunchable, isMeshNodeFreshEnoughToLaunch } from './mesh-node-identity.js';
|
|
23
25
|
import { queuePendingMeshCoordinatorEvent, retractPendingDispatchBlockedEvent, drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
24
26
|
import { isWorktreeBootstrapStaleRunning, shouldDeferDispatchForBootstrap } from './worktree-bootstrap-config.js';
|
|
25
27
|
import { isWithinCloneBootstrapGrace } from './mesh-clone-grace.js';
|
|
@@ -90,6 +92,25 @@ export function __resetIdleAutoFastForwardForTests(): void {
|
|
|
90
92
|
autoFastForwardWorkspaceLease.clear();
|
|
91
93
|
}
|
|
92
94
|
|
|
95
|
+
/**
|
|
96
|
+
* Load the repo-shared `.adhdev/mesh.json` for a node's workspace, tolerating a
|
|
97
|
+
* missing/invalid file (returns null → resolver falls back to provider-spec
|
|
98
|
+
* defaults, i.e. exactly the pre-providerDefaults behavior). Only the
|
|
99
|
+
* `providerDefaults` zone influences the delegated-worker MODE selection; it never
|
|
100
|
+
* touches the ENABLE decision. When a node carries no workspace path (should not
|
|
101
|
+
* happen for a launchable node, but be defensive), we skip the read entirely.
|
|
102
|
+
*/
|
|
103
|
+
function loadRepoConfigForNode(node: any): RepoMeshDeclarativeConfig | null {
|
|
104
|
+
const workspace = typeof node?.workspace === 'string' && node.workspace.trim() ? node.workspace.trim() : '';
|
|
105
|
+
if (!workspace) return null;
|
|
106
|
+
try {
|
|
107
|
+
const result = loadRepoMeshJsonConfig(workspace);
|
|
108
|
+
return result.sourceType === 'repo_file' && result.config ? result.config : null;
|
|
109
|
+
} catch {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
93
114
|
export function getMeshWithCache(components: DaemonComponents, meshId: string): any | undefined {
|
|
94
115
|
const localMesh = getMesh(meshId);
|
|
95
116
|
const cachedMesh = components.router?.getCachedInlineMesh(meshId);
|
|
@@ -690,7 +711,7 @@ export function tryAssignQueueTask(
|
|
|
690
711
|
if (inst && typeof inst.updateSettings === 'function') {
|
|
691
712
|
// Adopting a (possibly manually-opened) local session as a worker: apply the
|
|
692
713
|
// delegated-worker auto-approve policy here too, so a session that was launched
|
|
693
|
-
// without
|
|
714
|
+
// without an auto-approve boolean/mode still resolves the delegated policy once dispatched
|
|
694
715
|
// to it (the "approval notification fires only for certain delegated sessions"
|
|
695
716
|
// case). updateSettings preserves runtime mesh keys; passing autoApprove keeps it.
|
|
696
717
|
//
|
|
@@ -704,7 +725,13 @@ export function tryAssignQueueTask(
|
|
|
704
725
|
meshNodeFor: meshId,
|
|
705
726
|
meshNodeId: nodeId,
|
|
706
727
|
launchedByCoordinator: true,
|
|
707
|
-
|
|
728
|
+
...delegatedWorkerAutoApproveSettings(
|
|
729
|
+
mesh?.policy,
|
|
730
|
+
node?.policy,
|
|
731
|
+
components.providerLoader?.getMeta(providerType),
|
|
732
|
+
loadRepoConfigForNode(node),
|
|
733
|
+
providerType,
|
|
734
|
+
),
|
|
708
735
|
...(localDaemonId ? { meshCoordinatorDaemonId: localDaemonId } : {}),
|
|
709
736
|
// COMPLETION-PROPAGATION F5: (re)stamp the coordinator SESSION anchor from THIS
|
|
710
737
|
// task's sourceCoordinatorSessionId with PRIORITY — a manually-launched (or reused)
|
|
@@ -931,6 +958,105 @@ function isTargetNodeTransientlyUnresolved(mesh: any, task: MeshWorkQueueEntry):
|
|
|
931
958
|
return isWithinCloneBootstrapGrace(targetNodeId);
|
|
932
959
|
}
|
|
933
960
|
|
|
961
|
+
// ---------------------------------------------------------------------------
|
|
962
|
+
// DEAD-TARGET-SELFHEAL: unpin a queue task hard-pinned to a session/node that has
|
|
963
|
+
// died (absent from the live mesh) so a live idle session can claim it, instead of
|
|
964
|
+
// leaving it stranded 'pending' forever behind the target_session_constraint skip.
|
|
965
|
+
// ---------------------------------------------------------------------------
|
|
966
|
+
|
|
967
|
+
// Conservative age gate before a pinned-but-dead target is unpinned. A target that is
|
|
968
|
+
// merely briefly unassigned or momentarily reconnecting must not be reclaimed on the
|
|
969
|
+
// tick it drops out of view; we require the task to have been idle (no updatedAt bump)
|
|
970
|
+
// for at least this window first. Sized to comfortably outlast a transient P2P blip /
|
|
971
|
+
// reconnect while staying well inside the reclaim cadence of the rest of the file
|
|
972
|
+
// (AUTO_LAUNCH_AWAIT_CLAIM_MS is 90s; the stranded-reclaim watchdog fires on similar
|
|
973
|
+
// scales), so a real reconnect wins the race and the self-heal only fires on a target
|
|
974
|
+
// that is genuinely gone.
|
|
975
|
+
const DEAD_TARGET_GRACE_MS = 60_000;
|
|
976
|
+
|
|
977
|
+
interface DeadTargetVerdict {
|
|
978
|
+
/** The pinned target is confirmed dead and past the grace window → safe to unpin. */
|
|
979
|
+
dead: boolean;
|
|
980
|
+
/** True when the target NODE itself is absent from the live mesh (clear targetNodeId too). */
|
|
981
|
+
nodeDead: boolean;
|
|
982
|
+
/** Short reason string for the ledger/requeue. */
|
|
983
|
+
reason: string;
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
/**
|
|
987
|
+
* Decide whether a task's hard target pin (targetSessionId and/or targetNodeId) points at
|
|
988
|
+
* something that has DIED — i.e. is absent from the live mesh snapshot — and has been so
|
|
989
|
+
* long enough (DEAD_TARGET_GRACE_MS since the task's last update) that unpinning is safe.
|
|
990
|
+
*
|
|
991
|
+
* Two definitive death signals, deliberately conservative to never race a reconnecting node:
|
|
992
|
+
*
|
|
993
|
+
* (1) NODE dead — the task pins a targetNodeId that matches NO node in the live mesh
|
|
994
|
+
* (the same `meshNodeIdMatches`-over-mesh.nodes signal the targetPinUnmatched relabel
|
|
995
|
+
* uses at the empty-candidate site). A pinned session on an absent node is unreachable
|
|
996
|
+
* regardless, so the session pin is dead too. `nodeDead` ⇒ clear targetNodeId as well.
|
|
997
|
+
* Excluded: a target that is only TRANSIENTLY unresolved (a freshly-cloned worktree
|
|
998
|
+
* still propagating / bootstrapping) — isTargetNodeTransientlyUnresolved gates it out.
|
|
999
|
+
*
|
|
1000
|
+
* (2) SESSION dead on a LIVE LOCAL node — the target node IS present and is THIS daemon's
|
|
1001
|
+
* node, but the pinned session is absent from the local instance manager
|
|
1002
|
+
* (resolveSessionBusyVerdict === 'UNKNOWN'). Local session visibility is complete, so
|
|
1003
|
+
* absence here is genuine death, not a busy/generating flip. We KEEP targetNodeId (only
|
|
1004
|
+
* the session died; the node is healthy and can host a replacement claim).
|
|
1005
|
+
*
|
|
1006
|
+
* A live REMOTE node whose session is not in our idle view is NOT treated as dead: absence
|
|
1007
|
+
* from the remote-idle mirror is explicitly UNKNOWN liveness (the session may be busy or its
|
|
1008
|
+
* agent:ready pull merely lost), so unpinning it could race healthy in-flight work. Returns
|
|
1009
|
+
* dead=false in that case, leaving the existing skip in place.
|
|
1010
|
+
*/
|
|
1011
|
+
function resolveDeadTargetVerdict(components: DaemonComponents, meshId: string, mesh: any, task: MeshWorkQueueEntry): DeadTargetVerdict {
|
|
1012
|
+
const NOT_DEAD: DeadTargetVerdict = { dead: false, nodeDead: false, reason: '' };
|
|
1013
|
+
const targetSessionId = readNonEmptyString(task.targetSessionId);
|
|
1014
|
+
const targetNodeId = readNonEmptyString(task.targetNodeId);
|
|
1015
|
+
if (!targetSessionId && !targetNodeId) return NOT_DEAD;
|
|
1016
|
+
|
|
1017
|
+
// Age gate: never reclaim a pin younger than the grace window (guards against a target
|
|
1018
|
+
// that has only just dropped out of view for a momentary reconnect).
|
|
1019
|
+
const lastUpdateMs = Date.parse(task.updatedAt || task.createdAt || '');
|
|
1020
|
+
if (Number.isFinite(lastUpdateMs) && Date.now() - lastUpdateMs < DEAD_TARGET_GRACE_MS) return NOT_DEAD;
|
|
1021
|
+
|
|
1022
|
+
const nodes: any[] = Array.isArray(mesh?.nodes) ? mesh.nodes : [];
|
|
1023
|
+
|
|
1024
|
+
// (1) NODE-dead — a pinned node absent from the live mesh, and NOT merely transiently
|
|
1025
|
+
// unresolved (a propagating/bootstrapping clone). This is a permanent routing miss.
|
|
1026
|
+
if (targetNodeId) {
|
|
1027
|
+
const nodePresent = nodes.some(n => meshNodeIdMatches(n, targetNodeId));
|
|
1028
|
+
if (!nodePresent) {
|
|
1029
|
+
if (isTargetNodeTransientlyUnresolved(mesh, task)) return NOT_DEAD;
|
|
1030
|
+
return { dead: true, nodeDead: true, reason: 'dead_target_node_absent' };
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
// (2) SESSION-dead on a LIVE LOCAL node. Only meaningful when a session is pinned.
|
|
1035
|
+
if (targetSessionId) {
|
|
1036
|
+
// Resolve the pinned target's node (if any) to decide whether we can trust local
|
|
1037
|
+
// absence. Without a targetNodeId, fall back to whichever live node hosts the session
|
|
1038
|
+
// is unknowable here; treat that as a LOCAL check only (a session id we cannot see
|
|
1039
|
+
// locally on a node we cannot resolve remotely stays UNKNOWN → not dead).
|
|
1040
|
+
const node = targetNodeId
|
|
1041
|
+
? nodes.find(n => meshNodeIdMatches(n, targetNodeId))
|
|
1042
|
+
: undefined;
|
|
1043
|
+
// A pinned session on a REMOTE live node: absence from our view is UNKNOWN, not death.
|
|
1044
|
+
// Only a LOCAL node (or no node pin at all — same-daemon assumption) lets us conclude
|
|
1045
|
+
// death from local instance-manager absence.
|
|
1046
|
+
const nodeIsLocal = node ? isLocalAutoLaunchNode(node) : true;
|
|
1047
|
+
if (nodeIsLocal) {
|
|
1048
|
+
const verdict = resolveSessionBusyVerdict(components, targetSessionId);
|
|
1049
|
+
if (verdict === 'UNKNOWN') {
|
|
1050
|
+
// Session absent from the complete local session view → genuinely gone.
|
|
1051
|
+
return { dead: true, nodeDead: false, reason: 'dead_target_session_absent' };
|
|
1052
|
+
}
|
|
1053
|
+
// GENERATING / IDLE_CONFIRMED → the session is alive (possibly busy). Never disturb.
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
return NOT_DEAD;
|
|
1058
|
+
}
|
|
1059
|
+
|
|
934
1060
|
/**
|
|
935
1061
|
* FALSE-BLOCKER-CLONE-QUEUE (stale-event clear): once a task whose actionable blocker we
|
|
936
1062
|
* previously paged either gets claimed or transitions to a self-resolving state, re-arm the
|
|
@@ -1595,8 +1721,20 @@ function liveSessionCountForNode(components: DaemonComponents, meshId: string, n
|
|
|
1595
1721
|
* free claimer, and is excluded — so a read-only auto-launch onto a busy-but-no-idle node
|
|
1596
1722
|
* is still allowed. A node with NO live mesh session at all (dead, or never launched) does
|
|
1597
1723
|
* not match, preserving the legitimate first-session spawn.
|
|
1724
|
+
*
|
|
1725
|
+
* PROVIDER-MATCH gate (DISPATCH-DEADLOCK-PROVIDER-MISMATCH): a live session only suppresses
|
|
1726
|
+
* this launch if it could ACTUALLY claim THIS task. A session whose providerType does not
|
|
1727
|
+
* satisfy the task's requiredTags (e.g. a claude-cli coordinator/worker session on a node,
|
|
1728
|
+
* while the pending task is required_tags: [provider=codex-cli]) is NOT a pending claimer for
|
|
1729
|
+
* this task — claimNextQueueTask's nodeSatisfiesRequiredTags gate would reject its claim. Left
|
|
1730
|
+
* unchecked, such a mismatched session made this gate return true for a task it can never claim,
|
|
1731
|
+
* so the required-provider worker never auto-launched AND no session could claim → nobody made
|
|
1732
|
+
* progress → permanent silent deadlock. Mirror the claim path: build the session's own
|
|
1733
|
+
* capability tags (its providerType pinned onto the node) and only count it as a pending claimer
|
|
1734
|
+
* when those tags satisfy task.requiredTags. `node` is passed in so the tag set reflects this
|
|
1735
|
+
* node's os/arch/worktree/converge context, matching claimNextQueueTask exactly.
|
|
1598
1736
|
*/
|
|
1599
|
-
function nodeHasLiveSessionPendingClaim(components: DaemonComponents, meshId: string, nodeId: string): boolean {
|
|
1737
|
+
function nodeHasLiveSessionPendingClaim(components: DaemonComponents, meshId: string, nodeId: string, task: MeshWorkQueueEntry, node: any): boolean {
|
|
1600
1738
|
// (A) AUTOLAUNCH-CLAIM-CHURN remote-awareness: a task whose auto-launch record targets this
|
|
1601
1739
|
// node and is still inside its await-claim window (base or backoff) already has a session on
|
|
1602
1740
|
// its way to claim — even when that session is REMOTE and thus invisible to the local
|
|
@@ -1635,7 +1773,20 @@ function nodeHasLiveSessionPendingClaim(components: DaemonComponents, meshId: st
|
|
|
1635
1773
|
if (isTerminalSessionStatus(status)) return false; // dead → no claimer here, allow launch
|
|
1636
1774
|
const sessionId = readNonEmptyString(state.instanceId);
|
|
1637
1775
|
if (sessionId && busySessionIds.has(sessionId)) return false; // busy with its own assigned task
|
|
1638
|
-
|
|
1776
|
+
// PROVIDER-MATCH gate (DISPATCH-DEADLOCK-PROVIDER-MISMATCH): only count this session as a
|
|
1777
|
+
// pending claimer if its own provider could satisfy THIS task's requiredTags. A session
|
|
1778
|
+
// whose providerType does not match (e.g. a claude-cli session while task requires
|
|
1779
|
+
// provider=codex-cli) can never claim this task via claimNextQueueTask's
|
|
1780
|
+
// nodeSatisfiesRequiredTags gate, so it must not suppress the required-provider launch —
|
|
1781
|
+
// otherwise the task deadlocks (mismatched session blocks launch, yet cannot claim). Mirror
|
|
1782
|
+
// the claim path: pin the session's providerType onto this node and check the tags.
|
|
1783
|
+
if (task.requiredTags?.length) {
|
|
1784
|
+
const sessionProviderType = state.type || readNonEmptyString(settings.providerType);
|
|
1785
|
+
if (sessionProviderType && !nodeSatisfiesRequiredTags(task.requiredTags, buildMeshNodeCapabilityTags(node, sessionProviderType))) {
|
|
1786
|
+
return false; // provider mismatch → this session can't claim this task; not a pending claimer
|
|
1787
|
+
}
|
|
1788
|
+
}
|
|
1789
|
+
return true; // live + unassigned + provider-capable → will claim the pending task itself
|
|
1639
1790
|
});
|
|
1640
1791
|
}
|
|
1641
1792
|
|
|
@@ -1888,6 +2039,11 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
1888
2039
|
}
|
|
1889
2040
|
if (!pending.length) return false;
|
|
1890
2041
|
|
|
2042
|
+
// Launch-freshness threshold: reuse the auto-fast-forward policy's maxBehind (default
|
|
2043
|
+
// 0 → any behind blocks) so the launch gate and the repair path agree on how far
|
|
2044
|
+
// behind is tolerable. Resolved once per pass and shared across every candidate node.
|
|
2045
|
+
const freshnessGate = { maxBehind: resolveAutoFastForwardPolicy(mesh).maxBehind };
|
|
2046
|
+
|
|
1891
2047
|
// Write cap + read-only cap resolved through the shared helpers from the
|
|
1892
2048
|
// MACHINE-LOCAL stored mesh policy (no repo-file overlay). These are the same
|
|
1893
2049
|
// resolvers the observability projection uses, so the enforced and exposed
|
|
@@ -1929,6 +2085,32 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
1929
2085
|
continue;
|
|
1930
2086
|
}
|
|
1931
2087
|
if (task.targetSessionId) {
|
|
2088
|
+
// DEAD-TARGET-SELFHEAL: before the unconditional target_session_constraint skip,
|
|
2089
|
+
// check whether the pinned session/node has DIED (absent from the live mesh). A
|
|
2090
|
+
// hard-pinned task whose target is gone can NEVER re-enter 'assigned' (the claim
|
|
2091
|
+
// gate refuses every non-matching session) and this skip fires forever with no
|
|
2092
|
+
// liveness check — the triple-walled stranded-pending defect. If the pin is
|
|
2093
|
+
// confirmed dead past the grace window, requeue it (clearing the dead session
|
|
2094
|
+
// pin, and the node pin too when the NODE itself is gone) so a live idle session
|
|
2095
|
+
// can claim it. requeueTask counts toward maxTaskRetries → bounded self-heal that
|
|
2096
|
+
// auto-fails past the cap (the desired terminal state, unblocking dependents).
|
|
2097
|
+
const deadTarget = resolveDeadTargetVerdict(components, meshId, mesh, task);
|
|
2098
|
+
if (deadTarget.dead) {
|
|
2099
|
+
const requeued = requeueTask(meshId, task.id, {
|
|
2100
|
+
reason: deadTarget.reason,
|
|
2101
|
+
clearTargetSession: true,
|
|
2102
|
+
// Keep the node pin if only the SESSION died on a still-live node; clear it
|
|
2103
|
+
// when the NODE itself is absent (nothing to pin to).
|
|
2104
|
+
clearTargetNode: deadTarget.nodeDead,
|
|
2105
|
+
});
|
|
2106
|
+
if (requeued) {
|
|
2107
|
+
LOG.warn('MeshQueue', `DEAD-TARGET-SELFHEAL: task ${task.id} (mesh ${meshId}) was pinned to a dead target (${deadTarget.reason}); requeued${deadTarget.nodeDead ? ' and unpinned node' : ''} (requeueCount=${requeued.requeueCount ?? '?'}, status=${requeued.status}).`);
|
|
2108
|
+
}
|
|
2109
|
+
// Keep the skip for THIS tick (the requeue already flipped the row to
|
|
2110
|
+
// pending/failed); a later tick assigns/launches the now-unpinned task.
|
|
2111
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_dead_requeued' });
|
|
2112
|
+
continue;
|
|
2113
|
+
}
|
|
1932
2114
|
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_constraint' });
|
|
1933
2115
|
continue;
|
|
1934
2116
|
}
|
|
@@ -2085,6 +2267,18 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2085
2267
|
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_not_launch_ready', nodeId });
|
|
2086
2268
|
continue;
|
|
2087
2269
|
}
|
|
2270
|
+
// FRESHNESS gate (distinct from the health gate above): a clean-tree node that
|
|
2271
|
+
// is `behind` its upstream reads as 'online' and passes isLaunchableNode, so
|
|
2272
|
+
// without this it could win fitness routing and run a fresh worker against
|
|
2273
|
+
// stale code. Skip a node whose git telemetry proves it stale (behind >
|
|
2274
|
+
// maxBehind, or a submodule out of sync). Reuse the auto-fast-forward policy's
|
|
2275
|
+
// maxBehind threshold so "how far behind is tolerable" is configured in ONE
|
|
2276
|
+
// place. Telemetry-absent nodes pass (never block on missing data). The 4s
|
|
2277
|
+
// reconcile retries once the node's auto-ff repair path catches it up.
|
|
2278
|
+
if (!isMeshNodeFreshEnoughToLaunch(node, freshnessGate)) {
|
|
2279
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_stale_behind_upstream', nodeId });
|
|
2280
|
+
continue;
|
|
2281
|
+
}
|
|
2088
2282
|
const launchTarget = resolveAutoLaunchTarget(components, node);
|
|
2089
2283
|
if (launchTarget.mode === 'skip') {
|
|
2090
2284
|
// Remote node we can't reach (no transport / no coordinator daemonId).
|
|
@@ -2104,7 +2298,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2104
2298
|
// busy-but-no-idle node is still allowed. Skip with a transient (non-actionable)
|
|
2105
2299
|
// reason so the coordinator is not paged; the 4s reconcile retries, and once the
|
|
2106
2300
|
// existing session goes terminal this gate clears and a legitimate launch proceeds.
|
|
2107
|
-
if (nodeHasLiveSessionPendingClaim(components, meshId, nodeId)) {
|
|
2301
|
+
if (nodeHasLiveSessionPendingClaim(components, meshId, nodeId, task, node)) {
|
|
2108
2302
|
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_live_session_pending_claim', nodeId });
|
|
2109
2303
|
continue;
|
|
2110
2304
|
}
|
|
@@ -2176,8 +2370,14 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2176
2370
|
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
2177
2371
|
// Coordinator-dispatched worker: auto-approve unless mesh/node policy
|
|
2178
2372
|
// opts out (default true). Lands in settingsOverride and beats the
|
|
2179
|
-
// global per-provider-type
|
|
2180
|
-
|
|
2373
|
+
// global per-provider-type boolean/mode through explicit opposite-key clearing.
|
|
2374
|
+
...delegatedWorkerAutoApproveSettings(
|
|
2375
|
+
mesh?.policy,
|
|
2376
|
+
node?.policy,
|
|
2377
|
+
components.providerLoader?.getMeta(resolved.providerType),
|
|
2378
|
+
loadRepoConfigForNode(node),
|
|
2379
|
+
resolved.providerType,
|
|
2380
|
+
),
|
|
2181
2381
|
launchedByCoordinator: true,
|
|
2182
2382
|
autoLaunchedForQueueTaskId: task.id,
|
|
2183
2383
|
};
|
|
@@ -2842,4 +3042,3 @@ export function runIdleMaintenanceThenAssignQueue(components: DaemonComponents,
|
|
|
2842
3042
|
});
|
|
2843
3043
|
});
|
|
2844
3044
|
}
|
|
2845
|
-
|